From 6c7b80e1d942351dd8ec6f3354ea4b69a20c33e4 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Wed, 4 Mar 2020 11:09:08 +0100 Subject: [PATCH] engine: Goland-rename to storage No manual edits. Release note: None --- pkg/ccl/changefeedccl/changefeed_test.go | 6 +- pkg/ccl/changefeedccl/kvfeed/scanner.go | 2 +- .../changefeedccl/schemafeed/schema_feed.go | 6 +- pkg/ccl/cliccl/debug.go | 2 +- pkg/ccl/cmdccl/enc_utils/main.go | 2 +- .../followerreadsccl/followerreads_test.go | 2 +- pkg/ccl/importccl/bench_test.go | 8 +- pkg/ccl/importccl/load.go | 14 +- pkg/ccl/storageccl/bench_test.go | 8 +- pkg/ccl/storageccl/engineccl/bench_test.go | 26 ++-- pkg/ccl/storageccl/engineccl/ctr_stream.go | 2 +- .../storageccl/engineccl/ctr_stream_test.go | 2 +- pkg/ccl/storageccl/engineccl/encrypted_fs.go | 12 +- .../storageccl/engineccl/encrypted_fs_test.go | 24 ++-- .../engineccl/pebble_key_manager.go | 4 +- pkg/ccl/storageccl/engineccl/rocksdb.go | 16 +-- pkg/ccl/storageccl/engineccl/rocksdb_test.go | 20 +-- pkg/ccl/storageccl/export.go | 6 +- pkg/ccl/storageccl/export_test.go | 54 ++++---- pkg/ccl/storageccl/import.go | 12 +- pkg/ccl/storageccl/import_test.go | 32 ++--- pkg/ccl/storageccl/revision_reader.go | 6 +- pkg/ccl/storageccl/writebatch.go | 20 +-- pkg/ccl/storageccl/writebatch_test.go | 24 ++-- pkg/ccl/utilccl/sampledataccl/bankdata.go | 14 +- pkg/ccl/workloadccl/format/sstable.go | 8 +- pkg/cli/context.go | 8 +- pkg/cli/debug.go | 74 +++++----- pkg/cli/debug_check_store.go | 10 +- pkg/cli/debug_check_store_test.go | 4 +- pkg/cli/debug_synctest.go | 6 +- pkg/cli/debug_test.go | 10 +- pkg/cli/flags_util.go | 16 +-- pkg/cli/start.go | 14 +- pkg/cli/syncbench/syncbench.go | 10 +- pkg/internal/client/batch.go | 2 +- pkg/internal/client/client_test.go | 2 +- pkg/internal/client/db.go | 2 +- .../client/mock_transactional_sender.go | 2 +- pkg/internal/client/sender.go | 2 +- pkg/internal/client/txn.go | 2 +- pkg/kv/dist_sender_server_test.go | 4 +- pkg/kv/kvnemesis/engine.go | 8 +- pkg/kv/kvnemesis/engine_test.go | 6 +- pkg/kv/kvnemesis/validator.go | 22 +-- pkg/kv/kvnemesis/validator_test.go | 12 +- pkg/kv/kvnemesis/watcher.go | 4 +- pkg/kv/kvserver/abortspan/abortspan.go | 30 ++-- pkg/kv/kvserver/abortspan/abortspan_test.go | 8 +- pkg/kv/kvserver/addressing_test.go | 6 +- pkg/kv/kvserver/allocator_test.go | 2 +- pkg/kv/kvserver/api.pb.go | 2 +- pkg/kv/kvserver/batch_spanset_test.go | 84 ++++++------ pkg/kv/kvserver/batcheval/cmd_add_sstable.go | 24 ++-- .../batcheval/cmd_add_sstable_test.go | 68 +++++----- pkg/kv/kvserver/batcheval/cmd_clear_range.go | 16 +-- .../batcheval/cmd_clear_range_test.go | 16 +-- .../batcheval/cmd_compute_checksum.go | 4 +- .../kvserver/batcheval/cmd_conditional_put.go | 12 +- pkg/kv/kvserver/batcheval/cmd_delete.go | 6 +- pkg/kv/kvserver/batcheval/cmd_delete_range.go | 6 +- .../kvserver/batcheval/cmd_end_transaction.go | 52 +++---- .../batcheval/cmd_end_transaction_test.go | 24 ++-- pkg/kv/kvserver/batcheval/cmd_gc.go | 6 +- pkg/kv/kvserver/batcheval/cmd_get.go | 6 +- .../kvserver/batcheval/cmd_heartbeat_txn.go | 10 +- pkg/kv/kvserver/batcheval/cmd_increment.go | 6 +- pkg/kv/kvserver/batcheval/cmd_init_put.go | 10 +- pkg/kv/kvserver/batcheval/cmd_lease.go | 6 +- pkg/kv/kvserver/batcheval/cmd_lease_info.go | 4 +- .../kvserver/batcheval/cmd_lease_request.go | 4 +- .../kvserver/batcheval/cmd_lease_transfer.go | 4 +- pkg/kv/kvserver/batcheval/cmd_merge.go | 6 +- pkg/kv/kvserver/batcheval/cmd_push_txn.go | 8 +- pkg/kv/kvserver/batcheval/cmd_put.go | 10 +- pkg/kv/kvserver/batcheval/cmd_query_intent.go | 6 +- pkg/kv/kvserver/batcheval/cmd_query_txn.go | 8 +- pkg/kv/kvserver/batcheval/cmd_range_stats.go | 4 +- .../kvserver/batcheval/cmd_recompute_stats.go | 6 +- pkg/kv/kvserver/batcheval/cmd_recover_txn.go | 10 +- .../batcheval/cmd_recover_txn_test.go | 18 +-- pkg/kv/kvserver/batcheval/cmd_refresh.go | 6 +- .../kvserver/batcheval/cmd_refresh_range.go | 8 +- .../batcheval/cmd_refresh_range_test.go | 18 +-- .../kvserver/batcheval/cmd_resolve_intent.go | 6 +- .../batcheval/cmd_resolve_intent_range.go | 8 +- .../batcheval/cmd_resolve_intent_test.go | 14 +- pkg/kv/kvserver/batcheval/cmd_reverse_scan.go | 12 +- pkg/kv/kvserver/batcheval/cmd_revert_range.go | 12 +- .../batcheval/cmd_revert_range_test.go | 34 ++--- pkg/kv/kvserver/batcheval/cmd_scan.go | 12 +- pkg/kv/kvserver/batcheval/cmd_scan_test.go | 6 +- pkg/kv/kvserver/batcheval/cmd_subsume.go | 10 +- pkg/kv/kvserver/batcheval/cmd_truncate_log.go | 12 +- .../batcheval/cmd_truncate_log_test.go | 18 +-- pkg/kv/kvserver/batcheval/command.go | 10 +- pkg/kv/kvserver/batcheval/declare.go | 2 +- pkg/kv/kvserver/batcheval/eval_context.go | 8 +- pkg/kv/kvserver/batcheval/intent.go | 6 +- .../kvserver/batcheval/split_stats_helper.go | 2 +- pkg/kv/kvserver/batcheval/transaction.go | 6 +- pkg/kv/kvserver/batcheval/transaction_test.go | 128 +++++++++--------- pkg/kv/kvserver/below_raft_protos_test.go | 6 +- pkg/kv/kvserver/bulk/buffering_adder.go | 4 +- pkg/kv/kvserver/bulk/sst_batcher.go | 48 +++---- pkg/kv/kvserver/bulk/sst_batcher_test.go | 12 +- pkg/kv/kvserver/client_merge_test.go | 36 ++--- pkg/kv/kvserver/client_raft_test.go | 44 +++--- pkg/kv/kvserver/client_replica_gc_test.go | 6 +- pkg/kv/kvserver/client_replica_test.go | 24 ++-- pkg/kv/kvserver/client_split_test.go | 18 +-- pkg/kv/kvserver/client_test.go | 30 ++-- pkg/kv/kvserver/compactor/compactor.go | 26 ++-- pkg/kv/kvserver/compactor/compactor_test.go | 22 +-- .../concurrency/concurrency_control.go | 2 +- .../concurrency/concurrency_manager_test.go | 2 +- pkg/kv/kvserver/concurrency/lock_table.go | 2 +- .../kvserver/concurrency/lock_table_test.go | 2 +- .../kvserver/concurrency/lock_table_waiter.go | 2 +- .../concurrency/lock_table_waiter_test.go | 2 +- pkg/kv/kvserver/consistency_queue_test.go | 20 +-- pkg/kv/kvserver/debug_print.go | 48 +++---- pkg/kv/kvserver/debug_print_test.go | 6 +- pkg/kv/kvserver/gc/data_distribution_test.go | 18 +-- pkg/kv/kvserver/gc/gc.go | 18 +-- pkg/kv/kvserver/gc/gc_iterator.go | 18 +-- pkg/kv/kvserver/gc/gc_iterator_test.go | 16 +-- pkg/kv/kvserver/gc/gc_old_test.go | 24 ++-- pkg/kv/kvserver/gc/gc_random_test.go | 8 +- pkg/kv/kvserver/gc_queue.go | 2 +- pkg/kv/kvserver/gc_queue_test.go | 32 ++--- pkg/kv/kvserver/helpers_test.go | 18 +-- .../intentresolver/intent_resolver.go | 2 +- .../intentresolver/intent_resolver_test.go | 2 +- pkg/kv/kvserver/merge_queue.go | 2 +- pkg/kv/kvserver/merge_queue_test.go | 2 +- pkg/kv/kvserver/metrics.go | 8 +- pkg/kv/kvserver/node_liveness.go | 8 +- pkg/kv/kvserver/queue.go | 2 +- pkg/kv/kvserver/queue_test.go | 2 +- pkg/kv/kvserver/raft_log_queue_test.go | 6 +- pkg/kv/kvserver/raft_transport.go | 4 +- pkg/kv/kvserver/rangefeed/processor.go | 8 +- pkg/kv/kvserver/rangefeed/processor_test.go | 10 +- pkg/kv/kvserver/rangefeed/registry.go | 14 +- pkg/kv/kvserver/rangefeed/registry_test.go | 8 +- .../kvserver/rangefeed/resolved_timestamp.go | 2 +- pkg/kv/kvserver/rangefeed/task.go | 12 +- pkg/kv/kvserver/rangefeed/task_test.go | 36 ++--- pkg/kv/kvserver/rditer/replica_data_iter.go | 26 ++-- .../kvserver/rditer/replica_data_iter_test.go | 24 ++-- pkg/kv/kvserver/rditer/stats.go | 8 +- pkg/kv/kvserver/replica.go | 26 ++-- pkg/kv/kvserver/replica_application_result.go | 2 +- .../replica_application_state_machine.go | 8 +- pkg/kv/kvserver/replica_command.go | 10 +- pkg/kv/kvserver/replica_consistency.go | 12 +- pkg/kv/kvserver/replica_consistency_diff.go | 8 +- pkg/kv/kvserver/replica_destroy.go | 14 +- pkg/kv/kvserver/replica_eval_context_span.go | 6 +- pkg/kv/kvserver/replica_evaluate.go | 14 +- pkg/kv/kvserver/replica_evaluate_test.go | 10 +- pkg/kv/kvserver/replica_proposal.go | 8 +- pkg/kv/kvserver/replica_raft.go | 12 +- .../kvserver/replica_raft_truncation_test.go | 6 +- pkg/kv/kvserver/replica_raftstorage.go | 64 ++++----- pkg/kv/kvserver/replica_rangefeed.go | 24 ++-- pkg/kv/kvserver/replica_sideload_disk.go | 6 +- pkg/kv/kvserver/replica_sideload_inmem.go | 4 +- pkg/kv/kvserver/replica_sideload_test.go | 16 +-- .../kvserver/replica_sst_snapshot_storage.go | 8 +- .../replica_sst_snapshot_storage_test.go | 6 +- pkg/kv/kvserver/replica_test.go | 50 +++---- pkg/kv/kvserver/replica_write.go | 20 +-- pkg/kv/kvserver/scanner_test.go | 2 +- pkg/kv/kvserver/spanset/batch.go | 104 +++++++------- pkg/kv/kvserver/split_queue.go | 2 +- pkg/kv/kvserver/split_queue_test.go | 2 +- pkg/kv/kvserver/stateloader/initial.go | 8 +- pkg/kv/kvserver/stateloader/initial_test.go | 4 +- pkg/kv/kvserver/stateloader/stateloader.go | 128 +++++++++--------- pkg/kv/kvserver/stats_test.go | 2 +- pkg/kv/kvserver/storagepb/proposer_kv.pb.go | 2 +- pkg/kv/kvserver/storagepb/state.pb.go | 2 +- pkg/kv/kvserver/store.go | 64 ++++----- pkg/kv/kvserver/store_bootstrap.go | 20 +-- pkg/kv/kvserver/store_create_replica.go | 10 +- pkg/kv/kvserver/store_pool_test.go | 6 +- pkg/kv/kvserver/store_rebalancer_test.go | 2 +- pkg/kv/kvserver/store_snapshot.go | 22 +-- pkg/kv/kvserver/store_split.go | 6 +- pkg/kv/kvserver/store_test.go | 34 ++--- pkg/kv/kvserver/stores.go | 22 +-- pkg/kv/kvserver/stores_server.go | 4 +- pkg/kv/kvserver/stores_test.go | 6 +- pkg/kv/kvserver/syncing_write.go | 4 +- pkg/kv/kvserver/track_raft_protos.go | 2 +- pkg/kv/kvserver/ts_maintenance_queue.go | 4 +- pkg/kv/kvserver/ts_maintenance_queue_test.go | 4 +- pkg/kv/kvserver/txn_wait_queue_test.go | 8 +- pkg/kv/kvserver/txnwait/queue.go | 2 +- pkg/kv/kvserver/txnwait/queue_test.go | 2 +- pkg/kv/split_test.go | 6 +- pkg/kv/truncate_test.go | 2 +- pkg/kv/txn_coord_sender.go | 2 +- pkg/kv/txn_coord_sender_savepoints.go | 2 +- pkg/kv/txn_coord_sender_test.go | 8 +- pkg/kv/txn_correctness_test.go | 2 +- pkg/kv/txn_interceptor_pipeliner.go | 2 +- pkg/kv/txn_interceptor_pipeliner_test.go | 2 +- pkg/kv/txn_interceptor_seq_num_allocator.go | 2 +- .../txn_interceptor_seq_num_allocator_test.go | 2 +- pkg/kv/txn_test.go | 2 +- pkg/roachpb/api.pb.go | 2 +- pkg/roachpb/batch.go | 2 +- pkg/roachpb/batch_test.go | 2 +- pkg/roachpb/data.go | 2 +- pkg/roachpb/data.pb.go | 2 +- pkg/roachpb/data_test.go | 2 +- pkg/roachpb/ignored_seqnums.go | 2 +- pkg/roachpb/ignored_seqnums_test.go | 2 +- pkg/roachpb/string_test.go | 2 +- pkg/server/config.go | 38 +++--- pkg/server/config_unix.go | 14 +- pkg/server/config_windows.go | 4 +- pkg/server/node.go | 12 +- pkg/server/node_engine_health.go | 8 +- pkg/server/node_test.go | 46 +++---- pkg/server/server.go | 14 +- pkg/server/server_test.go | 8 +- pkg/server/serverpb/admin.pb.go | 2 +- pkg/server/serverpb/status.pb.go | 2 +- pkg/server/status.go | 4 +- pkg/server/status_test.go | 2 +- pkg/server/sticky_engine.go | 12 +- pkg/server/sticky_engine_test.go | 6 +- pkg/server/testserver.go | 6 +- pkg/sql/colcontainer/diskqueue.go | 2 +- pkg/sql/colcontainer/partitionedqueue_test.go | 2 +- pkg/sql/colflow/vectorized_flow_test.go | 4 +- pkg/sql/distsql/columnar_utils_test.go | 4 +- pkg/sql/execinfra/server_config.go | 2 +- pkg/sql/row/fetcher_mvcc_test.go | 8 +- pkg/sql/row/kv_fetcher.go | 2 +- .../rowcontainer/disk_row_container_test.go | 10 +- .../rowcontainer/hash_row_container_test.go | 6 +- pkg/sql/rowcontainer/row_container_test.go | 8 +- pkg/sql/rowexec/hashjoiner_test.go | 8 +- pkg/sql/rowexec/joinreader_test.go | 8 +- pkg/sql/rowexec/sorter_test.go | 4 +- pkg/sql/rowexec/windower_test.go | 4 +- pkg/sql/rowflow/routers_test.go | 4 +- pkg/storage/batch.go | 4 +- pkg/storage/batch_test.go | 4 +- pkg/storage/bench_pebble_test.go | 2 +- pkg/storage/bench_rocksdb_test.go | 2 +- pkg/storage/bench_test.go | 4 +- pkg/storage/disk_map.go | 2 +- pkg/storage/disk_map_test.go | 2 +- pkg/storage/doc.go | 2 +- pkg/storage/engine.go | 6 +- pkg/storage/engine_test.go | 6 +- pkg/storage/enginepb/decode_test.go | 6 +- pkg/storage/enginepb/mvcc_test.go | 2 +- pkg/storage/error.go | 2 +- pkg/storage/error_test.go | 2 +- pkg/storage/file_util.go | 2 +- pkg/storage/in_mem.go | 4 +- pkg/storage/main_test.go | 2 +- pkg/storage/merge.go | 4 +- pkg/storage/merge_test.go | 4 +- pkg/storage/metamorphic/generator.go | 34 ++--- pkg/storage/metamorphic/operands.go | 20 +-- pkg/storage/metamorphic/operations.go | 42 +++--- pkg/storage/multi_iterator.go | 2 +- pkg/storage/multi_iterator_test.go | 2 +- pkg/storage/mvcc.go | 4 +- pkg/storage/mvcc_history_test.go | 4 +- pkg/storage/mvcc_incremental_iterator.go | 4 +- pkg/storage/mvcc_incremental_iterator_test.go | 4 +- pkg/storage/mvcc_logical_ops.go | 4 +- pkg/storage/mvcc_logical_ops_test.go | 4 +- pkg/storage/mvcc_stats_test.go | 4 +- pkg/storage/mvcc_test.go | 4 +- pkg/storage/pebble.go | 6 +- pkg/storage/pebble_batch.go | 2 +- pkg/storage/pebble_file_registry.go | 4 +- pkg/storage/pebble_file_registry_test.go | 4 +- pkg/storage/pebble_iterator.go | 4 +- pkg/storage/pebble_merge.go | 4 +- pkg/storage/pebble_mvcc_scanner.go | 4 +- pkg/storage/pebble_test.go | 4 +- pkg/storage/rocksdb.go | 6 +- pkg/storage/rocksdb_32bit.go | 2 +- pkg/storage/rocksdb_64bit.go | 2 +- pkg/storage/rocksdb_error_dict.go | 2 +- pkg/storage/rocksdb_iter_stats_test.go | 2 +- pkg/storage/rocksdb_jemalloc.go | 2 +- pkg/storage/rocksdb_test.go | 2 +- pkg/storage/row_counter.go | 2 +- pkg/storage/slice.go | 2 +- pkg/storage/slice_gccgo.go | 2 +- pkg/storage/slice_go1.9.go | 2 +- pkg/storage/slice_test.go | 2 +- pkg/storage/sst_iterator.go | 2 +- pkg/storage/sst_iterator_test.go | 2 +- pkg/storage/sst_writer.go | 2 +- pkg/storage/sst_writer_test.go | 30 ++-- pkg/storage/tee.go | 6 +- pkg/storage/temp_dir.go | 2 +- pkg/storage/temp_dir_test.go | 2 +- pkg/storage/temp_engine.go | 6 +- pkg/storage/temp_engine_test.go | 2 +- pkg/storage/version.go | 2 +- pkg/storage/version_test.go | 2 +- .../colcontainerutils/diskqueuecfg.go | 8 +- .../localtestcluster/local_test_cluster.go | 10 +- pkg/ts/db_test.go | 4 +- pkg/ts/maintenance.go | 4 +- pkg/ts/pruning.go | 16 +-- pkg/ts/timeseries_test.go | 4 +- pkg/util/protoutil/clone_test.go | 2 +- 322 files changed, 1781 insertions(+), 1779 deletions(-) diff --git a/pkg/ccl/changefeedccl/changefeed_test.go b/pkg/ccl/changefeedccl/changefeed_test.go index ba0e9dc14034..fbb9b72fb25b 100644 --- a/pkg/ccl/changefeedccl/changefeed_test.go +++ b/pkg/ccl/changefeedccl/changefeed_test.go @@ -28,7 +28,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdctest" "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeedbase" "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" @@ -39,6 +38,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -821,12 +821,12 @@ func fetchDescVersionModificationTime( t.Fatal(pErr.GoError()) } for _, file := range res.(*roachpb.ExportResponse).Files { - it, err := engine.NewMemSSTIterator(file.SST, false /* verify */) + it, err := storage.NewMemSSTIterator(file.SST, false /* verify */) if err != nil { t.Fatal(err) } defer it.Close() - for it.SeekGE(engine.NilKey); ; it.Next() { + for it.SeekGE(storage.NilKey); ; it.Next() { if ok, err := it.Valid(); err != nil { t.Fatal(err) } else if !ok { diff --git a/pkg/ccl/changefeedccl/kvfeed/scanner.go b/pkg/ccl/changefeedccl/kvfeed/scanner.go index f34115021f5d..aac870063a73 100644 --- a/pkg/ccl/changefeedccl/kvfeed/scanner.go +++ b/pkg/ccl/changefeedccl/kvfeed/scanner.go @@ -13,7 +13,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -21,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/covering" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/ccl/changefeedccl/schemafeed/schema_feed.go b/pkg/ccl/changefeedccl/schemafeed/schema_feed.go index dedb7ce0c3db..59d9ec1df020 100644 --- a/pkg/ccl/changefeedccl/schemafeed/schema_feed.go +++ b/pkg/ccl/changefeedccl/schemafeed/schema_feed.go @@ -15,7 +15,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeedbase" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/keys" @@ -23,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -479,12 +479,12 @@ func fetchTableDescriptorVersions( var tableDescs []*sqlbase.TableDescriptor for _, file := range res.(*roachpb.ExportResponse).Files { if err := func() error { - it, err := engine.NewMemSSTIterator(file.SST, false /* verify */) + it, err := storage.NewMemSSTIterator(file.SST, false /* verify */) if err != nil { return err } defer it.Close() - for it.SeekGE(engine.NilKey); ; it.Next() { + for it.SeekGE(storage.NilKey); ; it.Next() { if ok, err := it.Valid(); err != nil { return err } else if !ok { diff --git a/pkg/ccl/cliccl/debug.go b/pkg/ccl/cliccl/debug.go index 072496bf1c9e..86652b8d3ab7 100644 --- a/pkg/ccl/cliccl/debug.go +++ b/pkg/ccl/cliccl/debug.go @@ -23,7 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/ccl/cliccl/cliflagsccl" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" "github.com/cockroachdb/cockroach/pkg/cli" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/timeutil" diff --git a/pkg/ccl/cmdccl/enc_utils/main.go b/pkg/ccl/cmdccl/enc_utils/main.go index ebb305cc79c0..cf577208f57e 100644 --- a/pkg/ccl/cmdccl/enc_utils/main.go +++ b/pkg/ccl/cmdccl/enc_utils/main.go @@ -19,7 +19,7 @@ import ( "path/filepath" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/pkg/errors" diff --git a/pkg/ccl/followerreadsccl/followerreads_test.go b/pkg/ccl/followerreadsccl/followerreads_test.go index 82551c59a25c..214a8bdd2e44 100644 --- a/pkg/ccl/followerreadsccl/followerreads_test.go +++ b/pkg/ccl/followerreadsccl/followerreads_test.go @@ -15,13 +15,13 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/physicalplan/replicaoracle" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/ccl/importccl/bench_test.go b/pkg/ccl/importccl/bench_test.go index bb98c9fe25b4..6540698c14a5 100644 --- a/pkg/ccl/importccl/bench_test.go +++ b/pkg/ccl/importccl/bench_test.go @@ -17,7 +17,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/ccl/importccl" "github.com/cockroachdb/cockroach/pkg/ccl/workloadccl/format" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" @@ -25,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" @@ -91,16 +91,16 @@ func benchmarkWriteAndLink(b *testing.B, dir string, tables []tableSSTable) { b.SetBytes(bytes) ctx := context.Background() - cache := engine.NewRocksDBCache(server.DefaultCacheSize) + cache := storage.NewRocksDBCache(server.DefaultCacheSize) defer cache.Release() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() - cfg := engine.RocksDBConfig{ + cfg := storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: filepath.Join(dir, `rocksdb`, timeutil.Now().String())}} - db, err := engine.NewRocksDB(cfg, cache) + db, err := storage.NewRocksDB(cfg, cache) if err != nil { b.Fatal(err) } diff --git a/pkg/ccl/importccl/load.go b/pkg/ccl/importccl/load.go index f77be10dffc6..606039f91eb0 100644 --- a/pkg/ccl/importccl/load.go +++ b/pkg/ccl/importccl/load.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/blobs" "github.com/cockroachdb/cockroach/pkg/ccl/backupccl" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/cloud" @@ -33,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/transform" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -143,7 +143,7 @@ func Load( var tableDesc *sqlbase.ImmutableTableDescriptor var tableName string var prevKey roachpb.Key - var kvs []engine.MVCCKeyValue + var kvs []storage.MVCCKeyValue var kvBytes int64 backup := backupccl.BackupManifest{ Descriptors: []sqlbase.Descriptor{ @@ -250,8 +250,8 @@ func Load( } prevKey = kv.Key kvBytes += int64(len(kv.Key) + len(kv.Value.RawBytes)) - kvs = append(kvs, engine.MVCCKeyValue{ - Key: engine.MVCCKey{Key: kv.Key, Timestamp: kv.Value.Timestamp}, + kvs = append(kvs, storage.MVCCKeyValue{ + Key: storage.MVCCKey{Key: kv.Key, Timestamp: kv.Value.Timestamp}, Value: kv.Value.RawBytes, }) }) @@ -376,7 +376,7 @@ func writeSST( backup *backupccl.BackupManifest, base cloud.ExternalStorage, tempPrefix string, - kvs []engine.MVCCKeyValue, + kvs []storage.MVCCKeyValue, ts hlc.Timestamp, ) error { if len(kvs) == 0 { @@ -386,8 +386,8 @@ func writeSST( filename := fmt.Sprintf("load-%d.sst", rand.Int63()) log.Info(ctx, "writesst ", filename) - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) defer sst.Close() for _, kv := range kvs { kv.Key.Timestamp = ts diff --git a/pkg/ccl/storageccl/bench_test.go b/pkg/ccl/storageccl/bench_test.go index da0cfd26a32b..dad72c2e674c 100644 --- a/pkg/ccl/storageccl/bench_test.go +++ b/pkg/ccl/storageccl/bench_test.go @@ -17,12 +17,12 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/ccl/utilccl/sampledataccl" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/cloud" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -54,8 +54,8 @@ func BenchmarkAddSSTable(b *testing.B) { b.StopTimer() b.ResetTimer() for i := 0; i < b.N; i++ { - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) id++ backup.ResetKeyValueIteration() @@ -107,7 +107,7 @@ func BenchmarkWriteBatch(b *testing.B) { kvDB := tc.Server(0).DB() id := sqlbase.ID(keys.MinUserDescID) - var batch engine.RocksDBBatchBuilder + var batch storage.RocksDBBatchBuilder var totalLen int64 b.StopTimer() diff --git a/pkg/ccl/storageccl/engineccl/bench_test.go b/pkg/ccl/storageccl/engineccl/bench_test.go index 928b06e03bf4..2f7fa4a58776 100644 --- a/pkg/ccl/storageccl/engineccl/bench_test.go +++ b/pkg/ccl/storageccl/engineccl/bench_test.go @@ -17,9 +17,9 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -43,7 +43,7 @@ import ( // whether to use a temporary or permanent location. func loadTestData( dir string, numKeys, numBatches, batchTimeSpan, valueBytes int, -) (engine.Engine, error) { +) (storage.Engine, error) { ctx := context.Background() exists := true @@ -51,14 +51,14 @@ func loadTestData( exists = false } - eng, err := engine.NewRocksDB( - engine.RocksDBConfig{ + eng, err := storage.NewRocksDB( + storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Settings: cluster.MakeTestingClusterSettings(), Dir: dir, }, }, - engine.RocksDBCache{}, + storage.RocksDBCache{}, ) if err != nil { return nil, err @@ -84,7 +84,7 @@ func loadTestData( sstTimestamps[i] = int64((i + 1) * batchTimeSpan) } - var batch engine.Batch + var batch storage.Batch var minWallTime int64 for i, key := range keys { if scaled := len(keys) / numBatches; (i % scaled) == 0 { @@ -104,7 +104,7 @@ func loadTestData( timestamp := hlc.Timestamp{WallTime: minWallTime + rand.Int63n(int64(batchTimeSpan))} value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueBytes)) value.InitChecksum(key) - if err := engine.MVCCPut(ctx, batch, nil, key, timestamp, value, nil); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, key, timestamp, value, nil); err != nil { return nil, err } } @@ -125,7 +125,7 @@ func loadTestData( func runIterate( b *testing.B, loadFactor float32, - makeIterator func(engine.Engine, hlc.Timestamp, hlc.Timestamp) engine.Iterator, + makeIterator func(storage.Engine, hlc.Timestamp, hlc.Timestamp) storage.Iterator, ) { const numKeys = 100000 const numBatches = 100 @@ -149,7 +149,7 @@ func runIterate( endTime := hlc.Timestamp{WallTime: int64(loadFactor * numBatches * batchTimeSpan)} it := makeIterator(eng, startTime, endTime) defer it.Close() - for it.SeekGE(engine.MVCCKey{}); ; it.Next() { + for it.SeekGE(storage.MVCCKey{}); ; it.Next() { if ok, err := it.Valid(); !ok { if err != nil { b.Fatal(err) @@ -170,13 +170,13 @@ func BenchmarkTimeBoundIterate(b *testing.B) { for _, loadFactor := range []float32{1.0, 0.5, 0.1, 0.05, 0.0} { b.Run(fmt.Sprintf("LoadFactor=%.2f", loadFactor), func(b *testing.B) { b.Run("NormalIterator", func(b *testing.B) { - runIterate(b, loadFactor, func(e engine.Engine, _, _ hlc.Timestamp) engine.Iterator { - return e.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + runIterate(b, loadFactor, func(e storage.Engine, _, _ hlc.Timestamp) storage.Iterator { + return e.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) }) }) b.Run("TimeBoundIterator", func(b *testing.B) { - runIterate(b, loadFactor, func(e engine.Engine, startTime, endTime hlc.Timestamp) engine.Iterator { - return e.NewIterator(engine.IterOptions{ + runIterate(b, loadFactor, func(e storage.Engine, startTime, endTime hlc.Timestamp) storage.Iterator { + return e.NewIterator(storage.IterOptions{ MinTimestampHint: startTime, MaxTimestampHint: endTime, UpperBound: roachpb.KeyMax, diff --git a/pkg/ccl/storageccl/engineccl/ctr_stream.go b/pkg/ccl/storageccl/engineccl/ctr_stream.go index 0567aaba3854..2910987a058f 100644 --- a/pkg/ccl/storageccl/engineccl/ctr_stream.go +++ b/pkg/ccl/storageccl/engineccl/ctr_stream.go @@ -17,7 +17,7 @@ import ( "fmt" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" ) // FileCipherStreamCreator wraps the KeyManager interface and provides functions to create a diff --git a/pkg/ccl/storageccl/engineccl/ctr_stream_test.go b/pkg/ccl/storageccl/engineccl/ctr_stream_test.go index 9de9965c0ff9..73571b2d2345 100644 --- a/pkg/ccl/storageccl/engineccl/ctr_stream_test.go +++ b/pkg/ccl/storageccl/engineccl/ctr_stream_test.go @@ -16,7 +16,7 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/kr/pretty" "github.com/stretchr/testify/require" diff --git a/pkg/ccl/storageccl/engineccl/encrypted_fs.go b/pkg/ccl/storageccl/engineccl/encrypted_fs.go index 3dd124d8f9f7..e1b670982545 100644 --- a/pkg/ccl/storageccl/engineccl/encrypted_fs.go +++ b/pkg/ccl/storageccl/engineccl/encrypted_fs.go @@ -14,8 +14,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/ccl/baseccl" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/pebble/vfs" @@ -107,7 +107,7 @@ func (f *encryptedFile) ReadAt(p []byte, off int64) (n int, err error) { // encryptedFS implements vfs.FS. type encryptedFS struct { vfs.FS - fileRegistry *engine.PebbleFileRegistry + fileRegistry *storage.PebbleFileRegistry streamCreator *FileCipherStreamCreator } @@ -255,7 +255,7 @@ func (e *encryptionStatsHandler) GetKeyIDFromSettings(settings []byte) (string, // Init initializes engine.NewEncryptedEncFunc. func init() { - engine.NewEncryptedEnvFunc = newEncryptedEnv + storage.NewEncryptedEnvFunc = newEncryptedEnv } // newEncryptedEnv creates an encrypted environment and returns the vfs.FS to use for reading and @@ -264,8 +264,8 @@ func init() { // // See the comment at the top of this file for the structure of this environment. func newEncryptedEnv( - fs vfs.FS, fr *engine.PebbleFileRegistry, dbDir string, readOnly bool, optionBytes []byte, -) (vfs.FS, engine.EncryptionStatsHandler, error) { + fs vfs.FS, fr *storage.PebbleFileRegistry, dbDir string, readOnly bool, optionBytes []byte, +) (vfs.FS, storage.EncryptionStatsHandler, error) { options := &baseccl.EncryptionOptions{} if err := protoutil.Unmarshal(optionBytes, options); err != nil { return nil, nil, err diff --git a/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go b/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go index 98af491de39f..c9399405907f 100644 --- a/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go +++ b/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go @@ -19,9 +19,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/ccl/baseccl" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/pebble" @@ -34,7 +34,7 @@ func TestEncryptedFS(t *testing.T) { memFS := vfs.NewMem() - fileRegistry := &engine.PebbleFileRegistry{FS: memFS, DBDir: "/bar"} + fileRegistry := &storage.PebbleFileRegistry{FS: memFS, DBDir: "/bar"} require.NoError(t, fileRegistry.Load()) // Using a StoreKeyManager for the test since it is easy to create. Write a key for the @@ -189,7 +189,7 @@ func TestEncryptedFS(t *testing.T) { func TestPebbleEncryption(t *testing.T) { defer leaktest.AfterTest(t)() - opts := engine.DefaultPebbleOptions() + opts := storage.DefaultPebbleOptions() opts.Cache = pebble.NewCache(1 << 20) defer opts.Cache.Unref() @@ -206,9 +206,9 @@ func TestPebbleEncryption(t *testing.T) { encOptions.DataKeyRotationPeriod = 1000 // arbitrary seconds encOptionsBytes, err := protoutil.Marshal(&encOptions) require.NoError(t, err) - db, err := engine.NewPebble( + db, err := storage.NewPebble( context.Background(), - engine.PebbleConfig{ + storage.PebbleConfig{ StorageConfig: base.StorageConfig{ Attrs: roachpb.Attributes{}, MaxSize: 512 << 20, @@ -230,22 +230,22 @@ func TestPebbleEncryption(t *testing.T) { t.Logf("EnvStats:\n%+v\n\n", *stats) batch := db.NewWriteOnlyBatch() - require.NoError(t, batch.Put(engine.MVCCKey{Key: roachpb.Key("a")}, []byte("a"))) + require.NoError(t, batch.Put(storage.MVCCKey{Key: roachpb.Key("a")}, []byte("a"))) require.NoError(t, batch.Commit(true)) require.NoError(t, db.Flush()) - val, err := db.Get(engine.MVCCKey{Key: roachpb.Key("a")}) + val, err := db.Get(storage.MVCCKey{Key: roachpb.Key("a")}) require.NoError(t, err) require.Equal(t, "a", string(val)) db.Close() - opts2 := engine.DefaultPebbleOptions() + opts2 := storage.DefaultPebbleOptions() opts2.Cache = pebble.NewCache(1 << 20) defer opts2.Cache.Unref() opts2.FS = memFS - db, err = engine.NewPebble( + db, err = storage.NewPebble( context.Background(), - engine.PebbleConfig{ + storage.PebbleConfig{ StorageConfig: base.StorageConfig{ Attrs: roachpb.Attributes{}, MaxSize: 512 << 20, @@ -255,7 +255,7 @@ func TestPebbleEncryption(t *testing.T) { Opts: opts2, }) require.NoError(t, err) - val, err = db.Get(engine.MVCCKey{Key: roachpb.Key("a")}) + val, err = db.Get(storage.MVCCKey{Key: roachpb.Key("a")}) require.NoError(t, err) require.Equal(t, "a", string(val)) db.Close() diff --git a/pkg/ccl/storageccl/engineccl/pebble_key_manager.go b/pkg/ccl/storageccl/engineccl/pebble_key_manager.go index b27ca007cd6f..80a50d8ccfed 100644 --- a/pkg/ccl/storageccl/engineccl/pebble_key_manager.go +++ b/pkg/ccl/storageccl/engineccl/pebble_key_manager.go @@ -18,7 +18,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" - "github.com/cockroachdb/cockroach/pkg/engine" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -391,7 +391,7 @@ func (m *DataKeyManager) rotateDataKeyAndWrite( if err != nil { return } - if err = engine.SafeWriteToFile(m.fs, m.dbDir, m.registryFilename, bytes); err != nil { + if err = storage.SafeWriteToFile(m.fs, m.dbDir, m.registryFilename, bytes); err != nil { return } m.mu.keyRegistry = keyRegistry diff --git a/pkg/ccl/storageccl/engineccl/rocksdb.go b/pkg/ccl/storageccl/engineccl/rocksdb.go index 39918b5f3444..f99ec37143a8 100644 --- a/pkg/ccl/storageccl/engineccl/rocksdb.go +++ b/pkg/ccl/storageccl/engineccl/rocksdb.go @@ -11,9 +11,9 @@ package engineccl import ( "unsafe" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/pkg/errors" ) @@ -34,13 +34,13 @@ import ( import "C" func init() { - engine.SetRocksDBOpenHook(C.DBOpenHookCCL) + storage.SetRocksDBOpenHook(C.DBOpenHookCCL) } // VerifyBatchRepr asserts that all keys in a BatchRepr are between the specified // start and end keys and computes the enginepb.MVCCStats for it. func VerifyBatchRepr( - repr []byte, start, end engine.MVCCKey, nowNanos int64, + repr []byte, start, end storage.MVCCKey, nowNanos int64, ) (enginepb.MVCCStats, error) { // We store a 4 byte checksum of each key/value entry in the value. Make // sure the all ones in this BatchRepr validate. @@ -50,13 +50,13 @@ func VerifyBatchRepr( // checksums in go and constructs the MVCCStats in c++. It'd be nice to move // one or the other. { - r, err := engine.NewRocksDBBatchReader(repr) + r, err := storage.NewRocksDBBatchReader(repr) if err != nil { return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums") } for r.Next() { switch r.BatchType() { - case engine.BatchTypeValue: + case storage.BatchTypeValue: mvccKey, err := r.MVCCKey() if err != nil { return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums") @@ -109,7 +109,7 @@ func goToCSlice(b []byte) C.DBSlice { } } -func goToCKey(key engine.MVCCKey) C.DBKey { +func goToCKey(key storage.MVCCKey) C.DBKey { return C.DBKey{ key: goToCSlice(key.Key), wall_time: C.int64_t(key.Timestamp.WallTime), @@ -122,7 +122,7 @@ func cSliceToUnsafeGoBytes(s C.DBSlice) []byte { return nil } // Interpret the C pointer as a pointer to a Go array, then slice. - return (*[engine.MaxArrayLen]byte)(unsafe.Pointer(s.data))[:s.len:s.len] + return (*[storage.MaxArrayLen]byte)(unsafe.Pointer(s.data))[:s.len:s.len] } func cStringToGoString(s C.DBString) string { diff --git a/pkg/ccl/storageccl/engineccl/rocksdb_test.go b/pkg/ccl/storageccl/engineccl/rocksdb_test.go index 6af285e8c8ad..aeb818fc746f 100644 --- a/pkg/ccl/storageccl/engineccl/rocksdb_test.go +++ b/pkg/ccl/storageccl/engineccl/rocksdb_test.go @@ -11,8 +11,8 @@ package engineccl import ( "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -21,14 +21,14 @@ import ( func TestVerifyBatchRepr(t *testing.T) { defer leaktest.AfterTest(t)() - keyA := engine.MVCCKey{Key: []byte("a")} - keyB := engine.MVCCKey{Key: []byte("b")} - keyC := engine.MVCCKey{Key: []byte("c")} - keyD := engine.MVCCKey{Key: []byte("d")} - keyE := engine.MVCCKey{Key: []byte("e")} + keyA := storage.MVCCKey{Key: []byte("a")} + keyB := storage.MVCCKey{Key: []byte("b")} + keyC := storage.MVCCKey{Key: []byte("c")} + keyD := storage.MVCCKey{Key: []byte("d")} + keyE := storage.MVCCKey{Key: []byte("e")} - var batch engine.RocksDBBatchBuilder - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + var batch storage.RocksDBBatchBuilder + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} batch.Put(key, roachpb.MakeValueFromString("1").RawBytes) data := batch.Finish() @@ -51,8 +51,8 @@ func TestVerifyBatchRepr(t *testing.T) { // Invalid key/value entry checksum. { - var batch engine.RocksDBBatchBuilder - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + var batch storage.RocksDBBatchBuilder + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} value := roachpb.MakeValueFromString("1") value.InitChecksum([]byte("foo")) batch.Put(key, value.RawBytes) diff --git a/pkg/ccl/storageccl/export.go b/pkg/ccl/storageccl/export.go index 6319662a82b5..3041e02f69f2 100644 --- a/pkg/ccl/storageccl/export.go +++ b/pkg/ccl/storageccl/export.go @@ -14,7 +14,6 @@ import ( "crypto/sha512" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" @@ -23,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/sql/sem/builtins" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/tracing" "github.com/pkg/errors" @@ -65,7 +65,7 @@ func declareKeysExport( // evalExport dumps the requested keys into files of non-overlapping key ranges // in a format suitable for bulk ingest. func evalExport( - ctx context.Context, batch engine.Reader, cArgs batcheval.CommandArgs, resp roachpb.Response, + ctx context.Context, batch storage.Reader, cArgs batcheval.CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ExportRequest) h := cArgs.Header @@ -139,7 +139,7 @@ func evalExport( return result.Result{}, errors.Errorf("unknown MVCC filter: %s", args.MVCCFilter) } - io := engine.IterOptions{ + io := storage.IterOptions{ UpperBound: args.EndKey, } diff --git a/pkg/ccl/storageccl/export_test.go b/pkg/ccl/storageccl/export_test.go index 1d6e3f3e0dae..aad3723db970 100644 --- a/pkg/ccl/storageccl/export_test.go +++ b/pkg/ccl/storageccl/export_test.go @@ -21,11 +21,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -64,22 +64,22 @@ func TestExportCmd(t *testing.T) { exportAndSlurpOne := func( t *testing.T, start hlc.Timestamp, mvccFilter roachpb.MVCCFilter, - ) ([]string, []engine.MVCCKeyValue) { + ) ([]string, []storage.MVCCKeyValue) { res, pErr := export(t, start, mvccFilter) if pErr != nil { t.Fatalf("%+v", pErr) } var paths []string - var kvs []engine.MVCCKeyValue - ingestFunc := func(kv engine.MVCCKeyValue) (bool, error) { + var kvs []storage.MVCCKeyValue + ingestFunc := func(kv storage.MVCCKeyValue) (bool, error) { kvs = append(kvs, kv) return false, nil } for _, file := range res.(*roachpb.ExportResponse).Files { paths = append(paths, file.Path) - sst := engine.MakeRocksDBSstFileReader() + sst := storage.MakeRocksDBSstFileReader() defer sst.Close() fileContents, err := ioutil.ReadFile(filepath.Join(dir, "foo", file.Path)) @@ -102,9 +102,9 @@ func TestExportCmd(t *testing.T) { type ExportAndSlurpResult struct { end hlc.Timestamp mvccLatestFiles []string - mvccLatestKVs []engine.MVCCKeyValue + mvccLatestKVs []storage.MVCCKeyValue mvccAllFiles []string - mvccAllKVs []engine.MVCCKeyValue + mvccAllKVs []storage.MVCCKeyValue } exportAndSlurp := func(t *testing.T, start hlc.Timestamp) ExportAndSlurpResult { var ret ExportAndSlurpResult @@ -285,41 +285,41 @@ func exportUsingGoIterator( startTime, endTime hlc.Timestamp, startKey, endKey roachpb.Key, enableTimeBoundIteratorOptimization bool, - reader engine.Reader, + reader storage.Reader, ) ([]byte, error) { - sst, err := engine.MakeRocksDBSstFileWriter() + sst, err := storage.MakeRocksDBSstFileWriter() if err != nil { return nil, nil //nolint:returnerrcheck } defer sst.Close() var skipTombstones bool - var iterFn func(*engine.MVCCIncrementalIterator) + var iterFn func(*storage.MVCCIncrementalIterator) switch filter { case roachpb.MVCCFilter_Latest: skipTombstones = true - iterFn = (*engine.MVCCIncrementalIterator).NextKey + iterFn = (*storage.MVCCIncrementalIterator).NextKey case roachpb.MVCCFilter_All: skipTombstones = false - iterFn = (*engine.MVCCIncrementalIterator).Next + iterFn = (*storage.MVCCIncrementalIterator).Next default: return nil, nil } - io := engine.IterOptions{ + io := storage.IterOptions{ UpperBound: endKey, } if enableTimeBoundIteratorOptimization { io.MaxTimestampHint = endTime io.MinTimestampHint = startTime.Next() } - iter := engine.NewMVCCIncrementalIterator(reader, engine.MVCCIncrementalIterOptions{ + iter := storage.NewMVCCIncrementalIterator(reader, storage.MVCCIncrementalIterOptions{ IterOptions: io, StartTime: startTime, EndTime: endTime, }) defer iter.Close() - for iter.SeekGE(engine.MakeMVCCMetadataKey(startKey)); ; iterFn(iter) { + for iter.SeekGE(storage.MakeMVCCMetadataKey(startKey)); ; iterFn(iter) { ok, err := iter.Valid() if err != nil { // The error may be a WriteIntentError. In which case, returning it will @@ -354,21 +354,21 @@ func exportUsingGoIterator( return sstContents, nil } -func loadSST(t *testing.T, data []byte, start, end roachpb.Key) []engine.MVCCKeyValue { +func loadSST(t *testing.T, data []byte, start, end roachpb.Key) []storage.MVCCKeyValue { t.Helper() if len(data) == 0 { return nil } - sst := engine.MakeRocksDBSstFileReader() + sst := storage.MakeRocksDBSstFileReader() defer sst.Close() if err := sst.IngestExternalFile(data); err != nil { t.Fatal(err) } - var kvs []engine.MVCCKeyValue - if err := sst.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) { + var kvs []storage.MVCCKeyValue + if err := sst.Iterate(start, end, func(kv storage.MVCCKeyValue) (bool, error) { kvs = append(kvs, kv) return false, nil }); err != nil { @@ -380,7 +380,7 @@ func loadSST(t *testing.T, data []byte, start, end roachpb.Key) []engine.MVCCKey func assertEqualKVs( ctx context.Context, - e engine.Engine, + e storage.Engine, startKey, endKey roachpb.Key, startTime, endTime hlc.Timestamp, exportAllRevisions bool, @@ -405,14 +405,14 @@ func assertEqualKVs( } // Run new C++ implementation of IncrementalIterator. - io := engine.IterOptions{ + io := storage.IterOptions{ UpperBound: endKey, } if enableTimeBoundIteratorOptimization { io.MaxTimestampHint = endTime io.MinTimestampHint = startTime.Next() } - var kvs []engine.MVCCKeyValue + var kvs []storage.MVCCKeyValue for start := startKey; start != nil; { var sst []byte var summary roachpb.BulkOpSummary @@ -486,9 +486,9 @@ func TestRandomKeyAndTimestampExport(t *testing.T) { ctx := context.Background() - mkEngine := func(t *testing.T) (e engine.Engine, cleanup func()) { + mkEngine := func(t *testing.T) (e storage.Engine, cleanup func()) { dir, cleanupDir := testutils.TempDir(t) - e, err := engine.NewDefaultEngine( + e, err := storage.NewDefaultEngine( 0, base.StorageConfig{ Settings: cluster.MakeTestingClusterSettings(), @@ -521,7 +521,7 @@ func TestRandomKeyAndTimestampExport(t *testing.T) { return numKeys } mkData := func( - t *testing.T, e engine.Engine, rnd *rand.Rand, numKeys int, + t *testing.T, e storage.Engine, rnd *rand.Rand, numKeys int, ) ([]roachpb.Key, []hlc.Timestamp) { // Store generated keys and timestamps. var keys []roachpb.Key @@ -545,7 +545,7 @@ func TestRandomKeyAndTimestampExport(t *testing.T) { value := roachpb.MakeValueFromBytes(randutil.RandBytes(rnd, 200)) value.InitChecksum(key) - if err := engine.MVCCPut(ctx, batch, nil, key, ts, value, nil); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, key, ts, value, nil); err != nil { t.Fatal(err) } @@ -556,7 +556,7 @@ func TestRandomKeyAndTimestampExport(t *testing.T) { ts = hlc.Timestamp{WallTime: int64(curWallTime), Logical: int32(curLogical)} value = roachpb.MakeValueFromBytes(randutil.RandBytes(rnd, 200)) value.InitChecksum(key) - if err := engine.MVCCPut(ctx, batch, nil, key, ts, value, nil); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, key, ts, value, nil); err != nil { t.Fatal(err) } } diff --git a/pkg/ccl/storageccl/import.go b/pkg/ccl/storageccl/import.go index 84e4a3797117..cb7c18c82e8b 100644 --- a/pkg/ccl/storageccl/import.go +++ b/pkg/ccl/storageccl/import.go @@ -15,13 +15,13 @@ import ( "io/ioutil" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/bulk" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -142,7 +142,7 @@ func evalImport(ctx context.Context, cArgs batcheval.CommandArgs) (*roachpb.Impo } defer cArgs.EvalCtx.GetLimiters().ConcurrentImportRequests.Finish() - var iters []engine.SimpleIterator + var iters []storage.SimpleIterator for _, file := range args.Files { log.VEventf(ctx, 2, "import file %s %s", file.Path, args.Key) @@ -189,7 +189,7 @@ func evalImport(ctx context.Context, cArgs batcheval.CommandArgs) (*roachpb.Impo } } - iter, err := engine.NewMemSSTIterator(fileContents, false) + iter, err := storage.NewMemSSTIterator(fileContents, false) if err != nil { return nil, err } @@ -204,8 +204,8 @@ func evalImport(ctx context.Context, cArgs batcheval.CommandArgs) (*roachpb.Impo } defer batcher.Close() - startKeyMVCC, endKeyMVCC := engine.MVCCKey{Key: args.DataSpan.Key}, engine.MVCCKey{Key: args.DataSpan.EndKey} - iter := engine.MakeMultiIterator(iters) + startKeyMVCC, endKeyMVCC := storage.MVCCKey{Key: args.DataSpan.Key}, storage.MVCCKey{Key: args.DataSpan.EndKey} + iter := storage.MakeMultiIterator(iters) defer iter.Close() var keyScratch, valueScratch []byte @@ -238,7 +238,7 @@ func evalImport(ctx context.Context, cArgs batcheval.CommandArgs) (*roachpb.Impo keyScratch = append(keyScratch[:0], iter.UnsafeKey().Key...) valueScratch = append(valueScratch[:0], iter.UnsafeValue()...) - key := engine.MVCCKey{Key: keyScratch, Timestamp: iter.UnsafeKey().Timestamp} + key := storage.MVCCKey{Key: keyScratch, Timestamp: iter.UnsafeKey().Timestamp} value := roachpb.Value{RawBytes: valueScratch} iter.NextKey() diff --git a/pkg/ccl/storageccl/import_test.go b/pkg/ccl/storageccl/import_test.go index a06125761633..f82b197d9078 100644 --- a/pkg/ccl/storageccl/import_test.go +++ b/pkg/ccl/storageccl/import_test.go @@ -21,7 +21,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -30,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" @@ -63,16 +63,16 @@ func TestMaxImportBatchSize(t *testing.T) { func slurpSSTablesLatestKey( t *testing.T, dir string, paths []string, kr prefixRewriter, -) []engine.MVCCKeyValue { - start, end := engine.MVCCKey{Key: keys.MinKey}, engine.MVCCKey{Key: keys.MaxKey} +) []storage.MVCCKeyValue { + start, end := storage.MVCCKey{Key: keys.MinKey}, storage.MVCCKey{Key: keys.MaxKey} - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() batch := e.NewBatch() defer batch.Close() for _, path := range paths { - sst := engine.MakeRocksDBSstFileReader() + sst := storage.MakeRocksDBSstFileReader() defer sst.Close() fileContents, err := ioutil.ReadFile(filepath.Join(dir, path)) @@ -82,7 +82,7 @@ func slurpSSTablesLatestKey( if err := sst.IngestExternalFile(fileContents); err != nil { t.Fatalf("%+v", err) } - if err := sst.Iterate(start.Key, end.Key, func(kv engine.MVCCKeyValue) (bool, error) { + if err := sst.Iterate(start.Key, end.Key, func(kv storage.MVCCKeyValue) (bool, error) { var ok bool kv.Key.Key, ok = kr.rewriteKey(kv.Key.Key) if !ok { @@ -100,8 +100,8 @@ func slurpSSTablesLatestKey( } } - var kvs []engine.MVCCKeyValue - it := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + var kvs []storage.MVCCKeyValue + it := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer it.Close() for it.SeekGE(start); ; it.NextKey() { if ok, err := it.Valid(); err != nil { @@ -109,22 +109,22 @@ func slurpSSTablesLatestKey( } else if !ok || !it.UnsafeKey().Less(end) { break } - kvs = append(kvs, engine.MVCCKeyValue{Key: it.Key(), Value: it.Value()}) + kvs = append(kvs, storage.MVCCKeyValue{Key: it.Key(), Value: it.Value()}) } return kvs } -func clientKVsToEngineKVs(kvs []client.KeyValue) []engine.MVCCKeyValue { - var ret []engine.MVCCKeyValue +func clientKVsToEngineKVs(kvs []client.KeyValue) []storage.MVCCKeyValue { + var ret []storage.MVCCKeyValue for _, kv := range kvs { if kv.Value == nil { continue } - k := engine.MVCCKey{ + k := storage.MVCCKey{ Key: kv.Key, Timestamp: kv.Value.Timestamp, } - ret = append(ret, engine.MVCCKeyValue{Key: k, Value: kv.Value.RawBytes}) + ret = append(ret, storage.MVCCKeyValue{Key: k, Value: kv.Value.RawBytes}) } return ret } @@ -170,8 +170,8 @@ func runTestImport(t *testing.T, init func(*cluster.Settings)) { writeSST := func(t *testing.T, offsets []int) string { path := strconv.FormatInt(hlc.UnixNano(), 10) - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) defer sst.Close() ts := hlc.NewClock(hlc.UnixNano, time.Nanosecond).Now() value := roachpb.MakeValueFromString("bar") @@ -179,7 +179,7 @@ func runTestImport(t *testing.T, init func(*cluster.Settings)) { key := keys[idx] value.ClearChecksum() value.InitChecksum(key) - if err := sst.Put(engine.MVCCKey{Key: key, Timestamp: ts}, value.RawBytes); err != nil { + if err := sst.Put(storage.MVCCKey{Key: key, Timestamp: ts}, value.RawBytes); err != nil { t.Fatalf("%+v", err) } } diff --git a/pkg/ccl/storageccl/revision_reader.go b/pkg/ccl/storageccl/revision_reader.go index a9f566313e5b..af39069d2057 100644 --- a/pkg/ccl/storageccl/revision_reader.go +++ b/pkg/ccl/storageccl/revision_reader.go @@ -11,9 +11,9 @@ package storageccl import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -50,13 +50,13 @@ func GetAllRevisions( var res []VersionedValues for _, file := range resp.(*roachpb.ExportResponse).Files { - sst := engine.MakeRocksDBSstFileReader() + sst := storage.MakeRocksDBSstFileReader() defer sst.Close() if err := sst.IngestExternalFile(file.SST); err != nil { return nil, err } - if err := sst.Iterate(startKey, endKey, func(kv engine.MVCCKeyValue) (bool, error) { + if err := sst.Iterate(startKey, endKey, func(kv storage.MVCCKeyValue) (bool, error) { if len(res) == 0 || !res[len(res)-1].Key.Equal(kv.Key.Key) { res = append(res, VersionedValues{Key: kv.Key.Key}) } diff --git a/pkg/ccl/storageccl/writebatch.go b/pkg/ccl/storageccl/writebatch.go index a009a5604634..6f757f4f4999 100644 --- a/pkg/ccl/storageccl/writebatch.go +++ b/pkg/ccl/storageccl/writebatch.go @@ -13,12 +13,12 @@ import ( "fmt" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/tracing" "github.com/pkg/errors" @@ -32,7 +32,7 @@ func init() { // data in the affected keyrange is first cleared (not tombstoned), which makes // this command idempotent. func evalWriteBatch( - ctx context.Context, batch engine.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, + ctx context.Context, batch storage.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.WriteBatchRequest) @@ -53,8 +53,8 @@ func evalWriteBatch( return result.Result{}, errors.New("data spans multiple ranges") } - mvccStartKey := engine.MVCCKey{Key: args.Key} - mvccEndKey := engine.MVCCKey{Key: args.EndKey} + mvccStartKey := storage.MVCCKey{Key: args.Key} + mvccEndKey := storage.MVCCKey{Key: args.EndKey} // Verify that the keys in the batch are within the range specified by the // request header. @@ -79,11 +79,11 @@ func evalWriteBatch( } func clearExistingData( - ctx context.Context, batch engine.ReadWriter, start, end roachpb.Key, nowNanos int64, + ctx context.Context, batch storage.ReadWriter, start, end roachpb.Key, nowNanos int64, ) (enginepb.MVCCStats, error) { { isEmpty := true - if err := batch.Iterate(start, end, func(_ engine.MVCCKeyValue) (bool, error) { + if err := batch.Iterate(start, end, func(_ storage.MVCCKeyValue) (bool, error) { isEmpty = false return true, nil // stop right away }); err != nil { @@ -95,13 +95,13 @@ func clearExistingData( } } - iter := batch.NewIterator(engine.IterOptions{UpperBound: end}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: end}) defer iter.Close() - iter.SeekGE(engine.MakeMVCCMetadataKey(start)) + iter.SeekGE(storage.MakeMVCCMetadataKey(start)) if ok, err := iter.Valid(); err != nil { return enginepb.MVCCStats{}, err - } else if ok && !iter.UnsafeKey().Less(engine.MakeMVCCMetadataKey(end)) { + } else if ok && !iter.UnsafeKey().Less(storage.MakeMVCCMetadataKey(end)) { return enginepb.MVCCStats{}, nil } diff --git a/pkg/ccl/storageccl/writebatch_test.go b/pkg/ccl/storageccl/writebatch_test.go index 8ab75e89a4a4..5f88d2860f1c 100644 --- a/pkg/ccl/storageccl/writebatch_test.go +++ b/pkg/ccl/storageccl/writebatch_test.go @@ -15,11 +15,11 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -41,8 +41,8 @@ func TestDBWriteBatch(t *testing.T) { } { - var batch engine.RocksDBBatchBuilder - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + var batch storage.RocksDBBatchBuilder + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} batch.Put(key, roachpb.MakeValueFromString("1").RawBytes) data := batch.Finish() @@ -71,8 +71,8 @@ func TestDBWriteBatch(t *testing.T) { // Key range in request span is not empty. { - var batch engine.RocksDBBatchBuilder - key := engine.MVCCKey{Key: []byte("bb2"), Timestamp: hlc.Timestamp{WallTime: 1}} + var batch storage.RocksDBBatchBuilder + key := storage.MVCCKey{Key: []byte("bb2"), Timestamp: hlc.Timestamp{WallTime: 1}} batch.Put(key, roachpb.MakeValueFromString("2").RawBytes) data := batch.Finish() if err := db.WriteBatch(ctx, "b", "c", data); err != nil { @@ -94,8 +94,8 @@ func TestDBWriteBatch(t *testing.T) { // Invalid key/value entry checksum. { - var batch engine.RocksDBBatchBuilder - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + var batch storage.RocksDBBatchBuilder + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} value := roachpb.MakeValueFromString("1") value.InitChecksum([]byte("foo")) batch.Put(key, value.RawBytes) @@ -111,12 +111,12 @@ func TestWriteBatchMVCCStats(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() - var batch engine.RocksDBBatchBuilder + var batch storage.RocksDBBatchBuilder { - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} batch.Put(key, roachpb.MakeValueFromString("1").RawBytes) } data := batch.Finish() @@ -127,7 +127,7 @@ func TestWriteBatchMVCCStats(t *testing.T) { // adjusted accordingly. const numInitialEntries = 100 for i := 0; i < numInitialEntries; i++ { - if err := e.Put(engine.MVCCKey{Key: append([]byte("b"), byte(i))}, nil); err != nil { + if err := e.Put(storage.MVCCKey{Key: append([]byte("b"), byte(i))}, nil); err != nil { t.Fatalf("%+v", err) } } diff --git a/pkg/ccl/utilccl/sampledataccl/bankdata.go b/pkg/ccl/utilccl/sampledataccl/bankdata.go index dcbcc91c4f84..d64e99e14abd 100644 --- a/pkg/ccl/utilccl/sampledataccl/bankdata.go +++ b/pkg/ccl/utilccl/sampledataccl/bankdata.go @@ -22,10 +22,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/ccl/backupccl" "github.com/cockroachdb/cockroach/pkg/ccl/importccl" "github.com/cockroachdb/cockroach/pkg/ccl/storageccl" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -93,7 +93,7 @@ func (b *Backup) ResetKeyValueIteration() { // remainer. func (b *Backup) NextKeyValues( count int, newTableID sqlbase.ID, -) ([]engine.MVCCKeyValue, roachpb.Span, error) { +) ([]storage.MVCCKeyValue, roachpb.Span, error) { var userTables []*sqlbase.TableDescriptor for _, d := range b.Desc.Descriptors { if t := d.Table(hlc.Timestamp{}); t != nil && t.ParentID != keys.SystemDatabaseID { @@ -113,12 +113,12 @@ func (b *Backup) NextKeyValues( return nil, roachpb.Span{}, err } - var kvs []engine.MVCCKeyValue + var kvs []storage.MVCCKeyValue span := roachpb.Span{Key: keys.MaxKey} for ; b.fileIdx < len(b.Desc.Files); b.fileIdx++ { file := b.Desc.Files[b.fileIdx] - sst := engine.MakeRocksDBSstFileReader() + sst := storage.MakeRocksDBSstFileReader() defer sst.Close() fileContents, err := ioutil.ReadFile(filepath.Join(b.BaseDir, file.Path)) if err != nil { @@ -128,9 +128,9 @@ func (b *Backup) NextKeyValues( return nil, roachpb.Span{}, err } - it := sst.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + it := sst.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer it.Close() - it.SeekGE(engine.MVCCKey{Key: file.Span.Key}) + it.SeekGE(storage.MVCCKey{Key: file.Span.Key}) iterIdx := 0 for ; ; it.Next() { @@ -166,7 +166,7 @@ func (b *Backup) NextKeyValues( v := roachpb.Value{RawBytes: it.Value()} v.ClearChecksum() v.InitChecksum(key.Key) - kvs = append(kvs, engine.MVCCKeyValue{Key: key, Value: v.RawBytes}) + kvs = append(kvs, storage.MVCCKeyValue{Key: key, Value: v.RawBytes}) if key.Key.Compare(span.Key) < 0 { span.Key = append(span.Key[:0], key.Key...) diff --git a/pkg/ccl/workloadccl/format/sstable.go b/pkg/ccl/workloadccl/format/sstable.go index 034c30c2bf61..98114a3a111c 100644 --- a/pkg/ccl/workloadccl/format/sstable.go +++ b/pkg/ccl/workloadccl/format/sstable.go @@ -14,13 +14,13 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/ccl/importccl" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/row" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/workload" @@ -75,12 +75,12 @@ func ToSSTable(t workload.Table, tableID sqlbase.ID, ts time.Time) ([]byte, erro var sst []byte g.GoCtx(func(ctx context.Context) error { sstTS := hlc.Timestamp{WallTime: ts.UnixNano()} - sstFile := &engine.MemFile{} - sw := engine.MakeIngestionSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sw := storage.MakeIngestionSSTWriter(sstFile) defer sw.Close() for kvBatch := range kvCh { for _, kv := range kvBatch.KVs { - mvccKey := engine.MVCCKey{Timestamp: sstTS, Key: kv.Key} + mvccKey := storage.MVCCKey{Timestamp: sstTS, Key: kv.Key} if err := sw.Put(mvccKey, kv.Value.RawBytes); err != nil { return err } diff --git a/pkg/cli/context.go b/pkg/cli/context.go index fee3ae329ef8..5ff0af1918ed 100644 --- a/pkg/cli/context.go +++ b/pkg/cli/context.go @@ -18,11 +18,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/mattn/go-isatty" "github.com/spf13/cobra" @@ -94,8 +94,8 @@ func initCLIDefaults() { dumpCtx.dumpMode = dumpBoth dumpCtx.asOf = "" - debugCtx.startKey = engine.NilKey - debugCtx.endKey = engine.MVCCKeyMax + debugCtx.startKey = storage.NilKey + debugCtx.endKey = storage.MVCCKeyMax debugCtx.values = false debugCtx.sizes = false debugCtx.replicated = false @@ -276,7 +276,7 @@ var authCtx struct { // debugCtx captures the command-line parameters of the `debug` command. // Defaults set by InitCLIDefaults() above. var debugCtx struct { - startKey, endKey engine.MVCCKey + startKey, endKey storage.MVCCKey values bool sizes bool replicated bool diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index d637782fb46d..8bd1cf43d8c8 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -30,8 +30,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cli/syncbench" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -45,6 +43,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/server/status/statuspb" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/ts/tspb" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/flagutil" @@ -116,13 +116,13 @@ type OpenEngineOptions struct { // OpenExistingStore opens the rocksdb engine rooted at 'dir'. // If 'readOnly' is true, opens the store in read-only mode. -func OpenExistingStore(dir string, stopper *stop.Stopper, readOnly bool) (engine.Engine, error) { +func OpenExistingStore(dir string, stopper *stop.Stopper, readOnly bool) (storage.Engine, error) { return OpenEngine(dir, stopper, OpenEngineOptions{ReadOnly: readOnly, MustExist: true}) } // OpenEngine opens the engine at 'dir'. Depending on the supplied options, // an empty engine might be initialized. -func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (engine.Engine, error) { +func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (storage.Engine, error) { maxOpenFiles, err := server.SetOpenFileLimitForOneStore() if err != nil { return nil, err @@ -139,14 +139,14 @@ func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (engi } } - var db engine.Engine - storageEngine := resolveStorageEngineType(engine.DefaultStorageEngine, storageConfig.Dir) + var db storage.Engine + storageEngine := resolveStorageEngineType(storage.DefaultStorageEngine, storageConfig.Dir) switch storageEngine { case enginepb.EngineTypePebble: - cfg := engine.PebbleConfig{ + cfg := storage.PebbleConfig{ StorageConfig: storageConfig, - Opts: engine.DefaultPebbleOptions(), + Opts: storage.DefaultPebbleOptions(), } cfg.Opts.Cache = pebble.NewCache(server.DefaultCacheSize) defer cfg.Opts.Cache.Unref() @@ -154,19 +154,19 @@ func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (engi cfg.Opts.MaxOpenFiles = int(maxOpenFiles) cfg.Opts.ReadOnly = opts.ReadOnly - db, err = engine.NewPebble(context.Background(), cfg) + db, err = storage.NewPebble(context.Background(), cfg) case enginepb.EngineTypeRocksDB: - cache := engine.NewRocksDBCache(server.DefaultCacheSize) + cache := storage.NewRocksDBCache(server.DefaultCacheSize) defer cache.Release() - cfg := engine.RocksDBConfig{ + cfg := storage.RocksDBConfig{ StorageConfig: storageConfig, MaxOpenFiles: maxOpenFiles, ReadOnly: opts.ReadOnly, } - db, err = engine.NewRocksDB(cfg, cache) + db, err = storage.NewRocksDB(cfg, cache) } if err != nil { @@ -177,7 +177,7 @@ func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (engi return db, nil } -func printKey(kv engine.MVCCKeyValue) (bool, error) { +func printKey(kv storage.MVCCKeyValue) (bool, error) { fmt.Printf("%s %s: ", kv.Key.Timestamp, kv.Key.Key) if debugCtx.sizes { fmt.Printf(" %d %d", len(kv.Key.Key), len(kv.Value)) @@ -197,7 +197,7 @@ func runDebugKeys(cmd *cobra.Command, args []string) error { printer := printKey if debugCtx.values { - printer = func(kv engine.MVCCKeyValue) (bool, error) { + printer = func(kv storage.MVCCKeyValue) (bool, error) { kvserver.PrintKeyValue(kv) return false, nil } @@ -299,7 +299,7 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error { } else if !ok { break } - kvserver.PrintKeyValue(engine.MVCCKeyValue{ + kvserver.PrintKeyValue(storage.MVCCKeyValue{ Key: iter.Key(), Value: iter.Value(), }) @@ -318,10 +318,10 @@ Prints all range descriptors in a store with a history of changes. } func loadRangeDescriptor( - db engine.Engine, rangeID roachpb.RangeID, + db storage.Engine, rangeID roachpb.RangeID, ) (roachpb.RangeDescriptor, error) { var desc roachpb.RangeDescriptor - handleKV := func(kv engine.MVCCKeyValue) (bool, error) { + handleKV := func(kv storage.MVCCKeyValue) (bool, error) { if kv.Key.Timestamp == (hlc.Timestamp{}) { // We only want values, not MVCCMetadata. return false, nil @@ -368,7 +368,7 @@ func runDebugRangeDescriptors(cmd *cobra.Command, args []string) error { start := keys.LocalRangePrefix end := keys.LocalRangeMax - return db.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) { + return db.Iterate(start, end, func(kv storage.MVCCKeyValue) (bool, error) { if kvserver.IsRangeDescriptorKey(kv.Key) != nil { return false, nil } @@ -393,7 +393,7 @@ Decode a hexadecimal-encoded key and pretty-print it. For example: if err != nil { return err } - k, err := engine.DecodeMVCCKey(b) + k, err := storage.DecodeMVCCKey(b) if err != nil { return err } @@ -424,7 +424,7 @@ Decode and print a hexadecimal-encoded key-value pair. } isTS := bytes.HasPrefix(bs[0], keys.TimeseriesPrefix) - k, err := engine.DecodeMVCCKey(bs[0]) + k, err := storage.DecodeMVCCKey(bs[0]) if err != nil { // Older versions of the consistency checker give you diffs with a raw_key that // is already a roachpb.Key, so make a half-assed attempt to support both. @@ -432,13 +432,13 @@ Decode and print a hexadecimal-encoded key-value pair. fmt.Printf("unable to decode key: %v, assuming it's a roachpb.Key with fake timestamp;\n"+ "if the result below looks like garbage, then it likely is:\n\n", err) } - k = engine.MVCCKey{ + k = storage.MVCCKey{ Key: bs[0], Timestamp: hlc.Timestamp{WallTime: 987654321}, } } - kvserver.PrintKeyValue(engine.MVCCKeyValue{ + kvserver.PrintKeyValue(storage.MVCCKeyValue{ Key: k, Value: bs[1], }) @@ -474,10 +474,10 @@ func runDebugRaftLog(cmd *cobra.Command, args []string) error { end := keys.RaftLogPrefix(rangeID).PrefixEnd() fmt.Printf("Printing keys %s -> %s (RocksDB keys: %#x - %#x )\n", start, end, - string(engine.EncodeKey(engine.MakeMVCCMetadataKey(start))), - string(engine.EncodeKey(engine.MakeMVCCMetadataKey(end)))) + string(storage.EncodeKey(storage.MakeMVCCMetadataKey(start))), + string(storage.EncodeKey(storage.MakeMVCCMetadataKey(end)))) - return db.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) { + return db.Iterate(start, end, func(kv storage.MVCCKeyValue) (bool, error) { kvserver.PrintKeyValue(kv) return false, nil }) @@ -532,8 +532,8 @@ func runDebugGCCmd(cmd *cobra.Command, args []string) error { var descs []roachpb.RangeDescriptor - if _, err := engine.MVCCIterate(context.Background(), db, start, end, hlc.MaxTimestamp, - engine.MVCCScanOptions{Inconsistent: true}, func(kv roachpb.KeyValue) (bool, error) { + if _, err := storage.MVCCIterate(context.Background(), db, start, end, hlc.MaxTimestamp, + storage.MVCCScanOptions{Inconsistent: true}, func(kv roachpb.KeyValue) (bool, error) { var desc roachpb.RangeDescriptor _, suffix, _, err := keys.DecodeRangeKey(kv.Key) if err != nil { @@ -594,7 +594,7 @@ https://github.com/facebook/rocksdb/wiki/Administration-and-Data-Access-Tool#ldb // TODO(mberhault): support encrypted stores. DisableFlagParsing: true, Run: func(cmd *cobra.Command, args []string) { - engine.RunLDB(args) + storage.RunLDB(args) }, } @@ -616,7 +616,7 @@ Runs the RocksDB 'sst_dump' tool // TODO(mberhault): support encrypted stores. DisableFlagParsing: true, Run: func(cmd *cobra.Command, args []string) { - engine.RunSSTDump(args) + storage.RunSSTDump(args) }, } @@ -987,8 +987,8 @@ func runDebugUnsafeRemoveDeadReplicas(cmd *cobra.Command, args []string) error { } func removeDeadReplicas( - db engine.Engine, deadStoreIDs map[roachpb.StoreID]struct{}, -) (engine.Batch, error) { + db storage.Engine, deadStoreIDs map[roachpb.StoreID]struct{}, +) (storage.Batch, error) { clock := hlc.NewClock(hlc.UnixNano, 0) ctx := context.Background() @@ -1106,7 +1106,7 @@ func removeDeadReplicas( if err != nil { return nil, errors.Wrap(err, "loading MVCCStats") } - err = engine.MVCCPutProto(ctx, batch, &ms, key, clock.Now(), nil /* txn */, &desc) + err = storage.MVCCPutProto(ctx, batch, &ms, key, clock.Now(), nil /* txn */, &desc) if wiErr, ok := err.(*roachpb.WriteIntentError); ok { if len(wiErr.Intents) != 1 { return nil, errors.Errorf("expected 1 intent, found %d: %s", len(wiErr.Intents), wiErr) @@ -1130,7 +1130,7 @@ func removeDeadReplicas( // A crude form of the intent resolution process: abort the // transaction by deleting its record. txnKey := keys.TransactionKey(intent.Txn.Key, intent.Txn.ID) - if err := engine.MVCCDelete(ctx, batch, &ms, txnKey, hlc.Timestamp{}, nil); err != nil { + if err := storage.MVCCDelete(ctx, batch, &ms, txnKey, hlc.Timestamp{}, nil); err != nil { return nil, err } update := roachpb.LockUpdate{ @@ -1139,11 +1139,11 @@ func removeDeadReplicas( Status: roachpb.ABORTED, Durability: lock.Replicated, } - if _, err := engine.MVCCResolveWriteIntent(ctx, batch, &ms, update); err != nil { + if _, err := storage.MVCCResolveWriteIntent(ctx, batch, &ms, update); err != nil { return nil, err } // With the intent resolved, we can try again. - if err := engine.MVCCPutProto(ctx, batch, &ms, key, clock.Now(), + if err := storage.MVCCPutProto(ctx, batch, &ms, key, clock.Now(), nil /* txn */, &desc); err != nil { return nil, err } @@ -1248,8 +1248,8 @@ func init() { // To be able to read Cockroach-written RocksDB manifests/SSTables, comparator // and merger functions must be specified to pebble that match the ones used // to write those files. - pebbleTool.RegisterMerger(engine.MVCCMerger) - pebbleTool.RegisterComparer(engine.MVCCComparer) + pebbleTool.RegisterMerger(storage.MVCCMerger) + pebbleTool.RegisterComparer(storage.MVCCComparer) debugPebbleCmd.AddCommand(pebbleTool.Commands...) DebugCmd.AddCommand(debugPebbleCmd) diff --git a/pkg/cli/debug_check_store.go b/pkg/cli/debug_check_store.go index 4ca56325b267..4f86f1ba8697 100644 --- a/pkg/cli/debug_check_store.go +++ b/pkg/cli/debug_check_store.go @@ -17,13 +17,13 @@ import ( "runtime" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -87,7 +87,7 @@ type replicaCheckInfo struct { } type checkInput struct { - eng engine.Engine + eng storage.Engine desc *roachpb.RangeDescriptor sl stateloader.StateLoader } @@ -240,8 +240,8 @@ func checkStoreRaftState( return replicaInfo[rangeID] } - if _, err := engine.MVCCIterate(ctx, db, start, end, hlc.MaxTimestamp, - engine.MVCCScanOptions{Inconsistent: true}, func(kv roachpb.KeyValue) (bool, error) { + if _, err := storage.MVCCIterate(ctx, db, start, end, hlc.MaxTimestamp, + storage.MVCCScanOptions{Inconsistent: true}, func(kv roachpb.KeyValue) (bool, error) { rangeID, _, suffix, detail, err := keys.DecodeRangeIDKey(kv.Key) if err != nil { return false, err diff --git a/pkg/cli/debug_check_store_test.go b/pkg/cli/debug_check_store_test.go index ee59e4b6fdb6..76f334017ff3 100644 --- a/pkg/cli/debug_check_store_test.go +++ b/pkg/cli/debug_check_store_test.go @@ -19,9 +19,9 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -79,7 +79,7 @@ func TestDebugCheckStore(t *testing.T) { // Introduce a stats divergence on s1. func() { - eng, err := engine.NewDefaultEngine( + eng, err := storage.NewDefaultEngine( 10<<20, /* 10mb */ base.StorageConfig{ Dir: storePaths[0], diff --git a/pkg/cli/debug_synctest.go b/pkg/cli/debug_synctest.go index 1b6ede7e9e79..ce026931c359 100644 --- a/pkg/cli/debug_synctest.go +++ b/pkg/cli/debug_synctest.go @@ -22,8 +22,8 @@ import ( "strconv" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/pkg/errors" @@ -133,7 +133,7 @@ func runSyncer( return encoding.EncodeUvarintAscending(buf[:0:0], uint64(seq)) } - check := func(kv engine.MVCCKeyValue) (bool, error) { + check := func(kv storage.MVCCKeyValue) (bool, error) { expKey := key() if !bytes.Equal(kv.Key.Key, expKey) { return false, errors.Errorf( @@ -184,7 +184,7 @@ func runSyncer( } }() - k, v := engine.MakeMVCCMetadataKey(key()), []byte("payload") + k, v := storage.MakeMVCCMetadataKey(key()), []byte("payload") switch seq % 2 { case 0: if err := db.Put(k, v); err != nil { diff --git a/pkg/cli/debug_test.go b/pkg/cli/debug_test.go index a5e2bf675677..abedafe715b2 100644 --- a/pkg/cli/debug_test.go +++ b/pkg/cli/debug_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -31,6 +30,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/server/serverpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -42,10 +42,10 @@ import ( func createStore(t *testing.T, path string) { t.Helper() - cache := engine.NewRocksDBCache(server.DefaultCacheSize) + cache := storage.NewRocksDBCache(server.DefaultCacheSize) defer cache.Release() - db, err := engine.NewRocksDB( - engine.RocksDBConfig{ + db, err := storage.NewRocksDB( + storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: path, MustExist: false, @@ -123,7 +123,7 @@ func TestOpenReadOnlyStore(t *testing.T) { t.Fatal(err) } - key := engine.MakeMVCCMetadataKey(roachpb.Key("key")) + key := storage.MakeMVCCMetadataKey(roachpb.Key("key")) val := []byte("value") err = db.Put(key, val) if !testutils.IsError(err, test.expErr) { diff --git a/pkg/cli/flags_util.go b/pkg/cli/flags_util.go index de02c323321e..85d5609a1ed3 100644 --- a/pkg/cli/flags_util.go +++ b/pkg/cli/flags_util.go @@ -18,10 +18,10 @@ import ( "strconv" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server/status" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/keysutil" @@ -168,14 +168,14 @@ func (m *dumpMode) Set(s string) error { return nil } -type mvccKey engine.MVCCKey +type mvccKey storage.MVCCKey // Type implements the pflag.Value interface. func (k *mvccKey) Type() string { return "engine.MVCCKey" } // String implements the pflag.Value interface. func (k *mvccKey) String() string { - return engine.MVCCKey(*k).String() + return storage.MVCCKey(*k).String() } // Set implements the pflag.Value interface. @@ -200,9 +200,9 @@ func (k *mvccKey) Set(value string) error { if err != nil { return err } - newK, err := engine.DecodeMVCCKey(b) + newK, err := storage.DecodeMVCCKey(b) if err != nil { - encoded := gohex.EncodeToString(engine.EncodeKey(engine.MakeMVCCMetadataKey(roachpb.Key(b)))) + encoded := gohex.EncodeToString(storage.EncodeKey(storage.MakeMVCCMetadataKey(roachpb.Key(b)))) return errors.Wrapf(err, "perhaps this is just a hex-encoded key; you need an "+ "encoded MVCCKey (i.e. with a timestamp component); here's one with a zero timestamp: %s", encoded) @@ -213,20 +213,20 @@ func (k *mvccKey) Set(value string) error { if err != nil { return err } - *k = mvccKey(engine.MakeMVCCMetadataKey(roachpb.Key(unquoted))) + *k = mvccKey(storage.MakeMVCCMetadataKey(roachpb.Key(unquoted))) case human: scanner := keysutil.MakePrettyScanner(nil /* tableParser */) key, err := scanner.Scan(keyStr) if err != nil { return err } - *k = mvccKey(engine.MakeMVCCMetadataKey(key)) + *k = mvccKey(storage.MakeMVCCMetadataKey(key)) case rangeID: fromID, err := parseRangeID(keyStr) if err != nil { return err } - *k = mvccKey(engine.MakeMVCCMetadataKey(keys.MakeRangeIDPrefix(fromID))) + *k = mvccKey(storage.MakeMVCCMetadataKey(keys.MakeRangeIDPrefix(fromID))) default: return fmt.Errorf("unknown key type %s", typ) } diff --git a/pkg/cli/start.go b/pkg/cli/start.go index 58912acbc155..d33bae693ad2 100644 --- a/pkg/cli/start.go +++ b/pkg/cli/start.go @@ -31,8 +31,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/build" "github.com/cockroachdb/cockroach/pkg/cli/cliflags" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/security" @@ -40,6 +38,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/server/status" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" "github.com/cockroachdb/cockroach/pkg/util/grpcutil" @@ -335,7 +335,7 @@ func initTempStorageConfig( // the temporary directory record file before creating any new // temporary directories in case the disk is completely full. if recordPath != "" { - if err = engine.CleanupTempDirs(recordPath); err != nil { + if err = storage.CleanupTempDirs(recordPath); err != nil { return base.TempStorageConfig{}, errors.Wrap(err, "could not cleanup temporary directories from record file") } } @@ -392,14 +392,14 @@ func initTempStorageConfig( tempDir = firstStore.Path } // Create the temporary subdirectory for the temp engine. - if tempStorageConfig.Path, err = engine.CreateTempDir(tempDir, server.TempDirPrefix, stopper); err != nil { + if tempStorageConfig.Path, err = storage.CreateTempDir(tempDir, server.TempDirPrefix, stopper); err != nil { return base.TempStorageConfig{}, errors.Wrap(err, "could not create temporary directory for temp storage") } // We record the new temporary directory in the record file (if it // exists) for cleanup in case the node crashes. if recordPath != "" { - if err = engine.RecordTempDir(recordPath, tempStorageConfig.Path); err != nil { + if err = storage.RecordTempDir(recordPath, tempStorageConfig.Path); err != nil { return base.TempStorageConfig{}, errors.Wrapf( err, "could not record temporary directory path to record file: %s", @@ -1184,8 +1184,8 @@ func setupAndInitializeLoggingAndProfiling( log.StartGCDaemon(ctx) // We have a valid logging directory. Configure Pebble/RocksDB to log into it. - engine.InitPebbleLogger(ctx) - engine.InitRocksDBLogger(ctx) + storage.InitPebbleLogger(ctx) + storage.InitRocksDBLogger(ctx) } outputDirectory := "." diff --git a/pkg/cli/syncbench/syncbench.go b/pkg/cli/syncbench/syncbench.go index ab66897284b0..b16e57451990 100644 --- a/pkg/cli/syncbench/syncbench.go +++ b/pkg/cli/syncbench/syncbench.go @@ -21,8 +21,8 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -51,7 +51,7 @@ func clampLatency(d, min, max time.Duration) time.Duration { } type worker struct { - db engine.Engine + db storage.Engine latency struct { syncutil.Mutex *hdrhistogram.WindowedHistogram @@ -59,7 +59,7 @@ type worker struct { logOnly bool } -func newWorker(db engine.Engine) *worker { +func newWorker(db storage.Engine) *worker { w := &worker{db: db} w.latency.WindowedHistogram = hdrhistogram.NewWindowed(1, minLatency.Nanoseconds(), maxLatency.Nanoseconds(), 1) @@ -93,7 +93,7 @@ func (w *worker) run(wg *sync.WaitGroup) { for j := 0; j < 5; j++ { block := randBlock(60, 80) key := encoding.EncodeUint32Ascending(buf, rand.Uint32()) - if err := b.Put(engine.MakeMVCCMetadataKey(key), block); err != nil { + if err := b.Put(storage.MakeMVCCMetadataKey(key), block); err != nil { log.Fatal(ctx, err) } buf = key[:0] @@ -139,7 +139,7 @@ func Run(opts Options) error { fmt.Printf("writing to %s\n", opts.Dir) - db, err := engine.NewDefaultEngine( + db, err := storage.NewDefaultEngine( 0, base.StorageConfig{ Settings: cluster.MakeTestingClusterSettings(), diff --git a/pkg/internal/client/batch.go b/pkg/internal/client/batch.go index 73db38e760d8..3817cd13ea2e 100644 --- a/pkg/internal/client/batch.go +++ b/pkg/internal/client/batch.go @@ -13,8 +13,8 @@ package client import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/pkg/errors" ) diff --git a/pkg/internal/client/client_test.go b/pkg/internal/client/client_test.go index 98f684ef799e..0c07e878f8aa 100644 --- a/pkg/internal/client/client_test.go +++ b/pkg/internal/client/client_test.go @@ -24,12 +24,12 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" diff --git a/pkg/internal/client/db.go b/pkg/internal/client/db.go index b538ce1ee611..00e8a9007a21 100644 --- a/pkg/internal/client/db.go +++ b/pkg/internal/client/db.go @@ -16,8 +16,8 @@ import ( "strings" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" diff --git a/pkg/internal/client/mock_transactional_sender.go b/pkg/internal/client/mock_transactional_sender.go index a94332fee829..d2f027c37647 100644 --- a/pkg/internal/client/mock_transactional_sender.go +++ b/pkg/internal/client/mock_transactional_sender.go @@ -13,8 +13,8 @@ package client import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/errors" ) diff --git a/pkg/internal/client/sender.go b/pkg/internal/client/sender.go index 79eb154a0597..e185df3886c6 100644 --- a/pkg/internal/client/sender.go +++ b/pkg/internal/client/sender.go @@ -13,8 +13,8 @@ package client import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) diff --git a/pkg/internal/client/txn.go b/pkg/internal/client/txn.go index bb1e954d15b3..f62a229e6e5c 100644 --- a/pkg/internal/client/txn.go +++ b/pkg/internal/client/txn.go @@ -15,8 +15,8 @@ import ( "fmt" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/dist_sender_server_test.go b/pkg/kv/dist_sender_server_test.go index ec6c4a23ab80..3a036f6fdc0a 100644 --- a/pkg/kv/dist_sender_server_test.go +++ b/pkg/kv/dist_sender_server_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -33,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -78,7 +78,7 @@ func TestRangeLookupWithOpenTransaction(t *testing.T) { key := testutils.MakeKey(keys.Meta1Prefix, roachpb.KeyMax) now := s.Clock().Now() txn := roachpb.MakeTransaction("txn", roachpb.Key("foobar"), 0, now, 0) - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( context.Background(), s.(*server.TestServer).Engines()[0], nil, key, now, &txn, &roachpb.RangeDescriptor{}); err != nil { t.Fatal(err) diff --git a/pkg/kv/kvnemesis/engine.go b/pkg/kv/kvnemesis/engine.go index d28e6cc2c000..e54368f13671 100644 --- a/pkg/kv/kvnemesis/engine.go +++ b/pkg/kv/kvnemesis/engine.go @@ -14,8 +14,8 @@ import ( "fmt" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/google/btree" ) @@ -43,7 +43,7 @@ func MakeEngine() *Engine { func (e *Engine) Get(key roachpb.Key, ts hlc.Timestamp) roachpb.Value { var value roachpb.Value e.kvs.AscendGreaterOrEqual( - btreeItem{Key: engine.MVCCKey{Key: key, Timestamp: ts}}, + btreeItem{Key: storage.MVCCKey{Key: key, Timestamp: ts}}, func(i btree.Item) bool { if kv := i.(btreeItem); kv.Key.Key.Equal(key) { value = roachpb.Value{ @@ -59,7 +59,7 @@ func (e *Engine) Get(key roachpb.Key, ts hlc.Timestamp) roachpb.Value { // Put inserts a key/value/timestamp tuple. If an exact key/timestamp pair is // Put again, it overwrites the previous value. -func (e *Engine) Put(key engine.MVCCKey, value []byte) { +func (e *Engine) Put(key storage.MVCCKey, value []byte) { e.kvs.ReplaceOrInsert(btreeItem{Key: key, Value: value}) } @@ -79,7 +79,7 @@ func (e *Engine) DebugPrint(indent string) string { return buf.String() } -type btreeItem engine.MVCCKeyValue +type btreeItem storage.MVCCKeyValue func (i btreeItem) Less(o btree.Item) bool { return i.Key.Less(o.(btreeItem).Key) diff --git a/pkg/kv/kvnemesis/engine_test.go b/pkg/kv/kvnemesis/engine_test.go index ada20b57da1a..16ef4cf2284d 100644 --- a/pkg/kv/kvnemesis/engine_test.go +++ b/pkg/kv/kvnemesis/engine_test.go @@ -14,8 +14,8 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/assert" @@ -24,8 +24,8 @@ import ( func TestEngine(t *testing.T) { defer leaktest.AfterTest(t)() - k := func(s string, ts hlc.Timestamp) engine.MVCCKey { - return engine.MVCCKey{Key: []byte(s), Timestamp: ts} + k := func(s string, ts hlc.Timestamp) storage.MVCCKey { + return storage.MVCCKey{Key: []byte(s), Timestamp: ts} } var missing roachpb.Value v := func(s string, ts hlc.Timestamp) roachpb.Value { diff --git a/pkg/kv/kvnemesis/validator.go b/pkg/kv/kvnemesis/validator.go index 338419cf6d15..8756ccd73ff6 100644 --- a/pkg/kv/kvnemesis/validator.go +++ b/pkg/kv/kvnemesis/validator.go @@ -17,8 +17,8 @@ import ( "sort" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" @@ -69,7 +69,7 @@ func Validate(steps []Step, kvs *Engine) []error { v.processOp(nil /* txnID */, s.Op) } - var extraKVs []engine.MVCCKeyValue + var extraKVs []storage.MVCCKeyValue for _, kv := range v.kvByValue { extraKVs = append(extraKVs, kv) } @@ -82,14 +82,14 @@ func Validate(steps []Step, kvs *Engine) []error { } type validator struct { - kvByValue map[string]engine.MVCCKeyValue - kvsByTxn map[string][]engine.MVCCKeyValue + kvByValue map[string]storage.MVCCKeyValue + kvsByTxn map[string][]storage.MVCCKeyValue failures []error } func makeValidator(kvs *Engine) *validator { - kvByValue := make(map[string]engine.MVCCKeyValue) + kvByValue := make(map[string]storage.MVCCKeyValue) kvs.kvs.Ascend(func(item btree.Item) bool { kv := item.(btreeItem) value := mustGetStringValue(kv.Value) @@ -99,12 +99,12 @@ func makeValidator(kvs *Engine) *validator { panic(errors.AssertionFailedf(`invariant violation: value %s was written by two operations %s and %s`, value, existing.Key, kv.Key)) } - kvByValue[value] = engine.MVCCKeyValue(kv) + kvByValue[value] = storage.MVCCKeyValue(kv) return true }) return &validator{ kvByValue: kvByValue, - kvsByTxn: make(map[string][]engine.MVCCKeyValue), + kvsByTxn: make(map[string][]storage.MVCCKeyValue), } } @@ -236,7 +236,7 @@ func (v *validator) checkAtomic(atomicType string, result Result, ops ...Operati } } -func (v *validator) checkCommittedTxn(atomicType string, txnKVs []engine.MVCCKeyValue) { +func (v *validator) checkCommittedTxn(atomicType string, txnKVs []storage.MVCCKeyValue) { // If the same key is written multiple times in a transaction, only the last // one makes it to kv. lastWriteIdxByKey := make(map[string]int, len(txnKVs)) @@ -274,7 +274,7 @@ func (v *validator) checkCommittedTxn(atomicType string, txnKVs []engine.MVCCKey } } -func (v *validator) checkAmbiguousTxn(atomicType string, txnKVs []engine.MVCCKeyValue) { +func (v *validator) checkAmbiguousTxn(atomicType string, txnKVs []storage.MVCCKeyValue) { var somethingCommitted bool for _, kv := range txnKVs { if !kv.Key.Timestamp.IsEmpty() { @@ -289,7 +289,7 @@ func (v *validator) checkAmbiguousTxn(atomicType string, txnKVs []engine.MVCCKey } } -func (v *validator) checkUncommittedTxn(atomicType string, txnKVs []engine.MVCCKeyValue) { +func (v *validator) checkUncommittedTxn(atomicType string, txnKVs []storage.MVCCKeyValue) { var failure string for _, kv := range txnKVs { if kv.Key.Timestamp.IsEmpty() { @@ -363,7 +363,7 @@ func mustGetStringValue(value []byte) string { return string(value) } -func printKVs(kvs ...engine.MVCCKeyValue) string { +func printKVs(kvs ...storage.MVCCKeyValue) string { sort.Slice(kvs, func(i, j int) bool { return kvs[i].Key.Less(kvs[j].Key) }) var buf strings.Builder diff --git a/pkg/kv/kvnemesis/validator_test.go b/pkg/kv/kvnemesis/validator_test.go index b81f8f79c2f6..7d21916e5cf1 100644 --- a/pkg/kv/kvnemesis/validator_test.go +++ b/pkg/kv/kvnemesis/validator_test.go @@ -14,8 +14,8 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -34,23 +34,23 @@ func withResult(op Operation, err error) Operation { func TestValidate(t *testing.T) { defer leaktest.AfterTest(t)() - kv := func(key string, ts int, value string) engine.MVCCKeyValue { - return engine.MVCCKeyValue{ - Key: engine.MVCCKey{ + kv := func(key string, ts int, value string) storage.MVCCKeyValue { + return storage.MVCCKeyValue{ + Key: storage.MVCCKey{ Key: []byte(key), Timestamp: hlc.Timestamp{WallTime: int64(ts)}, }, Value: roachpb.MakeValueFromString(value).RawBytes, } } - kvs := func(kvs ...engine.MVCCKeyValue) []engine.MVCCKeyValue { + kvs := func(kvs ...storage.MVCCKeyValue) []storage.MVCCKeyValue { return kvs } tests := []struct { name string steps []Step - kvs []engine.MVCCKeyValue + kvs []storage.MVCCKeyValue expected []string }{ { diff --git a/pkg/kv/kvnemesis/watcher.go b/pkg/kv/kvnemesis/watcher.go index 5c56f16d1473..12a98bda97a2 100644 --- a/pkg/kv/kvnemesis/watcher.go +++ b/pkg/kv/kvnemesis/watcher.go @@ -14,10 +14,10 @@ import ( "context" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -168,7 +168,7 @@ func (w *Watcher) processEvents(ctx context.Context, eventC chan *roachpb.RangeF // but it means that we'll won't catch it if we violate those semantics. // Consider first doing a Get and somehow failing if this exact key+ts // has previously been put with a different value. - w.mu.kvs.Put(engine.MVCCKey{Key: e.Key, Timestamp: e.Value.Timestamp}, e.Value.RawBytes) + w.mu.kvs.Put(storage.MVCCKey{Key: e.Key, Timestamp: e.Value.Timestamp}, e.Value.RawBytes) prevTs := e.Value.Timestamp.Prev() prevValue := w.mu.kvs.Get(e.Key, prevTs) diff --git a/pkg/kv/kvserver/abortspan/abortspan.go b/pkg/kv/kvserver/abortspan/abortspan.go index e67e06120ba4..e337875be5a1 100644 --- a/pkg/kv/kvserver/abortspan/abortspan.go +++ b/pkg/kv/kvserver/abortspan/abortspan.go @@ -13,11 +13,11 @@ package abortspan import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -75,8 +75,8 @@ func (sc *AbortSpan) max() roachpb.Key { } // ClearData removes all persisted items stored in the cache. -func (sc *AbortSpan) ClearData(e engine.Engine) error { - iter := e.NewIterator(engine.IterOptions{UpperBound: sc.max()}) +func (sc *AbortSpan) ClearData(e storage.Engine) error { + iter := e.NewIterator(storage.IterOptions{UpperBound: sc.max()}) defer iter.Close() b := e.NewWriteOnlyBatch() defer b.Close() @@ -90,20 +90,20 @@ func (sc *AbortSpan) ClearData(e engine.Engine) error { // Get looks up an AbortSpan entry recorded for this transaction ID. // Returns whether an abort record was found and any error. func (sc *AbortSpan) Get( - ctx context.Context, reader engine.Reader, txnID uuid.UUID, entry *roachpb.AbortSpanEntry, + ctx context.Context, reader storage.Reader, txnID uuid.UUID, entry *roachpb.AbortSpanEntry, ) (bool, error) { // Pull response from disk and read into reply if available. key := keys.AbortSpanKey(sc.rangeID, txnID) - ok, err := engine.MVCCGetProto(ctx, reader, key, hlc.Timestamp{}, entry, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, reader, key, hlc.Timestamp{}, entry, storage.MVCCGetOptions{}) return ok, err } // Iterate walks through the AbortSpan, invoking the given callback for // each unmarshaled entry with the MVCC key and the decoded entry. func (sc *AbortSpan) Iterate( - ctx context.Context, reader engine.Reader, f func(roachpb.Key, roachpb.AbortSpanEntry) error, + ctx context.Context, reader storage.Reader, f func(roachpb.Key, roachpb.AbortSpanEntry) error, ) error { - _, err := engine.MVCCIterate(ctx, reader, sc.min(), sc.max(), hlc.Timestamp{}, engine.MVCCScanOptions{}, + _, err := storage.MVCCIterate(ctx, reader, sc.min(), sc.max(), hlc.Timestamp{}, storage.MVCCScanOptions{}, func(kv roachpb.KeyValue) (bool, error) { var entry roachpb.AbortSpanEntry if _, err := keys.DecodeAbortSpanKey(kv.Key, nil); err != nil { @@ -119,22 +119,22 @@ func (sc *AbortSpan) Iterate( // Del removes all AbortSpan entries for the given transaction. func (sc *AbortSpan) Del( - ctx context.Context, reader engine.ReadWriter, ms *enginepb.MVCCStats, txnID uuid.UUID, + ctx context.Context, reader storage.ReadWriter, ms *enginepb.MVCCStats, txnID uuid.UUID, ) error { key := keys.AbortSpanKey(sc.rangeID, txnID) - return engine.MVCCDelete(ctx, reader, ms, key, hlc.Timestamp{}, nil /* txn */) + return storage.MVCCDelete(ctx, reader, ms, key, hlc.Timestamp{}, nil /* txn */) } // Put writes an entry for the specified transaction ID. func (sc *AbortSpan) Put( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, txnID uuid.UUID, entry *roachpb.AbortSpanEntry, ) error { key := keys.AbortSpanKey(sc.rangeID, txnID) - return engine.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, entry) + return storage.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, entry) } // CopyTo copies the abort span entries to the abort span for the range @@ -147,8 +147,8 @@ func (sc *AbortSpan) Put( // impossible, which is pretty bad (see #25233). func (sc *AbortSpan) CopyTo( ctx context.Context, - r engine.Reader, - w engine.ReadWriter, + r storage.Reader, + w storage.ReadWriter, ms *enginepb.MVCCStats, ts hlc.Timestamp, newRangeID roachpb.RangeID, @@ -177,7 +177,7 @@ func (sc *AbortSpan) CopyTo( if err != nil { return err } - return engine.MVCCPutProto(ctx, w, ms, + return storage.MVCCPutProto(ctx, w, ms, keys.AbortSpanKey(newRangeID, txnID), hlc.Timestamp{}, nil, &entry, ) diff --git a/pkg/kv/kvserver/abortspan/abortspan_test.go b/pkg/kv/kvserver/abortspan/abortspan_test.go index 40522a8491b8..ded9e9bd9ce0 100644 --- a/pkg/kv/kvserver/abortspan/abortspan_test.go +++ b/pkg/kv/kvserver/abortspan/abortspan_test.go @@ -15,9 +15,9 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -43,8 +43,8 @@ var ( // returns a AbortSpan using the supplied Range ID. func createTestAbortSpan( t *testing.T, rangeID roachpb.RangeID, stopper *stop.Stopper, -) (*AbortSpan, engine.Engine) { - eng := engine.NewDefaultInMem() +) (*AbortSpan, storage.Engine) { + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) return New(rangeID), eng } diff --git a/pkg/kv/kvserver/addressing_test.go b/pkg/kv/kvserver/addressing_test.go index 47c029eb0f41..ea33f3851b0e 100644 --- a/pkg/kv/kvserver/addressing_test.go +++ b/pkg/kv/kvserver/addressing_test.go @@ -19,11 +19,11 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -165,8 +165,8 @@ func TestUpdateRangeAddressing(t *testing.T) { // to RocksDB will be asynchronous. var kvs []roachpb.KeyValue testutils.SucceedsSoon(t, func() error { - res, err := engine.MVCCScan(ctx, store.Engine(), keys.MetaMin, keys.MetaMax, - hlc.MaxTimestamp, engine.MVCCScanOptions{}) + res, err := storage.MVCCScan(ctx, store.Engine(), keys.MetaMin, keys.MetaMax, + hlc.MaxTimestamp, storage.MVCCScanOptions{}) if err != nil { // Wait for the intent to be resolved. if _, ok := err.(*roachpb.WriteIntentError); ok { diff --git a/pkg/kv/kvserver/allocator_test.go b/pkg/kv/kvserver/allocator_test.go index 71b14dbf456d..490c5d664d51 100644 --- a/pkg/kv/kvserver/allocator_test.go +++ b/pkg/kv/kvserver/allocator_test.go @@ -24,7 +24,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/constraint" @@ -32,6 +31,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/gossiputil" "github.com/cockroachdb/cockroach/pkg/util" diff --git a/pkg/kv/kvserver/api.pb.go b/pkg/kv/kvserver/api.pb.go index 34e2d647f900..5d03a820c146 100644 --- a/pkg/kv/kvserver/api.pb.go +++ b/pkg/kv/kvserver/api.pb.go @@ -9,9 +9,9 @@ import ( io "io" math "math" - enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" github_com_cockroachdb_cockroach_pkg_roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" + enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" github_com_cockroachdb_cockroach_pkg_util_uuid "github.com/cockroachdb/cockroach/pkg/util/uuid" proto "github.com/gogo/protobuf/proto" grpc "google.golang.org/grpc" diff --git a/pkg/kv/kvserver/batch_spanset_test.go b/pkg/kv/kvserver/batch_spanset_test.go index cd20bd864023..ce3dd59121a5 100644 --- a/pkg/kv/kvserver/batch_spanset_test.go +++ b/pkg/kv/kvserver/batch_spanset_test.go @@ -16,11 +16,11 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -29,18 +29,18 @@ import ( func TestSpanSetBatchBoundaries(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() var ss spanset.SpanSet ss.AddNonMVCC(spanset.SpanReadWrite, roachpb.Span{Key: roachpb.Key("c"), EndKey: roachpb.Key("g")}) - outsideKey := engine.MakeMVCCMetadataKey(roachpb.Key("a")) - outsideKey2 := engine.MakeMVCCMetadataKey(roachpb.Key("b")) - outsideKey3 := engine.MakeMVCCMetadataKey(roachpb.Key("g")) - outsideKey4 := engine.MakeMVCCMetadataKey(roachpb.Key("m")) - insideKey := engine.MakeMVCCMetadataKey(roachpb.Key("c")) - insideKey2 := engine.MakeMVCCMetadataKey(roachpb.Key("d")) - insideKey3 := engine.MakeMVCCMetadataKey(roachpb.Key("f")) + outsideKey := storage.MakeMVCCMetadataKey(roachpb.Key("a")) + outsideKey2 := storage.MakeMVCCMetadataKey(roachpb.Key("b")) + outsideKey3 := storage.MakeMVCCMetadataKey(roachpb.Key("g")) + outsideKey4 := storage.MakeMVCCMetadataKey(roachpb.Key("m")) + insideKey := storage.MakeMVCCMetadataKey(roachpb.Key("c")) + insideKey2 := storage.MakeMVCCMetadataKey(roachpb.Key("d")) + insideKey3 := storage.MakeMVCCMetadataKey(roachpb.Key("f")) // Write values outside the range that we can try to read later. if err := eng.Put(outsideKey, []byte("value")); err != nil { @@ -77,7 +77,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { t.Errorf("ClearRange: unexpected error %v", err) } { - iter := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) err := batch.ClearIterRange(iter, outsideKey.Key, outsideKey2.Key) iter.Close() if !isWriteSpanErr(err) { @@ -100,7 +100,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { t.Errorf("ClearRange: unexpected error %v", err) } { - iter := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) err := batch.ClearIterRange(iter, insideKey2.Key, outsideKey4.Key) iter.Close() if !isWriteSpanErr(err) { @@ -127,7 +127,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { t.Errorf("GetProto: unexpected error %v", err) } if err := batch.Iterate(insideKey.Key, insideKey2.Key, - func(v engine.MVCCKeyValue) (bool, error) { + func(v storage.MVCCKeyValue) (bool, error) { return false, nil }, ); err != nil { @@ -150,7 +150,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { t.Errorf("GetProto: unexpected error %v", err) } if err := batch.Iterate(outsideKey.Key, insideKey2.Key, - func(v engine.MVCCKeyValue) (bool, error) { + func(v storage.MVCCKeyValue) (bool, error) { return false, errors.Errorf("unexpected callback: %v", v) }, ); !isReadSpanErr(err) { @@ -168,7 +168,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { t.Errorf("GetProto: unexpected error %v", err) } if err := batch.Iterate(insideKey2.Key, outsideKey4.Key, - func(v engine.MVCCKeyValue) (bool, error) { + func(v storage.MVCCKeyValue) (bool, error) { return false, errors.Errorf("unexpected callback: %v", v) }, ); !isReadSpanErr(err) { @@ -177,7 +177,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { }) t.Run("forward scans", func(t *testing.T) { - iter := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() // Iterators check boundaries on seek and next/prev @@ -222,7 +222,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { } t.Run("reverse scans", func(t *testing.T) { - iter := spanset.NewIterator(eng.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}), &ss) + iter := spanset.NewIterator(eng.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}), &ss) defer iter.Close() iter.SeekLT(outsideKey) if _, err := iter.Valid(); !isReadSpanErr(err) { @@ -273,7 +273,7 @@ func TestSpanSetBatchBoundaries(t *testing.T) { func TestSpanSetBatchTimestamps(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() var ss spanset.SpanSet @@ -282,8 +282,8 @@ func TestSpanSetBatchTimestamps(t *testing.T) { ss.AddMVCC(spanset.SpanReadWrite, roachpb.Span{Key: roachpb.Key("d"), EndKey: roachpb.Key("f")}, hlc.Timestamp{WallTime: 2}) - rkey := engine.MakeMVCCMetadataKey(roachpb.Key("b")) - wkey := engine.MakeMVCCMetadataKey(roachpb.Key("e")) + rkey := storage.MakeMVCCMetadataKey(roachpb.Key("b")) + wkey := storage.MakeMVCCMetadataKey(roachpb.Key("e")) value := []byte("value") @@ -309,7 +309,7 @@ func TestSpanSetBatchTimestamps(t *testing.T) { t.Fatalf("failed to write inside the range at same ts as latch declaration: %+v", err) } - for _, batch := range []engine.Batch{batchAfter, batchBefore, batchNonMVCC} { + for _, batch := range []storage.Batch{batchAfter, batchBefore, batchNonMVCC} { if err := batch.Put(wkey, value); err == nil { t.Fatalf("was able to write inside the range at ts greater than latch declaration: %+v", err) } @@ -322,12 +322,12 @@ func TestSpanSetBatchTimestamps(t *testing.T) { return testutils.IsError(err, "cannot write undeclared span") } - for _, batch := range []engine.Batch{batchAfter, batchBefore, batchNonMVCC} { + for _, batch := range []storage.Batch{batchAfter, batchBefore, batchNonMVCC} { if err := batch.Clear(wkey); !isWriteSpanErr(err) { t.Errorf("Clear: unexpected error %v", err) } { - iter := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) err := batch.ClearIterRange(iter, wkey.Key, wkey.Key) iter.Close() if !isWriteSpanErr(err) { @@ -343,7 +343,7 @@ func TestSpanSetBatchTimestamps(t *testing.T) { } // Reads. - for _, batch := range []engine.Batch{batchBefore, batchDuring} { + for _, batch := range []storage.Batch{batchBefore, batchDuring} { //lint:ignore SA1019 historical usage of deprecated batch.Get is OK if res, err := batch.Get(rkey); err != nil { t.Errorf("failed to read inside the range: %+v", err) @@ -356,7 +356,7 @@ func TestSpanSetBatchTimestamps(t *testing.T) { return testutils.IsError(err, "cannot read undeclared span") } - for _, batch := range []engine.Batch{batchAfter, batchNonMVCC} { + for _, batch := range []storage.Batch{batchAfter, batchNonMVCC} { //lint:ignore SA1019 historical usage of deprecated batch.Get is OK if _, err := batch.Get(rkey); !isReadSpanErr(err) { t.Errorf("Get: unexpected error %v", err) @@ -367,7 +367,7 @@ func TestSpanSetBatchTimestamps(t *testing.T) { t.Errorf("GetProto: unexpected error %v", err) } if err := batch.Iterate(rkey.Key, rkey.Key, - func(v engine.MVCCKeyValue) (bool, error) { + func(v storage.MVCCKeyValue) (bool, error) { return false, errors.Errorf("unexpected callback: %v", v) }, ); !isReadSpanErr(err) { @@ -378,7 +378,7 @@ func TestSpanSetBatchTimestamps(t *testing.T) { func TestSpanSetIteratorTimestamps(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() var ss spanset.SpanSet @@ -387,8 +387,8 @@ func TestSpanSetIteratorTimestamps(t *testing.T) { ss.AddMVCC(spanset.SpanReadOnly, roachpb.Span{ Key: roachpb.Key("c"), EndKey: roachpb.Key("e")}, hlc.Timestamp{WallTime: 2}) - k1, v1 := engine.MakeMVCCMetadataKey(roachpb.Key("b")), []byte("b-value") - k2, v2 := engine.MakeMVCCMetadataKey(roachpb.Key("d")), []byte("d-value") + k1, v1 := storage.MakeMVCCMetadataKey(roachpb.Key("b")), []byte("b-value") + k2, v2 := storage.MakeMVCCMetadataKey(roachpb.Key("d")), []byte("d-value") // Write values that we can try to read later. if err := eng.Put(k1, v1); err != nil { @@ -412,7 +412,7 @@ func TestSpanSetIteratorTimestamps(t *testing.T) { func() { // When accessing at t=1, we're able to read through latches declared at t=1 and t=2. - iter := batchAt1.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batchAt1.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() iter.SeekGE(k1) @@ -434,7 +434,7 @@ func TestSpanSetIteratorTimestamps(t *testing.T) { { // When accessing at t=2, we're only able to read through the latch declared at t=2. - iter := batchAt2.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batchAt2.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() iter.SeekGE(k1) @@ -451,10 +451,10 @@ func TestSpanSetIteratorTimestamps(t *testing.T) { } } - for _, batch := range []engine.Batch{batchAt3, batchNonMVCC} { + for _, batch := range []storage.Batch{batchAt3, batchNonMVCC} { // When accessing at t=3, we're unable to read through any of the declared latches. // Same is true when accessing without a timestamp. - iter := batch.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() iter.SeekGE(k1) @@ -471,15 +471,15 @@ func TestSpanSetIteratorTimestamps(t *testing.T) { func TestSpanSetNonMVCCBatch(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() var ss spanset.SpanSet ss.AddNonMVCC(spanset.SpanReadOnly, roachpb.Span{Key: roachpb.Key("a"), EndKey: roachpb.Key("c")}) ss.AddNonMVCC(spanset.SpanReadWrite, roachpb.Span{Key: roachpb.Key("d"), EndKey: roachpb.Key("f")}) - rkey := engine.MakeMVCCMetadataKey(roachpb.Key("b")) - wkey := engine.MakeMVCCMetadataKey(roachpb.Key("e")) + rkey := storage.MakeMVCCMetadataKey(roachpb.Key("b")) + wkey := storage.MakeMVCCMetadataKey(roachpb.Key("e")) value := []byte("value") @@ -495,14 +495,14 @@ func TestSpanSetNonMVCCBatch(t *testing.T) { defer batchMVCC.Close() // Writes. - for _, batch := range []engine.Batch{batchNonMVCC, batchMVCC} { + for _, batch := range []storage.Batch{batchNonMVCC, batchMVCC} { if err := batch.Put(wkey, value); err != nil { t.Fatalf("write disallowed through non-MVCC latch: %+v", err) } } // Reads. - for _, batch := range []engine.Batch{batchNonMVCC, batchMVCC} { + for _, batch := range []storage.Batch{batchNonMVCC, batchMVCC} { //lint:ignore SA1019 historical usage of deprecated batch.Get is OK if res, err := batch.Get(rkey); err != nil { t.Errorf("read disallowed through non-MVCC latch: %+v", err) @@ -519,14 +519,14 @@ func TestSpanSetNonMVCCBatch(t *testing.T) { // See #20894. func TestSpanSetMVCCResolveWriteIntentRangeUsingIter(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() ctx := context.Background() value := roachpb.MakeValueFromString("irrelevant") - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, eng, nil, // ms @@ -551,10 +551,10 @@ func TestSpanSetMVCCResolveWriteIntentRangeUsingIter(t *testing.T) { Durability: lock.Replicated, } - iterAndBuf := engine.GetIterAndBuf(batch, engine.IterOptions{UpperBound: intent.Span.EndKey}) + iterAndBuf := storage.GetIterAndBuf(batch, storage.IterOptions{UpperBound: intent.Span.EndKey}) defer iterAndBuf.Cleanup() - if _, _, err := engine.MVCCResolveWriteIntentRangeUsingIter( + if _, _, err := storage.MVCCResolveWriteIntentRangeUsingIter( ctx, batch, iterAndBuf, nil /* ms */, intent, 0, ); err != nil { t.Fatal(err) diff --git a/pkg/kv/kvserver/batcheval/cmd_add_sstable.go b/pkg/kv/kvserver/batcheval/cmd_add_sstable.go index 54c290b3edd0..bb1b3b354980 100644 --- a/pkg/kv/kvserver/batcheval/cmd_add_sstable.go +++ b/pkg/kv/kvserver/batcheval/cmd_add_sstable.go @@ -14,13 +14,13 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/kr/pretty" @@ -33,12 +33,12 @@ func init() { // EvalAddSSTable evaluates an AddSSTable command. func EvalAddSSTable( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, _ roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, _ roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.AddSSTableRequest) h := cArgs.Header ms := cArgs.Stats - mvccStartKey, mvccEndKey := engine.MVCCKey{Key: args.Key}, engine.MVCCKey{Key: args.EndKey} + mvccStartKey, mvccEndKey := storage.MVCCKey{Key: args.Key}, storage.MVCCKey{Key: args.EndKey} // TODO(tschottdorf): restore the below in some form (gets in the way of testing). // _, span := tracing.ChildSpan(ctx, fmt.Sprintf("AddSSTable [%s,%s)", args.Key, args.EndKey)) @@ -58,14 +58,14 @@ func EvalAddSSTable( // Verify that the keys in the sstable are within the range specified by the // request header, and if the request did not include pre-computed stats, // compute the expected MVCC stats delta of ingesting the SST. - dataIter, err := engine.NewMemSSTIterator(args.Data, true) + dataIter, err := storage.NewMemSSTIterator(args.Data, true) if err != nil { return result.Result{}, err } defer dataIter.Close() // Check that the first key is in the expected range. - dataIter.SeekGE(engine.MVCCKey{Key: keys.MinKey}) + dataIter.SeekGE(storage.MVCCKey{Key: keys.MinKey}) ok, err := dataIter.Valid() if err != nil { return result.Result{}, err @@ -89,7 +89,7 @@ func EvalAddSSTable( if args.MVCCStats == nil || verifyFastPath { log.VEventf(ctx, 2, "computing MVCCStats for SSTable [%s,%s)", mvccStartKey.Key, mvccEndKey.Key) - computed, err := engine.ComputeStatsGo( + computed, err := storage.ComputeStatsGo( dataIter, mvccStartKey.Key, mvccEndKey.Key, h.Timestamp.WallTime) if err != nil { return result.Result{}, errors.Wrap(err, "computing SSTable MVCC stats") @@ -181,7 +181,7 @@ func EvalAddSSTable( if args.IngestAsWrites { log.VEventf(ctx, 2, "ingesting SST (%d keys/%d bytes) via regular write batch", stats.KeyCount, len(args.Data)) - dataIter.SeekGE(engine.MVCCKey{Key: keys.MinKey}) + dataIter.SeekGE(storage.MVCCKey{Key: keys.MinKey}) for { ok, err := dataIter.Valid() if err != nil { @@ -212,9 +212,9 @@ func EvalAddSSTable( func checkForKeyCollisions( _ context.Context, - readWriter engine.ReadWriter, - mvccStartKey engine.MVCCKey, - mvccEndKey engine.MVCCKey, + readWriter storage.ReadWriter, + mvccStartKey storage.MVCCKey, + mvccEndKey storage.MVCCKey, data []byte, ) (enginepb.MVCCStats, error) { // We could get a spansetBatch so fetch the underlying db engine as @@ -225,7 +225,7 @@ func checkForKeyCollisions( emptyMVCCStats := enginepb.MVCCStats{} // Create iterator over the existing data. - existingDataIter := dbEngine.NewIterator(engine.IterOptions{UpperBound: mvccEndKey.Key}) + existingDataIter := dbEngine.NewIterator(storage.IterOptions{UpperBound: mvccEndKey.Key}) defer existingDataIter.Close() existingDataIter.SeekGE(mvccStartKey) if ok, err := existingDataIter.Valid(); err != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go b/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go index 7e1f3e7feee1..e03b29a40e1d 100644 --- a/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go @@ -20,13 +20,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -37,28 +37,28 @@ import ( // createTestRocksDBEngine returns a new in-memory RocksDB engine with 1MB of // storage capacity. -func createTestRocksDBEngine() engine.Engine { - return engine.NewInMem(context.Background(), +func createTestRocksDBEngine() storage.Engine { + return storage.NewInMem(context.Background(), enginepb.EngineTypeRocksDB, roachpb.Attributes{}, 1<<20) } // createTestPebbleEngine returns a new in-memory Pebble storage engine. -func createTestPebbleEngine() engine.Engine { - return engine.NewInMem(context.Background(), +func createTestPebbleEngine() storage.Engine { + return storage.NewInMem(context.Background(), enginepb.EngineTypePebble, roachpb.Attributes{}, 1<<20) } var engineImpls = []struct { name string - create func() engine.Engine + create func() storage.Engine }{ {"rocksdb", createTestRocksDBEngine}, {"pebble", createTestPebbleEngine}, } -func singleKVSSTable(key engine.MVCCKey, value []byte) ([]byte, error) { - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) +func singleKVSSTable(key storage.MVCCKey, value []byte) ([]byte, error) { + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) defer sst.Close() if err := sst.Put(key, value); err != nil { return nil, err @@ -101,7 +101,7 @@ func TestDBAddSSTable(t *testing.T) { // if store != nil, assume it is on-disk and check ingestion semantics. func runTestDBAddSSTable(ctx context.Context, t *testing.T, db *client.DB, store *kvserver.Store) { { - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 2}} + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 2}} data, err := singleKVSSTable(key, roachpb.MakeValueFromString("1").RawBytes) if err != nil { t.Fatalf("%+v", err) @@ -160,7 +160,7 @@ func runTestDBAddSSTable(ctx context.Context, t *testing.T, db *client.DB, store // Check that ingesting a key with an earlier mvcc timestamp doesn't affect // the value returned by Get. { - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} data, err := singleKVSSTable(key, roachpb.MakeValueFromString("2").RawBytes) if err != nil { t.Fatalf("%+v", err) @@ -187,7 +187,7 @@ func runTestDBAddSSTable(ctx context.Context, t *testing.T, db *client.DB, store // Key range in request span is not empty. First time through a different // key is present. Second time through checks the idempotency. { - key := engine.MVCCKey{Key: []byte("bc"), Timestamp: hlc.Timestamp{WallTime: 1}} + key := storage.MVCCKey{Key: []byte("bc"), Timestamp: hlc.Timestamp{WallTime: 1}} data, err := singleKVSSTable(key, roachpb.MakeValueFromString("3").RawBytes) if err != nil { t.Fatalf("%+v", err) @@ -242,7 +242,7 @@ func runTestDBAddSSTable(ctx context.Context, t *testing.T, db *client.DB, store // ... and doing the same thing but via write-batch works the same. { - key := engine.MVCCKey{Key: []byte("bd"), Timestamp: hlc.Timestamp{WallTime: 1}} + key := storage.MVCCKey{Key: []byte("bd"), Timestamp: hlc.Timestamp{WallTime: 1}} data, err := singleKVSSTable(key, roachpb.MakeValueFromString("3").RawBytes) if err != nil { t.Fatalf("%+v", err) @@ -290,7 +290,7 @@ func runTestDBAddSSTable(ctx context.Context, t *testing.T, db *client.DB, store // Invalid key/value entry checksum. { - key := engine.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} + key := storage.MVCCKey{Key: []byte("bb"), Timestamp: hlc.Timestamp{WallTime: 1}} value := roachpb.MakeValueFromString("1") value.InitChecksum([]byte("foo")) data, err := singleKVSSTable(key, value.RawBytes) @@ -312,8 +312,8 @@ type strKv struct { v string } -func mvccKVsFromStrs(in []strKv) []engine.MVCCKeyValue { - kvs := make([]engine.MVCCKeyValue, len(in)) +func mvccKVsFromStrs(in []strKv) []storage.MVCCKeyValue { + kvs := make([]storage.MVCCKeyValue, len(in)) for i := range kvs { kvs[i].Key.Key = []byte(in[i].k) kvs[i].Key.Timestamp.WallTime = in[i].ts @@ -381,7 +381,7 @@ func TestAddSSTableMVCCStats(t *testing.T) { ts, base.DefaultMaxClockOffset.Nanoseconds(), ) - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, e, nil, []byte("i"), ts, roachpb.MakeValueFromBytes([]byte("it")), &txn, @@ -395,18 +395,18 @@ func TestAddSSTableMVCCStats(t *testing.T) { // stats. Make sure recomputing from scratch gets the same answer as // applying the diff to the stats beforeStats := func() enginepb.MVCCStats { - iter := e.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := e.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() - beforeStats, err := engine.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 10) + beforeStats, err := storage.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 10) if err != nil { t.Fatalf("%+v", err) } return beforeStats }() - mkSST := func(kvs []engine.MVCCKeyValue) []byte { - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) + mkSST := func(kvs []storage.MVCCKeyValue) []byte { + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) defer sst.Close() for _, kv := range kvs { if err := sst.Put(kv.Key, kv.Value); err != nil { @@ -446,9 +446,9 @@ func TestAddSSTableMVCCStats(t *testing.T) { } afterStats := func() enginepb.MVCCStats { - iter := e.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := e.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() - afterStats, err := engine.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 10) + afterStats, err := storage.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 10) if err != nil { t.Fatalf("%+v", err) } @@ -464,8 +464,8 @@ func TestAddSSTableMVCCStats(t *testing.T) { Header: roachpb.Header{Timestamp: hlc.Timestamp{WallTime: 7}}, Args: &roachpb.AddSSTableRequest{ RequestHeader: roachpb.RequestHeader{Key: keys.MinKey, EndKey: keys.MaxKey}, - Data: mkSST([]engine.MVCCKeyValue{{ - Key: engine.MVCCKey{Key: roachpb.Key("zzzzzzz"), Timestamp: ts}, + Data: mkSST([]storage.MVCCKeyValue{{ + Key: storage.MVCCKey{Key: roachpb.Key("zzzzzzz"), Timestamp: ts}, Value: roachpb.MakeValueFromBytes([]byte("zzz")).RawBytes, }}), MVCCStats: &enginepb.MVCCStats{KeyCount: 10}, @@ -508,9 +508,9 @@ func TestAddSSTableDisallowShadowing(t *testing.T) { } } - getSSTBytes := func(sstKVs []engine.MVCCKeyValue) []byte { - sstFile := &engine.MemFile{} - sst := engine.MakeBackupSSTWriter(sstFile) + getSSTBytes := func(sstKVs []storage.MVCCKeyValue) []byte { + sstFile := &storage.MemFile{} + sst := storage.MakeBackupSSTWriter(sstFile) defer sst.Close() for _, kv := range sstKVs { if err := sst.Put(kv.Key, kv.Value); err != nil { @@ -524,13 +524,13 @@ func TestAddSSTableDisallowShadowing(t *testing.T) { } getStats := func(startKey, endKey roachpb.Key, data []byte) enginepb.MVCCStats { - dataIter, err := engine.NewMemSSTIterator(data, true) + dataIter, err := storage.NewMemSSTIterator(data, true) if err != nil { return enginepb.MVCCStats{} } defer dataIter.Close() - stats, err := engine.ComputeStatsGo(dataIter, startKey, endKey, 0) + stats, err := storage.ComputeStatsGo(dataIter, startKey, endKey, 0) if err != nil { t.Fatalf("%+v", err) } @@ -754,7 +754,7 @@ func TestAddSSTableDisallowShadowing(t *testing.T) { ts, base.DefaultMaxClockOffset.Nanoseconds(), ) - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, e, nil, []byte("t"), ts, roachpb.MakeValueFromBytes([]byte("tt")), &txn, @@ -794,7 +794,7 @@ func TestAddSSTableDisallowShadowing(t *testing.T) { // Add in an inline value. ts := hlc.Timestamp{} - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, e, nil, []byte("i"), ts, roachpb.MakeValueFromBytes([]byte("i")), nil, diff --git a/pkg/kv/kvserver/batcheval/cmd_clear_range.go b/pkg/kv/kvserver/batcheval/cmd_clear_range.go index bccad34dc224..d214714bd780 100644 --- a/pkg/kv/kvserver/batcheval/cmd_clear_range.go +++ b/pkg/kv/kvserver/batcheval/cmd_clear_range.go @@ -14,13 +14,13 @@ import ( "context" "errors" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/kr/pretty" @@ -56,7 +56,7 @@ func declareKeysClearRange( // or queried any more, such as after a DROP or TRUNCATE table, or // DROP index. func ClearRange( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { if cArgs.Header.Txn != nil { return result.Result{}, errors.New("cannot execute ClearRange within a transaction") @@ -82,7 +82,7 @@ func ClearRange( if total := statsDelta.Total(); total < ClearRangeBytesThreshold { log.VEventf(ctx, 2, "delta=%d < threshold=%d; using non-range clear", total, ClearRangeBytesThreshold) if err := readWriter.Iterate(from, to, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { return false, readWriter.Clear(kv.Key) }, ); err != nil { @@ -103,8 +103,8 @@ func ClearRange( }, }, } - if err := readWriter.ClearRange(engine.MakeMVCCMetadataKey(from), - engine.MakeMVCCMetadataKey(to)); err != nil { + if err := readWriter.ClearRange(storage.MakeMVCCMetadataKey(from), + storage.MakeMVCCMetadataKey(to)); err != nil { return result.Result{}, err } return pd, nil @@ -119,7 +119,7 @@ func ClearRange( // path of simply subtracting the non-system values is accurate. // Returns the delta stats. func computeStatsDelta( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, from, to roachpb.Key, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, from, to roachpb.Key, ) (enginepb.MVCCStats, error) { desc := cArgs.EvalCtx.Desc() var delta enginepb.MVCCStats @@ -140,7 +140,7 @@ func computeStatsDelta( // If we can't use the fast stats path, or race test is enabled, // compute stats across the key span to be cleared. if !fast || util.RaceEnabled { - iter := readWriter.NewIterator(engine.IterOptions{UpperBound: to}) + iter := readWriter.NewIterator(storage.IterOptions{UpperBound: to}) computed, err := iter.ComputeStats(from, to, delta.LastUpdateNanos) iter.Close() if err != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_clear_range_test.go b/pkg/kv/kvserver/batcheval/cmd_clear_range_test.go index 699d03dc397c..e93a8eb6b63d 100644 --- a/pkg/kv/kvserver/batcheval/cmd_clear_range_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_clear_range_test.go @@ -18,25 +18,25 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" ) type wrappedBatch struct { - engine.Batch + storage.Batch clearCount int clearRangeCount int } -func (wb *wrappedBatch) Clear(key engine.MVCCKey) error { +func (wb *wrappedBatch) Clear(key storage.MVCCKey) error { wb.clearCount++ return wb.Batch.Clear(key) } -func (wb *wrappedBatch) ClearRange(start, end engine.MVCCKey) error { +func (wb *wrappedBatch) ClearRange(start, end storage.MVCCKey) error { wb.clearRangeCount++ return wb.Batch.ClearRange(start, end) } @@ -86,13 +86,13 @@ func TestCmdClearRangeBytesThreshold(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { ctx := context.Background() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() var stats enginepb.MVCCStats for i := 0; i < test.keyCount; i++ { key := roachpb.Key(fmt.Sprintf("%04d", i)) - if err := engine.MVCCPut(ctx, eng, &stats, key, hlc.Timestamp{WallTime: int64(i % 2)}, value, nil); err != nil { + if err := storage.MVCCPut(ctx, eng, &stats, key, hlc.Timestamp{WallTime: int64(i % 2)}, value, nil); err != nil { t.Fatal(err) } } @@ -140,7 +140,7 @@ func TestCmdClearRangeBytesThreshold(t *testing.T) { t.Fatal(err) } if err := eng.Iterate(startKey, endKey, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { return true, errors.New("expected no data in underlying engine") }, ); err != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_compute_checksum.go b/pkg/kv/kvserver/batcheval/cmd_compute_checksum.go index 35de39b2c1bf..e61735d3ab2d 100644 --- a/pkg/kv/kvserver/batcheval/cmd_compute_checksum.go +++ b/pkg/kv/kvserver/batcheval/cmd_compute_checksum.go @@ -14,11 +14,11 @@ import ( "context" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) @@ -45,7 +45,7 @@ const ( // a particular snapshot. The checksum is later verified through a // CollectChecksumRequest. func ComputeChecksum( - _ context.Context, _ engine.Reader, cArgs CommandArgs, resp roachpb.Response, + _ context.Context, _ storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ComputeChecksumRequest) diff --git a/pkg/kv/kvserver/batcheval/cmd_conditional_put.go b/pkg/kv/kvserver/batcheval/cmd_conditional_put.go index 3e6c3f7321b5..42523aa7fb14 100644 --- a/pkg/kv/kvserver/batcheval/cmd_conditional_put.go +++ b/pkg/kv/kvserver/batcheval/cmd_conditional_put.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -26,13 +26,13 @@ func init() { // the expected value matches. If not, the return value contains // the actual value. func ConditionalPut( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ConditionalPutRequest) h := cArgs.Header if h.DistinctSpans { - if b, ok := readWriter.(engine.Batch); ok { + if b, ok := readWriter.(storage.Batch); ok { // Use the distinct batch for both blind and normal ops so that we don't // accidentally flush mutations to make them visible to the distinct // batch. @@ -40,12 +40,12 @@ func ConditionalPut( defer readWriter.Close() } } - handleMissing := engine.CPutMissingBehavior(args.AllowIfDoesNotExist) + handleMissing := storage.CPutMissingBehavior(args.AllowIfDoesNotExist) var err error if args.Blind { - err = engine.MVCCBlindConditionalPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.ExpValue, handleMissing, h.Txn) + err = storage.MVCCBlindConditionalPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.ExpValue, handleMissing, h.Txn) } else { - err = engine.MVCCConditionalPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.ExpValue, handleMissing, h.Txn) + err = storage.MVCCConditionalPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.ExpValue, handleMissing, h.Txn) } // NB: even if MVCC returns an error, it may still have written an intent // into the batch. This allows callers to consume errors like WriteTooOld diff --git a/pkg/kv/kvserver/batcheval/cmd_delete.go b/pkg/kv/kvserver/batcheval/cmd_delete.go index a80466c8992b..dac6d0c0926a 100644 --- a/pkg/kv/kvserver/batcheval/cmd_delete.go +++ b/pkg/kv/kvserver/batcheval/cmd_delete.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -24,12 +24,12 @@ func init() { // Delete deletes the key and value specified by key. func Delete( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.DeleteRequest) h := cArgs.Header - err := engine.MVCCDelete(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, h.Txn) + err := storage.MVCCDelete(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, h.Txn) // NB: even if MVCC returns an error, it may still have written an intent // into the batch. This allows callers to consume errors like WriteTooOld // without re-evaluating the batch. This behavior isn't particularly diff --git a/pkg/kv/kvserver/batcheval/cmd_delete_range.go b/pkg/kv/kvserver/batcheval/cmd_delete_range.go index 0fd492624c8b..04f6af62925f 100644 --- a/pkg/kv/kvserver/batcheval/cmd_delete_range.go +++ b/pkg/kv/kvserver/batcheval/cmd_delete_range.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -41,7 +41,7 @@ func declareKeysDeleteRange( // DeleteRange deletes the range of key/value pairs specified by // start and end keys. func DeleteRange( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.DeleteRangeRequest) h := cArgs.Header @@ -55,7 +55,7 @@ func DeleteRange( // written if we're evaluating the DeleteRange for a transaction so that we // can update the Result's WrittenIntents field. returnKeys := args.ReturnKeys || h.Txn != nil - deleted, resumeSpan, num, err := engine.MVCCDeleteRange( + deleted, resumeSpan, num, err := storage.MVCCDeleteRange( ctx, readWriter, cArgs.Stats, args.Key, args.EndKey, h.MaxSpanRequestKeys, timestamp, h.Txn, returnKeys, ) if err == nil && args.ReturnKeys { diff --git a/pkg/kv/kvserver/batcheval/cmd_end_transaction.go b/pkg/kv/kvserver/batcheval/cmd_end_transaction.go index cd9aa5138d89..318ed1754ced 100644 --- a/pkg/kv/kvserver/batcheval/cmd_end_transaction.go +++ b/pkg/kv/kvserver/batcheval/cmd_end_transaction.go @@ -18,8 +18,6 @@ import ( "sync/atomic" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" @@ -30,6 +28,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/tracing" @@ -168,7 +168,7 @@ func declareKeysEndTxn( // TODO(nvanbenschoten): rename this file to cmd_end_txn.go once some of andrei's // recent PRs have landed. func EndTxn( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.EndTxnRequest) h := cArgs.Header @@ -193,8 +193,8 @@ func EndTxn( // Fetch existing transaction. var existingTxn roachpb.Transaction - if ok, err := engine.MVCCGetProto( - ctx, readWriter, key, hlc.Timestamp{}, &existingTxn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, readWriter, key, hlc.Timestamp{}, &existingTxn, storage.MVCCGetOptions{}, ); err != nil { return result.Result{}, err } else if !ok { @@ -320,7 +320,7 @@ func EndTxn( // during startup, to infer that any lingering intents belong to in-progress // transactions and thus the pre-intent value can safely be used. if mt := args.InternalCommitTrigger.GetMergeTrigger(); mt != nil { - mergeResult, err := mergeTrigger(ctx, cArgs.EvalCtx, readWriter.(engine.Batch), + mergeResult, err := mergeTrigger(ctx, cArgs.EvalCtx, readWriter.(storage.Batch), ms, mt, reply.Txn.WriteTimestamp) if err != nil { return result.Result{}, err @@ -349,7 +349,7 @@ func EndTxn( // Run the rest of the commit triggers if successfully committed. if reply.Txn.Status == roachpb.COMMITTED { - triggerResult, err := RunCommitTrigger(ctx, cArgs.EvalCtx, readWriter.(engine.Batch), + triggerResult, err := RunCommitTrigger(ctx, cArgs.EvalCtx, readWriter.(storage.Batch), ms, args, reply.Txn) if err != nil { return result.Result{}, roachpb.NewReplicaCorruptionError(err) @@ -456,7 +456,7 @@ const intentResolutionBatchSize = 500 func resolveLocalIntents( ctx context.Context, desc *roachpb.RangeDescriptor, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, args *roachpb.EndTxnRequest, txn *roachpb.Transaction, @@ -469,10 +469,10 @@ func resolveLocalIntents( desc = &mergeTrigger.LeftDesc } - iter := readWriter.NewIterator(engine.IterOptions{ + iter := readWriter.NewIterator(storage.IterOptions{ UpperBound: desc.EndKey.AsRawKey(), }) - iterAndBuf := engine.GetBufUsingIter(iter) + iterAndBuf := storage.GetBufUsingIter(iter) defer iterAndBuf.Cleanup() var resolveAllowance int64 = intentResolutionBatchSize @@ -496,7 +496,7 @@ func resolveLocalIntents( return nil } resolveMS := ms - ok, err := engine.MVCCResolveWriteIntentUsingIter(ctx, readWriter, iterAndBuf, resolveMS, intent) + ok, err := storage.MVCCResolveWriteIntentUsingIter(ctx, readWriter, iterAndBuf, resolveMS, intent) if err != nil { return err } @@ -513,7 +513,7 @@ func resolveLocalIntents( externalIntents = append(externalIntents, outSpans...) if inSpan != nil { intent.Span = *inSpan - num, resumeSpan, err := engine.MVCCResolveWriteIntentRangeUsingIter(ctx, readWriter, iterAndBuf, ms, intent, resolveAllowance) + num, resumeSpan, err := storage.MVCCResolveWriteIntentRangeUsingIter(ctx, readWriter, iterAndBuf, ms, intent, resolveAllowance) if err != nil { return err } @@ -552,7 +552,7 @@ func resolveLocalIntents( // remote) intents. func updateStagingTxn( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, key []byte, args *roachpb.EndTxnRequest, @@ -561,7 +561,7 @@ func updateStagingTxn( txn.IntentSpans = args.IntentSpans txn.InFlightWrites = args.InFlightWrites txnRecord := txn.AsRecord() - return engine.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, &txnRecord) + return storage.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, &txnRecord) } // updateFinalizedTxn persists the COMMITTED or ABORTED transaction record with @@ -570,7 +570,7 @@ func updateStagingTxn( // it around. func updateFinalizedTxn( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, key []byte, args *roachpb.EndTxnRequest, @@ -581,19 +581,19 @@ func updateFinalizedTxn( if log.V(2) { log.Infof(ctx, "auto-gc'ed %s (%d intents)", txn.Short(), len(args.IntentSpans)) } - return engine.MVCCDelete(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */) + return storage.MVCCDelete(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */) } txn.IntentSpans = externalIntents txn.InFlightWrites = nil txnRecord := txn.AsRecord() - return engine.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, &txnRecord) + return storage.MVCCPutProto(ctx, readWriter, ms, key, hlc.Timestamp{}, nil /* txn */, &txnRecord) } // RunCommitTrigger runs the commit trigger from an end transaction request. func RunCommitTrigger( ctx context.Context, rec EvalContext, - batch engine.Batch, + batch storage.Batch, ms *enginepb.MVCCStats, args *roachpb.EndTxnRequest, txn *roachpb.Transaction, @@ -834,7 +834,7 @@ func RunCommitTrigger( func splitTrigger( ctx context.Context, rec EvalContext, - batch engine.Batch, + batch storage.Batch, bothDeltaMS enginepb.MVCCStats, split *roachpb.SplitTrigger, ts hlc.Timestamp, @@ -884,7 +884,7 @@ func splitTrigger( func splitTriggerHelper( ctx context.Context, rec EvalContext, - batch engine.Batch, + batch storage.Batch, statsInput splitStatsHelperInput, split *roachpb.SplitTrigger, ts hlc.Timestamp, @@ -903,7 +903,7 @@ func splitTriggerHelper( if err != nil { return enginepb.MVCCStats{}, result.Result{}, errors.Wrap(err, "unable to fetch last replica GC timestamp") } - if err := engine.MVCCPutProto(ctx, batch, nil, keys.RangeLastReplicaGCTimestampKey(split.RightDesc.RangeID), hlc.Timestamp{}, nil, &replicaGCTS); err != nil { + if err := storage.MVCCPutProto(ctx, batch, nil, keys.RangeLastReplicaGCTimestampKey(split.RightDesc.RangeID), hlc.Timestamp{}, nil, &replicaGCTS); err != nil { return enginepb.MVCCStats{}, result.Result{}, errors.Wrap(err, "unable to copy last replica GC timestamp") } @@ -984,13 +984,13 @@ func splitTriggerHelper( // // See VersionUnreplicatedRaftTruncatedState. truncStateType := stateloader.TruncatedStateUnreplicated - if found, err := engine.MVCCGetProto( + if found, err := storage.MVCCGetProto( ctx, batch, keys.RaftTruncatedStateLegacyKey(rec.GetRangeID()), hlc.Timestamp{}, nil, - engine.MVCCGetOptions{}, + storage.MVCCGetOptions{}, ); err != nil { return enginepb.MVCCStats{}, result.Result{}, errors.Wrap(err, "unable to load legacy truncated state") } else if found { @@ -1057,7 +1057,7 @@ func splitTriggerHelper( func mergeTrigger( ctx context.Context, rec EvalContext, - batch engine.Batch, + batch storage.Batch, ms *enginepb.MVCCStats, merge *roachpb.MergeTrigger, ts hlc.Timestamp, @@ -1085,7 +1085,7 @@ func mergeTrigger( ms.Add(merge.RightMVCCStats) { ridPrefix := keys.MakeRangeIDReplicatedPrefix(merge.RightDesc.RangeID) - iter := batch.NewIterator(engine.IterOptions{UpperBound: ridPrefix.PrefixEnd()}) + iter := batch.NewIterator(storage.IterOptions{UpperBound: ridPrefix.PrefixEnd()}) defer iter.Close() sysMS, err := iter.ComputeStats(ridPrefix, ridPrefix.PrefixEnd(), 0 /* nowNanos */) if err != nil { @@ -1102,7 +1102,7 @@ func mergeTrigger( } func changeReplicasTrigger( - _ context.Context, rec EvalContext, _ engine.Batch, change *roachpb.ChangeReplicasTrigger, + _ context.Context, rec EvalContext, _ storage.Batch, change *roachpb.ChangeReplicasTrigger, ) result.Result { var pd result.Result // After a successful replica addition or removal check to see if the diff --git a/pkg/kv/kvserver/batcheval/cmd_end_transaction_test.go b/pkg/kv/kvserver/batcheval/cmd_end_transaction_test.go index 8b5bf9c0ef97..82e5ca480f3f 100644 --- a/pkg/kv/kvserver/batcheval/cmd_end_transaction_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_end_transaction_test.go @@ -15,11 +15,11 @@ import ( "regexp" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -870,7 +870,7 @@ func TestEndTxnUpdatesTransactionRecord(t *testing.T) { } for _, c := range testCases { t.Run(c.name, func(t *testing.T) { - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() batch := db.NewBatch() defer batch.Close() @@ -878,7 +878,7 @@ func TestEndTxnUpdatesTransactionRecord(t *testing.T) { // Write the existing transaction record, if necessary. txnKey := keys.TransactionKey(txn.Key, txn.ID) if c.existingTxn != nil { - if err := engine.MVCCPutProto(ctx, batch, nil, txnKey, hlc.Timestamp{}, nil, c.existingTxn); err != nil { + if err := storage.MVCCPutProto(ctx, batch, nil, txnKey, hlc.Timestamp{}, nil, c.existingTxn); err != nil { t.Fatal(err) } } @@ -931,8 +931,8 @@ func TestEndTxnUpdatesTransactionRecord(t *testing.T) { // Assert that the txn record is written as expected. var resTxnRecord roachpb.TransactionRecord - if ok, err := engine.MVCCGetProto( - ctx, batch, txnKey, hlc.Timestamp{}, &resTxnRecord, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, batch, txnKey, hlc.Timestamp{}, &resTxnRecord, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } else if c.expTxn == nil { @@ -971,7 +971,7 @@ func TestPartialRollbackOnEndTransaction(t *testing.T) { defer TestingSetTxnAutoGC(false)() testutils.RunTrueAndFalse(t, "withStoredTxnRecord", func(t *testing.T, storeTxnBeforeEndTxn bool) { - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() batch := db.NewBatch() defer batch.Close() @@ -981,13 +981,13 @@ func TestPartialRollbackOnEndTransaction(t *testing.T) { // Write a first value at key. v.SetString("a") txn.Sequence = 1 - if err := engine.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { t.Fatal(err) } // Write another value. v.SetString("b") txn.Sequence = 2 - if err := engine.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { t.Fatal(err) } @@ -1000,7 +1000,7 @@ func TestPartialRollbackOnEndTransaction(t *testing.T) { txnKey := keys.TransactionKey(txn.Key, txn.ID) if storeTxnBeforeEndTxn { txnRec := txn.AsRecord() - if err := engine.MVCCPutProto(ctx, batch, nil, txnKey, hlc.Timestamp{}, nil, &txnRec); err != nil { + if err := storage.MVCCPutProto(ctx, batch, nil, txnKey, hlc.Timestamp{}, nil, &txnRec); err != nil { t.Fatal(err) } } @@ -1031,7 +1031,7 @@ func TestPartialRollbackOnEndTransaction(t *testing.T) { // The second write has been rolled back; verify that the remaining // value is from the first write. - res, i, err := engine.MVCCGet(ctx, batch, k, ts2, engine.MVCCGetOptions{}) + res, i, err := storage.MVCCGet(ctx, batch, k, ts2, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } @@ -1050,7 +1050,7 @@ func TestPartialRollbackOnEndTransaction(t *testing.T) { // Also verify that the txn record contains the ignore list. var txnRec roachpb.TransactionRecord - hasRec, err := engine.MVCCGetProto(ctx, batch, txnKey, hlc.Timestamp{}, &txnRec, engine.MVCCGetOptions{}) + hasRec, err := storage.MVCCGetProto(ctx, batch, txnKey, hlc.Timestamp{}, &txnRec, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/batcheval/cmd_gc.go b/pkg/kv/kvserver/batcheval/cmd_gc.go index 37c8e9395d9a..db400a578ddf 100644 --- a/pkg/kv/kvserver/batcheval/cmd_gc.go +++ b/pkg/kv/kvserver/batcheval/cmd_gc.go @@ -13,12 +13,12 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -56,7 +56,7 @@ func declareKeysGC( // listed key along with the expiration timestamp. The GC metadata // specified in the args is persisted after GC. func GC( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.GCRequest) h := cArgs.Header @@ -74,7 +74,7 @@ func GC( } // Garbage collect the specified keys by expiration timestamps. - if err := engine.MVCCGarbageCollect( + if err := storage.MVCCGarbageCollect( ctx, readWriter, cArgs.Stats, keys, h.Timestamp, ); err != nil { return result.Result{}, err diff --git a/pkg/kv/kvserver/batcheval/cmd_get.go b/pkg/kv/kvserver/batcheval/cmd_get.go index c5750a0026d8..7f778606a332 100644 --- a/pkg/kv/kvserver/batcheval/cmd_get.go +++ b/pkg/kv/kvserver/batcheval/cmd_get.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" ) @@ -25,13 +25,13 @@ func init() { // Get returns the value for a specified key. func Get( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.GetRequest) h := cArgs.Header reply := resp.(*roachpb.GetResponse) - val, intent, err := engine.MVCCGet(ctx, reader, args.Key, h.Timestamp, engine.MVCCGetOptions{ + val, intent, err := storage.MVCCGet(ctx, reader, args.Key, h.Timestamp, storage.MVCCGetOptions{ Inconsistent: h.ReadConsistency != roachpb.CONSISTENT, Txn: h.Txn, }) diff --git a/pkg/kv/kvserver/batcheval/cmd_heartbeat_txn.go b/pkg/kv/kvserver/batcheval/cmd_heartbeat_txn.go index a055ff8af8f4..868f9801e67b 100644 --- a/pkg/kv/kvserver/batcheval/cmd_heartbeat_txn.go +++ b/pkg/kv/kvserver/batcheval/cmd_heartbeat_txn.go @@ -14,11 +14,11 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -39,7 +39,7 @@ func declareKeysHeartbeatTransaction( // timestamp after receiving transaction heartbeat messages from // coordinator. Returns the updated transaction. func HeartbeatTxn( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.HeartbeatTxnRequest) h := cArgs.Header @@ -56,8 +56,8 @@ func HeartbeatTxn( key := keys.TransactionKey(h.Txn.Key, h.Txn.ID) var txn roachpb.Transaction - if ok, err := engine.MVCCGetProto( - ctx, readWriter, key, hlc.Timestamp{}, &txn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, readWriter, key, hlc.Timestamp{}, &txn, storage.MVCCGetOptions{}, ); err != nil { return result.Result{}, err } else if !ok { @@ -77,7 +77,7 @@ func HeartbeatTxn( // is up for debate. txn.LastHeartbeat.Forward(args.Now) txnRecord := txn.AsRecord() - if err := engine.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { + if err := storage.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { return result.Result{}, err } } diff --git a/pkg/kv/kvserver/batcheval/cmd_increment.go b/pkg/kv/kvserver/batcheval/cmd_increment.go index 64a92d7468c7..914ff571c591 100644 --- a/pkg/kv/kvserver/batcheval/cmd_increment.go +++ b/pkg/kv/kvserver/batcheval/cmd_increment.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -26,13 +26,13 @@ func init() { // returns the newly incremented value (encoded as varint64). If no value // exists for the key, zero is incremented. func Increment( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.IncrementRequest) h := cArgs.Header reply := resp.(*roachpb.IncrementResponse) - newVal, err := engine.MVCCIncrement(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, h.Txn, args.Increment) + newVal, err := storage.MVCCIncrement(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, h.Txn, args.Increment) reply.NewValue = newVal // NB: even if MVCC returns an error, it may still have written an intent // into the batch. This allows callers to consume errors like WriteTooOld diff --git a/pkg/kv/kvserver/batcheval/cmd_init_put.go b/pkg/kv/kvserver/batcheval/cmd_init_put.go index 366fa82f7747..8734475ed870 100644 --- a/pkg/kv/kvserver/batcheval/cmd_init_put.go +++ b/pkg/kv/kvserver/batcheval/cmd_init_put.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -27,13 +27,13 @@ func init() { // is different from the value provided. If FailOnTombstone is set to true, // tombstones count as mismatched values and will cause a ConditionFailedError. func InitPut( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.InitPutRequest) h := cArgs.Header if h.DistinctSpans { - if b, ok := readWriter.(engine.Batch); ok { + if b, ok := readWriter.(storage.Batch); ok { // Use the distinct batch for both blind and normal ops so that we don't // accidentally flush mutations to make them visible to the distinct // batch. @@ -43,9 +43,9 @@ func InitPut( } var err error if args.Blind { - err = engine.MVCCBlindInitPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.FailOnTombstones, h.Txn) + err = storage.MVCCBlindInitPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.FailOnTombstones, h.Txn) } else { - err = engine.MVCCInitPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.FailOnTombstones, h.Txn) + err = storage.MVCCInitPut(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value, args.FailOnTombstones, h.Txn) } // NB: even if MVCC returns an error, it may still have written an intent // into the batch. This allows callers to consume errors like WriteTooOld diff --git a/pkg/kv/kvserver/batcheval/cmd_lease.go b/pkg/kv/kvserver/batcheval/cmd_lease.go index 28455036638c..f44519f02e00 100644 --- a/pkg/kv/kvserver/batcheval/cmd_lease.go +++ b/pkg/kv/kvserver/batcheval/cmd_lease.go @@ -14,13 +14,13 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/errors" ) @@ -93,7 +93,7 @@ func checkCanReceiveLease(newLease *roachpb.Lease, rec EvalContext) error { func evalNewLease( ctx context.Context, rec EvalContext, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, lease roachpb.Lease, prevLease roachpb.Lease, diff --git a/pkg/kv/kvserver/batcheval/cmd_lease_info.go b/pkg/kv/kvserver/batcheval/cmd_lease_info.go index cc37ede12258..c94f510054b8 100644 --- a/pkg/kv/kvserver/batcheval/cmd_lease_info.go +++ b/pkg/kv/kvserver/batcheval/cmd_lease_info.go @@ -13,11 +13,11 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -35,7 +35,7 @@ func declareKeysLeaseInfo( // LeaseInfo returns information about the lease holder for the range. func LeaseInfo( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { reply := resp.(*roachpb.LeaseInfoResponse) lease, nextLease := cArgs.EvalCtx.GetLease() diff --git a/pkg/kv/kvserver/batcheval/cmd_lease_request.go b/pkg/kv/kvserver/batcheval/cmd_lease_request.go index 0fb0fe2a783e..323a140822a9 100644 --- a/pkg/kv/kvserver/batcheval/cmd_lease_request.go +++ b/pkg/kv/kvserver/batcheval/cmd_lease_request.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -30,7 +30,7 @@ func init() { // lease, all duties required of the range lease holder are commenced, including // releasing all latches and clearing the timestamp cache. func RequestLease( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { // When returning an error from this method, must always return a // newFailedLeaseTrigger() to satisfy stats. diff --git a/pkg/kv/kvserver/batcheval/cmd_lease_transfer.go b/pkg/kv/kvserver/batcheval/cmd_lease_transfer.go index 4a116cc0b819..deeb726228b2 100644 --- a/pkg/kv/kvserver/batcheval/cmd_lease_transfer.go +++ b/pkg/kv/kvserver/batcheval/cmd_lease_transfer.go @@ -13,11 +13,11 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" ) @@ -46,7 +46,7 @@ func init() { // ex-) lease holder which must have dropped all of its lease holder powers // before proposing. func TransferLease( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { // When returning an error from this method, must always return // a newFailedLeaseTrigger() to satisfy stats. diff --git a/pkg/kv/kvserver/batcheval/cmd_merge.go b/pkg/kv/kvserver/batcheval/cmd_merge.go index 33a7ecfc5ea7..b13a2c18a84b 100644 --- a/pkg/kv/kvserver/batcheval/cmd_merge.go +++ b/pkg/kv/kvserver/batcheval/cmd_merge.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -29,10 +29,10 @@ func init() { // transactional, merges are not currently exposed directly to // clients. Merged values are explicitly not MVCC data. func Merge( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.MergeRequest) h := cArgs.Header - return result.Result{}, engine.MVCCMerge(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value) + return result.Result{}, storage.MVCCMerge(ctx, readWriter, cArgs.Stats, args.Key, h.Timestamp, args.Value) } diff --git a/pkg/kv/kvserver/batcheval/cmd_push_txn.go b/pkg/kv/kvserver/batcheval/cmd_push_txn.go index f236e38cfdc3..5a17284145d6 100644 --- a/pkg/kv/kvserver/batcheval/cmd_push_txn.go +++ b/pkg/kv/kvserver/batcheval/cmd_push_txn.go @@ -14,12 +14,12 @@ import ( "bytes" "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" @@ -105,7 +105,7 @@ func declareKeysPushTransaction( // records for which the transaction coordinator must have found out via // its heartbeats that the transaction has failed. func PushTxn( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.PushTxnRequest) h := cArgs.Header @@ -138,7 +138,7 @@ func PushTxn( // Fetch existing transaction; if missing, we're allowed to abort. var existTxn roachpb.Transaction - ok, err := engine.MVCCGetProto(ctx, readWriter, key, hlc.Timestamp{}, &existTxn, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, readWriter, key, hlc.Timestamp{}, &existTxn, storage.MVCCGetOptions{}) if err != nil { return result.Result{}, err } else if !ok { @@ -306,7 +306,7 @@ func PushTxn( // in the timestamp cache. if ok { txnRecord := reply.PusheeTxn.AsRecord() - if err := engine.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { + if err := storage.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { return result.Result{}, err } } diff --git a/pkg/kv/kvserver/batcheval/cmd_put.go b/pkg/kv/kvserver/batcheval/cmd_put.go index 0f37b4fcf034..b95126023e7c 100644 --- a/pkg/kv/kvserver/batcheval/cmd_put.go +++ b/pkg/kv/kvserver/batcheval/cmd_put.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -40,7 +40,7 @@ func declareKeysPut( // Put sets the value for a specified key. func Put( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.PutRequest) h := cArgs.Header @@ -51,7 +51,7 @@ func Put( ts = h.Timestamp } if h.DistinctSpans { - if b, ok := readWriter.(engine.Batch); ok { + if b, ok := readWriter.(storage.Batch); ok { // Use the distinct batch for both blind and normal ops so that we don't // accidentally flush mutations to make them visible to the distinct // batch. @@ -61,9 +61,9 @@ func Put( } var err error if args.Blind { - err = engine.MVCCBlindPut(ctx, readWriter, ms, args.Key, ts, args.Value, h.Txn) + err = storage.MVCCBlindPut(ctx, readWriter, ms, args.Key, ts, args.Value, h.Txn) } else { - err = engine.MVCCPut(ctx, readWriter, ms, args.Key, ts, args.Value, h.Txn) + err = storage.MVCCPut(ctx, readWriter, ms, args.Key, ts, args.Value, h.Txn) } // NB: even if MVCC returns an error, it may still have written an intent // into the batch. This allows callers to consume errors like WriteTooOld diff --git a/pkg/kv/kvserver/batcheval/cmd_query_intent.go b/pkg/kv/kvserver/batcheval/cmd_query_intent.go index 332effe91c19..ce0b3d752823 100644 --- a/pkg/kv/kvserver/batcheval/cmd_query_intent.go +++ b/pkg/kv/kvserver/batcheval/cmd_query_intent.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -46,7 +46,7 @@ func declareKeysQueryIntent( // request is special-cased to return a SERIALIZABLE retry error if a transaction // queries its own intent and finds it has been pushed. func QueryIntent( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.QueryIntentRequest) h := cArgs.Header @@ -55,7 +55,7 @@ func QueryIntent( // Read at the specified key at the maximum timestamp. This ensures that we // see an intent if one exists, regardless of what timestamp it is written // at. - _, intent, err := engine.MVCCGet(ctx, reader, args.Key, hlc.MaxTimestamp, engine.MVCCGetOptions{ + _, intent, err := storage.MVCCGet(ctx, reader, args.Key, hlc.MaxTimestamp, storage.MVCCGetOptions{ // Perform an inconsistent read so that intents are returned instead of // causing WriteIntentErrors. Inconsistent: true, diff --git a/pkg/kv/kvserver/batcheval/cmd_query_txn.go b/pkg/kv/kvserver/batcheval/cmd_query_txn.go index 2899e7d00397..a62804eac07c 100644 --- a/pkg/kv/kvserver/batcheval/cmd_query_txn.go +++ b/pkg/kv/kvserver/batcheval/cmd_query_txn.go @@ -14,11 +14,11 @@ import ( "bytes" "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/pkg/errors" ) @@ -45,7 +45,7 @@ func declareKeysQueryTransaction( // other txns which are waiting on this transaction in order // to find dependency cycles. func QueryTxn( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.QueryTxnRequest) h := cArgs.Header @@ -64,8 +64,8 @@ func QueryTxn( key := keys.TransactionKey(args.Txn.Key, args.Txn.ID) // Fetch transaction record; if missing, attempt to synthesize one. - if ok, err := engine.MVCCGetProto( - ctx, reader, key, hlc.Timestamp{}, &reply.QueriedTxn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, reader, key, hlc.Timestamp{}, &reply.QueriedTxn, storage.MVCCGetOptions{}, ); err != nil { return result.Result{}, err } else if !ok { diff --git a/pkg/kv/kvserver/batcheval/cmd_range_stats.go b/pkg/kv/kvserver/batcheval/cmd_range_stats.go index 2a9abf5e5249..3f28db7d5436 100644 --- a/pkg/kv/kvserver/batcheval/cmd_range_stats.go +++ b/pkg/kv/kvserver/batcheval/cmd_range_stats.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -24,7 +24,7 @@ func init() { // RangeStats returns the MVCC statistics for a range. func RangeStats( - _ context.Context, _ engine.Reader, cArgs CommandArgs, resp roachpb.Response, + _ context.Context, _ storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { reply := resp.(*roachpb.RangeStatsResponse) reply.MVCCStats = cArgs.EvalCtx.GetMVCCStats() diff --git a/pkg/kv/kvserver/batcheval/cmd_recompute_stats.go b/pkg/kv/kvserver/batcheval/cmd_recompute_stats.go index 6e06734aff3a..cad470533ffc 100644 --- a/pkg/kv/kvserver/batcheval/cmd_recompute_stats.go +++ b/pkg/kv/kvserver/batcheval/cmd_recompute_stats.go @@ -14,13 +14,13 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/pkg/errors" ) @@ -57,7 +57,7 @@ func declareKeysRecomputeStats( // RecomputeStats recomputes the MVCCStats stored for this range and adjust them accordingly, // returning the MVCCStats delta obtained in the process. func RecomputeStats( - ctx context.Context, _ engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, _ storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { desc := cArgs.EvalCtx.Desc() args := cArgs.Args.(*roachpb.RecomputeStatsRequest) diff --git a/pkg/kv/kvserver/batcheval/cmd_recover_txn.go b/pkg/kv/kvserver/batcheval/cmd_recover_txn.go index 3a4273277ccb..ec9ed693e7c0 100644 --- a/pkg/kv/kvserver/batcheval/cmd_recover_txn.go +++ b/pkg/kv/kvserver/batcheval/cmd_recover_txn.go @@ -15,11 +15,11 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/pkg/errors" ) @@ -50,7 +50,7 @@ func declareKeysRecoverTransaction( // result of the recovery should be committing the abandoned transaction or // aborting it. func RecoverTxn( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.RecoverTxnRequest) h := cArgs.Header @@ -69,8 +69,8 @@ func RecoverTxn( key := keys.TransactionKey(args.Txn.Key, args.Txn.ID) // Fetch transaction record; if missing, attempt to synthesize one. - if ok, err := engine.MVCCGetProto( - ctx, readWriter, key, hlc.Timestamp{}, &reply.RecoveredTxn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, readWriter, key, hlc.Timestamp{}, &reply.RecoveredTxn, storage.MVCCGetOptions{}, ); err != nil { return result.Result{}, err } else if !ok { @@ -216,7 +216,7 @@ func RecoverTxn( reply.RecoveredTxn.Status = roachpb.ABORTED } txnRecord := reply.RecoveredTxn.AsRecord() - if err := engine.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { + if err := storage.MVCCPutProto(ctx, readWriter, cArgs.Stats, key, hlc.Timestamp{}, nil, &txnRecord); err != nil { return result.Result{}, err } diff --git a/pkg/kv/kvserver/batcheval/cmd_recover_txn_test.go b/pkg/kv/kvserver/batcheval/cmd_recover_txn_test.go index 0acb129d3724..4021bd1d1c62 100644 --- a/pkg/kv/kvserver/batcheval/cmd_recover_txn_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_recover_txn_test.go @@ -14,9 +14,9 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -40,13 +40,13 @@ func TestRecoverTxn(t *testing.T) { txn.InFlightWrites = []roachpb.SequencedWrite{{Key: k2, Sequence: 0}} testutils.RunTrueAndFalse(t, "missing write", func(t *testing.T, missingWrite bool) { - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() // Write the transaction record. txnKey := keys.TransactionKey(txn.Key, txn.ID) txnRecord := txn.AsRecord() - if err := engine.MVCCPutProto(ctx, db, nil, txnKey, hlc.Timestamp{}, nil, &txnRecord); err != nil { + if err := storage.MVCCPutProto(ctx, db, nil, txnKey, hlc.Timestamp{}, nil, &txnRecord); err != nil { t.Fatal(err) } @@ -81,8 +81,8 @@ func TestRecoverTxn(t *testing.T) { // Assert that the updated txn record was persisted correctly. var resTxnRecord roachpb.Transaction - if _, err := engine.MVCCGetProto( - ctx, db, txnKey, hlc.Timestamp{}, &resTxnRecord, engine.MVCCGetOptions{}, + if _, err := storage.MVCCGetProto( + ctx, db, txnKey, hlc.Timestamp{}, &resTxnRecord, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } @@ -191,7 +191,7 @@ func TestRecoverTxnRecordChanged(t *testing.T) { } for _, c := range testCases { t.Run(c.name, func(t *testing.T) { - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() // Write the modified transaction record, simulating a concurrent @@ -199,7 +199,7 @@ func TestRecoverTxnRecordChanged(t *testing.T) { // request is evaluated. txnKey := keys.TransactionKey(txn.Key, txn.ID) txnRecord := c.changedTxn.AsRecord() - if err := engine.MVCCPutProto(ctx, db, nil, txnKey, hlc.Timestamp{}, nil, &txnRecord); err != nil { + if err := storage.MVCCPutProto(ctx, db, nil, txnKey, hlc.Timestamp{}, nil, &txnRecord); err != nil { t.Fatal(err) } @@ -232,8 +232,8 @@ func TestRecoverTxnRecordChanged(t *testing.T) { // Assert that the txn record was not modified. var resTxnRecord roachpb.Transaction - if _, err := engine.MVCCGetProto( - ctx, db, txnKey, hlc.Timestamp{}, &resTxnRecord, engine.MVCCGetOptions{}, + if _, err := storage.MVCCGetProto( + ctx, db, txnKey, hlc.Timestamp{}, &resTxnRecord, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/batcheval/cmd_refresh.go b/pkg/kv/kvserver/batcheval/cmd_refresh.go index b532e2e6ec16..a7e6ded92086 100644 --- a/pkg/kv/kvserver/batcheval/cmd_refresh.go +++ b/pkg/kv/kvserver/batcheval/cmd_refresh.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" ) @@ -27,7 +27,7 @@ func init() { // Refresh checks whether the key has any values written in the interval // [args.RefreshFrom, header.Timestamp]. func Refresh( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.RefreshRequest) h := cArgs.Header @@ -55,7 +55,7 @@ func Refresh( // specifying consistent=false. Note that we include tombstones, // which must be considered as updates on refresh. log.VEventf(ctx, 2, "refresh %s @[%s-%s]", args.Span(), refreshFrom, refreshTo) - val, intent, err := engine.MVCCGet(ctx, reader, args.Key, refreshTo, engine.MVCCGetOptions{ + val, intent, err := storage.MVCCGet(ctx, reader, args.Key, refreshTo, storage.MVCCGetOptions{ Inconsistent: true, Tombstones: true, }) diff --git a/pkg/kv/kvserver/batcheval/cmd_refresh_range.go b/pkg/kv/kvserver/batcheval/cmd_refresh_range.go index 44ce2f3efd91..0da3617762e7 100644 --- a/pkg/kv/kvserver/batcheval/cmd_refresh_range.go +++ b/pkg/kv/kvserver/batcheval/cmd_refresh_range.go @@ -13,9 +13,9 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" ) @@ -27,7 +27,7 @@ func init() { // RefreshRange checks whether the key range specified has any values written in // the interval [args.RefreshFrom, header.Timestamp]. func RefreshRange( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.RefreshRangeRequest) h := cArgs.Header @@ -58,9 +58,9 @@ func RefreshRange( // committed version. Note also that we include tombstones, which must be // considered as updates on refresh. log.VEventf(ctx, 2, "refresh %s @[%s-%s]", args.Span(), refreshFrom, refreshTo) - intents, err := engine.MVCCIterate( + intents, err := storage.MVCCIterate( ctx, reader, args.Key, args.EndKey, refreshTo, - engine.MVCCScanOptions{ + storage.MVCCScanOptions{ Inconsistent: true, Tombstones: true, }, diff --git a/pkg/kv/kvserver/batcheval/cmd_refresh_range_test.go b/pkg/kv/kvserver/batcheval/cmd_refresh_range_test.go index cd1fabe8a4b3..bf34cbe966e4 100644 --- a/pkg/kv/kvserver/batcheval/cmd_refresh_range_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_refresh_range_test.go @@ -14,10 +14,10 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -53,7 +53,7 @@ func TestRefreshRangeTimeBoundIterator(t *testing.T) { ts3 := hlc.Timestamp{WallTime: 3} ts4 := hlc.Timestamp{WallTime: 4} - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() // Create an sstable containing an unresolved intent. @@ -66,10 +66,10 @@ func TestRefreshRangeTimeBoundIterator(t *testing.T) { }, ReadTimestamp: ts1, } - if err := engine.MVCCPut(ctx, db, nil, k, txn.ReadTimestamp, v, txn); err != nil { + if err := storage.MVCCPut(ctx, db, nil, k, txn.ReadTimestamp, v, txn); err != nil { t.Fatal(err) } - if err := engine.MVCCPut(ctx, db, nil, roachpb.Key("unused1"), ts4, v, nil); err != nil { + if err := storage.MVCCPut(ctx, db, nil, roachpb.Key("unused1"), ts4, v, nil); err != nil { t.Fatal(err) } if err := db.Flush(); err != nil { @@ -85,10 +85,10 @@ func TestRefreshRangeTimeBoundIterator(t *testing.T) { // would not have any timestamp bounds and would be selected for every read. intent := roachpb.MakeLockUpdate(txn, roachpb.Span{Key: k}) intent.Status = roachpb.COMMITTED - if _, err := engine.MVCCResolveWriteIntent(ctx, db, nil, intent); err != nil { + if _, err := storage.MVCCResolveWriteIntent(ctx, db, nil, intent); err != nil { t.Fatal(err) } - if err := engine.MVCCPut(ctx, db, nil, roachpb.Key("unused2"), ts1, v, nil); err != nil { + if err := storage.MVCCPut(ctx, db, nil, roachpb.Key("unused2"), ts1, v, nil); err != nil { t.Fatal(err) } if err := db.Flush(); err != nil { @@ -96,7 +96,7 @@ func TestRefreshRangeTimeBoundIterator(t *testing.T) { } // TODO(peter): Make this work for Pebble as well. - if rocksDB, ok := db.(*engine.RocksDB); ok { + if rocksDB, ok := db.(*storage.RocksDB); ok { // Double-check that we've created the SSTs we intended to. userProps, err := rocksDB.GetUserProperties() if err != nil { @@ -114,7 +114,7 @@ func TestRefreshRangeTimeBoundIterator(t *testing.T) { // have previously performed a consistent read at the lower time-bound to // prove that there are no intents present that would be missed by the time- // bound iterator. - if val, intent, err := engine.MVCCGet(ctx, db, k, ts1, engine.MVCCGetOptions{}); err != nil { + if val, intent, err := storage.MVCCGet(ctx, db, k, ts1, storage.MVCCGetOptions{}); err != nil { t.Fatal(err) } else if intent != nil { t.Fatalf("got unexpected intent: %v", intent) diff --git a/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go b/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go index 5cc031362c92..12d9648f9fa3 100644 --- a/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go +++ b/pkg/kv/kvserver/batcheval/cmd_resolve_intent.go @@ -13,11 +13,11 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) @@ -76,7 +76,7 @@ func resolveToMetricType(status roachpb.TransactionStatus, poison bool) *result. // ResolveIntent resolves a write intent from the specified key // according to the status of the transaction which created it. func ResolveIntent( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ResolveIntentRequest) h := cArgs.Header @@ -87,7 +87,7 @@ func ResolveIntent( } update := args.AsLockUpdate() - ok, err := engine.MVCCResolveWriteIntent(ctx, readWriter, ms, update) + ok, err := storage.MVCCResolveWriteIntent(ctx, readWriter, ms, update) if err != nil { return result.Result{}, err } diff --git a/pkg/kv/kvserver/batcheval/cmd_resolve_intent_range.go b/pkg/kv/kvserver/batcheval/cmd_resolve_intent_range.go index 212e2f6799bc..847bee9cab97 100644 --- a/pkg/kv/kvserver/batcheval/cmd_resolve_intent_range.go +++ b/pkg/kv/kvserver/batcheval/cmd_resolve_intent_range.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -35,7 +35,7 @@ func declareKeysResolveIntentRange( // ResolveIntentRange resolves write intents in the specified // key range according to the status of the transaction which created it. func ResolveIntentRange( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ResolveIntentRangeRequest) h := cArgs.Header @@ -47,10 +47,10 @@ func ResolveIntentRange( update := args.AsLockUpdate() - iterAndBuf := engine.GetIterAndBuf(readWriter, engine.IterOptions{UpperBound: args.EndKey}) + iterAndBuf := storage.GetIterAndBuf(readWriter, storage.IterOptions{UpperBound: args.EndKey}) defer iterAndBuf.Cleanup() - numKeys, resumeSpan, err := engine.MVCCResolveWriteIntentRangeUsingIter( + numKeys, resumeSpan, err := storage.MVCCResolveWriteIntentRangeUsingIter( ctx, readWriter, iterAndBuf, ms, update, h.MaxSpanRequestKeys, ) if err != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_resolve_intent_test.go b/pkg/kv/kvserver/batcheval/cmd_resolve_intent_test.go index 09da55539a11..25485492812d 100644 --- a/pkg/kv/kvserver/batcheval/cmd_resolve_intent_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_resolve_intent_test.go @@ -16,11 +16,11 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -73,7 +73,7 @@ func TestDeclareKeysResolveIntent(t *testing.T) { }, } ctx := context.Background() - engine := engine.NewDefaultInMem() + engine := storage.NewDefaultInMem() defer engine.Close() testutils.RunTrueAndFalse(t, "ranged", func(t *testing.T, ranged bool) { for _, test := range tests { @@ -150,7 +150,7 @@ func TestResolveIntentAfterPartialRollback(t *testing.T) { } testutils.RunTrueAndFalse(t, "ranged", func(t *testing.T, ranged bool) { - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() batch := db.NewBatch() defer batch.Close() @@ -159,13 +159,13 @@ func TestResolveIntentAfterPartialRollback(t *testing.T) { // Write a first value at key. v.SetString("a") txn.Sequence = 0 - if err := engine.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { t.Fatal(err) } // Write another value. v.SetString("b") txn.Sequence = 1 - if err := engine.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { + if err := storage.MVCCPut(ctx, batch, nil, k, ts, v, &txn); err != nil { t.Fatal(err) } if err := batch.Commit(true); err != nil { @@ -240,7 +240,7 @@ func TestResolveIntentAfterPartialRollback(t *testing.T) { // The second write has been rolled back; verify that the remaining // value is from the first write. - res, i, err := engine.MVCCGet(ctx, batch, k, ts2, engine.MVCCGetOptions{}) + res, i, err := storage.MVCCGet(ctx, batch, k, ts2, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/batcheval/cmd_reverse_scan.go b/pkg/kv/kvserver/batcheval/cmd_reverse_scan.go index 3a8a298dbe03..23712144dea3 100644 --- a/pkg/kv/kvserver/batcheval/cmd_reverse_scan.go +++ b/pkg/kv/kvserver/batcheval/cmd_reverse_scan.go @@ -14,9 +14,9 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -28,16 +28,16 @@ func init() { // maxKeys stores the number of scan results remaining for this batch // (MaxInt64 for no limit). func ReverseScan( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ReverseScanRequest) h := cArgs.Header reply := resp.(*roachpb.ReverseScanResponse) - var res engine.MVCCScanResult + var res storage.MVCCScanResult var err error - opts := engine.MVCCScanOptions{ + opts := storage.MVCCScanOptions{ Inconsistent: h.ReadConsistency != roachpb.CONSISTENT, Txn: h.Txn, MaxKeys: h.MaxSpanRequestKeys, @@ -47,14 +47,14 @@ func ReverseScan( switch args.ScanFormat { case roachpb.BATCH_RESPONSE: - res, err = engine.MVCCScanToBytes( + res, err = storage.MVCCScanToBytes( ctx, reader, args.Key, args.EndKey, h.Timestamp, opts) if err != nil { return result.Result{}, err } reply.BatchResponses = res.KVData case roachpb.KEY_VALUES: - res, err = engine.MVCCScan( + res, err = storage.MVCCScan( ctx, reader, args.Key, args.EndKey, h.Timestamp, opts) if err != nil { return result.Result{}, err diff --git a/pkg/kv/kvserver/batcheval/cmd_revert_range.go b/pkg/kv/kvserver/batcheval/cmd_revert_range.go index 56f179b7a569..1dd0a302ae0e 100644 --- a/pkg/kv/kvserver/batcheval/cmd_revert_range.go +++ b/pkg/kv/kvserver/batcheval/cmd_revert_range.go @@ -13,11 +13,11 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" @@ -42,18 +42,18 @@ func declareKeysRevertRange( // isEmptyKeyTimeRange checks if the span has no writes in (since,until]. func isEmptyKeyTimeRange( - readWriter engine.ReadWriter, from, to roachpb.Key, since, until hlc.Timestamp, + readWriter storage.ReadWriter, from, to roachpb.Key, since, until hlc.Timestamp, ) (bool, error) { // Use a TBI to check if there is anything to delete -- the first key Seek hits // may not be in the time range but the fact the TBI found any key indicates // that there is *a* key in the SST that is in the time range. Thus we should // proceed to iteration that actually checks timestamps on each key. - iter := readWriter.NewIterator(engine.IterOptions{ + iter := readWriter.NewIterator(storage.IterOptions{ LowerBound: from, UpperBound: to, MinTimestampHint: since.Next() /* make exclusive */, MaxTimestampHint: until, }) defer iter.Close() - iter.SeekGE(engine.MVCCKey{Key: from}) + iter.SeekGE(storage.MVCCKey{Key: from}) ok, err := iter.Valid() return !ok, err } @@ -65,7 +65,7 @@ func isEmptyKeyTimeRange( // Note: this should only be used when there is no user traffic writing to the // target span at or above the target time. func RevertRange( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { if cArgs.Header.Txn != nil { return result.Result{}, errors.New("cannot execute RevertRange within a transaction") @@ -91,7 +91,7 @@ func RevertRange( log.VEventf(ctx, 2, "clearing keys with timestamp (%v, %v]", args.TargetTime, cArgs.Header.Timestamp) - resume, err := engine.MVCCClearTimeRange(ctx, readWriter, cArgs.Stats, args.Key, args.EndKey, + resume, err := storage.MVCCClearTimeRange(ctx, readWriter, cArgs.Stats, args.Key, args.EndKey, args.TargetTime, cArgs.Header.Timestamp, cArgs.Header.MaxSpanRequestKeys) if err != nil { return result.Result{}, err diff --git a/pkg/kv/kvserver/batcheval/cmd_revert_range_test.go b/pkg/kv/kvserver/batcheval/cmd_revert_range_test.go index 26150d8266b8..0b34a3cb6846 100644 --- a/pkg/kv/kvserver/batcheval/cmd_revert_range_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_revert_range_test.go @@ -18,19 +18,19 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" ) -func hashRange(t *testing.T, reader engine.Reader, start, end roachpb.Key) []byte { +func hashRange(t *testing.T, reader storage.Reader, start, end roachpb.Key) []byte { t.Helper() h := sha256.New() if err := reader.Iterate(start, end, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { h.Write(kv.Key.Key) h.Write(kv.Value) return false, nil @@ -41,11 +41,11 @@ func hashRange(t *testing.T, reader engine.Reader, start, end roachpb.Key) []byt return h.Sum(nil) } -func getStats(t *testing.T, reader engine.Reader) enginepb.MVCCStats { +func getStats(t *testing.T, reader storage.Reader) enginepb.MVCCStats { t.Helper() - iter := reader.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + iter := reader.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() - s, err := engine.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 1100) + s, err := storage.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 1100) if err != nil { t.Fatalf("%+v", err) } @@ -54,18 +54,18 @@ func getStats(t *testing.T, reader engine.Reader) enginepb.MVCCStats { // createTestRocksDBEngine returns a new in-memory RocksDB engine with 1MB of // storage capacity. -func createTestRocksDBEngine(ctx context.Context) engine.Engine { - return engine.NewInMem(ctx, enginepb.EngineTypeRocksDB, roachpb.Attributes{}, 1<<20) +func createTestRocksDBEngine(ctx context.Context) storage.Engine { + return storage.NewInMem(ctx, enginepb.EngineTypeRocksDB, roachpb.Attributes{}, 1<<20) } // createTestPebbleEngine returns a new in-memory Pebble storage engine. -func createTestPebbleEngine(ctx context.Context) engine.Engine { - return engine.NewInMem(ctx, enginepb.EngineTypePebble, roachpb.Attributes{}, 1<<20) +func createTestPebbleEngine(ctx context.Context) storage.Engine { + return storage.NewInMem(ctx, enginepb.EngineTypePebble, roachpb.Attributes{}, 1<<20) } var engineImpls = []struct { name string - create func(context.Context) engine.Engine + create func(context.Context) storage.Engine }{ {"rocksdb", createTestRocksDBEngine}, {"pebble", createTestPebbleEngine}, @@ -95,7 +95,7 @@ func TestCmdRevertRange(t *testing.T) { key := roachpb.Key(fmt.Sprintf("%04d", i)) var value roachpb.Value value.SetString(fmt.Sprintf("%d", i)) - if err := engine.MVCCPut(ctx, eng, &stats, key, baseTime.Add(int64(i%10), 0), value, nil); err != nil { + if err := storage.MVCCPut(ctx, eng, &stats, key, baseTime.Add(int64(i%10), 0), value, nil); err != nil { t.Fatal(err) } } @@ -109,7 +109,7 @@ func TestCmdRevertRange(t *testing.T) { key := roachpb.Key(fmt.Sprintf("%04d", i)) var value roachpb.Value value.SetString(fmt.Sprintf("%d-rev-a", i)) - if err := engine.MVCCPut(ctx, eng, &stats, key, tsA.Add(int64(i%5), 1), value, nil); err != nil { + if err := storage.MVCCPut(ctx, eng, &stats, key, tsA.Add(int64(i%5), 1), value, nil); err != nil { t.Fatal(err) } } @@ -122,7 +122,7 @@ func TestCmdRevertRange(t *testing.T) { key := roachpb.Key(fmt.Sprintf("%04d", i)) var value roachpb.Value value.SetString(fmt.Sprintf("%d-rev-b", i)) - if err := engine.MVCCPut(ctx, eng, &stats, key, tsB.Add(1, int32(i%5)), value, nil); err != nil { + if err := storage.MVCCPut(ctx, eng, &stats, key, tsB.Add(1, int32(i%5)), value, nil); err != nil { t.Fatal(err) } } @@ -200,7 +200,7 @@ func TestCmdRevertRange(t *testing.T) { }) txn := roachpb.MakeTransaction("test", nil, roachpb.NormalUserPriority, tsC, 1) - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, eng, &stats, []byte("0012"), tsC, roachpb.MakeValueFromBytes([]byte("i")), &txn, ); err != nil { t.Fatal(err) @@ -212,7 +212,7 @@ func TestCmdRevertRange(t *testing.T) { key := roachpb.Key(fmt.Sprintf("%04d", i)) var value roachpb.Value value.SetString(fmt.Sprintf("%d-rev-b", i)) - if err := engine.MVCCPut(ctx, eng, &stats, key, tsC.Add(10, int32(i%5)), value, nil); err != nil { + if err := storage.MVCCPut(ctx, eng, &stats, key, tsC.Add(10, int32(i%5)), value, nil); err != nil { t.Fatalf("writing key %s: %+v", key, err) } } diff --git a/pkg/kv/kvserver/batcheval/cmd_scan.go b/pkg/kv/kvserver/batcheval/cmd_scan.go index e73a350b64ae..17b6e45b0193 100644 --- a/pkg/kv/kvserver/batcheval/cmd_scan.go +++ b/pkg/kv/kvserver/batcheval/cmd_scan.go @@ -14,9 +14,9 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) func init() { @@ -28,16 +28,16 @@ func init() { // stores the number of scan results remaining for this batch // (MaxInt64 for no limit). func Scan( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.ScanRequest) h := cArgs.Header reply := resp.(*roachpb.ScanResponse) - var res engine.MVCCScanResult + var res storage.MVCCScanResult var err error - opts := engine.MVCCScanOptions{ + opts := storage.MVCCScanOptions{ Inconsistent: h.ReadConsistency != roachpb.CONSISTENT, Txn: h.Txn, MaxKeys: h.MaxSpanRequestKeys, @@ -47,14 +47,14 @@ func Scan( switch args.ScanFormat { case roachpb.BATCH_RESPONSE: - res, err = engine.MVCCScanToBytes( + res, err = storage.MVCCScanToBytes( ctx, reader, args.Key, args.EndKey, h.Timestamp, opts) if err != nil { return result.Result{}, err } reply.BatchResponses = res.KVData case roachpb.KEY_VALUES: - res, err = engine.MVCCScan( + res, err = storage.MVCCScan( ctx, reader, args.Key, args.EndKey, h.Timestamp, opts) if err != nil { return result.Result{}, err diff --git a/pkg/kv/kvserver/batcheval/cmd_scan_test.go b/pkg/kv/kvserver/batcheval/cmd_scan_test.go index 3700c294429b..78f3c820d1b0 100644 --- a/pkg/kv/kvserver/batcheval/cmd_scan_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_scan_test.go @@ -15,8 +15,8 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -56,12 +56,12 @@ func testScanReverseScanInner( k1, k2 := roachpb.Key("a"), roachpb.Key("b") ts := hlc.Timestamp{WallTime: 1} - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() // Write to k1 and k2. for _, k := range []roachpb.Key{k1, k2} { - err := engine.MVCCPut(ctx, eng, nil, k, ts, roachpb.MakeValueFromString("value-"+string(k)), nil) + err := storage.MVCCPut(ctx, eng, nil, k, ts, roachpb.MakeValueFromString("value-"+string(k)), nil) require.NoError(t, err) } diff --git a/pkg/kv/kvserver/batcheval/cmd_subsume.go b/pkg/kv/kvserver/batcheval/cmd_subsume.go index 421e6f244ece..fd679d786b6c 100644 --- a/pkg/kv/kvserver/batcheval/cmd_subsume.go +++ b/pkg/kv/kvserver/batcheval/cmd_subsume.go @@ -15,11 +15,11 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/pkg/errors" ) @@ -87,7 +87,7 @@ func declareKeysSubsume( // The period of time after intents have been placed but before the merge // transaction is complete is called the merge's "critical phase". func Subsume( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.SubsumeRequest) reply := resp.(*roachpb.SubsumeResponse) @@ -112,14 +112,14 @@ func Subsume( // Sanity check the caller has initiated a merge transaction by checking for // a deletion intent on the local range descriptor. descKey := keys.RangeDescriptorKey(desc.StartKey) - _, intent, err := engine.MVCCGet(ctx, readWriter, descKey, cArgs.Header.Timestamp, - engine.MVCCGetOptions{Inconsistent: true}) + _, intent, err := storage.MVCCGet(ctx, readWriter, descKey, cArgs.Header.Timestamp, + storage.MVCCGetOptions{Inconsistent: true}) if err != nil { return result.Result{}, fmt.Errorf("fetching local range descriptor: %s", err) } else if intent == nil { return result.Result{}, errors.New("range missing intent on its local descriptor") } - val, _, err := engine.MVCCGetAsTxn(ctx, readWriter, descKey, cArgs.Header.Timestamp, intent.Txn) + val, _, err := storage.MVCCGetAsTxn(ctx, readWriter, descKey, cArgs.Header.Timestamp, intent.Txn) if err != nil { return result.Result{}, fmt.Errorf("fetching local range descriptor as txn: %s", err) } else if val != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_truncate_log.go b/pkg/kv/kvserver/batcheval/cmd_truncate_log.go index ae1b83afb028..35673db0c589 100644 --- a/pkg/kv/kvserver/batcheval/cmd_truncate_log.go +++ b/pkg/kv/kvserver/batcheval/cmd_truncate_log.go @@ -13,12 +13,12 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" @@ -43,7 +43,7 @@ func declareKeysTruncateLog( // has already been truncated has no effect. If this range is not the one // specified within the request body, the request will also be ignored. func TruncateLog( - ctx context.Context, readWriter engine.ReadWriter, cArgs CommandArgs, resp roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs CommandArgs, resp roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.TruncateLogRequest) @@ -58,9 +58,9 @@ func TruncateLog( } var legacyTruncatedState roachpb.RaftTruncatedState - legacyKeyFound, err := engine.MVCCGetProto( + legacyKeyFound, err := storage.MVCCGetProto( ctx, readWriter, keys.RaftTruncatedStateLegacyKey(cArgs.EvalCtx.GetRangeID()), - hlc.Timestamp{}, &legacyTruncatedState, engine.MVCCGetOptions{}, + hlc.Timestamp{}, &legacyTruncatedState, storage.MVCCGetOptions{}, ) if err != nil { return result.Result{}, err @@ -113,7 +113,7 @@ func TruncateLog( // bugs that let it diverge. It might be easier to compute the stats // from scratch, stopping when 4mb (defaultRaftLogTruncationThreshold) // is reached as at that point we'll truncate aggressively anyway. - iter := readWriter.NewIterator(engine.IterOptions{UpperBound: end}) + iter := readWriter.NewIterator(storage.IterOptions{UpperBound: end}) defer iter.Close() // We can pass zero as nowNanos because we're only interested in SysBytes. ms, err := iter.ComputeStats(start, end, 0 /* nowNanos */) @@ -138,7 +138,7 @@ func TruncateLog( // Time to migrate by deleting the legacy key. The downstream-of-Raft // code will atomically rewrite the truncated state (supplied via the // side effect) into the new unreplicated key. - if err := engine.MVCCDelete( + if err := storage.MVCCDelete( ctx, readWriter, cArgs.Stats, keys.RaftTruncatedStateLegacyKey(cArgs.EvalCtx.GetRangeID()), hlc.Timestamp{}, nil, /* txn */ ); err != nil { diff --git a/pkg/kv/kvserver/batcheval/cmd_truncate_log_test.go b/pkg/kv/kvserver/batcheval/cmd_truncate_log_test.go index a3c4d3b1f0cc..bf5848b27943 100644 --- a/pkg/kv/kvserver/batcheval/cmd_truncate_log_test.go +++ b/pkg/kv/kvserver/batcheval/cmd_truncate_log_test.go @@ -15,9 +15,9 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/assert" @@ -25,7 +25,7 @@ import ( func putTruncatedState( t *testing.T, - eng engine.Engine, + eng storage.Engine, rangeID roachpb.RangeID, truncState roachpb.RaftTruncatedState, legacy bool, @@ -34,7 +34,7 @@ func putTruncatedState( if legacy { key = keys.RaftTruncatedStateLegacyKey(rangeID) } - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( context.Background(), eng, nil, key, hlc.Timestamp{}, nil /* txn */, &truncState, ); err != nil { @@ -43,12 +43,12 @@ func putTruncatedState( } func readTruncStates( - t *testing.T, eng engine.Engine, rangeID roachpb.RangeID, + t *testing.T, eng storage.Engine, rangeID roachpb.RangeID, ) (legacy roachpb.RaftTruncatedState, unreplicated roachpb.RaftTruncatedState) { t.Helper() - legacyFound, err := engine.MVCCGetProto( + legacyFound, err := storage.MVCCGetProto( context.Background(), eng, keys.RaftTruncatedStateLegacyKey(rangeID), - hlc.Timestamp{}, &legacy, engine.MVCCGetOptions{}, + hlc.Timestamp{}, &legacy, storage.MVCCGetOptions{}, ) if err != nil { t.Fatal(err) @@ -57,9 +57,9 @@ func readTruncStates( t.Fatalf("legacy key found=%t but state is %+v", legacyFound, legacy) } - unreplicatedFound, err := engine.MVCCGetProto( + unreplicatedFound, err := storage.MVCCGetProto( context.Background(), eng, keys.RaftTruncatedStateKey(rangeID), - hlc.Timestamp{}, &unreplicated, engine.MVCCGetOptions{}, + hlc.Timestamp{}, &unreplicated, storage.MVCCGetOptions{}, ) if err != nil { t.Fatal(err) @@ -124,7 +124,7 @@ func runUnreplicatedTruncatedState(t *testing.T, tc unreplicatedTruncStateTest) FirstIndex: firstIndex, } - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() truncState := roachpb.RaftTruncatedState{ diff --git a/pkg/kv/kvserver/batcheval/command.go b/pkg/kv/kvserver/batcheval/command.go index 876f0abba376..cb4d13183ca0 100644 --- a/pkg/kv/kvserver/batcheval/command.go +++ b/pkg/kv/kvserver/batcheval/command.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" ) @@ -46,8 +46,8 @@ type Command struct { // should treat the provided request as immutable. // // Only one of these is ever set at a time. - EvalRW func(context.Context, engine.ReadWriter, CommandArgs, roachpb.Response) (result.Result, error) - EvalRO func(context.Context, engine.Reader, CommandArgs, roachpb.Response) (result.Result, error) + EvalRW func(context.Context, storage.ReadWriter, CommandArgs, roachpb.Response) (result.Result, error) + EvalRO func(context.Context, storage.Reader, CommandArgs, roachpb.Response) (result.Result, error) } var cmds = make(map[roachpb.Method]Command) @@ -57,7 +57,7 @@ var cmds = make(map[roachpb.Method]Command) func RegisterReadWriteCommand( method roachpb.Method, declare declareKeysFunc, - impl func(context.Context, engine.ReadWriter, CommandArgs, roachpb.Response) (result.Result, error), + impl func(context.Context, storage.ReadWriter, CommandArgs, roachpb.Response) (result.Result, error), ) { register(method, Command{ DeclareKeys: declare, @@ -70,7 +70,7 @@ func RegisterReadWriteCommand( func RegisterReadOnlyCommand( method roachpb.Method, declare declareKeysFunc, - impl func(context.Context, engine.Reader, CommandArgs, roachpb.Response) (result.Result, error), + impl func(context.Context, storage.Reader, CommandArgs, roachpb.Response) (result.Result, error), ) { register(method, Command{ DeclareKeys: declare, diff --git a/pkg/kv/kvserver/batcheval/declare.go b/pkg/kv/kvserver/batcheval/declare.go index 5652f3674014..f2bbb1d69433 100644 --- a/pkg/kv/kvserver/batcheval/declare.go +++ b/pkg/kv/kvserver/batcheval/declare.go @@ -13,10 +13,10 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" ) // DefaultDeclareKeys is the default implementation of Command.DeclareKeys. diff --git a/pkg/kv/kvserver/batcheval/eval_context.go b/pkg/kv/kvserver/batcheval/eval_context.go index 9bf42cb10200..e07f788481f0 100644 --- a/pkg/kv/kvserver/batcheval/eval_context.go +++ b/pkg/kv/kvserver/batcheval/eval_context.go @@ -14,8 +14,6 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/cloud" @@ -23,6 +21,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/limit" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -49,7 +49,7 @@ type EvalContext interface { ClusterSettings() *cluster.Settings EvalKnobs() storagebase.BatchEvalTestingKnobs - Engine() engine.Engine + Engine() storage.Engine Clock() *hlc.Clock DB() *client.DB AbortSpan() *abortspan.AbortSpan @@ -134,7 +134,7 @@ func (m *mockEvalCtxImpl) ClusterSettings() *cluster.Settings { func (m *mockEvalCtxImpl) EvalKnobs() storagebase.BatchEvalTestingKnobs { return storagebase.BatchEvalTestingKnobs{} } -func (m *mockEvalCtxImpl) Engine() engine.Engine { +func (m *mockEvalCtxImpl) Engine() storage.Engine { panic("unimplemented") } func (m *mockEvalCtxImpl) Clock() *hlc.Clock { diff --git a/pkg/kv/kvserver/batcheval/intent.go b/pkg/kv/kvserver/batcheval/intent.go index f335ce6bd4c2..34508bbe81ad 100644 --- a/pkg/kv/kvserver/batcheval/intent.go +++ b/pkg/kv/kvserver/batcheval/intent.go @@ -13,8 +13,8 @@ package batcheval import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" ) // CollectIntentRows collects the key-value pairs for each intent provided. It @@ -25,14 +25,14 @@ import ( // RangeLookups and since this is how they currently collect intent values, this // is ok for now. func CollectIntentRows( - ctx context.Context, reader engine.Reader, cArgs CommandArgs, intents []roachpb.Intent, + ctx context.Context, reader storage.Reader, cArgs CommandArgs, intents []roachpb.Intent, ) ([]roachpb.KeyValue, error) { if len(intents) == 0 { return nil, nil } res := make([]roachpb.KeyValue, 0, len(intents)) for _, intent := range intents { - val, _, err := engine.MVCCGetAsTxn( + val, _, err := storage.MVCCGetAsTxn( ctx, reader, intent.Key, intent.Txn.WriteTimestamp, intent.Txn, ) if err != nil { diff --git a/pkg/kv/kvserver/batcheval/split_stats_helper.go b/pkg/kv/kvserver/batcheval/split_stats_helper.go index d55415f2b982..6e9d76874dc5 100644 --- a/pkg/kv/kvserver/batcheval/split_stats_helper.go +++ b/pkg/kv/kvserver/batcheval/split_stats_helper.go @@ -10,7 +10,7 @@ package batcheval -import "github.com/cockroachdb/cockroach/pkg/engine/enginepb" +import "github.com/cockroachdb/cockroach/pkg/storage/enginepb" // splitStatsHelper codifies and explains the stats computations related to a // split. The quantities known during a split (i.e. while the split trigger diff --git a/pkg/kv/kvserver/batcheval/transaction.go b/pkg/kv/kvserver/batcheval/transaction.go index 733de959fe11..90a4c0dca9fc 100644 --- a/pkg/kv/kvserver/batcheval/transaction.go +++ b/pkg/kv/kvserver/batcheval/transaction.go @@ -15,9 +15,9 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" @@ -81,7 +81,7 @@ func WriteAbortSpanOnResolve(status roachpb.TransactionStatus, poison, removedIn func UpdateAbortSpan( ctx context.Context, rec EvalContext, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, txn enginepb.TxnMeta, poison bool, diff --git a/pkg/kv/kvserver/batcheval/transaction_test.go b/pkg/kv/kvserver/batcheval/transaction_test.go index d16d3a95bceb..ec0442fcca15 100644 --- a/pkg/kv/kvserver/batcheval/transaction_test.go +++ b/pkg/kv/kvserver/batcheval/transaction_test.go @@ -14,9 +14,9 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" @@ -58,16 +58,16 @@ func TestUpdateAbortSpan(t *testing.T) { } // Setup helpers. - type evalFn func(engine.ReadWriter, EvalContext) error - addIntent := func(b engine.ReadWriter, _ EvalContext) error { + type evalFn func(storage.ReadWriter, EvalContext) error + addIntent := func(b storage.ReadWriter, _ EvalContext) error { val := roachpb.MakeValueFromString("val") - return engine.MVCCPut(ctx, b, nil /* ms */, intentKey, txn.ReadTimestamp, val, &txn) + return storage.MVCCPut(ctx, b, nil /* ms */, intentKey, txn.ReadTimestamp, val, &txn) } - addPrevAbortSpanEntry := func(b engine.ReadWriter, rec EvalContext) error { + addPrevAbortSpanEntry := func(b storage.ReadWriter, rec EvalContext) error { return UpdateAbortSpan(ctx, rec, b, nil /* ms */, prevTxn.TxnMeta, true /* poison */) } compose := func(fns ...evalFn) evalFn { - return func(b engine.ReadWriter, rec EvalContext) error { + return func(b storage.ReadWriter, rec EvalContext) error { for _, fn := range fns { if err := fn(b, rec); err != nil { return err @@ -78,7 +78,7 @@ func TestUpdateAbortSpan(t *testing.T) { } // Request helpers. - endTxn := func(b engine.ReadWriter, rec EvalContext, commit bool, poison bool) error { + endTxn := func(b storage.ReadWriter, rec EvalContext, commit bool, poison bool) error { req := roachpb.EndTxnRequest{ RequestHeader: roachpb.RequestHeader{Key: txnKey}, Commit: commit, @@ -99,7 +99,7 @@ func TestUpdateAbortSpan(t *testing.T) { return err } resolveIntent := func( - b engine.ReadWriter, rec EvalContext, status roachpb.TransactionStatus, poison bool, + b storage.ReadWriter, rec EvalContext, status roachpb.TransactionStatus, poison bool, ) error { req := roachpb.ResolveIntentRequest{ RequestHeader: roachpb.RequestHeader{Key: intentKey}, @@ -117,7 +117,7 @@ func TestUpdateAbortSpan(t *testing.T) { return err } resolveIntentRange := func( - b engine.ReadWriter, rec EvalContext, status roachpb.TransactionStatus, poison bool, + b storage.ReadWriter, rec EvalContext, status roachpb.TransactionStatus, poison bool, ) error { req := roachpb.ResolveIntentRangeRequest{ RequestHeader: roachpb.RequestHeader{Key: startKey, EndKey: endKey}, @@ -147,7 +147,7 @@ func TestUpdateAbortSpan(t *testing.T) { /////////////////////////////////////////////////////////////////////// { name: "end txn, rollback, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -156,7 +156,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, no poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -164,7 +164,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "end txn, rollback, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, true /* poison */) }, // Poisoning, but no intents found, should not add an abort span entry. @@ -173,7 +173,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, true /* poison */) }, // Poisoning, but no intents found, don't touch abort span. @@ -181,7 +181,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "end txn, commit, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -195,7 +195,7 @@ func TestUpdateAbortSpan(t *testing.T) { { // It is an error for EndTxn to pass Commit = true and Poison = true. name: "end txn, commit, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, true /* poison */) }, expErr: "cannot poison during a committing EndTxn request", @@ -204,7 +204,7 @@ func TestUpdateAbortSpan(t *testing.T) { // It is an error for EndTxn to pass Commit = true and Poison = true. name: "end txn, commit, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, true /* poison */) }, expErr: "cannot poison during a committing EndTxn request", @@ -212,7 +212,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -221,7 +221,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, no poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -230,7 +230,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, true /* poison */) }, // Poisoning, should add an abort span entry. @@ -239,7 +239,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, rollback, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, false /* commit */, true /* poison */) }, // Poisoning, should update abort span entry. @@ -248,7 +248,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "end txn, commit, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -263,7 +263,7 @@ func TestUpdateAbortSpan(t *testing.T) { // It is an error for EndTxn to pass Commit = true and Poison = true. name: "end txn, commit, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, true /* poison */) }, expErr: "cannot poison during a committing EndTxn request", @@ -272,7 +272,7 @@ func TestUpdateAbortSpan(t *testing.T) { // It is an error for EndTxn to pass Commit = true and Poison = true. name: "end txn, commit, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return endTxn(b, rec, true /* commit */, true /* poison */) }, expErr: "cannot poison during a committing EndTxn request", @@ -282,7 +282,7 @@ func TestUpdateAbortSpan(t *testing.T) { /////////////////////////////////////////////////////////////////////// { name: "resolve intent, txn pending, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -291,7 +291,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, no poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, false /* poison */) }, // Not aborted, don't touch abort span. @@ -300,7 +300,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -309,7 +309,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, no poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, false /* poison */) }, // Not aborted, don't touch abort span. @@ -317,7 +317,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent, txn pending, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -326,7 +326,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, true /* poison */) }, // Not aborted, don't touch abort span. @@ -335,7 +335,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -344,7 +344,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn pending, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.PENDING, true /* poison */) }, // Not aborted, don't touch abort span. @@ -352,7 +352,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent, txn aborted, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -361,7 +361,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, no poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -370,7 +370,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -379,7 +379,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, no poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -387,7 +387,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent, txn aborted, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, but no intents found, should not add an abort span entry. @@ -396,7 +396,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, but no intents found, don't touch abort span. @@ -405,7 +405,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, should add an abort span entry. @@ -414,7 +414,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn aborted, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, should update abort span entry. @@ -422,7 +422,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent, txn committed, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.COMMITTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -436,7 +436,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn committed, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.COMMITTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -449,7 +449,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent, txn committed, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.COMMITTED, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -463,7 +463,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent, txn committed, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntent(b, rec, roachpb.COMMITTED, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -479,7 +479,7 @@ func TestUpdateAbortSpan(t *testing.T) { /////////////////////////////////////////////////////////////////////// { name: "resolve intent range, txn pending, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -488,7 +488,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, no poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, false /* poison */) }, // Not aborted, don't touch abort span. @@ -497,7 +497,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -506,7 +506,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, no poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, false /* poison */) }, // Not aborted, don't touch abort span. @@ -514,7 +514,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent range, txn pending, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -523,7 +523,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, true /* poison */) }, // Not aborted, don't touch abort span. @@ -532,7 +532,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -541,7 +541,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn pending, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.PENDING, true /* poison */) }, // Not aborted, don't touch abort span. @@ -549,7 +549,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent range, txn aborted, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -558,7 +558,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, no poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -567,7 +567,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -576,7 +576,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, no poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, false /* poison */) }, // Not poisoning, should clean up abort span entry. @@ -584,7 +584,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent range, txn aborted, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, but no intents found, should not add an abort span entry. @@ -593,7 +593,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, poison, intent missing, abort span present", before: addPrevAbortSpanEntry, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, but no intents found, don't touch abort span. @@ -602,7 +602,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, should add an abort span entry. @@ -611,7 +611,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn aborted, poison, intent present, abort span present", before: compose(addIntent, addPrevAbortSpanEntry), - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.ABORTED, true /* poison */) }, // Poisoning, should update abort span entry. @@ -619,7 +619,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent range, txn committed, no poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.COMMITTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -633,7 +633,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn committed, no poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.COMMITTED, false /* poison */) }, // Not poisoning, should not add an abort span entry. @@ -646,7 +646,7 @@ func TestUpdateAbortSpan(t *testing.T) { }, { name: "resolve intent range, txn committed, poison, intent missing, abort span missing", - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.COMMITTED, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -660,7 +660,7 @@ func TestUpdateAbortSpan(t *testing.T) { { name: "resolve intent range, txn committed, poison, intent present, abort span missing", before: addIntent, - run: func(b engine.ReadWriter, rec EvalContext) error { + run: func(b storage.ReadWriter, rec EvalContext) error { return resolveIntentRange(b, rec, roachpb.COMMITTED, true /* poison */) }, // Poisoning but not aborted, should not add an abort span entry. @@ -678,7 +678,7 @@ func TestUpdateAbortSpan(t *testing.T) { t.Skip("invalid test case") } - db := engine.NewDefaultInMem() + db := storage.NewDefaultInMem() defer db.Close() batch := db.NewBatch() defer batch.Close() diff --git a/pkg/kv/kvserver/below_raft_protos_test.go b/pkg/kv/kvserver/below_raft_protos_test.go index 1a84e9827792..e908a1e75218 100644 --- a/pkg/kv/kvserver/below_raft_protos_test.go +++ b/pkg/kv/kvserver/below_raft_protos_test.go @@ -17,10 +17,10 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "go.etcd.io/etcd/raft/raftpb" @@ -120,7 +120,7 @@ var belowRaftGoldenProtos = map[reflect.Type]fixture{ } func init() { - if engine.DefaultStorageEngine != enginepb.EngineTypeRocksDB && engine.DefaultStorageEngine != enginepb.EngineTypeDefault { + if storage.DefaultStorageEngine != enginepb.EngineTypeRocksDB && storage.DefaultStorageEngine != enginepb.EngineTypeDefault { // This is marshaled below Raft by the Pebble merge operator. The Pebble // merge operator can be called below Raft whenever a Pebble Iterator is // used. Note that we only see this proto marshaled below Raft when the diff --git a/pkg/kv/kvserver/bulk/buffering_adder.go b/pkg/kv/kvserver/bulk/buffering_adder.go index 94e123c2da68..dc91c78f1044 100644 --- a/pkg/kv/kvserver/bulk/buffering_adder.go +++ b/pkg/kv/kvserver/bulk/buffering_adder.go @@ -15,11 +15,11 @@ import ( "sort" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/mon" @@ -218,7 +218,7 @@ func (b *BufferingAdder) Flush(ctx context.Context) error { beforeSort := timeutil.Now() sort.Sort(&b.curBuf) - mvccKey := engine.MVCCKey{Timestamp: b.timestamp} + mvccKey := storage.MVCCKey{Timestamp: b.timestamp} beforeFlush := timeutil.Now() b.flushCounts.totalSort += beforeFlush.Sub(beforeSort) diff --git a/pkg/kv/kvserver/bulk/sst_batcher.go b/pkg/kv/kvserver/bulk/sst_batcher.go index 6ed521837eee..ab0217062281 100644 --- a/pkg/kv/kvserver/bulk/sst_batcher.go +++ b/pkg/kv/kvserver/bulk/sst_batcher.go @@ -16,14 +16,14 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -91,8 +91,8 @@ type SSTBatcher struct { // The rest of the fields are per-batch and are reset via Reset() before each // batch is started. - sstWriter engine.SSTWriter - sstFile *engine.MemFile + sstWriter storage.SSTWriter + sstFile *storage.MemFile batchStartKey []byte batchEndKey []byte batchEndValue []byte @@ -101,7 +101,7 @@ type SSTBatcher struct { // stores on-the-fly stats for the SST if disallowShadowing is true. ms enginepb.MVCCStats // rows written in the current batch. - rowCounter engine.RowCounter + rowCounter storage.RowCounter } // MakeSSTBatcher makes a ready-to-use SSTBatcher. @@ -113,7 +113,7 @@ func MakeSSTBatcher( return b, err } -func (b *SSTBatcher) updateMVCCStats(key engine.MVCCKey, value []byte) { +func (b *SSTBatcher) updateMVCCStats(key storage.MVCCKey, value []byte) { metaKeySize := int64(len(key.Key)) + 1 metaValSize := int64(0) b.ms.LiveBytes += metaKeySize @@ -122,9 +122,9 @@ func (b *SSTBatcher) updateMVCCStats(key engine.MVCCKey, value []byte) { b.ms.ValBytes += metaValSize b.ms.KeyCount++ - totalBytes := int64(len(value)) + engine.MVCCVersionTimestampSize + totalBytes := int64(len(value)) + storage.MVCCVersionTimestampSize b.ms.LiveBytes += totalBytes - b.ms.KeyBytes += engine.MVCCVersionTimestampSize + b.ms.KeyBytes += storage.MVCCVersionTimestampSize b.ms.ValBytes += int64(len(value)) b.ms.ValCount++ } @@ -133,7 +133,7 @@ func (b *SSTBatcher) updateMVCCStats(key engine.MVCCKey, value []byte) { // This is only for callers that want to control the timestamp on individual // keys -- like RESTORE where we want the restored data to look the like backup. // Keys must be added in order. -func (b *SSTBatcher) AddMVCCKey(ctx context.Context, key engine.MVCCKey, value []byte) error { +func (b *SSTBatcher) AddMVCCKey(ctx context.Context, key storage.MVCCKey, value []byte) error { if len(b.batchEndKey) > 0 && bytes.Equal(b.batchEndKey, key.Key) { if b.skipDuplicates && bytes.Equal(b.batchEndValue, value) { return nil @@ -177,14 +177,14 @@ func (b *SSTBatcher) AddMVCCKey(ctx context.Context, key engine.MVCCKey, value [ // Reset clears all state in the batcher and prepares it for reuse. func (b *SSTBatcher) Reset(ctx context.Context) error { b.sstWriter.Close() - b.sstFile = &engine.MemFile{} + b.sstFile = &storage.MemFile{} // Create "Ingestion" SSTs in the newer RocksDBv2 format only if all nodes // in the cluster can support it. Until then, for backward compatibility, // create SSTs in the leveldb format ("backup" ones). if b.settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { - b.sstWriter = engine.MakeIngestionSSTWriter(b.sstFile) + b.sstWriter = storage.MakeIngestionSSTWriter(b.sstFile) } else { - b.sstWriter = engine.MakeBackupSSTWriter(b.sstFile) + b.sstWriter = storage.MakeBackupSSTWriter(b.sstFile) } b.batchStartKey = b.batchStartKey[:0] b.batchEndKey = b.batchEndKey[:0] @@ -381,7 +381,7 @@ func AddSSTable( ) (int, error) { var files int now := timeutil.Now() - iter, err := engine.NewMemSSTIterator(sstBytes, true) + iter, err := storage.NewMemSSTIterator(sstBytes, true) if err != nil { return 0, err } @@ -389,7 +389,7 @@ func AddSSTable( var stats enginepb.MVCCStats if (ms == enginepb.MVCCStats{}) { - stats, err = engine.ComputeStatsGo(iter, start, end, now.UnixNano()) + stats, err = storage.ComputeStatsGo(iter, start, end, now.UnixNano()) if err != nil { return 0, errors.Wrapf(err, "computing stats for SST [%s, %s)", start, end) } @@ -437,7 +437,7 @@ func AddSSTable( return err } - right.stats, err = engine.ComputeStatsGo( + right.stats, err = storage.ComputeStatsGo( iter, right.start, right.end, now.UnixNano(), ) if err != nil { @@ -476,15 +476,15 @@ func createSplitSSTable( db SSTSender, start, splitKey roachpb.Key, disallowShadowing bool, - iter engine.SimpleIterator, + iter storage.SimpleIterator, settings *cluster.Settings, ) (*sstSpan, *sstSpan, error) { - sstFile := &engine.MemFile{} - var w engine.SSTWriter + sstFile := &storage.MemFile{} + var w storage.SSTWriter if settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { - w = engine.MakeIngestionSSTWriter(sstFile) + w = storage.MakeIngestionSSTWriter(sstFile) } else { - w = engine.MakeBackupSSTWriter(sstFile) + w = storage.MakeBackupSSTWriter(sstFile) } defer w.Close() @@ -492,7 +492,7 @@ func createSplitSSTable( var first, last roachpb.Key var left, right *sstSpan - iter.SeekGE(engine.MVCCKey{Key: start}) + iter.SeekGE(storage.MVCCKey{Key: start}) for { if ok, err := iter.Valid(); err != nil { return nil, nil, err @@ -514,11 +514,11 @@ func createSplitSSTable( sstBytes: sstFile.Data(), disallowShadowing: disallowShadowing, } - *sstFile = engine.MemFile{} + *sstFile = storage.MemFile{} if settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { - w = engine.MakeIngestionSSTWriter(sstFile) + w = storage.MakeIngestionSSTWriter(sstFile) } else { - w = engine.MakeBackupSSTWriter(sstFile) + w = storage.MakeBackupSSTWriter(sstFile) } split = true first = nil diff --git a/pkg/kv/kvserver/bulk/sst_batcher_test.go b/pkg/kv/kvserver/bulk/sst_batcher_test.go index 7ab70860fca5..b704ed612b09 100644 --- a/pkg/kv/kvserver/bulk/sst_batcher_test.go +++ b/pkg/kv/kvserver/bulk/sst_batcher_test.go @@ -20,8 +20,6 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" @@ -29,6 +27,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -38,9 +38,9 @@ import ( "github.com/stretchr/testify/require" ) -func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []engine.MVCCKeyValue { +func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []storage.MVCCKeyValue { prefix := encoding.EncodeUvarintAscending(keys.MakeTablePrefix(uint32(100)), uint64(1)) - kvs := make([]engine.MVCCKeyValue, numKeys) + kvs := make([]storage.MVCCKeyValue, numKeys) r, _ := randutil.NewPseudoRand() var k int @@ -64,8 +64,8 @@ func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []engine.MVCCKeyValue return kvs } -func makeRocksSST(t testing.TB, kvs []engine.MVCCKeyValue) []byte { - w, err := engine.MakeRocksDBSstFileWriter() +func makeRocksSST(t testing.TB, kvs []storage.MVCCKeyValue) []byte { + w, err := storage.MakeRocksDBSstFileWriter() require.NoError(t, err) defer w.Close() diff --git a/pkg/kv/kvserver/client_merge_test.go b/pkg/kv/kvserver/client_merge_test.go index e406b84504a6..b57f856a3cec 100644 --- a/pkg/kv/kvserver/client_merge_test.go +++ b/pkg/kv/kvserver/client_merge_test.go @@ -27,8 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -42,6 +40,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -128,9 +128,9 @@ func TestStoreRangeMergeTwoEmptyRanges(t *testing.T) { } } -func getEngineKeySet(t *testing.T, e engine.Engine) map[string]struct{} { +func getEngineKeySet(t *testing.T, e storage.Engine) map[string]struct{} { t.Helper() - kvs, err := engine.Scan(e, roachpb.KeyMin, roachpb.KeyMax, 0 /* max */) + kvs, err := storage.Scan(e, roachpb.KeyMin, roachpb.KeyMax, 0 /* max */) if err != nil { t.Fatal(err) } @@ -316,8 +316,8 @@ func mergeWithData(t *testing.T, retries int64) { // Verify no intents remains on range descriptor keys. for _, key := range []roachpb.Key{keys.RangeDescriptorKey(lhsDesc.StartKey), keys.RangeDescriptorKey(rhsDesc.StartKey)} { - if _, _, err := engine.MVCCGet( - ctx, store1.Engine(), key, store1.Clock().Now(), engine.MVCCGetOptions{}, + if _, _, err := storage.MVCCGet( + ctx, store1.Engine(), key, store1.Clock().Now(), storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } @@ -1764,11 +1764,11 @@ func TestStoreReplicaGCAfterMerge(t *testing.T) { }) // Be extra paranoid and verify the exact value of the replica tombstone. - checkTombstone := func(eng engine.Engine) { + checkTombstone := func(eng storage.Engine) { var rhsTombstone roachpb.RangeTombstone rhsTombstoneKey := keys.RangeTombstoneKey(rhsDesc.RangeID) - ok, err = engine.MVCCGetProto(ctx, eng, rhsTombstoneKey, hlc.Timestamp{}, - &rhsTombstone, engine.MVCCGetOptions{}) + ok, err = storage.MVCCGetProto(ctx, eng, rhsTombstoneKey, hlc.Timestamp{}, + &rhsTombstone, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } else if !ok { @@ -2855,7 +2855,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { defer leaktest.AfterTest(t)() // We will be testing the SSTs written on store2's engine. - var receivingEng, sendingEng engine.Engine + var receivingEng, sendingEng storage.Engine ctx := context.Background() storeCfg := kvserver.TestStoreConfig(nil) storeCfg.TestingKnobs.DisableReplicateQueue = true @@ -2913,8 +2913,8 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { // Write a range deletion tombstone to each of the SSTs then put in the // kv entries from the sender of the snapshot. for _, r := range keyRanges { - sstFile := &engine.MemFile{} - sst := engine.MakeIngestionSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeIngestionSSTWriter(sstFile) if err := sst.ClearRange(r.Start, r.End); err != nil { return err } @@ -2944,8 +2944,8 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { // Construct SSTs #5 and #6: range-id local keys of subsumed replicas // with RangeIDs 3 and 4. for _, rangeID := range []roachpb.RangeID{roachpb.RangeID(3), roachpb.RangeID(4)} { - sstFile := &engine.MemFile{} - sst := engine.MakeIngestionSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeIngestionSSTWriter(sstFile) defer sst.Close() r := rditer.MakeRangeIDLocalKeyRange(rangeID, false /* replicatedOnly */) if err := sst.ClearRange(r.Start, r.End); err != nil { @@ -2953,7 +2953,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { } tombstoneKey := keys.RangeTombstoneKey(rangeID) tombstoneValue := &roachpb.RangeTombstone{NextReplicaID: math.MaxInt32} - if err := engine.MVCCBlindPutProto(context.TODO(), &sst, nil, tombstoneKey, hlc.Timestamp{}, tombstoneValue, nil); err != nil { + if err := storage.MVCCBlindPutProto(context.TODO(), &sst, nil, tombstoneKey, hlc.Timestamp{}, tombstoneValue, nil); err != nil { return err } err := sst.Finish() @@ -2964,15 +2964,15 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { } // Construct SST #7: user key range of subsumed replicas. - sstFile := &engine.MemFile{} - sst := engine.MakeIngestionSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeIngestionSSTWriter(sstFile) defer sst.Close() desc := roachpb.RangeDescriptor{ StartKey: roachpb.RKey("d"), EndKey: roachpb.RKeyMax, } r := rditer.MakeUserKeyRange(&desc) - if err := engine.ClearRangeWithHeuristic(receivingEng, &sst, r.Start.Key, r.End.Key); err != nil { + if err := storage.ClearRangeWithHeuristic(receivingEng, &sst, r.Start.Key, r.End.Key); err != nil { return err } err := sst.Finish() diff --git a/pkg/kv/kvserver/client_raft_test.go b/pkg/kv/kvserver/client_raft_test.go index cb486c58fb4b..a86a32815f4a 100644 --- a/pkg/kv/kvserver/client_raft_test.go +++ b/pkg/kv/kvserver/client_raft_test.go @@ -25,8 +25,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -39,6 +37,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -88,7 +88,7 @@ func TestStoreRecoverFromEngine(t *testing.T) { engineStopper := stop.NewStopper() defer engineStopper.Stop(context.TODO()) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() engineStopper.AddCloser(eng) var rangeID2 roachpb.RangeID @@ -194,7 +194,7 @@ func TestStoreRecoverWithErrors(t *testing.T) { // and trying to handle that complicates the test without providing any // added benefit. storeCfg.TestingKnobs.DisableSplitQueue = true - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() numIncrements := 0 @@ -294,8 +294,8 @@ func TestReplicateRange(t *testing.T) { // Verify no intent remains on range descriptor key. key := keys.RangeDescriptorKey(repl.Desc().StartKey) desc := roachpb.RangeDescriptor{} - if ok, err := engine.MVCCGetProto(context.Background(), mtc.stores[0].Engine(), key, - mtc.stores[0].Clock().Now(), &desc, engine.MVCCGetOptions{}); err != nil { + if ok, err := storage.MVCCGetProto(context.Background(), mtc.stores[0].Engine(), key, + mtc.stores[0].Clock().Now(), &desc, storage.MVCCGetOptions{}); err != nil { t.Fatal(err) } else if !ok { t.Fatalf("range descriptor key %s was not found", key) @@ -307,8 +307,8 @@ func TestReplicateRange(t *testing.T) { meta1 := keys.RangeMetaKey(meta2) for _, key := range []roachpb.RKey{meta2, meta1} { metaDesc := roachpb.RangeDescriptor{} - if ok, err := engine.MVCCGetProto(context.Background(), mtc.stores[0].Engine(), key.AsRawKey(), - mtc.stores[0].Clock().Now(), &metaDesc, engine.MVCCGetOptions{}); err != nil { + if ok, err := storage.MVCCGetProto(context.Background(), mtc.stores[0].Engine(), key.AsRawKey(), + mtc.stores[0].Clock().Now(), &metaDesc, storage.MVCCGetOptions{}); err != nil { return err } else if !ok { return errors.Errorf("failed to resolve %s", key.AsRawKey()) @@ -1083,7 +1083,7 @@ func TestConcurrentRaftSnapshots(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -1719,7 +1719,7 @@ func TestProgressWithDownNode(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } mtc := &multiTestContext{ @@ -1744,8 +1744,8 @@ func TestProgressWithDownNode(t *testing.T) { testutils.SucceedsSoon(t, func() error { values := []int64{} for _, eng := range mtc.engines { - val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), mtc.clock.Now(), - engine.MVCCGetOptions{}) + val, _, err := storage.MVCCGet(context.Background(), eng, roachpb.Key("a"), mtc.clock.Now(), + storage.MVCCGetOptions{}) if err != nil { return err } @@ -1891,7 +1891,7 @@ func testReplicaAddRemove(t *testing.T, addFirst bool) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } sc := kvserver.TestStoreConfig(nil) @@ -1915,8 +1915,8 @@ func testReplicaAddRemove(t *testing.T, addFirst bool) { return func() error { values := make([]int64, len(mtc.engines)) for i, eng := range mtc.engines { - val, _, err := engine.MVCCGet(context.Background(), eng, key, mtc.clock.Now(), - engine.MVCCGetOptions{}) + val, _, err := storage.MVCCGet(context.Background(), eng, key, mtc.clock.Now(), + storage.MVCCGetOptions{}) if err != nil { return err } @@ -2703,9 +2703,9 @@ func TestRaftRemoveRace(t *testing.T) { // Verify the tombstone key does not exist. See #12130. tombstoneKey := keys.RangeTombstoneKey(rangeID) var tombstone roachpb.RangeTombstone - if ok, err := engine.MVCCGetProto( + if ok, err := storage.MVCCGetProto( context.Background(), mtc.stores[2].Engine(), tombstoneKey, - hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } else if ok { @@ -3003,7 +3003,7 @@ func TestDecommission(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -4424,7 +4424,7 @@ func TestDefaultConnectionDisruptionDoesNotInterfereWithSystemTraffic(t *testing // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -4713,7 +4713,7 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } sc := kvserver.TestStoreConfig(nil) @@ -4759,8 +4759,8 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { ensureNoTombstone := func(t *testing.T, store *kvserver.Store, rangeID roachpb.RangeID) { var tombstone roachpb.RangeTombstone tombstoneKey := keys.RangeTombstoneKey(rangeID) - ok, err := engine.MVCCGetProto( - ctx, store.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + ok, err := storage.MVCCGetProto( + ctx, store.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ) require.NoError(t, err) require.False(t, ok) diff --git a/pkg/kv/kvserver/client_replica_gc_test.go b/pkg/kv/kvserver/client_replica_gc_test.go index 460192fc1c07..801f8a8dead0 100644 --- a/pkg/kv/kvserver/client_replica_gc_test.go +++ b/pkg/kv/kvserver/client_replica_gc_test.go @@ -20,10 +20,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/pkg/errors" @@ -42,10 +42,10 @@ func TestReplicaGCQueueDropReplicaDirect(t *testing.T) { // would probably look at an empty sideloaded directory and fail. tempDir, cleanup := testutils.TempDir(t) defer cleanup() - cache := engine.NewRocksDBCache(1 << 20) + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() for i := 0; i < 3; i++ { - eng, err := engine.NewRocksDB(engine.RocksDBConfig{ + eng, err := storage.NewRocksDB(storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: filepath.Join(tempDir, strconv.Itoa(i)), }, diff --git a/pkg/kv/kvserver/client_replica_test.go b/pkg/kv/kvserver/client_replica_test.go index 47f837ab175f..c823334571b0 100644 --- a/pkg/kv/kvserver/client_replica_test.go +++ b/pkg/kv/kvserver/client_replica_test.go @@ -24,8 +24,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -34,6 +32,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -95,8 +95,8 @@ func TestRangeCommandClockUpdate(t *testing.T) { testutils.SucceedsSoon(t, func() error { values := []int64{} for _, eng := range mtc.engines { - val, _, err := engine.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), - engine.MVCCGetOptions{}) + val, _, err := storage.MVCCGet(context.Background(), eng, roachpb.Key("a"), clocks[0].Now(), + storage.MVCCGetOptions{}) if err != nil { return err } @@ -162,8 +162,8 @@ func TestRejectFutureCommand(t *testing.T) { if advance := ts3.GoTime().Sub(ts2.GoTime()); advance != 0 { t.Fatalf("expected clock not to advance, but it advanced by %s", advance) } - val, _, err := engine.MVCCGet(context.Background(), mtc.engines[0], key, ts3, - engine.MVCCGetOptions{}) + val, _, err := storage.MVCCGet(context.Background(), mtc.engines[0], key, ts3, + storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } @@ -234,7 +234,7 @@ func TestTxnPutOutOfOrder(t *testing.T) { } return nil } - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) store := createTestStoreWithOpts(t, testStoreOpts{eng: eng, cfg: &cfg}, @@ -1700,7 +1700,7 @@ func TestSystemZoneConfigs(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -1824,7 +1824,7 @@ func TestClearRange(t *testing.T) { t.Helper() start := prefix end := prefix.PrefixEnd() - kvs, err := engine.Scan(store.Engine(), start, end, 0 /* maxRows */) + kvs, err := storage.Scan(store.Engine(), start, end, 0 /* maxRows */) if err != nil { t.Fatal(err) } @@ -2174,7 +2174,7 @@ func TestReplicaTombstone(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -2558,8 +2558,8 @@ func TestReplicaTombstone(t *testing.T) { return err } tombstoneKey := keys.RangeTombstoneKey(rhsDesc.RangeID) - ok, err := engine.MVCCGetProto( - context.TODO(), store.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + ok, err := storage.MVCCGetProto( + context.TODO(), store.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ) require.NoError(t, err) if !ok { diff --git a/pkg/kv/kvserver/client_split_test.go b/pkg/kv/kvserver/client_split_test.go index f87a78c49248..4e5e7898058d 100644 --- a/pkg/kv/kvserver/client_split_test.go +++ b/pkg/kv/kvserver/client_split_test.go @@ -27,8 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -41,6 +39,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -384,8 +384,8 @@ func TestStoreRangeSplitIntents(t *testing.T) { t.Fatal(err) } for _, key := range []roachpb.Key{keys.RangeDescriptorKey(roachpb.RKeyMin), keys.RangeDescriptorKey(splitKeyAddr)} { - if _, _, err := engine.MVCCGet( - context.Background(), store.Engine(), key, store.Clock().Now(), engine.MVCCGetOptions{}, + if _, _, err := storage.MVCCGet( + context.Background(), store.Engine(), key, store.Clock().Now(), storage.MVCCGetOptions{}, ); err != nil { t.Errorf("failed to read consistent range descriptor for key %s: %+v", key, err) } @@ -399,9 +399,9 @@ func TestStoreRangeSplitIntents(t *testing.T) { return keys.MakeRangeKey(rk, keys.LocalTransactionSuffix, nil) } // Verify the transaction record is gone. - start := engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(roachpb.RKeyMin)) - end := engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(roachpb.RKeyMax)) - iter := store.Engine().NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + start := storage.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(roachpb.RKeyMin)) + end := storage.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(roachpb.RKeyMax)) + iter := store.Engine().NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer iter.Close() for iter.SeekGE(start); ; iter.Next() { @@ -604,8 +604,8 @@ func TestStoreRangeSplitIdempotency(t *testing.T) { t.Fatal(err) } for _, key := range []roachpb.Key{keys.RangeDescriptorKey(roachpb.RKeyMin), keys.RangeDescriptorKey(splitKeyAddr)} { - if _, _, err := engine.MVCCGet( - context.Background(), store.Engine(), key, store.Clock().Now(), engine.MVCCGetOptions{}, + if _, _, err := storage.MVCCGet( + context.Background(), store.Engine(), key, store.Clock().Now(), storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/client_test.go b/pkg/kv/kvserver/client_test.go index 309920fdcda5..52e659cfe037 100644 --- a/pkg/kv/kvserver/client_test.go +++ b/pkg/kv/kvserver/client_test.go @@ -35,8 +35,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/gossip/resolver" "github.com/cockroachdb/cockroach/pkg/internal/client" @@ -50,6 +48,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -100,7 +100,7 @@ type testStoreOpts struct { dontCreateSystemRanges bool cfg *kvserver.StoreConfig - eng engine.Engine + eng storage.Engine } // createTestStoreWithOpts creates a test store using the given engine and clock. @@ -118,7 +118,7 @@ func createTestStoreWithOpts( } eng := opts.eng if eng == nil { - eng = engine.NewDefaultInMem() + eng = storage.NewDefaultInMem() stopper.AddCloser(eng) } @@ -277,7 +277,7 @@ type multiTestContext struct { // multiTestContext.clock, but it may be populated before Start() to // use distinct clocks per store. clocks []*hlc.Clock - engines []engine.Engine + engines []storage.Engine grpcServers []*grpc.Server distSenders []*kv.DistSender dbs []*client.DB @@ -793,7 +793,7 @@ func (m *multiTestContext) addStore(idx int) { clock = m.clock m.clocks = append(m.clocks, clock) } - var eng engine.Engine + var eng storage.Engine var needBootstrap bool if len(m.engines) > idx { eng = m.engines[idx] @@ -806,7 +806,7 @@ func (m *multiTestContext) addStore(idx int) { } else { engineStopper := stop.NewStopper() m.engineStoppers = append(m.engineStoppers, engineStopper) - eng = engine.NewDefaultInMem() + eng = storage.NewDefaultInMem() engineStopper.AddCloser(eng) m.engines = append(m.engines, eng) needBootstrap = true @@ -1275,8 +1275,8 @@ func (m *multiTestContext) waitForUnreplicated(rangeID roachpb.RangeID, dest int func (m *multiTestContext) readIntFromEngines(key roachpb.Key) []int64 { results := make([]int64, len(m.engines)) for i, eng := range m.engines { - val, _, err := engine.MVCCGet(context.Background(), eng, key, m.clocks[i].Now(), - engine.MVCCGetOptions{}) + val, _, err := storage.MVCCGet(context.Background(), eng, key, m.clocks[i].Now(), + storage.MVCCGetOptions{}) if err != nil { log.VEventf(context.TODO(), 1, "engine %d: error reading from key %s: %s", i, key, err) } else if val == nil { @@ -1300,7 +1300,7 @@ func (m *multiTestContext) waitForValuesT(t testing.TB, key roachpb.Key, expecte // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } testutils.SucceedsSoon(t, func() error { @@ -1563,7 +1563,7 @@ func TestSortRangeDescByAge(t *testing.T) { } func verifyRangeStats( - reader engine.Reader, rangeID roachpb.RangeID, expMS enginepb.MVCCStats, + reader storage.Reader, rangeID roachpb.RangeID, expMS enginepb.MVCCStats, ) error { ms, err := stateloader.Make(rangeID).LoadMVCCStats(context.Background(), reader) if err != nil { @@ -1578,7 +1578,7 @@ func verifyRangeStats( } func verifyRecomputedStats( - reader engine.Reader, d *roachpb.RangeDescriptor, expMS enginepb.MVCCStats, nowNanos int64, + reader storage.Reader, d *roachpb.RangeDescriptor, expMS enginepb.MVCCStats, nowNanos int64, ) error { if ms, err := rditer.ComputeStatsForRange(d, reader, nowNanos); err != nil { return err @@ -1589,12 +1589,12 @@ func verifyRecomputedStats( } func waitForTombstone( - t *testing.T, reader engine.Reader, rangeID roachpb.RangeID, + t *testing.T, reader storage.Reader, rangeID roachpb.RangeID, ) (tombstone roachpb.RangeTombstone) { testutils.SucceedsSoon(t, func() error { tombstoneKey := keys.RangeTombstoneKey(rangeID) - ok, err := engine.MVCCGetProto( - context.TODO(), reader, tombstoneKey, hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + ok, err := storage.MVCCGetProto( + context.TODO(), reader, tombstoneKey, hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ) if err != nil { t.Fatalf("failed to read tombstone: %v", err) diff --git a/pkg/kv/kvserver/compactor/compactor.go b/pkg/kv/kvserver/compactor/compactor.go index 951854d758b8..aa9f979b1f63 100644 --- a/pkg/kv/kvserver/compactor/compactor.go +++ b/pkg/kv/kvserver/compactor/compactor.go @@ -15,11 +15,11 @@ import ( "fmt" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -38,7 +38,7 @@ type doneCompactingFunc func(ctx context.Context) // makes requests to the engine to reclaim storage space. type Compactor struct { st *cluster.Settings - eng engine.Engine + eng storage.Engine capFn storeCapacityFunc doneFn doneCompactingFunc ch chan struct{} @@ -47,7 +47,7 @@ type Compactor struct { // NewCompactor returns a compactor for the specified storage engine. func NewCompactor( - st *cluster.Settings, eng engine.Engine, capFn storeCapacityFunc, doneFn doneCompactingFunc, + st *cluster.Settings, eng storage.Engine, capFn storeCapacityFunc, doneFn doneCompactingFunc, ) *Compactor { return &Compactor{ st: st, @@ -228,7 +228,7 @@ func (c *Compactor) processSuggestions(ctx context.Context) (bool, error) { } // Get information about SSTables in the underlying RocksDB instance. - ssti := engine.NewSSTableInfosByLevel(c.eng.GetSSTables()) + ssti := storage.NewSSTableInfosByLevel(c.eng.GetSSTables()) // Update the bytes queued metric based, periodically querying the persisted // suggestions so that we pick up newly added suggestions in the case where @@ -286,7 +286,7 @@ func (c *Compactor) processSuggestions(ctx context.Context) (bool, error) { func (c *Compactor) fetchSuggestions( ctx context.Context, ) (suggestions []storagepb.SuggestedCompaction, totalBytes int64, err error) { - dataIter := c.eng.NewIterator(engine.IterOptions{ + dataIter := c.eng.NewIterator(storage.IterOptions{ UpperBound: roachpb.KeyMax, // refined before every seek }) defer dataIter.Close() @@ -297,7 +297,7 @@ func (c *Compactor) fetchSuggestions( err = c.eng.Iterate( keys.LocalStoreSuggestedCompactionsMin, keys.LocalStoreSuggestedCompactionsMax, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { var sc storagepb.SuggestedCompaction var err error sc.StartKey, sc.EndKey, err = keys.DecodeStoreSuggestedCompactionKey(kv.Key.Key) @@ -309,10 +309,10 @@ func (c *Compactor) fetchSuggestions( } dataIter.SetUpperBound(sc.EndKey) - dataIter.SeekGE(engine.MakeMVCCMetadataKey(sc.StartKey)) + dataIter.SeekGE(storage.MakeMVCCMetadataKey(sc.StartKey)) if ok, err := dataIter.Valid(); err != nil { return false, err - } else if ok && dataIter.UnsafeKey().Less(engine.MakeMVCCMetadataKey(sc.EndKey)) { + } else if ok && dataIter.UnsafeKey().Less(storage.MakeMVCCMetadataKey(sc.EndKey)) { // The suggested compaction span has live keys remaining. This is a // strong indicator that compacting this range will be significantly // more expensive than we expected when the compaction was suggested, as @@ -402,7 +402,7 @@ func (c *Compactor) processCompaction( c.Metrics.BytesSkipped.Inc(sc.Bytes) } key := keys.StoreSuggestedCompactionKey(sc.StartKey, sc.EndKey) - if err := delBatch.Clear(engine.MVCCKey{Key: key}); err != nil { + if err := delBatch.Clear(storage.MVCCKey{Key: key}); err != nil { log.Fatal(ctx, err) // should never happen on a batch } } @@ -427,7 +427,7 @@ func (c *Compactor) processCompaction( // continue aggregating suggested compactions. func (c *Compactor) aggregateCompaction( ctx context.Context, - ssti engine.SSTableInfosByLevel, + ssti storage.SSTableInfosByLevel, aggr *aggregatedCompaction, sc storagepb.SuggestedCompaction, ) (done bool) { @@ -465,7 +465,7 @@ func (c *Compactor) examineQueue(ctx context.Context) (int64, error) { if err := c.eng.Iterate( keys.LocalStoreSuggestedCompactionsMin, keys.LocalStoreSuggestedCompactionsMax, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { var c storagepb.Compaction if err := protoutil.Unmarshal(kv.Value, &c); err != nil { return false, err @@ -489,7 +489,7 @@ func (c *Compactor) Suggest(ctx context.Context, sc storagepb.SuggestedCompactio key := keys.StoreSuggestedCompactionKey(sc.StartKey, sc.EndKey) var existing storagepb.Compaction //lint:ignore SA1019 historical usage of deprecated c.eng.GetProto is OK - ok, _, _, err := c.eng.GetProto(engine.MVCCKey{Key: key}, &existing) + ok, _, _, err := c.eng.GetProto(storage.MVCCKey{Key: key}, &existing) if err != nil { log.VErrEventf(ctx, 2, "unable to record suggested compaction: %s", err) return @@ -506,7 +506,7 @@ func (c *Compactor) Suggest(ctx context.Context, sc storagepb.SuggestedCompactio // Store the new compaction. //lint:ignore SA1019 historical usage of deprecated engine.PutProto is OK - if _, _, err = engine.PutProto(c.eng, engine.MVCCKey{Key: key}, &sc.Compaction); err != nil { + if _, _, err = storage.PutProto(c.eng, storage.MVCCKey{Key: key}, &sc.Compaction); err != nil { log.Warningf(ctx, "unable to record suggested compaction: %+v", err) } diff --git a/pkg/kv/kvserver/compactor/compactor_test.go b/pkg/kv/kvserver/compactor/compactor_test.go index 2634d5f6ff16..181cc70cce0e 100644 --- a/pkg/kv/kvserver/compactor/compactor_test.go +++ b/pkg/kv/kvserver/compactor/compactor_test.go @@ -19,12 +19,12 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -36,7 +36,7 @@ import ( const testCompactionLatency = 1 * time.Millisecond type wrappedEngine struct { - engine.Engine + storage.Engine mu struct { syncutil.Mutex compactions []roachpb.Span @@ -45,15 +45,15 @@ type wrappedEngine struct { func newWrappedEngine() *wrappedEngine { return &wrappedEngine{ - Engine: engine.NewDefaultInMem(), + Engine: storage.NewDefaultInMem(), } } -func (we *wrappedEngine) GetSSTables() engine.SSTableInfos { - key := func(s string) engine.MVCCKey { - return engine.MakeMVCCMetadataKey([]byte(s)) +func (we *wrappedEngine) GetSSTables() storage.SSTableInfos { + key := func(s string) storage.MVCCKey { + return storage.MakeMVCCMetadataKey([]byte(s)) } - ssti := engine.SSTableInfos{ + ssti := storage.SSTableInfos{ // Level 0. {Level: 0, Size: 20, Start: key("a"), End: key("z")}, {Level: 0, Size: 15, Start: key("a"), End: key("k")}, @@ -113,7 +113,7 @@ func TestCompactorThresholds(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } @@ -536,7 +536,7 @@ func TestCompactorThresholds(t *testing.T) { // Add a key so we can test that suggestions that span live data are // ignored. - if err := we.Put(engine.MakeMVCCMetadataKey(key("5")), nil); err != nil { + if err := we.Put(storage.MakeMVCCMetadataKey(key("5")), nil); err != nil { t.Fatal(err) } @@ -581,7 +581,7 @@ func TestCompactorThresholds(t *testing.T) { return we.Iterate( keys.LocalStoreSuggestedCompactionsMin, keys.LocalStoreSuggestedCompactionsMax, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { start, end, err := keys.DecodeStoreSuggestedCompactionKey(kv.Key.Key) if err != nil { t.Fatalf("failed to decode suggested compaction key: %+v", err) diff --git a/pkg/kv/kvserver/concurrency/concurrency_control.go b/pkg/kv/kvserver/concurrency/concurrency_control.go index bfb8785512f3..9b50a7d53782 100644 --- a/pkg/kv/kvserver/concurrency/concurrency_control.go +++ b/pkg/kv/kvserver/concurrency/concurrency_control.go @@ -16,12 +16,12 @@ package concurrency import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) diff --git a/pkg/kv/kvserver/concurrency/concurrency_manager_test.go b/pkg/kv/kvserver/concurrency/concurrency_manager_test.go index a4ab499eb558..4a1918070309 100644 --- a/pkg/kv/kvserver/concurrency/concurrency_manager_test.go +++ b/pkg/kv/kvserver/concurrency/concurrency_manager_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" @@ -33,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait" "github.com/cockroachdb/cockroach/pkg/roachpb" clustersettings "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/concurrency/lock_table.go b/pkg/kv/kvserver/concurrency/lock_table.go index 212a03df6c15..b312017eacc1 100644 --- a/pkg/kv/kvserver/concurrency/lock_table.go +++ b/pkg/kv/kvserver/concurrency/lock_table.go @@ -18,11 +18,11 @@ import ( "sync" "sync/atomic" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" diff --git a/pkg/kv/kvserver/concurrency/lock_table_test.go b/pkg/kv/kvserver/concurrency/lock_table_test.go index a344523de908..76788d0f4d3d 100644 --- a/pkg/kv/kvserver/concurrency/lock_table_test.go +++ b/pkg/kv/kvserver/concurrency/lock_table_test.go @@ -20,11 +20,11 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/concurrency/lock_table_waiter.go b/pkg/kv/kvserver/concurrency/lock_table_waiter.go index 8d61724ed149..dad138ecae8b 100644 --- a/pkg/kv/kvserver/concurrency/lock_table_waiter.go +++ b/pkg/kv/kvserver/concurrency/lock_table_waiter.go @@ -15,12 +15,12 @@ import ( "math" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/timeutil" ) diff --git a/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go b/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go index 686651928598..c00f08b3223e 100644 --- a/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go +++ b/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go @@ -14,11 +14,11 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/consistency_queue_test.go b/pkg/kv/kvserver/consistency_queue_test.go index 9edb9363b617..aba35618f458 100644 --- a/pkg/kv/kvserver/consistency_queue_test.go +++ b/pkg/kv/kvserver/consistency_queue_test.go @@ -22,13 +22,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -188,13 +188,13 @@ func TestCheckConsistencyInconsistent(t *testing.T) { dir, cleanup := testutils.TempDir(t) defer cleanup() - cache := engine.NewRocksDBCache(1 << 20) + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() // Use on-disk stores because we want to take a RocksDB checkpoint and be // able to find it. for i := 0; i < numStores; i++ { - eng, err := engine.NewRocksDB(engine.RocksDBConfig{ + eng, err := storage.NewRocksDB(storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: filepath.Join(dir, fmt.Sprintf("%d", i)), }, @@ -313,7 +313,7 @@ func TestCheckConsistencyInconsistent(t *testing.T) { var val roachpb.Value val.SetInt(42) diffTimestamp = mtc.stores[1].Clock().Now() - if err := engine.MVCCPut( + if err := storage.MVCCPut( context.Background(), mtc.stores[1].Engine(), nil, diffKey, diffTimestamp, val, nil, ); err != nil { t.Fatal(err) @@ -337,7 +337,7 @@ func TestCheckConsistencyInconsistent(t *testing.T) { for i := 0; i < numStores; i++ { cps := checkpoints(i) assert.Len(t, cps, 1) - cpEng, err := engine.NewRocksDB(engine.RocksDBConfig{ + cpEng, err := storage.NewRocksDB(storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: cps[0], }, @@ -345,10 +345,10 @@ func TestCheckConsistencyInconsistent(t *testing.T) { assert.NoError(t, err) defer cpEng.Close() - iter := cpEng.NewIterator(engine.IterOptions{UpperBound: []byte("\xff")}) + iter := cpEng.NewIterator(storage.IterOptions{UpperBound: []byte("\xff")}) defer iter.Close() - ms, err := engine.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 0 /* nowNanos */) + ms, err := storage.ComputeStatsGo(iter, roachpb.KeyMin, roachpb.KeyMax, 0 /* nowNanos */) assert.NoError(t, err) assert.NotZero(t, ms.KeyBytes) @@ -468,9 +468,9 @@ func testConsistencyQueueRecomputeStatsImpl(t *testing.T, hadEstimates bool) { const sysCountGarbage = 123000 func() { - cache := engine.NewRocksDBCache(1 << 20) + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() - eng, err := engine.NewRocksDB(engine.RocksDBConfig{ + eng, err := storage.NewRocksDB(storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: path, MustExist: true, diff --git a/pkg/kv/kvserver/debug_print.go b/pkg/kv/kvserver/debug_print.go index dafc81ff3a9d..7f9a4567990f 100644 --- a/pkg/kv/kvserver/debug_print.go +++ b/pkg/kv/kvserver/debug_print.go @@ -17,11 +17,11 @@ import ( "strconv" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "go.etcd.io/etcd/raft/raftpb" @@ -29,24 +29,24 @@ import ( // PrintKeyValue attempts to pretty-print the specified MVCCKeyValue to // os.Stdout, falling back to '%q' formatting. -func PrintKeyValue(kv engine.MVCCKeyValue) { +func PrintKeyValue(kv storage.MVCCKeyValue) { fmt.Println(SprintKeyValue(kv, true /* printKey */)) } // SprintKey pretty-prings the specified MVCCKey. -func SprintKey(key engine.MVCCKey) string { - return fmt.Sprintf("%s %s (%#x): ", key.Timestamp, key.Key, engine.EncodeKey(key)) +func SprintKey(key storage.MVCCKey) string { + return fmt.Sprintf("%s %s (%#x): ", key.Timestamp, key.Key, storage.EncodeKey(key)) } // SprintKeyValue is like PrintKeyValue, but returns a string. If // printKey is true, prints the key and the value together; otherwise, // prints just the value. -func SprintKeyValue(kv engine.MVCCKeyValue, printKey bool) string { +func SprintKeyValue(kv storage.MVCCKeyValue, printKey bool) string { var sb strings.Builder if printKey { sb.WriteString(SprintKey(kv.Key)) } - decoders := []func(kv engine.MVCCKeyValue) (string, error){ + decoders := []func(kv storage.MVCCKeyValue) (string, error){ tryRaftLogEntry, tryRangeDescriptor, tryMeta, @@ -54,7 +54,7 @@ func SprintKeyValue(kv engine.MVCCKeyValue, printKey bool) string { tryRangeIDKey, tryTimeSeries, tryIntent, - func(kv engine.MVCCKeyValue) (string, error) { + func(kv storage.MVCCKeyValue) (string, error) { // No better idea, just print raw bytes and hope that folks use `less -S`. return fmt.Sprintf("%q", kv.Value), nil }, @@ -70,7 +70,7 @@ func SprintKeyValue(kv engine.MVCCKeyValue, printKey bool) string { panic("unreachable") } -func tryRangeDescriptor(kv engine.MVCCKeyValue) (string, error) { +func tryRangeDescriptor(kv storage.MVCCKeyValue) (string, error) { if err := IsRangeDescriptorKey(kv.Key); err != nil { return "", err } @@ -81,7 +81,7 @@ func tryRangeDescriptor(kv engine.MVCCKeyValue) (string, error) { return descStr(desc), nil } -func tryIntent(kv engine.MVCCKeyValue) (string, error) { +func tryIntent(kv storage.MVCCKeyValue) (string, error) { if len(kv.Value) == 0 { return "", errors.New("empty") } @@ -101,7 +101,7 @@ func decodeWriteBatch(writeBatch *storagepb.WriteBatch) (string, error) { return "\n", nil } - r, err := engine.NewRocksDBBatchReader(writeBatch.Data) + r, err := storage.NewRocksDBBatchReader(writeBatch.Data) if err != nil { return "", err } @@ -111,37 +111,37 @@ func decodeWriteBatch(writeBatch *storagepb.WriteBatch) (string, error) { var sb strings.Builder for r.Next() { switch r.BatchType() { - case engine.BatchTypeDeletion: + case storage.BatchTypeDeletion: mvccKey, err := r.MVCCKey() if err != nil { return sb.String(), err } sb.WriteString(fmt.Sprintf("Delete: %s\n", SprintKey(mvccKey))) - case engine.BatchTypeValue: + case storage.BatchTypeValue: mvccKey, err := r.MVCCKey() if err != nil { return sb.String(), err } - sb.WriteString(fmt.Sprintf("Put: %s\n", SprintKeyValue(engine.MVCCKeyValue{ + sb.WriteString(fmt.Sprintf("Put: %s\n", SprintKeyValue(storage.MVCCKeyValue{ Key: mvccKey, Value: r.Value(), }, true /* printKey */))) - case engine.BatchTypeMerge: + case storage.BatchTypeMerge: mvccKey, err := r.MVCCKey() if err != nil { return sb.String(), err } - sb.WriteString(fmt.Sprintf("Merge: %s\n", SprintKeyValue(engine.MVCCKeyValue{ + sb.WriteString(fmt.Sprintf("Merge: %s\n", SprintKeyValue(storage.MVCCKeyValue{ Key: mvccKey, Value: r.Value(), }, true /* printKey */))) - case engine.BatchTypeSingleDeletion: + case storage.BatchTypeSingleDeletion: mvccKey, err := r.MVCCKey() if err != nil { return sb.String(), err } sb.WriteString(fmt.Sprintf("Single Delete: %s\n", SprintKey(mvccKey))) - case engine.BatchTypeRangeDeletion: + case storage.BatchTypeRangeDeletion: mvccStartKey, err := r.MVCCKey() if err != nil { return sb.String(), err @@ -160,7 +160,7 @@ func decodeWriteBatch(writeBatch *storagepb.WriteBatch) (string, error) { return sb.String(), r.Error() } -func tryRaftLogEntry(kv engine.MVCCKeyValue) (string, error) { +func tryRaftLogEntry(kv storage.MVCCKeyValue) (string, error) { var ent raftpb.Entry if err := maybeUnmarshalInline(kv.Value, &ent); err != nil { return "", err @@ -220,7 +220,7 @@ func tryRaftLogEntry(kv engine.MVCCKeyValue) (string, error) { return fmt.Sprintf("%s by %s\n%s\nwrite batch:\n%s", &ent, leaseStr, &cmd, wbStr), nil } -func tryTxn(kv engine.MVCCKeyValue) (string, error) { +func tryTxn(kv storage.MVCCKeyValue) (string, error) { var txn roachpb.Transaction if err := maybeUnmarshalInline(kv.Value, &txn); err != nil { return "", err @@ -228,7 +228,7 @@ func tryTxn(kv engine.MVCCKeyValue) (string, error) { return txn.String() + "\n", nil } -func tryRangeIDKey(kv engine.MVCCKeyValue) (string, error) { +func tryRangeIDKey(kv storage.MVCCKeyValue) (string, error) { if kv.Key.Timestamp != (hlc.Timestamp{}) { return "", fmt.Errorf("range ID keys shouldn't have timestamps: %s", kv.Key) } @@ -294,7 +294,7 @@ func tryRangeIDKey(kv engine.MVCCKeyValue) (string, error) { return msg.String(), nil } -func tryMeta(kv engine.MVCCKeyValue) (string, error) { +func tryMeta(kv storage.MVCCKeyValue) (string, error) { if !bytes.HasPrefix(kv.Key.Key, keys.Meta1Prefix) && !bytes.HasPrefix(kv.Key.Key, keys.Meta2Prefix) { return "", errors.New("not a meta key") } @@ -309,7 +309,7 @@ func tryMeta(kv engine.MVCCKeyValue) (string, error) { return descStr(desc), nil } -func tryTimeSeries(kv engine.MVCCKeyValue) (string, error) { +func tryTimeSeries(kv storage.MVCCKeyValue) (string, error) { if len(kv.Value) == 0 || !bytes.HasPrefix(kv.Key.Key, keys.TimeseriesPrefix) { return "", errors.New("empty or not TS") } @@ -326,7 +326,7 @@ func tryTimeSeries(kv engine.MVCCKeyValue) (string, error) { } // IsRangeDescriptorKey returns nil if the key decodes as a RangeDescriptor. -func IsRangeDescriptorKey(key engine.MVCCKey) error { +func IsRangeDescriptorKey(key storage.MVCCKey) error { _, suffix, _, err := keys.DecodeRangeKey(key.Key) if err != nil { return err diff --git a/pkg/kv/kvserver/debug_print_test.go b/pkg/kv/kvserver/debug_print_test.go index ab5f4958ec52..3f0bb567afd4 100644 --- a/pkg/kv/kvserver/debug_print_test.go +++ b/pkg/kv/kvserver/debug_print_test.go @@ -14,9 +14,9 @@ import ( "math" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" ) @@ -30,8 +30,8 @@ func TestStringifyWriteBatch(t *testing.T) { t.Errorf("expected %q for stringified write batch; got %q", expStr, str) } - builder := engine.RocksDBBatchBuilder{} - builder.Put(engine.MVCCKey{ + builder := storage.RocksDBBatchBuilder{} + builder.Put(storage.MVCCKey{ Key: roachpb.Key("/db1"), Timestamp: hlc.Timestamp{WallTime: math.MaxInt64}, }, []byte("test value")) diff --git a/pkg/kv/kvserver/gc/data_distribution_test.go b/pkg/kv/kvserver/gc/data_distribution_test.go index 21f5c9e96ab1..c791fd37c434 100644 --- a/pkg/kv/kvserver/gc/data_distribution_test.go +++ b/pkg/kv/kvserver/gc/data_distribution_test.go @@ -18,11 +18,11 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -33,12 +33,12 @@ import ( // MVCCKeyValues. The stream may indicate that a value is an intent by returning // a non-nil transaction. If an intent is returned it must have a higher // timestamp than any other version written for the key. -type dataDistribution func() (engine.MVCCKeyValue, *roachpb.Transaction, bool) +type dataDistribution func() (storage.MVCCKeyValue, *roachpb.Transaction, bool) // setupTest writes the data from this distribution into eng. All data should // be a part of the range represented by desc. func (ds dataDistribution) setupTest( - t testing.TB, eng engine.Engine, desc roachpb.RangeDescriptor, + t testing.TB, eng storage.Engine, desc roachpb.RangeDescriptor, ) enginepb.MVCCStats { ctx := context.TODO() var maxTs hlc.Timestamp @@ -59,7 +59,7 @@ func (ds dataDistribution) setupTest( if txn.WriteTimestamp == (hlc.Timestamp{}) { txn.WriteTimestamp = ts } - err := engine.MVCCPut(ctx, eng, &ms, kv.Key.Key, ts, + err := storage.MVCCPut(ctx, eng, &ms, kv.Key.Key, ts, roachpb.Value{RawBytes: kv.Value}, txn) require.NoError(t, err) } @@ -96,9 +96,9 @@ func newDataDistribution( timestamps []hlc.Timestamp haveIntent bool ) - return func() (engine.MVCCKeyValue, *roachpb.Transaction, bool) { + return func() (storage.MVCCKeyValue, *roachpb.Transaction, bool) { if remaining == 0 { - return engine.MVCCKeyValue{}, nil, false + return storage.MVCCKeyValue{}, nil, false } defer func() { remaining-- }() for len(timestamps) == 0 { @@ -140,8 +140,8 @@ func newDataDistribution( txn.WriteTimestamp = ts txn.Key = keyDist() } - return engine.MVCCKeyValue{ - Key: engine.MVCCKey{Key: key, Timestamp: ts}, + return storage.MVCCKeyValue{ + Key: storage.MVCCKey{Key: key, Timestamp: ts}, Value: valueDist(), }, txn, true } diff --git a/pkg/kv/kvserver/gc/gc.go b/pkg/kv/kvserver/gc/gc.go index c01239f37023..5edd3769dd77 100644 --- a/pkg/kv/kvserver/gc/gc.go +++ b/pkg/kv/kvserver/gc/gc.go @@ -22,13 +22,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -158,7 +158,7 @@ type CleanupTxnIntentsAsyncFunc func(context.Context, *roachpb.Transaction, []ro func Run( ctx context.Context, desc *roachpb.RangeDescriptor, - snap engine.Reader, + snap storage.Reader, now, newThreshold hlc.Timestamp, policy zonepb.GCPolicy, gcer GCer, @@ -223,7 +223,7 @@ func Run( func processReplicatedKeyRange( ctx context.Context, desc *roachpb.RangeDescriptor, - snap engine.Reader, + snap storage.Reader, now hlc.Timestamp, threshold hlc.Timestamp, gcer GCer, @@ -234,7 +234,7 @@ func processReplicatedKeyRange( var alloc bufalloc.ByteAllocator // Compute intent expiration (intent age at which we attempt to resolve). intentExp := now.Add(-IntentAgeThreshold.Nanoseconds(), 0) - handleIntent := func(md *engine.MVCCKeyValue) { + handleIntent := func(md *storage.MVCCKeyValue) { meta := &enginepb.MVCCMetadata{} if err := protoutil.Unmarshal(md.Value, meta); err != nil { log.Errorf(ctx, "unable to unmarshal MVCC metadata for key %q: %+v", md.Key, err) @@ -363,7 +363,7 @@ func processReplicatedKeyRange( // guaranteed as described above. However if this were the only rule, then if // the most recent write was a delete, it would never be removed. Thus, when a // deleted value is the most recent before expiration, it can be deleted. -func isGarbage(threshold hlc.Timestamp, cur, next *engine.MVCCKeyValue, isNewest bool) bool { +func isGarbage(threshold hlc.Timestamp, cur, next *storage.MVCCKeyValue, isNewest bool) bool { // If the value is not at or below the threshold then it's not garbage. if belowThreshold := cur.Key.Timestamp.LessEq(threshold); !belowThreshold { return false @@ -397,7 +397,7 @@ func isGarbage(threshold hlc.Timestamp, cur, next *engine.MVCCKeyValue, isNewest // this range's start key. This can happen on range merges. func processLocalKeyRange( ctx context.Context, - snap engine.Reader, + snap storage.Reader, desc *roachpb.RangeDescriptor, cutoff hlc.Timestamp, info *Info, @@ -473,7 +473,7 @@ func processLocalKeyRange( startKey := keys.MakeRangeKeyPrefix(desc.StartKey) endKey := keys.MakeRangeKeyPrefix(desc.EndKey) - _, err := engine.MVCCIterate(ctx, snap, startKey, endKey, hlc.Timestamp{}, engine.MVCCScanOptions{}, + _, err := storage.MVCCIterate(ctx, snap, startKey, endKey, hlc.Timestamp{}, storage.MVCCScanOptions{}, func(kv roachpb.KeyValue) (bool, error) { return false, handleOne(kv) }) @@ -487,7 +487,7 @@ func processLocalKeyRange( // multiple of the heartbeat timeout used by the coordinator. func processAbortSpan( ctx context.Context, - snap engine.Reader, + snap storage.Reader, rangeID roachpb.RangeID, threshold hlc.Timestamp, info *Info, diff --git a/pkg/kv/kvserver/gc/gc_iterator.go b/pkg/kv/kvserver/gc/gc_iterator.go index 38f5f3667bc6..2b7ed66292b1 100644 --- a/pkg/kv/kvserver/gc/gc_iterator.go +++ b/pkg/kv/kvserver/gc/gc_iterator.go @@ -11,9 +11,9 @@ package gc import ( - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" ) @@ -26,7 +26,7 @@ type gcIterator struct { buf gcIteratorRingBuf } -func makeGCIterator(desc *roachpb.RangeDescriptor, snap engine.Reader) gcIterator { +func makeGCIterator(desc *roachpb.RangeDescriptor, snap storage.Reader) gcIterator { return gcIterator{ it: rditer.NewReplicaDataIterator(desc, snap, true /* replicatedOnly */, true /* seekEnd */), @@ -34,7 +34,7 @@ func makeGCIterator(desc *roachpb.RangeDescriptor, snap engine.Reader) gcIterato } type gcIteratorState struct { - cur, next, afterNext *engine.MVCCKeyValue + cur, next, afterNext *storage.MVCCKeyValue } // curIsNewest returns true if the current MVCCKeyValue in the gcIteratorState @@ -98,7 +98,7 @@ func (it *gcIterator) step() { it.buf.removeFront() } -func (it *gcIterator) peekAt(i int) (*engine.MVCCKeyValue, bool) { +func (it *gcIterator) peekAt(i int) (*storage.MVCCKeyValue, bool) { if it.buf.len <= i { if !it.fillTo(i + 1) { return nil, false @@ -130,12 +130,12 @@ const gcIteratorRingBufSize = 3 type gcIteratorRingBuf struct { allocs [gcIteratorRingBufSize]bufalloc.ByteAllocator - buf [gcIteratorRingBufSize]engine.MVCCKeyValue + buf [gcIteratorRingBufSize]storage.MVCCKeyValue len int head int } -func (b *gcIteratorRingBuf) at(i int) *engine.MVCCKeyValue { +func (b *gcIteratorRingBuf) at(i int) *storage.MVCCKeyValue { if i >= b.len { panic("index out of range") } @@ -146,13 +146,13 @@ func (b *gcIteratorRingBuf) removeFront() { if b.len == 0 { panic("cannot remove from empty gcIteratorRingBuf") } - b.buf[b.head] = engine.MVCCKeyValue{} + b.buf[b.head] = storage.MVCCKeyValue{} b.head = (b.head + 1) % gcIteratorRingBufSize b.len-- } type iterator interface { - UnsafeKey() engine.MVCCKey + UnsafeKey() storage.MVCCKey UnsafeValue() []byte } @@ -166,7 +166,7 @@ func (b *gcIteratorRingBuf) pushBack(it iterator) { v := it.UnsafeValue() b.allocs[i], k.Key = b.allocs[i].Copy(k.Key, len(v)) b.allocs[i], v = b.allocs[i].Copy(v, 0) - b.buf[i] = engine.MVCCKeyValue{ + b.buf[i] = storage.MVCCKeyValue{ Key: k, Value: v, } diff --git a/pkg/kv/kvserver/gc/gc_iterator_test.go b/pkg/kv/kvserver/gc/gc_iterator_test.go index ffa56f4ed817..ebf5f758d94c 100644 --- a/pkg/kv/kvserver/gc/gc_iterator_test.go +++ b/pkg/kv/kvserver/gc/gc_iterator_test.go @@ -16,9 +16,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/stretchr/testify/require" @@ -31,14 +31,14 @@ func TestGCIterator(t *testing.T) { // dataItem represents a version in the storage engine and optionally a // corresponding transaction which will make the MVCCKeyValue an intent. type dataItem struct { - engine.MVCCKeyValue + storage.MVCCKeyValue txn *roachpb.Transaction } // makeDataItem is a shorthand to construct dataItems. makeDataItem := func(k roachpb.Key, val []byte, ts int64, txn *roachpb.Transaction) dataItem { return dataItem{ - MVCCKeyValue: engine.MVCCKeyValue{ - Key: engine.MVCCKey{ + MVCCKeyValue: storage.MVCCKeyValue{ + Key: storage.MVCCKey{ Key: k, Timestamp: hlc.Timestamp{WallTime: ts * time.Nanosecond.Nanoseconds()}, }, @@ -50,9 +50,9 @@ func TestGCIterator(t *testing.T) { // makeLiteralDistribution adapts dataItems for use with the data distribution // infrastructure. makeLiteralDataDistribution := func(items ...dataItem) dataDistribution { - return func() (engine.MVCCKeyValue, *roachpb.Transaction, bool) { + return func() (storage.MVCCKeyValue, *roachpb.Transaction, bool) { if len(items) == 0 { - return engine.MVCCKeyValue{}, nil, false + return storage.MVCCKeyValue{}, nil, false } item := items[0] defer func() { items = items[1:] }() @@ -104,7 +104,7 @@ func TestGCIterator(t *testing.T) { checkExpectations := func( t *testing.T, data []dataItem, ex stateExpectations, s gcIteratorState, ) { - check := func(ex int, kv *engine.MVCCKeyValue) { + check := func(ex int, kv *storage.MVCCKeyValue) { switch { case ex >= 0: require.EqualValues(t, &data[ex].MVCCKeyValue, kv) @@ -122,7 +122,7 @@ func TestGCIterator(t *testing.T) { } makeTest := func(tc testCase) func(t *testing.T) { return func(t *testing.T) { - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() ds := makeLiteralDataDistribution(tc.data...) ds.setupTest(t, eng, desc) diff --git a/pkg/kv/kvserver/gc/gc_old_test.go b/pkg/kv/kvserver/gc/gc_old_test.go index 69cc3195a5bf..a809ec82af50 100644 --- a/pkg/kv/kvserver/gc/gc_old_test.go +++ b/pkg/kv/kvserver/gc/gc_old_test.go @@ -16,11 +16,11 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -41,7 +41,7 @@ import ( func runGCOld( ctx context.Context, desc *roachpb.RangeDescriptor, - snap engine.Reader, + snap storage.Reader, now hlc.Timestamp, _ hlc.Timestamp, // exists to make signature match RunGC policy zonepb.GCPolicy, @@ -70,7 +70,7 @@ func runGCOld( var batchGCKeys []roachpb.GCRequest_GCKey var batchGCKeysBytes int64 var expBaseKey roachpb.Key - var keys []engine.MVCCKey + var keys []storage.MVCCKey var vals [][]byte var keyBytes int64 var valBytes int64 @@ -191,12 +191,12 @@ func runGCOld( processKeysAndValues() expBaseKey = iterKey.Key if !iterKey.IsValue() { - keys = []engine.MVCCKey{iter.Key()} + keys = []storage.MVCCKey{iter.Key()} vals = [][]byte{iter.Value()} continue } // An implicit metadata. - keys = []engine.MVCCKey{engine.MakeMVCCMetadataKey(iterKey.Key)} + keys = []storage.MVCCKey{storage.MakeMVCCMetadataKey(iterKey.Key)} // A nil value for the encoded MVCCMetadata. This will unmarshal to an // empty MVCCMetadata which is sufficient for processKeysAndValues to // determine that there is no intent. @@ -274,7 +274,7 @@ func MakeGarbageCollector(now hlc.Timestamp, policy zonepb.GCPolicy) GarbageColl // deleted value is the most recent before expiration, it can be deleted. This // would still allow for the tombstone bugs in #6227, so in the future we will // add checks that disallow writes before the last GC expiration time. -func (gc GarbageCollector) Filter(keys []engine.MVCCKey, values [][]byte) (int, hlc.Timestamp) { +func (gc GarbageCollector) Filter(keys []storage.MVCCKey, values [][]byte) (int, hlc.Timestamp) { if gc.policy.TTLSeconds <= 0 { return -1, hlc.Timestamp{} } @@ -306,19 +306,19 @@ func (gc GarbageCollector) Filter(keys []engine.MVCCKey, values [][]byte) (int, return -1, hlc.Timestamp{} } -func mvccVersionKey(key roachpb.Key, ts hlc.Timestamp) engine.MVCCKey { - return engine.MVCCKey{Key: key, Timestamp: ts} +func mvccVersionKey(key roachpb.Key, ts hlc.Timestamp) storage.MVCCKey { + return storage.MVCCKey{Key: key, Timestamp: ts} } var ( aKey = roachpb.Key("a") bKey = roachpb.Key("b") - aKeys = []engine.MVCCKey{ + aKeys = []storage.MVCCKey{ mvccVersionKey(aKey, hlc.Timestamp{WallTime: 2e9, Logical: 0}), mvccVersionKey(aKey, hlc.Timestamp{WallTime: 1e9, Logical: 1}), mvccVersionKey(aKey, hlc.Timestamp{WallTime: 1e9, Logical: 0}), } - bKeys = []engine.MVCCKey{ + bKeys = []storage.MVCCKey{ mvccVersionKey(bKey, hlc.Timestamp{WallTime: 2e9, Logical: 0}), mvccVersionKey(bKey, hlc.Timestamp{WallTime: 1e9, Logical: 0}), } @@ -335,7 +335,7 @@ func TestGarbageCollectorFilter(t *testing.T) { testData := []struct { gc GarbageCollector time hlc.Timestamp - keys []engine.MVCCKey + keys []storage.MVCCKey values [][]byte expIdx int expDelTS hlc.Timestamp diff --git a/pkg/kv/kvserver/gc/gc_random_test.go b/pkg/kv/kvserver/gc/gc_random_test.go index 083db468f3d6..3b30e242c2f7 100644 --- a/pkg/kv/kvserver/gc/gc_random_test.go +++ b/pkg/kv/kvserver/gc/gc_random_test.go @@ -19,8 +19,8 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/stretchr/testify/require" ) @@ -87,7 +87,7 @@ func TestRunNewVsOld(t *testing.T) { }, } { t.Run(fmt.Sprintf("%v@%v,ttl=%v", tc.ds, tc.now, tc.ttl), func(t *testing.T) { - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() tc.ds.dist(N, rng).setupTest(t, eng, *tc.ds.desc()) @@ -124,7 +124,7 @@ func TestRunNewVsOld(t *testing.T) { func BenchmarkRun(b *testing.B) { rng := rand.New(rand.NewSource(1)) ctx := context.Background() - runGC := func(eng engine.Engine, old bool, spec randomRunGCTestSpec) (Info, error) { + runGC := func(eng storage.Engine, old bool, spec randomRunGCTestSpec) (Info, error) { runGCFunc := Run if old { runGCFunc = runGCOld @@ -144,7 +144,7 @@ func BenchmarkRun(b *testing.B) { } makeTest := func(old bool, spec randomRunGCTestSpec) func(b *testing.B) { return func(b *testing.B) { - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() ms := spec.ds.dist(b.N, rng).setupTest(b, eng, *spec.ds.desc()) b.SetBytes(int64(float64(ms.Total()) / float64(b.N))) diff --git a/pkg/kv/kvserver/gc_queue.go b/pkg/kv/kvserver/gc_queue.go index 82a9a2121041..780866145ad7 100644 --- a/pkg/kv/kvserver/gc_queue.go +++ b/pkg/kv/kvserver/gc_queue.go @@ -20,10 +20,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/gc" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/gc_queue_test.go b/pkg/kv/kvserver/gc_queue_test.go index d9beba20ec06..9a027b3622dc 100644 --- a/pkg/kv/kvserver/gc_queue_test.go +++ b/pkg/kv/kvserver/gc_queue_test.go @@ -21,12 +21,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/gc" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" @@ -180,14 +180,14 @@ func (cws *cachedWriteSimulator) value(size int) roachpb.Value { func (cws *cachedWriteSimulator) multiKey( numOps int, size int, txn *roachpb.Transaction, ms *enginepb.MVCCStats, ) { - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() t, ctx := cws.t, context.Background() ts := hlc.Timestamp{}.Add(ms.LastUpdateNanos, 0) key, value := []byte("multikey"), cws.value(size) var eachMS enginepb.MVCCStats - if err := engine.MVCCPut(ctx, eng, &eachMS, key, ts, value, txn); err != nil { + if err := storage.MVCCPut(ctx, eng, &eachMS, key, ts, value, txn); err != nil { t.Fatal(err) } for i := 1; i < numOps; i++ { @@ -198,7 +198,7 @@ func (cws *cachedWriteSimulator) multiKey( func (cws *cachedWriteSimulator) singleKeySteady( qps int, duration time.Duration, size int, ms *enginepb.MVCCStats, ) { - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() t, ctx := cws.t, context.Background() cacheKey := fmt.Sprintf("%d-%s-%s", qps, duration, humanizeutil.IBytes(int64(size))) @@ -216,7 +216,7 @@ func (cws *cachedWriteSimulator) singleKeySteady( for i := 0; i < qps; i++ { now := initialNow.Add(elapsed.Nanoseconds(), int32(i)) - if err := engine.MVCCPut(ctx, eng, ms, key, now, value, nil /* txn */); err != nil { + if err := storage.MVCCPut(ctx, eng, ms, key, now, value, nil /* txn */); err != nil { t.Fatal(err) } if len(firstSl) < cacheFirstLen { @@ -569,7 +569,7 @@ func TestGCQueueProcess(t *testing.T) { // However, because the GC processing pushes transactions and // resolves intents asynchronously, we use a SucceedsSoon loop. testutils.SucceedsSoon(t, func() error { - kvs, err := engine.Scan(tc.store.Engine(), key1, keys.MaxKey, 0) + kvs, err := storage.Scan(tc.store.Engine(), key1, keys.MaxKey, 0) if err != nil { return err } @@ -752,7 +752,7 @@ func TestGCQueueTransactionTable(t *testing.T) { txns[strKey] = *txn for _, addrKey := range []roachpb.Key{baseKey, outsideKey} { key := keys.TransactionKey(addrKey, txn.ID) - if err := engine.MVCCPutProto(ctx, tc.engine, nil, key, hlc.Timestamp{}, nil, txn); err != nil { + if err := storage.MVCCPutProto(ctx, tc.engine, nil, key, hlc.Timestamp{}, nil, txn); err != nil { t.Fatal(err) } } @@ -779,8 +779,8 @@ func TestGCQueueTransactionTable(t *testing.T) { txnKey := keys.TransactionKey(roachpb.Key(strKey), txns[strKey].ID) txnTombstoneTSCacheKey := transactionTombstoneMarker( roachpb.Key(strKey), txns[strKey].ID) - ok, err := engine.MVCCGetProto(ctx, tc.engine, txnKey, hlc.Timestamp{}, txn, - engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, tc.engine, txnKey, hlc.Timestamp{}, txn, + storage.MVCCGetOptions{}) if err != nil { return err } @@ -831,8 +831,8 @@ func TestGCQueueTransactionTable(t *testing.T) { outsideTxnPrefix := keys.TransactionKey(outsideKey, uuid.UUID{}) outsideTxnPrefixEnd := keys.TransactionKey(outsideKey.Next(), uuid.UUID{}) var count int - if _, err := engine.MVCCIterate(ctx, tc.store.Engine(), outsideTxnPrefix, outsideTxnPrefixEnd, hlc.Timestamp{}, - engine.MVCCScanOptions{}, func(roachpb.KeyValue) (bool, error) { + if _, err := storage.MVCCIterate(ctx, tc.store.Engine(), outsideTxnPrefix, outsideTxnPrefixEnd, hlc.Timestamp{}, + storage.MVCCScanOptions{}, func(roachpb.KeyValue) (bool, error) { count++ return false, nil }); err != nil { @@ -905,7 +905,7 @@ func TestGCQueueIntentResolution(t *testing.T) { testutils.SucceedsSoon(t, func() error { meta := &enginepb.MVCCMetadata{} return tc.store.Engine().Iterate(roachpb.KeyMin, roachpb.KeyMax, - func(kv engine.MVCCKeyValue) (bool, error) { + func(kv storage.MVCCKeyValue) (bool, error) { if !kv.Key.IsValue() { if err := protoutil.Unmarshal(kv.Value, meta); err != nil { return false, err @@ -942,7 +942,7 @@ func TestGCQueueLastProcessedTimestamps(t *testing.T) { ts := tc.Clock().Now() for _, lpv := range lastProcessedVals { - if err := engine.MVCCPutProto(ctx, tc.engine, nil, lpv.key, hlc.Timestamp{}, nil, &ts); err != nil { + if err := storage.MVCCPutProto(ctx, tc.engine, nil, lpv.key, hlc.Timestamp{}, nil, &ts); err != nil { t.Fatal(err) } } @@ -961,8 +961,8 @@ func TestGCQueueLastProcessedTimestamps(t *testing.T) { // Verify GC. testutils.SucceedsSoon(t, func() error { for _, lpv := range lastProcessedVals { - ok, err := engine.MVCCGetProto(ctx, tc.engine, lpv.key, hlc.Timestamp{}, &ts, - engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, tc.engine, lpv.key, hlc.Timestamp{}, &ts, + storage.MVCCGetOptions{}) if err != nil { return err } diff --git a/pkg/kv/kvserver/helpers_test.go b/pkg/kv/kvserver/helpers_test.go index 5365546ce8e7..8e023f7b1957 100644 --- a/pkg/kv/kvserver/helpers_test.go +++ b/pkg/kv/kvserver/helpers_test.go @@ -26,8 +26,6 @@ import ( circuit "github.com/cockroachdb/circuitbreaker" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" @@ -35,6 +33,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -239,7 +239,7 @@ func NewTestStorePool(cfg StoreConfig) *StorePool { ) } -func (r *Replica) AssertState(ctx context.Context, reader engine.Reader) { +func (r *Replica) AssertState(ctx context.Context, reader storage.Reader) { r.raftMu.Lock() defer r.raftMu.Unlock() r.mu.Lock() @@ -380,16 +380,16 @@ func (r *Replica) SideloadedRaftMuLocked() SideloadStorage { return r.raftMu.sideloaded } -func MakeSSTable(key, value string, ts hlc.Timestamp) ([]byte, engine.MVCCKeyValue) { - sstFile := &engine.MemFile{} - sst := engine.MakeIngestionSSTWriter(sstFile) +func MakeSSTable(key, value string, ts hlc.Timestamp) ([]byte, storage.MVCCKeyValue) { + sstFile := &storage.MemFile{} + sst := storage.MakeIngestionSSTWriter(sstFile) defer sst.Close() v := roachpb.MakeValueFromBytes([]byte(value)) v.InitChecksum([]byte(key)) - kv := engine.MVCCKeyValue{ - Key: engine.MVCCKey{ + kv := storage.MVCCKeyValue{ + Key: storage.MVCCKey{ Key: []byte(key), Timestamp: ts, }, @@ -428,7 +428,7 @@ func SetMockAddSSTable() (undo func()) { // TODO(tschottdorf): this already does nontrivial work. Worth open-sourcing the relevant // subparts of the real evalAddSSTable to make this test less likely to rot. evalAddSSTable := func( - ctx context.Context, _ engine.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, + ctx context.Context, _ storage.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, ) (result.Result, error) { log.Event(ctx, "evaluated testing-only AddSSTable mock") args := cArgs.Args.(*roachpb.AddSSTableRequest) diff --git a/pkg/kv/kvserver/intentresolver/intent_resolver.go b/pkg/kv/kvserver/intentresolver/intent_resolver.go index 1ec27a222fb9..ae569cb25e46 100644 --- a/pkg/kv/kvserver/intentresolver/intent_resolver.go +++ b/pkg/kv/kvserver/intentresolver/intent_resolver.go @@ -16,7 +16,6 @@ import ( "sort" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher" "github.com/cockroachdb/cockroach/pkg/keys" @@ -26,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/intentresolver/intent_resolver_test.go b/pkg/kv/kvserver/intentresolver/intent_resolver_test.go index a338566a846d..f22f07f09667 100644 --- a/pkg/kv/kvserver/intentresolver/intent_resolver_test.go +++ b/pkg/kv/kvserver/intentresolver/intent_resolver_test.go @@ -20,11 +20,11 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/merge_queue.go b/pkg/kv/kvserver/merge_queue.go index d63298b59cae..2cd86a952a34 100644 --- a/pkg/kv/kvserver/merge_queue.go +++ b/pkg/kv/kvserver/merge_queue.go @@ -17,13 +17,13 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/merge_queue_test.go b/pkg/kv/kvserver/merge_queue_test.go index 2b3c71a4a9aa..e65131f8caf3 100644 --- a/pkg/kv/kvserver/merge_queue_test.go +++ b/pkg/kv/kvserver/merge_queue_test.go @@ -16,11 +16,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index be1502866ea5..efd9831685f2 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -14,10 +14,10 @@ import ( "context" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/metric" "go.etcd.io/etcd/raft/raftpb" @@ -1435,7 +1435,7 @@ func (sm *StoreMetrics) subtractMVCCStats(delta enginepb.MVCCStats) { sm.incMVCCGauges(neg) } -func (sm *StoreMetrics) updateRocksDBStats(stats engine.Stats) { +func (sm *StoreMetrics) updateRocksDBStats(stats storage.Stats) { // We do not grab a lock here, because it's not possible to get a point-in- // time snapshot of RocksDB stats. Retrieving RocksDB stats doesn't grab any // locks, and there's no way to retrieve multiple stats in a single operation. @@ -1451,7 +1451,7 @@ func (sm *StoreMetrics) updateRocksDBStats(stats engine.Stats) { sm.RdbTableReadersMemEstimate.Update(stats.TableReadersMemEstimate) } -func (sm *StoreMetrics) updateEnvStats(stats engine.EnvStats) { +func (sm *StoreMetrics) updateEnvStats(stats storage.EnvStats) { sm.EncryptionAlgorithm.Update(int64(stats.EncryptionType)) } diff --git a/pkg/kv/kvserver/node_liveness.go b/pkg/kv/kvserver/node_liveness.go index e93b9177a04a..5529039cfdd5 100644 --- a/pkg/kv/kvserver/node_liveness.go +++ b/pkg/kv/kvserver/node_liveness.go @@ -17,7 +17,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -26,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -151,7 +151,7 @@ type NodeLiveness struct { ambientCtx log.AmbientContext clock *hlc.Clock db *client.DB - engines []engine.Engine + engines []storage.Engine gossip *gossip.Gossip livenessThreshold time.Duration heartbeatInterval time.Duration @@ -179,7 +179,7 @@ func NewNodeLiveness( ambient log.AmbientContext, clock *hlc.Clock, db *client.DB, - engines []engine.Engine, + engines []storage.Engine, g *gossip.Gossip, livenessThreshold time.Duration, renewalDuration time.Duration, @@ -794,7 +794,7 @@ func (nl *NodeLiveness) updateLiveness( // don't want any excessively slow disks to prevent leases from being // shifted to other nodes. A slow/stalled disk would block here and cause // the node to lose its leases. - if err := engine.WriteSyncNoop(ctx, eng); err != nil { + if err := storage.WriteSyncNoop(ctx, eng); err != nil { return errors.Wrapf(err, "couldn't update node liveness because disk write failed") } } diff --git a/pkg/kv/kvserver/queue.go b/pkg/kv/kvserver/queue.go index 16e5adca8165..b038f15ecef8 100644 --- a/pkg/kv/kvserver/queue.go +++ b/pkg/kv/kvserver/queue.go @@ -18,12 +18,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/causer" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" diff --git a/pkg/kv/kvserver/queue_test.go b/pkg/kv/kvserver/queue_test.go index 8667f98e889f..de0fac377b63 100644 --- a/pkg/kv/kvserver/queue_test.go +++ b/pkg/kv/kvserver/queue_test.go @@ -22,12 +22,12 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/raft_log_queue_test.go b/pkg/kv/kvserver/raft_log_queue_test.go index e762476be966..409476b4dbbe 100644 --- a/pkg/kv/kvserver/raft_log_queue_test.go +++ b/pkg/kv/kvserver/raft_log_queue_test.go @@ -20,10 +20,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -839,9 +839,9 @@ func TestTruncateLogRecompute(t *testing.T) { dir, cleanup := testutils.TempDir(t) defer cleanup() - cache := engine.NewRocksDBCache(1 << 20) + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() - eng, err := engine.NewRocksDB(engine.RocksDBConfig{StorageConfig: base.StorageConfig{Dir: dir}}, cache) + eng, err := storage.NewRocksDB(storage.RocksDBConfig{StorageConfig: base.StorageConfig{Dir: dir}}, cache) if err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/raft_transport.go b/pkg/kv/kvserver/raft_transport.go index 25c047536fd4..d6b7ef297195 100644 --- a/pkg/kv/kvserver/raft_transport.go +++ b/pkg/kv/kvserver/raft_transport.go @@ -21,11 +21,11 @@ import ( "unsafe" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -657,7 +657,7 @@ func (t *RaftTransport) SendSnapshot( storePool *StorePool, header SnapshotRequest_Header, snap *OutgoingSnapshot, - newBatch func() engine.Batch, + newBatch func() storage.Batch, sent func(), ) error { var stream MultiRaft_RaftSnapshotClient diff --git a/pkg/kv/kvserver/rangefeed/processor.go b/pkg/kv/kvserver/rangefeed/processor.go index beecdd274499..15d9d7ad62f7 100644 --- a/pkg/kv/kvserver/rangefeed/processor.go +++ b/pkg/kv/kvserver/rangefeed/processor.go @@ -15,9 +15,9 @@ import ( "fmt" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -167,7 +167,7 @@ func NewProcessor(cfg Config) *Processor { // calling its Close method when it is finished. If the iterator is nil then // no initialization scan will be performed and the resolved timestamp will // immediately be considered initialized. -func (p *Processor) Start(stopper *stop.Stopper, rtsIter engine.SimpleIterator) { +func (p *Processor) Start(stopper *stop.Stopper, rtsIter storage.SimpleIterator) { ctx := p.AnnotateCtx(context.Background()) stopper.RunWorker(ctx, func(ctx context.Context) { defer close(p.stoppedC) @@ -357,7 +357,7 @@ func (p *Processor) sendStop(pErr *roachpb.Error) { func (p *Processor) Register( span roachpb.RSpan, startTS hlc.Timestamp, - catchupIter engine.SimpleIterator, + catchupIter storage.SimpleIterator, withDiff bool, stream Stream, errC chan<- *roachpb.Error, diff --git a/pkg/kv/kvserver/rangefeed/processor_test.go b/pkg/kv/kvserver/rangefeed/processor_test.go index 645b73fc1252..ead0c1f24f54 100644 --- a/pkg/kv/kvserver/rangefeed/processor_test.go +++ b/pkg/kv/kvserver/rangefeed/processor_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -132,7 +132,7 @@ func rangeFeedCheckpoint(span roachpb.Span, ts hlc.Timestamp) *roachpb.RangeFeed const testProcessorEventCCap = 16 func newTestProcessorWithTxnPusher( - rtsIter engine.SimpleIterator, txnPusher TxnPusher, + rtsIter storage.SimpleIterator, txnPusher TxnPusher, ) (*Processor, *stop.Stopper) { stopper := stop.NewStopper() @@ -156,7 +156,7 @@ func newTestProcessorWithTxnPusher( return p, stopper } -func newTestProcessor(rtsIter engine.SimpleIterator) (*Processor, *stop.Stopper) { +func newTestProcessor(rtsIter storage.SimpleIterator) (*Processor, *stop.Stopper) { return newTestProcessorWithTxnPusher(rtsIter, nil /* pusher */) } @@ -528,7 +528,7 @@ func TestProcessorInitializeResolvedTimestamp(t *testing.T) { defer leaktest.AfterTest(t)() txn1, txn2 := uuid.MakeV4(), uuid.MakeV4() - rtsIter := newTestIterator([]engine.MVCCKeyValue{ + rtsIter := newTestIterator([]storage.MVCCKeyValue{ makeKV("a", "val1", 10), makeInline("b", "val2"), makeIntent("c", txn1, "txnKey1", 15), diff --git a/pkg/kv/kvserver/rangefeed/registry.go b/pkg/kv/kvserver/rangefeed/registry.go index 4a010ddb066d..77d99cde69b5 100644 --- a/pkg/kv/kvserver/rangefeed/registry.go +++ b/pkg/kv/kvserver/rangefeed/registry.go @@ -17,9 +17,9 @@ import ( "sync" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/interval" @@ -55,7 +55,7 @@ type registration struct { // Input. span roachpb.Span catchupTimestamp hlc.Timestamp - catchupIter engine.SimpleIterator + catchupIter storage.SimpleIterator withDiff bool metrics *Metrics @@ -86,7 +86,7 @@ type registration struct { func newRegistration( span roachpb.Span, startTS hlc.Timestamp, - catchupIter engine.SimpleIterator, + catchupIter storage.SimpleIterator, withDiff bool, bufferSz int, metrics *Metrics, @@ -290,8 +290,8 @@ func (r *registration) runCatchupScan() error { }() var a bufalloc.ByteAllocator - startKey := engine.MakeMVCCMetadataKey(r.span.Key) - endKey := engine.MakeMVCCMetadataKey(r.span.EndKey) + startKey := storage.MakeMVCCMetadataKey(r.span.Key) + endKey := storage.MakeMVCCMetadataKey(r.span.EndKey) // Iterator will encounter historical values for each key in // reverse-chronological order. To output in chronological order, store @@ -344,7 +344,7 @@ func (r *registration) runCatchupScan() error { // past the corresponding provisional key-value. To do this, // scan to the timestamp immediately before (i.e. the key // immediately after) the provisional key. - r.catchupIter.SeekGE(engine.MVCCKey{ + r.catchupIter.SeekGE(storage.MVCCKey{ Key: unsafeKey.Key, Timestamp: hlc.Timestamp(meta.Timestamp).Prev(), }) diff --git a/pkg/kv/kvserver/rangefeed/registry_test.go b/pkg/kv/kvserver/rangefeed/registry_test.go index a06a8b108c13..d003b488030b 100644 --- a/pkg/kv/kvserver/rangefeed/registry_test.go +++ b/pkg/kv/kvserver/rangefeed/registry_test.go @@ -15,9 +15,9 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" _ "github.com/cockroachdb/cockroach/pkg/keys" // hook up pretty printer "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -96,7 +96,7 @@ type testRegistration struct { } func newTestRegistration( - span roachpb.Span, ts hlc.Timestamp, catchup engine.SimpleIterator, withDiff bool, + span roachpb.Span, ts hlc.Timestamp, catchup storage.SimpleIterator, withDiff bool, ) *testRegistration { s := newTestStream() errC := make(chan *roachpb.Error, 1) @@ -149,7 +149,7 @@ func TestRegistrationBasic(t *testing.T) { <-noCatchupReg.errC // Registration with catchup scan. - catchupReg := newTestRegistration(spBC, hlc.Timestamp{WallTime: 1}, newTestIterator([]engine.MVCCKeyValue{ + catchupReg := newTestRegistration(spBC, hlc.Timestamp{WallTime: 1}, newTestIterator([]storage.MVCCKeyValue{ makeKV("b", "val1", 10), makeInline("ba", "val2"), makeKV("bc", "val3", 11), @@ -212,7 +212,7 @@ func TestRegistrationCatchUpScan(t *testing.T) { // Run a catch-up scan for a registration over a test // iterator with the following keys. txn1, txn2 := uuid.MakeV4(), uuid.MakeV4() - iter := newTestIterator([]engine.MVCCKeyValue{ + iter := newTestIterator([]storage.MVCCKeyValue{ makeKV("a", "valA1", 10), makeInline("b", "valB1"), makeIntent("c", txn1, "txnKeyC", 15), diff --git a/pkg/kv/kvserver/rangefeed/resolved_timestamp.go b/pkg/kv/kvserver/rangefeed/resolved_timestamp.go index bb728801ff6a..b5c4ecf5ff11 100644 --- a/pkg/kv/kvserver/rangefeed/resolved_timestamp.go +++ b/pkg/kv/kvserver/rangefeed/resolved_timestamp.go @@ -15,8 +15,8 @@ import ( "container/heap" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) diff --git a/pkg/kv/kvserver/rangefeed/task.go b/pkg/kv/kvserver/rangefeed/task.go index 90d265d4fc35..e4b710029bd3 100644 --- a/pkg/kv/kvserver/rangefeed/task.go +++ b/pkg/kv/kvserver/rangefeed/task.go @@ -13,9 +13,9 @@ package rangefeed import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -46,10 +46,10 @@ type runnable interface { // type initResolvedTSScan struct { p *Processor - it engine.SimpleIterator + it storage.SimpleIterator } -func newInitResolvedTSScan(p *Processor, it engine.SimpleIterator) runnable { +func newInitResolvedTSScan(p *Processor, it storage.SimpleIterator) runnable { return &initResolvedTSScan{p: p, it: it} } @@ -66,8 +66,8 @@ func (s *initResolvedTSScan) Run(ctx context.Context) { } func (s *initResolvedTSScan) iterateAndConsume(ctx context.Context) error { - startKey := engine.MakeMVCCMetadataKey(s.p.Span.Key.AsRawKey()) - endKey := engine.MakeMVCCMetadataKey(s.p.Span.EndKey.AsRawKey()) + startKey := storage.MakeMVCCMetadataKey(s.p.Span.Key.AsRawKey()) + endKey := storage.MakeMVCCMetadataKey(s.p.Span.EndKey.AsRawKey()) // Iterate through all keys using NextKey. This will look at the first MVCC // version for each key. We're only looking for MVCCMetadata versions, which diff --git a/pkg/kv/kvserver/rangefeed/task_test.go b/pkg/kv/kvserver/rangefeed/task_test.go index d6e71c99945d..fb46476a181e 100644 --- a/pkg/kv/kvserver/rangefeed/task_test.go +++ b/pkg/kv/kvserver/rangefeed/task_test.go @@ -15,9 +15,9 @@ import ( "sort" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -25,9 +25,9 @@ import ( "github.com/stretchr/testify/require" ) -func makeKV(key, val string, ts int64) engine.MVCCKeyValue { - return engine.MVCCKeyValue{ - Key: engine.MVCCKey{ +func makeKV(key, val string, ts int64) storage.MVCCKeyValue { + return storage.MVCCKeyValue{ + Key: storage.MVCCKey{ Key: roachpb.Key(key), Timestamp: hlc.Timestamp{WallTime: ts}, }, @@ -35,30 +35,30 @@ func makeKV(key, val string, ts int64) engine.MVCCKeyValue { } } -func makeProvisionalKV(key, val string, ts int64) engine.MVCCKeyValue { +func makeProvisionalKV(key, val string, ts int64) storage.MVCCKeyValue { return makeKV(key, val, ts) } -func makeMetaKV(key string, meta enginepb.MVCCMetadata) engine.MVCCKeyValue { +func makeMetaKV(key string, meta enginepb.MVCCMetadata) storage.MVCCKeyValue { b, err := protoutil.Marshal(&meta) if err != nil { panic(err) } - return engine.MVCCKeyValue{ - Key: engine.MVCCKey{ + return storage.MVCCKeyValue{ + Key: storage.MVCCKey{ Key: roachpb.Key(key), }, Value: b, } } -func makeInline(key, val string) engine.MVCCKeyValue { +func makeInline(key, val string) storage.MVCCKeyValue { return makeMetaKV(key, enginepb.MVCCMetadata{ RawBytes: []byte(val), }) } -func makeIntent(key string, txnID uuid.UUID, txnKey string, txnTS int64) engine.MVCCKeyValue { +func makeIntent(key string, txnID uuid.UUID, txnKey string, txnTS int64) storage.MVCCKeyValue { return makeMetaKV(key, enginepb.MVCCMetadata{ Txn: &enginepb.TxnMeta{ ID: txnID, @@ -71,7 +71,7 @@ func makeIntent(key string, txnID uuid.UUID, txnKey string, txnTS int64) engine. } type testIterator struct { - kvs []engine.MVCCKeyValue + kvs []storage.MVCCKeyValue cur int closed bool @@ -80,7 +80,7 @@ type testIterator struct { done chan struct{} } -func newTestIterator(kvs []engine.MVCCKeyValue) *testIterator { +func newTestIterator(kvs []storage.MVCCKeyValue) *testIterator { // Ensure that the key-values are sorted. if !sort.SliceIsSorted(kvs, func(i, j int) bool { return kvs[i].Key.Less(kvs[j].Key) @@ -103,7 +103,7 @@ func newTestIterator(kvs []engine.MVCCKeyValue) *testIterator { if i == len(kvs) { panic(missingErr) } - expNextKey := engine.MVCCKey{ + expNextKey := storage.MVCCKey{ Key: kv.Key.Key, Timestamp: hlc.Timestamp(meta.Timestamp), } @@ -126,7 +126,7 @@ func (s *testIterator) Close() { close(s.done) } -func (s *testIterator) SeekGE(key engine.MVCCKey) { +func (s *testIterator) SeekGE(key storage.MVCCKey) { if s.closed { panic("testIterator closed") } @@ -171,7 +171,7 @@ func (s *testIterator) NextKey() { } } -func (s *testIterator) UnsafeKey() engine.MVCCKey { +func (s *testIterator) UnsafeKey() storage.MVCCKey { return s.curKV().Key } @@ -179,7 +179,7 @@ func (s *testIterator) UnsafeValue() []byte { return s.curKV().Value } -func (s *testIterator) curKV() engine.MVCCKeyValue { +func (s *testIterator) curKV() storage.MVCCKeyValue { return s.kvs[s.cur] } @@ -199,7 +199,7 @@ func TestInitResolvedTSScan(t *testing.T) { // Run an init rts scan over a test iterator with the following keys. txn1, txn2 := uuid.MakeV4(), uuid.MakeV4() - iter := newTestIterator([]engine.MVCCKeyValue{ + iter := newTestIterator([]storage.MVCCKeyValue{ makeKV("a", "val1", 10), makeInline("b", "val2"), makeIntent("c", txn1, "txnKey1", 15), diff --git a/pkg/kv/kvserver/rditer/replica_data_iter.go b/pkg/kv/kvserver/rditer/replica_data_iter.go index cf872ebcc211..42d846757277 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter.go @@ -11,15 +11,15 @@ package rditer import ( - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" ) // KeyRange is a helper struct for the ReplicaDataIterator. type KeyRange struct { - Start, End engine.MVCCKey + Start, End storage.MVCCKey } // ReplicaDataIterator provides a complete iteration over all key / value @@ -37,7 +37,7 @@ type KeyRange struct { type ReplicaDataIterator struct { curIndex int ranges []KeyRange - it engine.Iterator + it storage.Iterator a bufalloc.ByteAllocator } @@ -79,8 +79,8 @@ func MakeRangeIDLocalKeyRange(rangeID roachpb.RangeID, replicatedOnly bool) KeyR } sysRangeIDKey := prefixFn(rangeID) return KeyRange{ - Start: engine.MakeMVCCMetadataKey(sysRangeIDKey), - End: engine.MakeMVCCMetadataKey(sysRangeIDKey.PrefixEnd()), + Start: storage.MakeMVCCMetadataKey(sysRangeIDKey), + End: storage.MakeMVCCMetadataKey(sysRangeIDKey.PrefixEnd()), } } @@ -90,8 +90,8 @@ func MakeRangeIDLocalKeyRange(rangeID roachpb.RangeID, replicatedOnly bool) KeyR // /System), but it actually belongs to [/Table/1, /Table/2). func MakeRangeLocalKeyRange(d *roachpb.RangeDescriptor) KeyRange { return KeyRange{ - Start: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.StartKey)), - End: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.EndKey)), + Start: storage.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.StartKey)), + End: storage.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.EndKey)), } } @@ -105,16 +105,16 @@ func MakeUserKeyRange(d *roachpb.RangeDescriptor) KeyRange { dataStartKey = keys.LocalMax } return KeyRange{ - Start: engine.MakeMVCCMetadataKey(dataStartKey), - End: engine.MakeMVCCMetadataKey(d.EndKey.AsRawKey()), + Start: storage.MakeMVCCMetadataKey(dataStartKey), + End: storage.MakeMVCCMetadataKey(d.EndKey.AsRawKey()), } } // NewReplicaDataIterator creates a ReplicaDataIterator for the given replica. func NewReplicaDataIterator( - d *roachpb.RangeDescriptor, reader engine.Reader, replicatedOnly bool, seekEnd bool, + d *roachpb.RangeDescriptor, reader storage.Reader, replicatedOnly bool, seekEnd bool, ) *ReplicaDataIterator { - it := reader.NewIterator(engine.IterOptions{UpperBound: d.EndKey.AsRawKey()}) + it := reader.NewIterator(storage.IterOptions{UpperBound: d.EndKey.AsRawKey()}) rangeFunc := MakeAllKeyRanges if replicatedOnly { @@ -204,7 +204,7 @@ func (ri *ReplicaDataIterator) Valid() (bool, error) { } // Key returns the current key. -func (ri *ReplicaDataIterator) Key() engine.MVCCKey { +func (ri *ReplicaDataIterator) Key() storage.MVCCKey { key := ri.it.UnsafeKey() ri.a, key.Key = ri.a.Copy(key.Key, 0) return key @@ -219,7 +219,7 @@ func (ri *ReplicaDataIterator) Value() []byte { // UnsafeKey returns the same value as Key, but the memory is invalidated on // the next call to {Next,Prev,Close}. -func (ri *ReplicaDataIterator) UnsafeKey() engine.MVCCKey { +func (ri *ReplicaDataIterator) UnsafeKey() storage.MVCCKey { return ri.it.UnsafeKey() } diff --git a/pkg/kv/kvserver/rditer/replica_data_iter_test.go b/pkg/kv/kvserver/rditer/replica_data_iter_test.go index 7a9f1524a842..c7fb3da302c3 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter_test.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter_test.go @@ -16,10 +16,10 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -64,8 +64,8 @@ func uuidFromString(input string) uuid.UUID { // the key space. Returns a slice of the encoded keys of all created // data. func createRangeData( - t *testing.T, eng engine.Engine, desc roachpb.RangeDescriptor, -) []engine.MVCCKey { + t *testing.T, eng storage.Engine, desc roachpb.RangeDescriptor, +) []storage.MVCCKey { testTxnID := uuidFromString("0ce61c17-5eb4-4587-8c36-dcf4062ada4c") testTxnID2 := uuidFromString("9855a1ef-8eb9-4c06-a106-cab1dda78a2b") @@ -103,12 +103,12 @@ func createRangeData( {fakePrevKey(desc.EndKey), ts}, } - keys := []engine.MVCCKey{} + keys := []storage.MVCCKey{} for _, keyTS := range keyTSs { - if err := engine.MVCCPut(context.Background(), eng, nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil { + if err := storage.MVCCPut(context.Background(), eng, nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil { t.Fatal(err) } - keys = append(keys, engine.MVCCKey{Key: keyTS.key, Timestamp: keyTS.ts}) + keys = append(keys, storage.MVCCKey{Key: keyTS.key, Timestamp: keyTS.ts}) } return keys } @@ -116,9 +116,9 @@ func createRangeData( func verifyRDIter( t *testing.T, desc *roachpb.RangeDescriptor, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, replicatedOnly bool, - expectedKeys []engine.MVCCKey, + expectedKeys []storage.MVCCKey, ) { t.Helper() verify := func(t *testing.T, useSpanSet, reverse bool) { @@ -185,7 +185,7 @@ func verifyRDIter( func TestReplicaDataIteratorEmptyRange(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() desc := &roachpb.RangeDescriptor{ @@ -194,7 +194,7 @@ func TestReplicaDataIteratorEmptyRange(t *testing.T) { EndKey: roachpb.RKey("z"), } - verifyRDIter(t, desc, eng, false /* replicatedOnly */, []engine.MVCCKey{}) + verifyRDIter(t, desc, eng, false /* replicatedOnly */, []storage.MVCCKey{}) } // TestReplicaDataIterator creates three ranges {"a"-"b" (pre), "b"-"c" @@ -206,7 +206,7 @@ func TestReplicaDataIteratorEmptyRange(t *testing.T) { func TestReplicaDataIterator(t *testing.T) { defer leaktest.AfterTest(t)() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() descPre := roachpb.RangeDescriptor{ @@ -255,7 +255,7 @@ func TestReplicaDataIterator(t *testing.T) { for _, test := range []struct { name string desc *roachpb.RangeDescriptor - keys []engine.MVCCKey + keys []storage.MVCCKey }{ {"pre", &descPre, preKeys}, {"post", &descPost, postKeys}, diff --git a/pkg/kv/kvserver/rditer/stats.go b/pkg/kv/kvserver/rditer/stats.go index 7a6e1604b51f..f2704b1b7aa4 100644 --- a/pkg/kv/kvserver/rditer/stats.go +++ b/pkg/kv/kvserver/rditer/stats.go @@ -11,18 +11,18 @@ package rditer import ( - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" ) // ComputeStatsForRange computes the stats for a given range by // iterating over all key ranges for the given range that should // be accounted for in its stats. func ComputeStatsForRange( - d *roachpb.RangeDescriptor, reader engine.Reader, nowNanos int64, + d *roachpb.RangeDescriptor, reader storage.Reader, nowNanos int64, ) (enginepb.MVCCStats, error) { - iter := reader.NewIterator(engine.IterOptions{UpperBound: d.EndKey.AsRawKey()}) + iter := reader.NewIterator(storage.IterOptions{UpperBound: d.EndKey.AsRawKey()}) defer iter.Close() ms := enginepb.MVCCStats{} diff --git a/pkg/kv/kvserver/replica.go b/pkg/kv/kvserver/replica.go index f3b92cbf7bb7..b5d20df78897 100644 --- a/pkg/kv/kvserver/replica.go +++ b/pkg/kv/kvserver/replica.go @@ -20,8 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" @@ -38,6 +36,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -643,7 +643,7 @@ func (r *Replica) DB() *client.DB { // Engine returns the Replica's underlying Engine. In most cases the // evaluation Batch should be used instead. -func (r *Replica) Engine() engine.Engine { +func (r *Replica) Engine() storage.Engine { return r.store.Engine() } @@ -783,8 +783,8 @@ func (r *Replica) ContainsKeyRange(start, end roachpb.Key) bool { func (r *Replica) GetLastReplicaGCTimestamp(ctx context.Context) (hlc.Timestamp, error) { key := keys.RangeLastReplicaGCTimestampKey(r.RangeID) var timestamp hlc.Timestamp - _, err := engine.MVCCGetProto(ctx, r.store.Engine(), key, hlc.Timestamp{}, ×tamp, - engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, r.store.Engine(), key, hlc.Timestamp{}, ×tamp, + storage.MVCCGetOptions{}) if err != nil { return hlc.Timestamp{}, err } @@ -793,7 +793,7 @@ func (r *Replica) GetLastReplicaGCTimestamp(ctx context.Context) (hlc.Timestamp, func (r *Replica) setLastReplicaGCTimestamp(ctx context.Context, timestamp hlc.Timestamp) error { key := keys.RangeLastReplicaGCTimestampKey(r.RangeID) - return engine.MVCCPutProto(ctx, r.store.Engine(), nil, key, hlc.Timestamp{}, nil, ×tamp) + return storage.MVCCPutProto(ctx, r.store.Engine(), nil, key, hlc.Timestamp{}, nil, ×tamp) } // getQueueLastProcessed returns the last processed timestamp for the @@ -802,8 +802,8 @@ func (r *Replica) getQueueLastProcessed(ctx context.Context, queue string) (hlc. key := keys.QueueLastProcessedKey(r.Desc().StartKey, queue) var timestamp hlc.Timestamp if r.store != nil { - _, err := engine.MVCCGetProto(ctx, r.store.Engine(), key, hlc.Timestamp{}, ×tamp, - engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, r.store.Engine(), key, hlc.Timestamp{}, ×tamp, + storage.MVCCGetOptions{}) if err != nil { log.VErrEventf(ctx, 2, "last processed timestamp unavailable: %s", err) return hlc.Timestamp{}, err @@ -900,7 +900,7 @@ func (r *Replica) State() storagepb.RangeInfo { // Requires that both r.raftMu and r.mu are held. // // TODO(tschottdorf): Consider future removal (for example, when #7224 is resolved). -func (r *Replica) assertStateLocked(ctx context.Context, reader engine.Reader) { +func (r *Replica) assertStateLocked(ctx context.Context, reader storage.Reader) { diskState, err := r.mu.stateLoader.Load(ctx, reader, r.mu.state.Desc) if err != nil { log.Fatal(ctx, err) @@ -1162,14 +1162,14 @@ func (ec *endCmds) done( func (r *Replica) maybeWatchForMerge(ctx context.Context) error { desc := r.Desc() descKey := keys.RangeDescriptorKey(desc.StartKey) - _, intent, err := engine.MVCCGet(ctx, r.Engine(), descKey, r.Clock().Now(), - engine.MVCCGetOptions{Inconsistent: true}) + _, intent, err := storage.MVCCGet(ctx, r.Engine(), descKey, r.Clock().Now(), + storage.MVCCGetOptions{Inconsistent: true}) if err != nil { return err } else if intent == nil { return nil } - val, _, err := engine.MVCCGetAsTxn( + val, _, err := storage.MVCCGetAsTxn( ctx, r.Engine(), descKey, intent.Txn.WriteTimestamp, intent.Txn) if err != nil { return err @@ -1382,7 +1382,7 @@ func (r *Replica) getReplicaDescriptorByIDRLocked( // transaction. In case the transaction has been aborted, return a // transaction abort error. func checkIfTxnAborted( - ctx context.Context, rec batcheval.EvalContext, reader engine.Reader, txn roachpb.Transaction, + ctx context.Context, rec batcheval.EvalContext, reader storage.Reader, txn roachpb.Transaction, ) *roachpb.Error { var entry roachpb.AbortSpanEntry aborted, err := rec.AbortSpan().Get(ctx, reader, txn.ID, &entry) diff --git a/pkg/kv/kvserver/replica_application_result.go b/pkg/kv/kvserver/replica_application_result.go index 55a21c37c3eb..d25238ad5438 100644 --- a/pkg/kv/kvserver/replica_application_result.go +++ b/pkg/kv/kvserver/replica_application_result.go @@ -13,11 +13,11 @@ package kvserver import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" ) diff --git a/pkg/kv/kvserver/replica_application_state_machine.go b/pkg/kv/kvserver/replica_application_state_machine.go index 395f7b52989a..13776878202e 100644 --- a/pkg/kv/kvserver/replica_application_state_machine.go +++ b/pkg/kv/kvserver/replica_application_state_machine.go @@ -16,12 +16,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -360,7 +360,7 @@ type replicaAppBatch struct { sm *replicaStateMachine // batch accumulates writes implied by the raft entries in this batch. - batch engine.Batch + batch storage.Batch // state is this batch's view of the replica's state. It is copied from // under the Replica.mu when the batch is initialized and is updated in // stageTrivialReplicatedEvalResult. @@ -515,7 +515,7 @@ func (b *replicaAppBatch) stageWriteBatch(ctx context.Context, cmd *replicatedCm if wb == nil { return nil } - if mutations, err := engine.RocksDBBatchCount(wb.Data); err != nil { + if mutations, err := storage.RocksDBBatchCount(wb.Data); err != nil { log.Errorf(ctx, "unable to read header of committed WriteBatch: %+v", err) } else { b.mutations += mutations diff --git a/pkg/kv/kvserver/replica_command.go b/pkg/kv/kvserver/replica_command.go index 9bf25d491786..450ce5f89119 100644 --- a/pkg/kv/kvserver/replica_command.go +++ b/pkg/kv/kvserver/replica_command.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" @@ -30,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/causer" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" @@ -318,7 +318,7 @@ func (r *Replica) adminSplitWithDescriptor( // Find a key to split by size. var err error targetSize := r.GetMaxBytes() / 2 - foundSplitKey, err = engine.MVCCFindSplitKey( + foundSplitKey, err = storage.MVCCFindSplitKey( ctx, r.store.engine, desc.StartKey, desc.EndKey, targetSize) if err != nil { return reply, errors.Errorf("unable to determine split key: %s", err) @@ -349,7 +349,7 @@ func (r *Replica) adminSplitWithDescriptor( if !splitKey.Equal(foundSplitKey) { return reply, errors.Errorf("cannot split range at range-local key %s", splitKey) } - if !engine.IsValidSplitKey(foundSplitKey) { + if !storage.IsValidSplitKey(foundSplitKey) { return reply, errors.Errorf("cannot split range at key %s", splitKey) } } @@ -1808,8 +1808,8 @@ func (r *Replica) sendSnapshot( return &benignError{errors.New("raft status not initialized")} } - usesReplicatedTruncatedState, err := engine.MVCCGetProto( - ctx, snap.EngineSnap, keys.RaftTruncatedStateLegacyKey(r.RangeID), hlc.Timestamp{}, nil, engine.MVCCGetOptions{}, + usesReplicatedTruncatedState, err := storage.MVCCGetProto( + ctx, snap.EngineSnap, keys.RaftTruncatedStateLegacyKey(r.RangeID), hlc.Timestamp{}, nil, storage.MVCCGetOptions{}, ) if err != nil { return errors.Wrap(err, "loading legacy truncated state") diff --git a/pkg/kv/kvserver/replica_consistency.go b/pkg/kv/kvserver/replica_consistency.go index dbe400066605..b086343f870d 100644 --- a/pkg/kv/kvserver/replica_consistency.go +++ b/pkg/kv/kvserver/replica_consistency.go @@ -22,8 +22,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" @@ -31,6 +29,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -502,14 +502,14 @@ type replicaHash struct { func (r *Replica) sha512( ctx context.Context, desc roachpb.RangeDescriptor, - snap engine.Reader, + snap storage.Reader, snapshot *roachpb.RaftSnapshotData, mode roachpb.ChecksumMode, ) (*replicaHash, error) { statsOnly := mode == roachpb.ChecksumMode_CHECK_STATS // Iterate over all the data in the range. - iter := snap.NewIterator(engine.IterOptions{UpperBound: desc.EndKey.AsRawKey()}) + iter := snap.NewIterator(storage.IterOptions{UpperBound: desc.EndKey.AsRawKey()}) defer iter.Close() var alloc bufalloc.ByteAllocator @@ -518,7 +518,7 @@ func (r *Replica) sha512( var timestampBuf []byte hasher := sha512.New() - visitor := func(unsafeKey engine.MVCCKey, unsafeValue []byte) error { + visitor := func(unsafeKey storage.MVCCKey, unsafeValue []byte) error { if snapshot != nil { // Add (a copy of) the kv pair into the debug message. kv := roachpb.RaftSnapshotData_KeyValue{ @@ -562,7 +562,7 @@ func (r *Replica) sha512( // all of the replicated key space. if !statsOnly { for _, span := range rditer.MakeReplicatedKeyRanges(&desc) { - spanMS, err := engine.ComputeStatsGo( + spanMS, err := storage.ComputeStatsGo( iter, span.Start.Key, span.End.Key, 0 /* nowNanos */, visitor, ) if err != nil { diff --git a/pkg/kv/kvserver/replica_consistency_diff.go b/pkg/kv/kvserver/replica_consistency_diff.go index d4a39cfae324..e9de24e70e9f 100644 --- a/pkg/kv/kvserver/replica_consistency_diff.go +++ b/pkg/kv/kvserver/replica_consistency_diff.go @@ -15,8 +15,8 @@ import ( "fmt" "io" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -59,12 +59,12 @@ func (rsds ReplicaSnapshotDiffSlice) WriteTo(w io.Writer) (int64, error) { } else { prettyTime = d.Timestamp.GoTime().UTC().String() } - mvccKey := engine.MVCCKey{Key: d.Key, Timestamp: ts} + mvccKey := storage.MVCCKey{Key: d.Key, Timestamp: ts} num, err := fmt.Fprintf(w, format, prefix, ts.WallTime/1e9, ts.WallTime%1e9, ts.Logical, d.Key, prefix, prettyTime, - prefix, SprintKeyValue(engine.MVCCKeyValue{Key: mvccKey, Value: d.Value}, false /* printKey */), - prefix, engine.EncodeKey(mvccKey), d.Value) + prefix, SprintKeyValue(storage.MVCCKeyValue{Key: mvccKey, Value: d.Value}, false /* printKey */), + prefix, storage.EncodeKey(mvccKey), d.Value) if err != nil { return 0, err } diff --git a/pkg/kv/kvserver/replica_destroy.go b/pkg/kv/kvserver/replica_destroy.go index 6e51db7c493f..527aaf332e24 100644 --- a/pkg/kv/kvserver/replica_destroy.go +++ b/pkg/kv/kvserver/replica_destroy.go @@ -15,11 +15,11 @@ import ( "fmt" "math" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -126,8 +126,8 @@ const mergedTombstoneReplicaID roachpb.ReplicaID = math.MaxInt32 func (r *Replica) preDestroyRaftMuLocked( ctx context.Context, - reader engine.Reader, - writer engine.Writer, + reader storage.Reader, + writer storage.Writer, nextReplicaID roachpb.ReplicaID, clearRangeIDLocalOnly bool, mustUseClearRange bool, @@ -264,7 +264,7 @@ func (r *Replica) disconnectReplicationRaftMuLocked(ctx context.Context) { // ID that it hasn't yet received a RangeDescriptor for if it receives raft // requests for that replica ID (as seen in #14231). func (r *Replica) setTombstoneKey( - ctx context.Context, writer engine.Writer, externalNextReplicaID roachpb.ReplicaID, + ctx context.Context, writer storage.Writer, externalNextReplicaID roachpb.ReplicaID, ) error { r.mu.Lock() nextReplicaID := r.mu.state.Desc.NextReplicaID @@ -280,7 +280,7 @@ func (r *Replica) setTombstoneKey( func writeTombstoneKey( ctx context.Context, - writer engine.Writer, + writer storage.Writer, rangeID roachpb.RangeID, nextReplicaID roachpb.ReplicaID, ) error { @@ -289,6 +289,6 @@ func writeTombstoneKey( NextReplicaID: nextReplicaID, } // "Blind" because ms == nil and timestamp == hlc.Timestamp{}. - return engine.MVCCBlindPutProto(ctx, writer, nil, tombstoneKey, + return storage.MVCCBlindPutProto(ctx, writer, nil, tombstoneKey, hlc.Timestamp{}, tombstone, nil) } diff --git a/pkg/kv/kvserver/replica_eval_context_span.go b/pkg/kv/kvserver/replica_eval_context_span.go index 337718ede7d1..afdebb533228 100644 --- a/pkg/kv/kvserver/replica_eval_context_span.go +++ b/pkg/kv/kvserver/replica_eval_context_span.go @@ -13,8 +13,6 @@ package kvserver import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan" @@ -25,6 +23,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) @@ -90,7 +90,7 @@ func (rec *SpanSetReplicaEvalContext) GetNodeLocality() roachpb.Locality { } // Engine returns the engine. -func (rec *SpanSetReplicaEvalContext) Engine() engine.Engine { +func (rec *SpanSetReplicaEvalContext) Engine() storage.Engine { return rec.i.Engine() } diff --git a/pkg/kv/kvserver/replica_evaluate.go b/pkg/kv/kvserver/replica_evaluate.go index 160cf5c4ad86..ac4b17380d0a 100644 --- a/pkg/kv/kvserver/replica_evaluate.go +++ b/pkg/kv/kvserver/replica_evaluate.go @@ -14,12 +14,12 @@ import ( "bytes" "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" "github.com/kr/pretty" @@ -35,7 +35,7 @@ import ( // the input slice, or has been shallow-copied appropriately to avoid // mutating the original requests). func optimizePuts( - reader engine.Reader, origReqs []roachpb.RequestUnion, distinctSpans bool, + reader storage.Reader, origReqs []roachpb.RequestUnion, distinctSpans bool, ) []roachpb.RequestUnion { var minKey, maxKey roachpb.Key var unique map[string]struct{} @@ -83,7 +83,7 @@ func optimizePuts( if firstUnoptimizedIndex < optimizePutThreshold { // don't bother if below this threshold return origReqs } - iter := reader.NewIterator(engine.IterOptions{ + iter := reader.NewIterator(storage.IterOptions{ // We want to include maxKey in our scan. Since UpperBound is exclusive, we // need to set it to the key after maxKey. UpperBound: maxKey.Next(), @@ -94,7 +94,7 @@ func optimizePuts( // we can determine whether any part of the range being written // is "virgin" and set the puts to write blindly. // Find the first non-empty key in the run. - iter.SeekGE(engine.MakeMVCCMetadataKey(minKey)) + iter.SeekGE(storage.MakeMVCCMetadataKey(minKey)) var iterKey roachpb.Key if ok, err := iter.Valid(); err != nil { // TODO(bdarnell): return an error here instead of silently @@ -137,7 +137,7 @@ func optimizePuts( func evaluateBatch( ctx context.Context, idKey storagebase.CmdIDKey, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, rec batcheval.EvalContext, ms *enginepb.MVCCStats, ba *roachpb.BatchRequest, @@ -396,7 +396,7 @@ func evaluateCommand( ctx context.Context, raftCmdID storagebase.CmdIDKey, index int, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, rec batcheval.EvalContext, ms *enginepb.MVCCStats, h roachpb.Header, diff --git a/pkg/kv/kvserver/replica_evaluate_test.go b/pkg/kv/kvserver/replica_evaluate_test.go index 8bd2c514ab4b..de898403c3a2 100644 --- a/pkg/kv/kvserver/replica_evaluate_test.go +++ b/pkg/kv/kvserver/replica_evaluate_test.go @@ -15,12 +15,12 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" @@ -249,7 +249,7 @@ func TestEvaluateBatch(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() d := &data{ @@ -281,7 +281,7 @@ type data struct { batcheval.MockEvalCtx ba roachpb.BatchRequest idKey storagebase.CmdIDKey - eng engine.Engine + eng storage.Engine ms enginepb.MVCCStats readOnly bool } @@ -301,7 +301,7 @@ type testCase struct { func writeABCDEF(t *testing.T, d *data) { for _, k := range []string{"a", "b", "c", "d", "e", "f"} { - require.NoError(t, engine.MVCCPut( + require.NoError(t, storage.MVCCPut( context.Background(), d.eng, nil /* ms */, roachpb.Key(k), d.ba.Timestamp, roachpb.MakeValueFromString("value-"+k), nil /* txn */)) } diff --git a/pkg/kv/kvserver/replica_proposal.go b/pkg/kv/kvserver/replica_proposal.go index e31e88fd7b36..51223961c431 100644 --- a/pkg/kv/kvserver/replica_proposal.go +++ b/pkg/kv/kvserver/replica_proposal.go @@ -22,8 +22,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" @@ -33,6 +31,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -467,7 +467,7 @@ func (r *Replica) leasePostApply(ctx context.Context, newLease roachpb.Lease, pe func addSSTablePreApply( ctx context.Context, st *cluster.Settings, - eng engine.Engine, + eng storage.Engine, sideloaded SideloadStorage, term, index uint64, sst storagepb.ReplicatedEvalResult_AddSSTable, @@ -548,7 +548,7 @@ func addSSTablePreApply( // going to be surfaced. ingestErrMsg := ingestErr.Error() isSeqNoErr := strings.Contains(ingestErrMsg, seqNoMsg) || strings.Contains(ingestErrMsg, seqNoOnReIngest) - if _, ok := ingestErr.(*engine.Error); !ok || !isSeqNoErr { + if _, ok := ingestErr.(*storage.Error); !ok || !isSeqNoErr { log.Fatalf(ctx, "while ingesting %s: %s", ingestPath, ingestErr) } } diff --git a/pkg/kv/kvserver/replica_raft.go b/pkg/kv/kvserver/replica_raft.go index 05d0b9f46d01..875c7e3869b4 100644 --- a/pkg/kv/kvserver/replica_raft.go +++ b/pkg/kv/kvserver/replica_raft.go @@ -17,7 +17,6 @@ import ( "sort" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency" @@ -25,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -1602,7 +1602,7 @@ func handleTruncatedStateBelowRaft( ctx context.Context, oldTruncatedState, newTruncatedState *roachpb.RaftTruncatedState, loader stateloader.StateLoader, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ) (_apply bool, _ error) { // If this is a log truncation, load the resulting unreplicated or legacy // replicated truncated state (in that order). If the migration is happening @@ -1634,7 +1634,7 @@ func handleTruncatedStateBelowRaft( // NB: RangeIDPrefixBufs have sufficient capacity (32 bytes) to // avoid allocating when constructing Raft log keys (16 bytes). unsafeKey := prefixBuf.RaftLogKey(idx) - if err := readWriter.Clear(engine.MakeMVCCMetadataKey(unsafeKey)); err != nil { + if err := readWriter.Clear(storage.MakeMVCCMetadataKey(unsafeKey)); err != nil { return false, errors.Wrapf(err, "unable to clear truncated Raft entries for %+v", newTruncatedState) } } @@ -1651,7 +1651,7 @@ func handleTruncatedStateBelowRaft( // new key (which wasn't touched by the batch), and we need to // overwrite it if this truncation "moves it forward". - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, readWriter, nil /* ms */, prefixBuf.RaftTruncatedStateKey(), hlc.Timestamp{}, nil /* txn */, newTruncatedState, ); err != nil { @@ -1676,11 +1676,11 @@ func handleTruncatedStateBelowRaft( // // The sideloaded storage may be nil, in which case it is treated as empty. func ComputeRaftLogSize( - ctx context.Context, rangeID roachpb.RangeID, reader engine.Reader, sideloaded SideloadStorage, + ctx context.Context, rangeID roachpb.RangeID, reader storage.Reader, sideloaded SideloadStorage, ) (int64, error) { prefix := keys.RaftLogPrefix(rangeID) prefixEnd := prefix.PrefixEnd() - iter := reader.NewIterator(engine.IterOptions{ + iter := reader.NewIterator(storage.IterOptions{ LowerBound: prefix, UpperBound: prefixEnd, }) diff --git a/pkg/kv/kvserver/replica_raft_truncation_test.go b/pkg/kv/kvserver/replica_raft_truncation_test.go index d2a723e6aaaa..d24b66554172 100644 --- a/pkg/kv/kvserver/replica_raft_truncation_test.go +++ b/pkg/kv/kvserver/replica_raft_truncation_test.go @@ -16,10 +16,10 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/datadriven" @@ -44,7 +44,7 @@ func TestHandleTruncatedStateBelowRaft(t *testing.T) { datadriven.Walk(t, "testdata/truncated_state_migration", func(t *testing.T, path string) { const rangeID = 12 loader := stateloader.Make(rangeID) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { @@ -96,7 +96,7 @@ func TestHandleTruncatedStateBelowRaft(t *testing.T) { keys.RaftTruncatedStateKey(rangeID), } { var truncatedState roachpb.RaftTruncatedState - ok, err := engine.MVCCGetProto(ctx, eng, key, hlc.Timestamp{}, &truncatedState, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, eng, key, hlc.Timestamp{}, &truncatedState, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } diff --git a/pkg/kv/kvserver/replica_raftstorage.go b/pkg/kv/kvserver/replica_raftstorage.go index d04381cbff57..c105574f6629 100644 --- a/pkg/kv/kvserver/replica_raftstorage.go +++ b/pkg/kv/kvserver/replica_raftstorage.go @@ -17,8 +17,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftentry" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" @@ -26,6 +24,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -95,7 +95,7 @@ func (r *Replica) raftEntriesLocked(lo, hi, maxBytes uint64) ([]raftpb.Entry, er func entries( ctx context.Context, rsl stateloader.StateLoader, - reader engine.Reader, + reader storage.Reader, rangeID roachpb.RangeID, eCache *raftentry.Cache, sideloaded SideloadStorage, @@ -218,17 +218,17 @@ func entries( func iterateEntries( ctx context.Context, - reader engine.Reader, + reader storage.Reader, rangeID roachpb.RangeID, lo, hi uint64, scanFunc func(roachpb.KeyValue) (bool, error), ) error { - _, err := engine.MVCCIterate( + _, err := storage.MVCCIterate( ctx, reader, keys.RaftLogKey(rangeID, lo), keys.RaftLogKey(rangeID, hi), hlc.Timestamp{}, - engine.MVCCScanOptions{}, + storage.MVCCScanOptions{}, scanFunc, ) return err @@ -264,7 +264,7 @@ func (r *Replica) raftTermRLocked(i uint64) (uint64, error) { func term( ctx context.Context, rsl stateloader.StateLoader, - reader engine.Reader, + reader storage.Reader, rangeID roachpb.RangeID, eCache *raftentry.Cache, i uint64, @@ -446,7 +446,7 @@ type OutgoingSnapshot struct { // The Raft snapshot message to send. Contains SnapUUID as its data. RaftSnap raftpb.Snapshot // The RocksDB snapshot that will be streamed from. - EngineSnap engine.Reader + EngineSnap storage.Reader // The complete range iterator for the snapshot to stream. Iter *rditer.ReplicaDataIterator // The replica state within the snapshot. @@ -506,7 +506,7 @@ func snapshot( snapUUID uuid.UUID, rsl stateloader.StateLoader, snapType SnapshotRequest_Type, - snap engine.Reader, + snap storage.Reader, rangeID roachpb.RangeID, eCache *raftentry.Cache, withSideloaded func(func(SideloadStorage) error) error, @@ -516,8 +516,8 @@ func snapshot( // We ignore intents on the range descriptor (consistent=false) because we // know they cannot be committed yet; operations that modify range // descriptors resolve their own intents when they commit. - ok, err := engine.MVCCGetProto(ctx, snap, keys.RangeDescriptorKey(startKey), - hlc.MaxTimestamp, &desc, engine.MVCCGetOptions{Inconsistent: true}) + ok, err := storage.MVCCGetProto(ctx, snap, keys.RangeDescriptorKey(startKey), + hlc.MaxTimestamp, &desc, storage.MVCCGetOptions{Inconsistent: true}) if err != nil { return OutgoingSnapshot{}, errors.Errorf("failed to get desc: %s", err) } @@ -583,7 +583,7 @@ func snapshot( // engine.ReadWriter must be passed in. func (r *Replica) append( ctx context.Context, - writer engine.Writer, + writer storage.Writer, prevLastIndex uint64, prevLastTerm uint64, prevRaftLogSize int64, @@ -604,15 +604,15 @@ func (r *Replica) append( value.InitChecksum(key) var err error if ent.Index > prevLastIndex { - err = engine.MVCCBlindPut(ctx, writer, &diff, key, hlc.Timestamp{}, value, nil /* txn */) + err = storage.MVCCBlindPut(ctx, writer, &diff, key, hlc.Timestamp{}, value, nil /* txn */) } else { // We type assert `writer` to also be an engine.ReadWriter only in // the case where we're replacing existing entries. - eng, ok := writer.(engine.ReadWriter) + eng, ok := writer.(storage.ReadWriter) if !ok { panic("expected writer to be a engine.ReadWriter when overwriting log entries") } - err = engine.MVCCPut(ctx, eng, &diff, key, hlc.Timestamp{}, value, nil /* txn */) + err = storage.MVCCPut(ctx, eng, &diff, key, hlc.Timestamp{}, value, nil /* txn */) } if err != nil { return 0, 0, 0, err @@ -625,14 +625,14 @@ func (r *Replica) append( if prevLastIndex > 0 { // We type assert `writer` to also be an engine.ReadWriter only in the // case where we're deleting existing entries. - eng, ok := writer.(engine.ReadWriter) + eng, ok := writer.(storage.ReadWriter) if !ok { panic("expected writer to be a engine.ReadWriter when deleting log entries") } for i := lastIndex + 1; i <= prevLastIndex; i++ { // Note that the caller is in charge of deleting any sideloaded payloads // (which they must only do *after* the batch has committed). - err := engine.MVCCDelete(ctx, eng, &diff, r.raftMu.stateLoader.RaftLogKey(i), + err := storage.MVCCDelete(ctx, eng, &diff, r.raftMu.stateLoader.RaftLogKey(i), hlc.Timestamp{}, nil /* txn */) if err != nil { return 0, 0, 0, err @@ -683,8 +683,8 @@ func (r *Replica) updateRangeInfo(desc *roachpb.RangeDescriptor) error { // range. func clearRangeData( desc *roachpb.RangeDescriptor, - reader engine.Reader, - writer engine.Writer, + reader storage.Reader, + writer storage.Writer, rangeIDLocalOnly bool, mustClearRange bool, ) error { @@ -694,13 +694,13 @@ func clearRangeData( } else { keyRanges = rditer.MakeAllKeyRanges(desc) } - var clearRangeFn func(engine.Reader, engine.Writer, roachpb.Key, roachpb.Key) error + var clearRangeFn func(storage.Reader, storage.Writer, roachpb.Key, roachpb.Key) error if mustClearRange { - clearRangeFn = func(reader engine.Reader, writer engine.Writer, start, end roachpb.Key) error { - return writer.ClearRange(engine.MakeMVCCMetadataKey(start), engine.MakeMVCCMetadataKey(end)) + clearRangeFn = func(reader storage.Reader, writer storage.Writer, start, end roachpb.Key) error { + return writer.ClearRange(storage.MakeMVCCMetadataKey(start), storage.MakeMVCCMetadataKey(end)) } } else { - clearRangeFn = engine.ClearRangeWithHeuristic + clearRangeFn = storage.ClearRangeWithHeuristic } for _, keyRange := range keyRanges { @@ -803,14 +803,14 @@ func (r *Replica) applySnapshot( inSnap.SnapUUID.Short(), snap.Metadata.Index) }(timeutil.Now()) - unreplicatedSSTFile := &engine.MemFile{} - unreplicatedSST := engine.MakeIngestionSSTWriter(unreplicatedSSTFile) + unreplicatedSSTFile := &storage.MemFile{} + unreplicatedSST := storage.MakeIngestionSSTWriter(unreplicatedSSTFile) defer unreplicatedSST.Close() // Clearing the unreplicated state. unreplicatedPrefixKey := keys.MakeRangeIDUnreplicatedPrefix(r.RangeID) - unreplicatedStart := engine.MakeMVCCMetadataKey(unreplicatedPrefixKey) - unreplicatedEnd := engine.MakeMVCCMetadataKey(unreplicatedPrefixKey.PrefixEnd()) + unreplicatedStart := storage.MakeMVCCMetadataKey(unreplicatedPrefixKey) + unreplicatedEnd := storage.MakeMVCCMetadataKey(unreplicatedPrefixKey.PrefixEnd()) if err = unreplicatedSST.ClearRange(unreplicatedStart, unreplicatedEnd); err != nil { return errors.Wrapf(err, "error clearing range of unreplicated SST writer") } @@ -1007,8 +1007,8 @@ func (r *Replica) clearSubsumedReplicaDiskData( totalKeyRanges := append([]rditer.KeyRange(nil), keyRanges[:]...) for _, sr := range subsumedRepls { // We have to create an SST for the subsumed replica's range-id local keys. - subsumedReplSSTFile := &engine.MemFile{} - subsumedReplSST := engine.MakeIngestionSSTWriter(subsumedReplSSTFile) + subsumedReplSSTFile := &storage.MemFile{} + subsumedReplSST := storage.MakeIngestionSSTWriter(subsumedReplSSTFile) defer subsumedReplSST.Close() // NOTE: We set mustClearRange to true because we are setting // RangeTombstoneKey. Since Clears and Puts need to be done in increasing @@ -1063,10 +1063,10 @@ func (r *Replica) clearSubsumedReplicaDiskData( // subsume both r1 and r2 in S1. for i := range keyRanges { if totalKeyRanges[i].End.Key.Compare(keyRanges[i].End.Key) > 0 { - subsumedReplSSTFile := &engine.MemFile{} - subsumedReplSST := engine.MakeIngestionSSTWriter(subsumedReplSSTFile) + subsumedReplSSTFile := &storage.MemFile{} + subsumedReplSST := storage.MakeIngestionSSTWriter(subsumedReplSSTFile) defer subsumedReplSST.Close() - if err := engine.ClearRangeWithHeuristic( + if err := storage.ClearRangeWithHeuristic( r.store.Engine(), &subsumedReplSST, keyRanges[i].End.Key, diff --git a/pkg/kv/kvserver/replica_rangefeed.go b/pkg/kv/kvserver/replica_rangefeed.go index 7e708b8b1a8a..4540dff73b0b 100644 --- a/pkg/kv/kvserver/replica_rangefeed.go +++ b/pkg/kv/kvserver/replica_rangefeed.go @@ -16,8 +16,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts" @@ -26,6 +24,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -114,7 +114,7 @@ func (tp *rangefeedTxnPusher) CleanupTxnIntentsAsync( } type iteratorWithCloser struct { - engine.SimpleIterator + storage.SimpleIterator close func() } @@ -200,9 +200,9 @@ func (r *Replica) RangeFeed( } // Register the stream with a catch-up iterator. - var catchUpIter engine.SimpleIterator + var catchUpIter storage.SimpleIterator if usingCatchupIter { - innerIter := r.Engine().NewIterator(engine.IterOptions{ + innerIter := r.Engine().NewIterator(storage.IterOptions{ UpperBound: args.Span.EndKey, // RangeFeed originally intended to use the time-bound iterator // performance optimization. However, they've had correctness issues in @@ -302,7 +302,7 @@ func (r *Replica) registerWithRangefeedRaftMuLocked( ctx context.Context, span roachpb.RSpan, startTS hlc.Timestamp, - catchupIter engine.SimpleIterator, + catchupIter storage.SimpleIterator, withDiff bool, stream rangefeed.Stream, errC chan<- *roachpb.Error, @@ -347,7 +347,7 @@ func (r *Replica) registerWithRangefeedRaftMuLocked( p = rangefeed.NewProcessor(cfg) // Start it with an iterator to initialize the resolved timestamp. - rtsIter := r.Engine().NewIterator(engine.IterOptions{ + rtsIter := r.Engine().NewIterator(storage.IterOptions{ UpperBound: desc.EndKey.AsRawKey(), // TODO(nvanbenschoten): To facilitate fast restarts of rangefeed // we should periodically persist the resolved timestamp so that we @@ -440,7 +440,7 @@ func (r *Replica) numRangefeedRegistrations() int { // the state of the Replica before the operations in the logical op log are // applied. No-op if a rangefeed is not active. Requires raftMu to be locked. func (r *Replica) populatePrevValsInLogicalOpLogRaftMuLocked( - ctx context.Context, ops *storagepb.LogicalOpLog, prevReader engine.Reader, + ctx context.Context, ops *storagepb.LogicalOpLog, prevReader storage.Reader, ) { p, filter := r.getRangefeedProcessorAndFilter() if p == nil { @@ -475,8 +475,8 @@ func (r *Replica) populatePrevValsInLogicalOpLogRaftMuLocked( // Read the previous value from the prev Reader. Unlike the new value // (see handleLogicalOpLogRaftMuLocked), this one may be missing. - prevVal, _, err := engine.MVCCGet( - ctx, prevReader, key, ts, engine.MVCCGetOptions{Tombstones: true, Inconsistent: true}, + prevVal, _, err := storage.MVCCGet( + ctx, prevReader, key, ts, storage.MVCCGetOptions{Tombstones: true, Inconsistent: true}, ) if err != nil { r.disconnectRangefeedWithErr(p, roachpb.NewErrorf( @@ -498,7 +498,7 @@ func (r *Replica) populatePrevValsInLogicalOpLogRaftMuLocked( // them to the rangefeed processor. No-op if a rangefeed is not active. Requires // raftMu to be locked. func (r *Replica) handleLogicalOpLogRaftMuLocked( - ctx context.Context, ops *storagepb.LogicalOpLog, reader engine.Reader, + ctx context.Context, ops *storagepb.LogicalOpLog, reader storage.Reader, ) { p, filter := r.getRangefeedProcessorAndFilter() if p == nil { @@ -556,7 +556,7 @@ func (r *Replica) handleLogicalOpLogRaftMuLocked( // Read the value directly from the Reader. This is performed in the // same raftMu critical section that the logical op's corresponding // WriteBatch is applied, so the value should exist. - val, _, err := engine.MVCCGet(ctx, reader, key, ts, engine.MVCCGetOptions{Tombstones: true}) + val, _, err := storage.MVCCGet(ctx, reader, key, ts, storage.MVCCGetOptions{Tombstones: true}) if val == nil && err == nil { err = errors.New("value missing in reader") } diff --git a/pkg/kv/kvserver/replica_sideload_disk.go b/pkg/kv/kvserver/replica_sideload_disk.go index b81fc5c355b2..c8e635653b39 100644 --- a/pkg/kv/kvserver/replica_sideload_disk.go +++ b/pkg/kv/kvserver/replica_sideload_disk.go @@ -18,9 +18,9 @@ import ( "strconv" "strings" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/pkg/errors" "golang.org/x/time/rate" ) @@ -32,7 +32,7 @@ type diskSideloadStorage struct { limiter *rate.Limiter dir string dirCreated bool - eng engine.Engine + eng storage.Engine } func deprecatedSideloadedPath( @@ -78,7 +78,7 @@ func newDiskSideloadStorage( replicaID roachpb.ReplicaID, baseDir string, limiter *rate.Limiter, - eng engine.Engine, + eng storage.Engine, ) (*diskSideloadStorage, error) { path := deprecatedSideloadedPath(baseDir, rangeID, replicaID) newPath := sideloadedPath(baseDir, rangeID) diff --git a/pkg/kv/kvserver/replica_sideload_inmem.go b/pkg/kv/kvserver/replica_sideload_inmem.go index ab5232665edf..55e550292070 100644 --- a/pkg/kv/kvserver/replica_sideload_inmem.go +++ b/pkg/kv/kvserver/replica_sideload_inmem.go @@ -15,9 +15,9 @@ import ( "fmt" "path/filepath" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" ) type slKey struct { @@ -44,7 +44,7 @@ func newInMemSideloadStorage( rangeID roachpb.RangeID, replicaID roachpb.ReplicaID, baseDir string, - eng engine.Engine, + eng storage.Engine, ) (SideloadStorage, error) { return &inMemSideloadStorage{ prefix: filepath.Join(baseDir, fmt.Sprintf("%d.%d", rangeID, replicaID)), diff --git a/pkg/kv/kvserver/replica_sideload_test.go b/pkg/kv/kvserver/replica_sideload_test.go index 64e0db732a49..9ddefd122dbe 100644 --- a/pkg/kv/kvserver/replica_sideload_test.go +++ b/pkg/kv/kvserver/replica_sideload_test.go @@ -26,7 +26,6 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftentry" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" @@ -34,6 +33,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -89,7 +89,7 @@ func TestSideloadingSideloadedStorage(t *testing.T) { }) t.Run("Disk", func(t *testing.T) { maker := func( - s *cluster.Settings, rangeID roachpb.RangeID, rep roachpb.ReplicaID, name string, eng engine.Engine, + s *cluster.Settings, rangeID roachpb.RangeID, rep roachpb.ReplicaID, name string, eng storage.Engine, ) (SideloadStorage, error) { return newDiskSideloadStorage(s, rangeID, rep, name, rate.NewLimiter(rate.Inf, math.MaxInt64), eng) } @@ -99,7 +99,7 @@ func TestSideloadingSideloadedStorage(t *testing.T) { func testSideloadingSideloadedStorage( t *testing.T, - maker func(*cluster.Settings, roachpb.RangeID, roachpb.ReplicaID, string, engine.Engine) (SideloadStorage, error), + maker func(*cluster.Settings, roachpb.RangeID, roachpb.ReplicaID, string, storage.Engine) (SideloadStorage, error), ) { dir, cleanup := testutils.TempDir(t) defer cleanup() @@ -612,16 +612,16 @@ func testRaftSSTableSideloadingProposal(t *testing.T, engineInMem, mockSideloade stopper := stop.NewStopper() tc := testContext{} if !engineInMem { - cfg := engine.RocksDBConfig{ + cfg := storage.RocksDBConfig{ StorageConfig: base.StorageConfig{ Dir: dir, Settings: cluster.MakeTestingClusterSettings(), }, } var err error - cache := engine.NewRocksDBCache(1 << 20) + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() - tc.engine, err = engine.NewRocksDB(cfg, cache) + tc.engine, err = storage.NewRocksDB(cfg, cache) if err != nil { t.Fatal(err) } @@ -752,9 +752,9 @@ func (mr *mockSender) Recv() (*SnapshotResponse, error) { return &SnapshotResponse{Status: status}, nil } -func newEngine(t *testing.T) (func(), engine.Engine) { +func newEngine(t *testing.T) (func(), storage.Engine) { dir, cleanup := testutils.TempDir(t) - eng, err := engine.NewDefaultEngine( + eng, err := storage.NewDefaultEngine( 1<<20, base.StorageConfig{ Dir: dir, diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage.go b/pkg/kv/kvserver/replica_sst_snapshot_storage.go index 2668ac5f8479..8bebc02570f0 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage.go @@ -17,9 +17,9 @@ import ( "path/filepath" "strconv" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/pkg/errors" "golang.org/x/time/rate" @@ -29,13 +29,13 @@ import ( // directory of scratches created. A scratch manages the SSTs created during a // specific snapshot. type SSTSnapshotStorage struct { - engine engine.Engine + engine storage.Engine limiter *rate.Limiter dir string } // NewSSTSnapshotStorage creates a new SST snapshot storage. -func NewSSTSnapshotStorage(engine engine.Engine, limiter *rate.Limiter) SSTSnapshotStorage { +func NewSSTSnapshotStorage(engine storage.Engine, limiter *rate.Limiter) SSTSnapshotStorage { return SSTSnapshotStorage{ engine: engine, limiter: limiter, diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go index b71ac867f520..d309ebb5f0fd 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go @@ -15,9 +15,9 @@ import ( "os" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/stretchr/testify/require" @@ -142,8 +142,8 @@ func TestMultiSSTWriterInitSST(t *testing.T) { var expectedSSTs [][]byte for _, r := range keyRanges { func() { - sstFile := &engine.MemFile{} - sst := engine.MakeIngestionSSTWriter(sstFile) + sstFile := &storage.MemFile{} + sst := storage.MakeIngestionSSTWriter(sstFile) defer sst.Close() err := sst.ClearRange(r.Start, r.End) require.NoError(t, err) diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index dea66774d4ca..1265311ff0bc 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -31,8 +31,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -51,6 +49,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -182,7 +182,7 @@ type testContext struct { repl *Replica rangeID roachpb.RangeID gossip *gossip.Gossip - engine engine.Engine + engine storage.Engine manualClock *hlc.ManualClock bootstrapMode bootstrapMode } @@ -220,7 +220,7 @@ func (tc *testContext) StartWithStoreConfigAndVersion( tc.gossip = gossip.NewTest(1, rpcContext, server, stopper, metric.NewRegistry(), cfg.DefaultZoneConfig) } if tc.engine == nil { - tc.engine = engine.NewInMem(context.Background(), engine.DefaultStorageEngine, + tc.engine = storage.NewInMem(context.Background(), storage.DefaultStorageEngine, roachpb.Attributes{Attrs: []string{"dc1", "mem"}}, 1<<20) stopper.AddCloser(tc.engine) } @@ -1210,7 +1210,7 @@ func TestReplicaGossipConfigsOnLease(t *testing.T) { key := keys.MakeTablePrefix(keys.MaxSystemConfigDescID) var val roachpb.Value val.SetInt(42) - if err := engine.MVCCPut(context.Background(), tc.engine, nil, key, hlc.Timestamp{}, val, nil); err != nil { + if err := storage.MVCCPut(context.Background(), tc.engine, nil, key, hlc.Timestamp{}, val, nil); err != nil { t.Fatal(err) } @@ -1969,7 +1969,7 @@ func TestOptimizePuts(t *testing.T) { for i, c := range testCases { if c.exKey != nil { - if err := engine.MVCCPut(context.Background(), tc.engine, nil, c.exKey, + if err := storage.MVCCPut(context.Background(), tc.engine, nil, c.exKey, hlc.Timestamp{}, roachpb.MakeValueFromString("foo"), nil); err != nil { t.Fatal(err) } @@ -2016,7 +2016,7 @@ func TestOptimizePuts(t *testing.T) { t.Errorf("%d: expected %+v; got %+v", i, c.expBlind, blind) } if c.exKey != nil { - if err := tc.engine.Clear(engine.MakeMVCCMetadataKey(c.exKey)); err != nil { + if err := tc.engine.Clear(storage.MakeMVCCMetadataKey(c.exKey)); err != nil { t.Fatal(err) } } @@ -2799,9 +2799,9 @@ func TestReplicaTSCacheForwardsIntentTS(t *testing.T) { if _, pErr := tc.SendWrappedWith(roachpb.Header{Txn: txnOld}, &pArgs); pErr != nil { t.Fatal(pErr) } - iter := tc.engine.NewIterator(engine.IterOptions{Prefix: true}) + iter := tc.engine.NewIterator(storage.IterOptions{Prefix: true}) defer iter.Close() - mvccKey := engine.MakeMVCCMetadataKey(key) + mvccKey := storage.MakeMVCCMetadataKey(key) iter.SeekGE(mvccKey) var keyMeta enginepb.MVCCMetadata if ok, err := iter.Valid(); !ok || !iter.UnsafeKey().Equal(mvccKey) { @@ -3182,7 +3182,7 @@ func TestReplicaAbortSpanReadError(t *testing.T) { // Overwrite Abort span entry with garbage for the last op. key := keys.AbortSpanKey(tc.repl.RangeID, txn.ID) - err := engine.MVCCPut(context.Background(), tc.engine, nil, key, hlc.Timestamp{}, roachpb.MakeValueFromString("never read in this test"), nil) + err := storage.MVCCPut(context.Background(), tc.engine, nil, key, hlc.Timestamp{}, roachpb.MakeValueFromString("never read in this test"), nil) if err != nil { t.Fatal(err) } @@ -4238,7 +4238,7 @@ func TestEndTxnWithErrors(t *testing.T) { existTxn.Epoch = test.existEpoch existTxnRecord := existTxn.AsRecord() txnKey := keys.TransactionKey(test.key, txn.ID) - if err := engine.MVCCPutProto(ctx, tc.repl.store.Engine(), nil, txnKey, hlc.Timestamp{}, + if err := storage.MVCCPutProto(ctx, tc.repl.store.Engine(), nil, txnKey, hlc.Timestamp{}, nil, &existTxnRecord); err != nil { t.Fatal(err) } @@ -4307,9 +4307,9 @@ func TestEndTxnRollbackAbortedTransaction(t *testing.T) { if populateAbortSpan { var txnRecord roachpb.Transaction txnKey := keys.TransactionKey(txn.Key, txn.ID) - if ok, err := engine.MVCCGetProto( + if ok, err := storage.MVCCGetProto( context.TODO(), tc.repl.store.Engine(), - txnKey, hlc.Timestamp{}, &txnRecord, engine.MVCCGetOptions{}, + txnKey, hlc.Timestamp{}, &txnRecord, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } else if ok { @@ -4539,8 +4539,8 @@ func TestBatchRetryCantCommitIntents(t *testing.T) { // Verify txn record is cleaned. var readTxn roachpb.Transaction txnKey := keys.TransactionKey(txn.Key, txn.ID) - ok, err := engine.MVCCGetProto(context.Background(), tc.repl.store.Engine(), txnKey, - hlc.Timestamp{}, &readTxn, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(context.Background(), tc.repl.store.Engine(), txnKey, + hlc.Timestamp{}, &readTxn, storage.MVCCGetOptions{}) if err != nil || ok { t.Errorf("expected transaction record to be cleared (%t): %+v", ok, err) } @@ -4632,8 +4632,8 @@ func TestEndTxnLocalGC(t *testing.T) { } var readTxn roachpb.Transaction txnKey := keys.TransactionKey(txn.Key, txn.ID) - ok, err := engine.MVCCGetProto(context.Background(), tc.repl.store.Engine(), txnKey, hlc.Timestamp{}, - &readTxn, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(context.Background(), tc.repl.store.Engine(), txnKey, hlc.Timestamp{}, + &readTxn, storage.MVCCGetOptions{}) if err != nil { t.Fatal(err) } @@ -5952,7 +5952,7 @@ func TestReplicaResolveIntentRange(t *testing.T) { } func verifyRangeStats( - reader engine.Reader, rangeID roachpb.RangeID, expMS enginepb.MVCCStats, + reader storage.Reader, rangeID roachpb.RangeID, expMS enginepb.MVCCStats, ) error { ms, err := stateloader.Make(rangeID).LoadMVCCStats(context.Background(), reader) if err != nil { @@ -6778,7 +6778,7 @@ func TestReplicaLoadSystemConfigSpanIntent(t *testing.T) { // there and verify that we can now load the data as expected. v := roachpb.MakeValueFromString("foo") testutils.SucceedsSoon(t, func() error { - if err := engine.MVCCPut(context.Background(), repl.store.Engine(), &enginepb.MVCCStats{}, + if err := storage.MVCCPut(context.Background(), repl.store.Engine(), &enginepb.MVCCStats{}, keys.SystemConfigSpan.Key, repl.store.Clock().Now(), v, nil); err != nil { return err } @@ -8145,7 +8145,7 @@ func TestGCWithoutThreshold(t *testing.T) { t.Fatalf("expected %d declared keys, found %d", expSpans, numSpans) } - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() defer eng.Close() batch := eng.NewBatch() @@ -9996,7 +9996,7 @@ func TestReplicaPushed1PC(t *testing.T) { // Write a value outside the transaction. tc.manualClock.Increment(10) ts2 := tc.Clock().Now() - if err := engine.MVCCPut(ctx, tc.engine, nil, k, ts2, roachpb.MakeValueFromString("one"), nil); err != nil { + if err := storage.MVCCPut(ctx, tc.engine, nil, k, ts2, roachpb.MakeValueFromString("one"), nil); err != nil { t.Fatalf("writing interfering value: %+v", err) } @@ -11606,9 +11606,9 @@ func TestTxnRecordLifecycleTransitions(t *testing.T) { } var foundRecord roachpb.TransactionRecord - if found, err := engine.MVCCGetProto( + if found, err := storage.MVCCGetProto( ctx, tc.repl.store.Engine(), keys.TransactionKey(txn.Key, txn.ID), - hlc.Timestamp{}, &foundRecord, engine.MVCCGetOptions{}, + hlc.Timestamp{}, &foundRecord, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } else if found { @@ -12215,13 +12215,13 @@ func setMockPutWithEstimates(containsEstimatesDelta int64) (undo func()) { prev, _ := batcheval.LookupCommand(roachpb.Put) mockPut := func( - ctx context.Context, readWriter engine.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, + ctx context.Context, readWriter storage.ReadWriter, cArgs batcheval.CommandArgs, _ roachpb.Response, ) (result.Result, error) { args := cArgs.Args.(*roachpb.PutRequest) ms := cArgs.Stats ms.ContainsEstimates += containsEstimatesDelta ts := cArgs.Header.Timestamp - return result.Result{}, engine.MVCCBlindPut(ctx, readWriter, ms, args.Key, ts, args.Value, cArgs.Header.Txn) + return result.Result{}, storage.MVCCBlindPut(ctx, readWriter, ms, args.Key, ts, args.Value, cArgs.Header.Txn) } batcheval.UnregisterCommand(roachpb.Put) diff --git a/pkg/kv/kvserver/replica_write.go b/pkg/kv/kvserver/replica_write.go index ca00377a73f5..385892d3ecee 100644 --- a/pkg/kv/kvserver/replica_write.go +++ b/pkg/kv/kvserver/replica_write.go @@ -16,8 +16,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb" @@ -26,6 +24,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -307,7 +307,7 @@ func (r *Replica) canAttempt1PCEvaluation( // retryable error. func (r *Replica) evaluateWriteBatch( ctx context.Context, idKey storagebase.CmdIDKey, ba *roachpb.BatchRequest, spans *spanset.SpanSet, -) (engine.Batch, enginepb.MVCCStats, *roachpb.BatchResponse, result.Result, *roachpb.Error) { +) (storage.Batch, enginepb.MVCCStats, *roachpb.BatchResponse, result.Result, *roachpb.Error) { // If the transaction has been pushed but it can commit at the higher // timestamp, let's evaluate the batch at the bumped timestamp. This will @@ -365,7 +365,7 @@ type onePCResult struct { stats enginepb.MVCCStats br *roachpb.BatchResponse res result.Result - batch engine.Batch + batch storage.Batch } // evaluate1PC attempts to evaluate the batch as a 1PC transaction - meaning it @@ -379,7 +379,7 @@ func (r *Replica) evaluate1PC( ) (onePCRes onePCResult) { log.VEventf(ctx, 2, "attempting 1PC execution") - var batch engine.Batch + var batch storage.Batch defer func() { // Close the batch unless it's passed to the caller (when the evaluation // succeeds). @@ -501,7 +501,7 @@ func (r *Replica) evaluateWriteBatchWithServersideRefreshes( ba *roachpb.BatchRequest, spans *spanset.SpanSet, deadline *hlc.Timestamp, -) (batch engine.Batch, br *roachpb.BatchResponse, res result.Result, pErr *roachpb.Error) { +) (batch storage.Batch, br *roachpb.BatchResponse, res result.Result, pErr *roachpb.Error) { goldenMS := *ms for retries := 0; ; retries++ { if retries > 0 { @@ -542,7 +542,7 @@ func (r *Replica) evaluateWriteBatchWrapper( ms *enginepb.MVCCStats, ba *roachpb.BatchRequest, spans *spanset.SpanSet, -) (engine.Batch, *roachpb.BatchResponse, result.Result, *roachpb.Error) { +) (storage.Batch, *roachpb.BatchResponse, result.Result, *roachpb.Error) { batch, opLogger := r.newBatchedEngine(spans) br, res, pErr := evaluateBatch(ctx, idKey, batch, rec, ms, ba, false /* readOnly */) if pErr == nil { @@ -627,9 +627,9 @@ func canDoServersideRetry( // are enabled, it also returns an engine.OpLoggerBatch. If non-nil, then this // OpLogger is attached to the returned engine.Batch, recording all operations. // Its recording should be attached to the Result of request evaluation. -func (r *Replica) newBatchedEngine(spans *spanset.SpanSet) (engine.Batch, *engine.OpLoggerBatch) { +func (r *Replica) newBatchedEngine(spans *spanset.SpanSet) (storage.Batch, *storage.OpLoggerBatch) { batch := r.store.Engine().NewBatch() - var opLogger *engine.OpLoggerBatch + var opLogger *storage.OpLoggerBatch if RangefeedEnabled.Get(&r.store.cfg.Settings.SV) { // TODO(nvanbenschoten): once we get rid of the RangefeedEnabled // cluster setting we'll need a way to turn this on when any @@ -663,7 +663,7 @@ func (r *Replica) newBatchedEngine(spans *spanset.SpanSet) (engine.Batch, *engin // // Another alternative is to make the setting table/zone-scoped // instead of a fine-grained per-replica state. - opLogger = engine.NewOpLoggerBatch(batch) + opLogger = storage.NewOpLoggerBatch(batch) batch = opLogger } if util.RaceEnabled { diff --git a/pkg/kv/kvserver/scanner_test.go b/pkg/kv/kvserver/scanner_test.go index 058ff9dccf7b..2ad6ce81391c 100644 --- a/pkg/kv/kvserver/scanner_test.go +++ b/pkg/kv/kvserver/scanner_test.go @@ -16,8 +16,8 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/spanset/batch.go b/pkg/kv/kvserver/spanset/batch.go index acf8aed952ed..5e6c757b41f2 100644 --- a/pkg/kv/kvserver/spanset/batch.go +++ b/pkg/kv/kvserver/spanset/batch.go @@ -11,9 +11,9 @@ package spanset import ( - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" ) @@ -21,7 +21,7 @@ import ( // Iterator wraps an engine.Iterator and ensures that it can // only be used to access spans in a SpanSet. type Iterator struct { - i engine.Iterator + i storage.Iterator spans *SpanSet // spansOnly controls whether or not timestamps associated with the @@ -40,19 +40,19 @@ type Iterator struct { invalid bool } -var _ engine.Iterator = &Iterator{} -var _ engine.MVCCIterator = &Iterator{} +var _ storage.Iterator = &Iterator{} +var _ storage.MVCCIterator = &Iterator{} // NewIterator constructs an iterator that verifies access of the underlying // iterator against the given SpanSet. Timestamps associated with the spans // in the spanset are not considered, only the span boundaries are checked. -func NewIterator(iter engine.Iterator, spans *SpanSet) *Iterator { +func NewIterator(iter storage.Iterator, spans *SpanSet) *Iterator { return &Iterator{i: iter, spans: spans, spansOnly: true} } // NewIteratorAt constructs an iterator that verifies access of the underlying // iterator against the given SpanSet at the given timestamp. -func NewIteratorAt(iter engine.Iterator, spans *SpanSet, ts hlc.Timestamp) *Iterator { +func NewIteratorAt(iter storage.Iterator, spans *SpanSet, ts hlc.Timestamp) *Iterator { return &Iterator{i: iter, spans: spans, ts: ts} } @@ -62,12 +62,12 @@ func (i *Iterator) Close() { } // Iterator returns the underlying engine.Iterator. -func (i *Iterator) Iterator() engine.Iterator { +func (i *Iterator) Iterator() storage.Iterator { return i.i } // SeekGE is part of the engine.Iterator interface. -func (i *Iterator) SeekGE(key engine.MVCCKey) { +func (i *Iterator) SeekGE(key storage.MVCCKey) { if i.spansOnly { i.err = i.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: key.Key}) } else { @@ -80,7 +80,7 @@ func (i *Iterator) SeekGE(key engine.MVCCKey) { } // SeekLT is part of the engine.Iterator interface. -func (i *Iterator) SeekLT(key engine.MVCCKey) { +func (i *Iterator) SeekLT(key storage.MVCCKey) { // CheckAllowed{At} supports the span representation of [,key), which // corresponds to the span [key.Prev(),). revSpan := roachpb.Span{EndKey: key.Key} @@ -150,7 +150,7 @@ func (i *Iterator) NextKey() { } // Key is part of the engine.Iterator interface. -func (i *Iterator) Key() engine.MVCCKey { +func (i *Iterator) Key() storage.MVCCKey { return i.i.Key() } @@ -165,7 +165,7 @@ func (i *Iterator) ValueProto(msg protoutil.Message) error { } // UnsafeKey is part of the engine.Iterator interface. -func (i *Iterator) UnsafeKey() engine.MVCCKey { +func (i *Iterator) UnsafeKey() storage.MVCCKey { return i.i.UnsafeKey() } @@ -193,14 +193,14 @@ func (i *Iterator) ComputeStats( // FindSplitKey is part of the engine.Iterator interface. func (i *Iterator) FindSplitKey( start, end, minSplitKey roachpb.Key, targetSize int64, -) (engine.MVCCKey, error) { +) (storage.MVCCKey, error) { if i.spansOnly { if err := i.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: start, EndKey: end}); err != nil { - return engine.MVCCKey{}, err + return storage.MVCCKey{}, err } } else { if err := i.spans.CheckAllowedAt(SpanReadOnly, roachpb.Span{Key: start, EndKey: end}, i.ts); err != nil { - return engine.MVCCKey{}, err + return storage.MVCCKey{}, err } } return i.i.FindSplitKey(start, end, minSplitKey, targetSize) @@ -219,13 +219,13 @@ func (i *Iterator) SetUpperBound(key roachpb.Key) { } // Stats is part of the engine.Iterator interface. -func (i *Iterator) Stats() engine.IteratorStats { +func (i *Iterator) Stats() storage.IteratorStats { return i.i.Stats() } // MVCCOpsSpecialized is part of the engine.MVCCIterator interface. func (i *Iterator) MVCCOpsSpecialized() bool { - if mvccIt, ok := i.i.(engine.MVCCIterator); ok { + if mvccIt, ok := i.i.(storage.MVCCIterator); ok { return mvccIt.MVCCOpsSpecialized() } return false @@ -233,7 +233,7 @@ func (i *Iterator) MVCCOpsSpecialized() bool { // MVCCGet is part of the engine.MVCCIterator interface. func (i *Iterator) MVCCGet( - key roachpb.Key, timestamp hlc.Timestamp, opts engine.MVCCGetOptions, + key roachpb.Key, timestamp hlc.Timestamp, opts storage.MVCCGetOptions, ) (*roachpb.Value, *roachpb.Intent, error) { if i.spansOnly { if err := i.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: key}); err != nil { @@ -244,34 +244,34 @@ func (i *Iterator) MVCCGet( return nil, nil, err } } - return i.i.(engine.MVCCIterator).MVCCGet(key, timestamp, opts) + return i.i.(storage.MVCCIterator).MVCCGet(key, timestamp, opts) } // MVCCScan is part of the engine.MVCCIterator interface. func (i *Iterator) MVCCScan( - start, end roachpb.Key, timestamp hlc.Timestamp, opts engine.MVCCScanOptions, -) (engine.MVCCScanResult, error) { + start, end roachpb.Key, timestamp hlc.Timestamp, opts storage.MVCCScanOptions, +) (storage.MVCCScanResult, error) { if i.spansOnly { if err := i.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: start, EndKey: end}); err != nil { - return engine.MVCCScanResult{}, err + return storage.MVCCScanResult{}, err } } else { if err := i.spans.CheckAllowedAt(SpanReadOnly, roachpb.Span{Key: start, EndKey: end}, timestamp); err != nil { - return engine.MVCCScanResult{}, err + return storage.MVCCScanResult{}, err } } - return i.i.(engine.MVCCIterator).MVCCScan(start, end, timestamp, opts) + return i.i.(storage.MVCCIterator).MVCCScan(start, end, timestamp, opts) } type spanSetReader struct { - r engine.Reader + r storage.Reader spans *SpanSet spansOnly bool ts hlc.Timestamp } -var _ engine.Reader = spanSetReader{} +var _ storage.Reader = spanSetReader{} func (s spanSetReader) Close() { s.r.Close() @@ -287,12 +287,12 @@ func (s spanSetReader) ExportToSst( startTS, endTS hlc.Timestamp, exportAllRevisions bool, targetSize, maxSize uint64, - io engine.IterOptions, + io storage.IterOptions, ) ([]byte, roachpb.BulkOpSummary, roachpb.Key, error) { return s.r.ExportToSst(startKey, endKey, startTS, endTS, exportAllRevisions, targetSize, maxSize, io) } -func (s spanSetReader) Get(key engine.MVCCKey) ([]byte, error) { +func (s spanSetReader) Get(key storage.MVCCKey) ([]byte, error) { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: key.Key}); err != nil { return nil, err @@ -307,7 +307,7 @@ func (s spanSetReader) Get(key engine.MVCCKey) ([]byte, error) { } func (s spanSetReader) GetProto( - key engine.MVCCKey, msg protoutil.Message, + key storage.MVCCKey, msg protoutil.Message, ) (bool, int64, int64, error) { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: key.Key}); err != nil { @@ -323,7 +323,7 @@ func (s spanSetReader) GetProto( } func (s spanSetReader) Iterate( - start, end roachpb.Key, f func(engine.MVCCKeyValue) (bool, error), + start, end roachpb.Key, f func(storage.MVCCKeyValue) (bool, error), ) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadOnly, roachpb.Span{Key: start, EndKey: end}); err != nil { @@ -337,7 +337,7 @@ func (s spanSetReader) Iterate( return s.r.Iterate(start, end, f) } -func (s spanSetReader) NewIterator(opts engine.IterOptions) engine.Iterator { +func (s spanSetReader) NewIterator(opts storage.IterOptions) storage.Iterator { if s.spansOnly { return NewIterator(s.r.NewIterator(opts), s.spans) } @@ -345,7 +345,7 @@ func (s spanSetReader) NewIterator(opts engine.IterOptions) engine.Iterator { } // GetDBEngine recursively searches for the underlying rocksDB engine. -func GetDBEngine(reader engine.Reader, span roachpb.Span) engine.Reader { +func GetDBEngine(reader storage.Reader, span roachpb.Span) storage.Reader { switch v := reader.(type) { case ReadWriter: return GetDBEngine(getSpanReader(v, span), span) @@ -358,7 +358,7 @@ func GetDBEngine(reader engine.Reader, span roachpb.Span) engine.Reader { // getSpanReader is a getter to access the engine.Reader field of the // spansetReader. -func getSpanReader(r ReadWriter, span roachpb.Span) engine.Reader { +func getSpanReader(r ReadWriter, span roachpb.Span) storage.Reader { if err := r.spanSetReader.spans.CheckAllowed(SpanReadOnly, span); err != nil { panic("Not in the span") } @@ -367,21 +367,21 @@ func getSpanReader(r ReadWriter, span roachpb.Span) engine.Reader { } type spanSetWriter struct { - w engine.Writer + w storage.Writer spans *SpanSet spansOnly bool ts hlc.Timestamp } -var _ engine.Writer = spanSetWriter{} +var _ storage.Writer = spanSetWriter{} func (s spanSetWriter) ApplyBatchRepr(repr []byte, sync bool) error { // Assume that the constructor of the batch has bounded it correctly. return s.w.ApplyBatchRepr(repr, sync) } -func (s spanSetWriter) Clear(key engine.MVCCKey) error { +func (s spanSetWriter) Clear(key storage.MVCCKey) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: key.Key}); err != nil { return err @@ -394,7 +394,7 @@ func (s spanSetWriter) Clear(key engine.MVCCKey) error { return s.w.Clear(key) } -func (s spanSetWriter) SingleClear(key engine.MVCCKey) error { +func (s spanSetWriter) SingleClear(key storage.MVCCKey) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: key.Key}); err != nil { return err @@ -407,7 +407,7 @@ func (s spanSetWriter) SingleClear(key engine.MVCCKey) error { return s.w.SingleClear(key) } -func (s spanSetWriter) ClearRange(start, end engine.MVCCKey) error { +func (s spanSetWriter) ClearRange(start, end storage.MVCCKey) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: start.Key, EndKey: end.Key}); err != nil { return err @@ -420,7 +420,7 @@ func (s spanSetWriter) ClearRange(start, end engine.MVCCKey) error { return s.w.ClearRange(start, end) } -func (s spanSetWriter) ClearIterRange(iter engine.Iterator, start, end roachpb.Key) error { +func (s spanSetWriter) ClearIterRange(iter storage.Iterator, start, end roachpb.Key) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: start, EndKey: end}); err != nil { return err @@ -433,7 +433,7 @@ func (s spanSetWriter) ClearIterRange(iter engine.Iterator, start, end roachpb.K return s.w.ClearIterRange(iter, start, end) } -func (s spanSetWriter) Merge(key engine.MVCCKey, value []byte) error { +func (s spanSetWriter) Merge(key storage.MVCCKey, value []byte) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: key.Key}); err != nil { return err @@ -446,7 +446,7 @@ func (s spanSetWriter) Merge(key engine.MVCCKey, value []byte) error { return s.w.Merge(key, value) } -func (s spanSetWriter) Put(key engine.MVCCKey, value []byte) error { +func (s spanSetWriter) Put(key storage.MVCCKey, value []byte) error { if s.spansOnly { if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: key.Key}); err != nil { return err @@ -464,7 +464,7 @@ func (s spanSetWriter) LogData(data []byte) error { } func (s spanSetWriter) LogLogicalOp( - op engine.MVCCLogicalOpType, details engine.MVCCLogicalOpDetails, + op storage.MVCCLogicalOpType, details storage.MVCCLogicalOpDetails, ) { s.w.LogLogicalOp(op, details) } @@ -475,16 +475,16 @@ type ReadWriter struct { spanSetWriter } -var _ engine.ReadWriter = ReadWriter{} +var _ storage.ReadWriter = ReadWriter{} -func makeSpanSetReadWriter(rw engine.ReadWriter, spans *SpanSet) ReadWriter { +func makeSpanSetReadWriter(rw storage.ReadWriter, spans *SpanSet) ReadWriter { return ReadWriter{ spanSetReader: spanSetReader{r: rw, spans: spans, spansOnly: true}, spanSetWriter: spanSetWriter{w: rw, spans: spans, spansOnly: true}, } } -func makeSpanSetReadWriterAt(rw engine.ReadWriter, spans *SpanSet, ts hlc.Timestamp) ReadWriter { +func makeSpanSetReadWriterAt(rw storage.ReadWriter, spans *SpanSet, ts hlc.Timestamp) ReadWriter { return ReadWriter{ spanSetReader: spanSetReader{r: rw, spans: spans, ts: ts}, spanSetWriter: spanSetWriter{w: rw, spans: spans, ts: ts}, @@ -493,33 +493,33 @@ func makeSpanSetReadWriterAt(rw engine.ReadWriter, spans *SpanSet, ts hlc.Timest // NewReadWriter returns an engine.ReadWriter that asserts access of the // underlying ReadWriter against the given SpanSet. -func NewReadWriter(rw engine.ReadWriter, spans *SpanSet) engine.ReadWriter { +func NewReadWriter(rw storage.ReadWriter, spans *SpanSet) storage.ReadWriter { return makeSpanSetReadWriter(rw, spans) } // NewReadWriterAt returns an engine.ReadWriter that asserts access of the // underlying ReadWriter against the given SpanSet at a given timestamp. // If zero timestamp is provided, accesses are considered non-MVCC. -func NewReadWriterAt(rw engine.ReadWriter, spans *SpanSet, ts hlc.Timestamp) engine.ReadWriter { +func NewReadWriterAt(rw storage.ReadWriter, spans *SpanSet, ts hlc.Timestamp) storage.ReadWriter { return makeSpanSetReadWriterAt(rw, spans, ts) } type spanSetBatch struct { ReadWriter - b engine.Batch + b storage.Batch spans *SpanSet spansOnly bool ts hlc.Timestamp } -var _ engine.Batch = spanSetBatch{} +var _ storage.Batch = spanSetBatch{} func (s spanSetBatch) Commit(sync bool) error { return s.b.Commit(sync) } -func (s spanSetBatch) Distinct() engine.ReadWriter { +func (s spanSetBatch) Distinct() storage.ReadWriter { if s.spansOnly { return NewReadWriter(s.b.Distinct(), s.spans) } @@ -541,7 +541,7 @@ func (s spanSetBatch) Repr() []byte { // NewBatch returns an engine.Batch that asserts access of the underlying // Batch against the given SpanSet. We only consider span boundaries, associated // timestamps are not considered. -func NewBatch(b engine.Batch, spans *SpanSet) engine.Batch { +func NewBatch(b storage.Batch, spans *SpanSet) storage.Batch { return &spanSetBatch{ ReadWriter: makeSpanSetReadWriter(b, spans), b: b, @@ -553,7 +553,7 @@ func NewBatch(b engine.Batch, spans *SpanSet) engine.Batch { // NewBatchAt returns an engine.Batch that asserts access of the underlying // Batch against the given SpanSet at the given timestamp. // If the zero timestamp is used, all accesses are considered non-MVCC. -func NewBatchAt(b engine.Batch, spans *SpanSet, ts hlc.Timestamp) engine.Batch { +func NewBatchAt(b storage.Batch, spans *SpanSet, ts hlc.Timestamp) storage.Batch { return &spanSetBatch{ ReadWriter: makeSpanSetReadWriterAt(b, spans, ts), b: b, diff --git a/pkg/kv/kvserver/split_queue.go b/pkg/kv/kvserver/split_queue.go index bb3702e47eee..ec98c18087d4 100644 --- a/pkg/kv/kvserver/split_queue.go +++ b/pkg/kv/kvserver/split_queue.go @@ -16,11 +16,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server/telemetry" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/split_queue_test.go b/pkg/kv/kvserver/split_queue_test.go index 8a65072e7c10..5599ef6cbd38 100644 --- a/pkg/kv/kvserver/split_queue_test.go +++ b/pkg/kv/kvserver/split_queue_test.go @@ -17,9 +17,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/gogo/protobuf/proto" diff --git a/pkg/kv/kvserver/stateloader/initial.go b/pkg/kv/kvserver/stateloader/initial.go index 268b603d17a1..710a2ce2ccf5 100644 --- a/pkg/kv/kvserver/stateloader/initial.go +++ b/pkg/kv/kvserver/stateloader/initial.go @@ -13,10 +13,10 @@ package stateloader import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" @@ -40,7 +40,7 @@ const ( // are returned. func WriteInitialReplicaState( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms enginepb.MVCCStats, desc roachpb.RangeDescriptor, lease roachpb.Lease, @@ -88,7 +88,7 @@ func WriteInitialReplicaState( // state itself, and the updated stats are returned. func WriteInitialState( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms enginepb.MVCCStats, desc roachpb.RangeDescriptor, lease roachpb.Lease, diff --git a/pkg/kv/kvserver/stateloader/initial_test.go b/pkg/kv/kvserver/stateloader/initial_test.go index bf2a704bc423..3424b80f578b 100644 --- a/pkg/kv/kvserver/stateloader/initial_test.go +++ b/pkg/kv/kvserver/stateloader/initial_test.go @@ -15,8 +15,8 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -27,7 +27,7 @@ func TestSynthesizeHardState(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() defer stopper.Stop(context.TODO()) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) tHS := raftpb.HardState{Term: 2, Vote: 3, Commit: 4} diff --git a/pkg/kv/kvserver/stateloader/stateloader.go b/pkg/kv/kvserver/stateloader/stateloader.go index ef2e8c2f36d5..f2b0588b6818 100644 --- a/pkg/kv/kvserver/stateloader/stateloader.go +++ b/pkg/kv/kvserver/stateloader/stateloader.go @@ -14,11 +14,11 @@ import ( "context" "math" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -56,7 +56,7 @@ func Make(rangeID roachpb.RangeID) StateLoader { // updated transactionally, and is populated from the supplied RangeDescriptor // under the convention that that is the latest committed version. func (rsl StateLoader) Load( - ctx context.Context, reader engine.Reader, desc *roachpb.RangeDescriptor, + ctx context.Context, reader storage.Reader, desc *roachpb.RangeDescriptor, ) (storagepb.ReplicaState, error) { var s storagepb.ReplicaState // TODO(tschottdorf): figure out whether this is always synchronous with @@ -130,7 +130,7 @@ const ( // strictly for use in Result. Do before merge. func (rsl StateLoader) Save( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, state storagepb.ReplicaState, truncStateType TruncatedStateType, ) (enginepb.MVCCStats, error) { @@ -169,29 +169,31 @@ func (rsl StateLoader) Save( } // LoadLease loads the lease. -func (rsl StateLoader) LoadLease(ctx context.Context, reader engine.Reader) (roachpb.Lease, error) { +func (rsl StateLoader) LoadLease( + ctx context.Context, reader storage.Reader, +) (roachpb.Lease, error) { var lease roachpb.Lease - _, err := engine.MVCCGetProto(ctx, reader, rsl.RangeLeaseKey(), - hlc.Timestamp{}, &lease, engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, reader, rsl.RangeLeaseKey(), + hlc.Timestamp{}, &lease, storage.MVCCGetOptions{}) return lease, err } // SetLease persists a lease. func (rsl StateLoader) SetLease( - ctx context.Context, readWriter engine.ReadWriter, ms *enginepb.MVCCStats, lease roachpb.Lease, + ctx context.Context, readWriter storage.ReadWriter, ms *enginepb.MVCCStats, lease roachpb.Lease, ) error { - return engine.MVCCPutProto(ctx, readWriter, ms, rsl.RangeLeaseKey(), + return storage.MVCCPutProto(ctx, readWriter, ms, rsl.RangeLeaseKey(), hlc.Timestamp{}, nil, &lease) } // LoadRangeAppliedState loads the Range applied state. The returned pointer // will be nil if the applied state key is not found. func (rsl StateLoader) LoadRangeAppliedState( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (*enginepb.RangeAppliedState, error) { var as enginepb.RangeAppliedState - found, err := engine.MVCCGetProto(ctx, reader, rsl.RangeAppliedStateKey(), hlc.Timestamp{}, &as, - engine.MVCCGetOptions{}) + found, err := storage.MVCCGetProto(ctx, reader, rsl.RangeAppliedStateKey(), hlc.Timestamp{}, &as, + storage.MVCCGetOptions{}) if !found { return nil, err } @@ -199,7 +201,7 @@ func (rsl StateLoader) LoadRangeAppliedState( } // AssertNoRangeAppliedState asserts that no Range applied state key is present. -func (rsl StateLoader) AssertNoRangeAppliedState(ctx context.Context, reader engine.Reader) error { +func (rsl StateLoader) AssertNoRangeAppliedState(ctx context.Context, reader storage.Reader) error { if as, err := rsl.LoadRangeAppliedState(ctx, reader); err != nil { return err } else if as != nil { @@ -210,7 +212,7 @@ func (rsl StateLoader) AssertNoRangeAppliedState(ctx context.Context, reader eng // LoadAppliedIndex returns the Raft applied index and the lease applied index. func (rsl StateLoader) LoadAppliedIndex( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (raftAppliedIndex uint64, leaseAppliedIndex uint64, err error) { // Check the applied state key. if as, err := rsl.LoadRangeAppliedState(ctx, reader); err != nil { @@ -222,8 +224,8 @@ func (rsl StateLoader) LoadAppliedIndex( // If the range applied state is not found, check the legacy Raft applied // index and the lease applied index keys. This is where these indices were // stored before the range applied state was introduced. - v, _, err := engine.MVCCGet(ctx, reader, rsl.RaftAppliedIndexLegacyKey(), - hlc.Timestamp{}, engine.MVCCGetOptions{}) + v, _, err := storage.MVCCGet(ctx, reader, rsl.RaftAppliedIndexLegacyKey(), + hlc.Timestamp{}, storage.MVCCGetOptions{}) if err != nil { return 0, 0, err } @@ -235,8 +237,8 @@ func (rsl StateLoader) LoadAppliedIndex( raftAppliedIndex = uint64(int64AppliedIndex) } // TODO(tschottdorf): code duplication. - v, _, err = engine.MVCCGet(ctx, reader, rsl.LeaseAppliedIndexLegacyKey(), - hlc.Timestamp{}, engine.MVCCGetOptions{}) + v, _, err = storage.MVCCGet(ctx, reader, rsl.LeaseAppliedIndexLegacyKey(), + hlc.Timestamp{}, storage.MVCCGetOptions{}) if err != nil { return 0, 0, err } @@ -252,7 +254,7 @@ func (rsl StateLoader) LoadAppliedIndex( // LoadMVCCStats loads the MVCC stats. func (rsl StateLoader) LoadMVCCStats( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (enginepb.MVCCStats, error) { // Check the applied state key. if as, err := rsl.LoadRangeAppliedState(ctx, reader); err != nil { @@ -265,8 +267,8 @@ func (rsl StateLoader) LoadMVCCStats( // key. This is where stats were stored before the range applied // state was introduced. var ms enginepb.MVCCStats - _, err := engine.MVCCGetProto(ctx, reader, rsl.RangeStatsLegacyKey(), hlc.Timestamp{}, &ms, - engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, reader, rsl.RangeStatsLegacyKey(), hlc.Timestamp{}, &ms, + storage.MVCCGetOptions{}) return ms, err } @@ -278,7 +280,7 @@ func (rsl StateLoader) LoadMVCCStats( // by the range applied state key. func (rsl StateLoader) SetRangeAppliedState( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, appliedIndex, leaseAppliedIndex uint64, newMS *enginepb.MVCCStats, ) error { @@ -290,22 +292,22 @@ func (rsl StateLoader) SetRangeAppliedState( // The RangeAppliedStateKey is not included in stats. This is also reflected // in C.MVCCComputeStats and ComputeStatsGo. ms := (*enginepb.MVCCStats)(nil) - return engine.MVCCPutProto(ctx, readWriter, ms, rsl.RangeAppliedStateKey(), hlc.Timestamp{}, nil, &as) + return storage.MVCCPutProto(ctx, readWriter, ms, rsl.RangeAppliedStateKey(), hlc.Timestamp{}, nil, &as) } // MigrateToRangeAppliedStateKey deletes the keys that were replaced by the // RangeAppliedState key. func (rsl StateLoader) MigrateToRangeAppliedStateKey( - ctx context.Context, readWriter engine.ReadWriter, ms *enginepb.MVCCStats, + ctx context.Context, readWriter storage.ReadWriter, ms *enginepb.MVCCStats, ) error { noTS := hlc.Timestamp{} - if err := engine.MVCCDelete(ctx, readWriter, ms, rsl.RaftAppliedIndexLegacyKey(), noTS, nil); err != nil { + if err := storage.MVCCDelete(ctx, readWriter, ms, rsl.RaftAppliedIndexLegacyKey(), noTS, nil); err != nil { return err } - if err := engine.MVCCDelete(ctx, readWriter, ms, rsl.LeaseAppliedIndexLegacyKey(), noTS, nil); err != nil { + if err := storage.MVCCDelete(ctx, readWriter, ms, rsl.LeaseAppliedIndexLegacyKey(), noTS, nil); err != nil { return err } - return engine.MVCCDelete(ctx, readWriter, ms, rsl.RangeStatsLegacyKey(), noTS, nil) + return storage.MVCCDelete(ctx, readWriter, ms, rsl.RangeStatsLegacyKey(), noTS, nil) } // SetLegacyAppliedIndex sets the legacy {raft,lease} applied index values, @@ -315,7 +317,7 @@ func (rsl StateLoader) MigrateToRangeAppliedStateKey( // triggered. See comment on SetRangeAppliedState for why this is "legacy". func (rsl StateLoader) SetLegacyAppliedIndex( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, appliedIndex, leaseAppliedIndex uint64, ) error { @@ -325,7 +327,7 @@ func (rsl StateLoader) SetLegacyAppliedIndex( var value roachpb.Value value.SetInt(int64(appliedIndex)) - if err := engine.MVCCPut(ctx, readWriter, ms, + if err := storage.MVCCPut(ctx, readWriter, ms, rsl.RaftAppliedIndexLegacyKey(), hlc.Timestamp{}, value, @@ -333,7 +335,7 @@ func (rsl StateLoader) SetLegacyAppliedIndex( return err } value.SetInt(int64(leaseAppliedIndex)) - return engine.MVCCPut(ctx, readWriter, ms, + return storage.MVCCPut(ctx, readWriter, ms, rsl.LeaseAppliedIndexLegacyKey(), hlc.Timestamp{}, value, @@ -350,7 +352,7 @@ func (rsl StateLoader) SetLegacyAppliedIndex( // triggered. See comment on SetRangeAppliedState for why this is "legacy". func (rsl StateLoader) SetLegacyAppliedIndexBlind( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, appliedIndex, leaseAppliedIndex uint64, ) error { @@ -360,7 +362,7 @@ func (rsl StateLoader) SetLegacyAppliedIndexBlind( var value roachpb.Value value.SetInt(int64(appliedIndex)) - if err := engine.MVCCBlindPut(ctx, readWriter, ms, + if err := storage.MVCCBlindPut(ctx, readWriter, ms, rsl.RaftAppliedIndexLegacyKey(), hlc.Timestamp{}, value, @@ -368,7 +370,7 @@ func (rsl StateLoader) SetLegacyAppliedIndexBlind( return err } value.SetInt(int64(leaseAppliedIndex)) - return engine.MVCCBlindPut(ctx, readWriter, ms, + return storage.MVCCBlindPut(ctx, readWriter, ms, rsl.LeaseAppliedIndexLegacyKey(), hlc.Timestamp{}, value, @@ -385,20 +387,20 @@ func inlineValueIntEncodedSize(v int64) int { // CalcAppliedIndexSysBytes calculates the size (MVCCStats.SysBytes) of the {raft,lease} applied // index keys/values. func (rsl StateLoader) CalcAppliedIndexSysBytes(appliedIndex, leaseAppliedIndex uint64) int64 { - return int64(engine.MakeMVCCMetadataKey(rsl.RaftAppliedIndexLegacyKey()).EncodedSize() + - engine.MakeMVCCMetadataKey(rsl.LeaseAppliedIndexLegacyKey()).EncodedSize() + + return int64(storage.MakeMVCCMetadataKey(rsl.RaftAppliedIndexLegacyKey()).EncodedSize() + + storage.MakeMVCCMetadataKey(rsl.LeaseAppliedIndexLegacyKey()).EncodedSize() + inlineValueIntEncodedSize(int64(appliedIndex)) + inlineValueIntEncodedSize(int64(leaseAppliedIndex))) } func (rsl StateLoader) writeLegacyMVCCStatsInternal( - ctx context.Context, readWriter engine.ReadWriter, newMS *enginepb.MVCCStats, + ctx context.Context, readWriter storage.ReadWriter, newMS *enginepb.MVCCStats, ) error { // NB: newMS is copied to prevent conditional calls to this method from // causing the stats argument to escape. This is legacy code which does // not need to be optimized for performance. newMSCopy := *newMS - return engine.MVCCPutProto(ctx, readWriter, nil, rsl.RangeStatsLegacyKey(), hlc.Timestamp{}, nil, &newMSCopy) + return storage.MVCCPutProto(ctx, readWriter, nil, rsl.RangeStatsLegacyKey(), hlc.Timestamp{}, nil, &newMSCopy) } // SetLegacyMVCCStats overwrites the legacy MVCC stats key. @@ -406,7 +408,7 @@ func (rsl StateLoader) writeLegacyMVCCStatsInternal( // The range applied state key cannot already exist or an assetion will be // triggered. See comment on SetRangeAppliedState for why this is "legacy". func (rsl StateLoader) SetLegacyMVCCStats( - ctx context.Context, readWriter engine.ReadWriter, newMS *enginepb.MVCCStats, + ctx context.Context, readWriter storage.ReadWriter, newMS *enginepb.MVCCStats, ) error { if err := rsl.AssertNoRangeAppliedState(ctx, readWriter); err != nil { return err @@ -419,7 +421,7 @@ func (rsl StateLoader) SetLegacyMVCCStats( // RangeAppliedState key before overwriting the stats. Use SetRangeAppliedState // when performance is important. func (rsl StateLoader) SetMVCCStats( - ctx context.Context, readWriter engine.ReadWriter, newMS *enginepb.MVCCStats, + ctx context.Context, readWriter storage.ReadWriter, newMS *enginepb.MVCCStats, ) error { if as, err := rsl.LoadRangeAppliedState(ctx, readWriter); err != nil { return err @@ -433,51 +435,51 @@ func (rsl StateLoader) SetMVCCStats( // SetLegacyRaftTruncatedState overwrites the truncated state. func (rsl StateLoader) SetLegacyRaftTruncatedState( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, truncState *roachpb.RaftTruncatedState, ) error { if (*truncState == roachpb.RaftTruncatedState{}) { return errors.New("cannot persist empty RaftTruncatedState") } - return engine.MVCCPutProto(ctx, readWriter, ms, + return storage.MVCCPutProto(ctx, readWriter, ms, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, nil, truncState) } // LoadGCThreshold loads the GC threshold. func (rsl StateLoader) LoadGCThreshold( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (*hlc.Timestamp, error) { var t hlc.Timestamp - _, err := engine.MVCCGetProto(ctx, reader, rsl.RangeLastGCKey(), - hlc.Timestamp{}, &t, engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, reader, rsl.RangeLastGCKey(), + hlc.Timestamp{}, &t, storage.MVCCGetOptions{}) return &t, err } // SetGCThreshold sets the GC threshold. func (rsl StateLoader) SetGCThreshold( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, ms *enginepb.MVCCStats, threshold *hlc.Timestamp, ) error { if threshold == nil { return errors.New("cannot persist nil GCThreshold") } - return engine.MVCCPutProto(ctx, readWriter, ms, + return storage.MVCCPutProto(ctx, readWriter, ms, rsl.RangeLastGCKey(), hlc.Timestamp{}, nil, threshold) } // The rest is not technically part of ReplicaState. // LoadLastIndex loads the last index. -func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader engine.Reader) (uint64, error) { +func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader storage.Reader) (uint64, error) { prefix := rsl.RaftLogPrefix() - iter := reader.NewIterator(engine.IterOptions{LowerBound: prefix}) + iter := reader.NewIterator(storage.IterOptions{LowerBound: prefix}) defer iter.Close() var lastIndex uint64 - iter.SeekLT(engine.MakeMVCCMetadataKey(rsl.RaftLogKey(math.MaxUint64))) + iter.SeekLT(storage.MakeMVCCMetadataKey(rsl.RaftLogKey(math.MaxUint64))) if ok, _ := iter.Valid(); ok { key := iter.Key() var err error @@ -505,11 +507,11 @@ func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader engine.Reader) // // See VersionUnreplicatedRaftTruncatedState. func (rsl StateLoader) LoadRaftTruncatedState( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (_ roachpb.RaftTruncatedState, isLegacy bool, _ error) { var truncState roachpb.RaftTruncatedState - if found, err := engine.MVCCGetProto( - ctx, reader, rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + if found, err := storage.MVCCGetProto( + ctx, reader, rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, &truncState, storage.MVCCGetOptions{}, ); err != nil { return roachpb.RaftTruncatedState{}, false, err } else if found { @@ -521,8 +523,8 @@ func (rsl StateLoader) LoadRaftTruncatedState( // assuming the cluster version has advanced sufficiently. // // See VersionUnreplicatedRaftTruncatedState. - legacyFound, err := engine.MVCCGetProto( - ctx, reader, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + legacyFound, err := storage.MVCCGetProto( + ctx, reader, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, &truncState, storage.MVCCGetOptions{}, ) if err != nil { return roachpb.RaftTruncatedState{}, false, err @@ -532,13 +534,13 @@ func (rsl StateLoader) LoadRaftTruncatedState( // SetRaftTruncatedState overwrites the truncated state. func (rsl StateLoader) SetRaftTruncatedState( - ctx context.Context, writer engine.Writer, truncState *roachpb.RaftTruncatedState, + ctx context.Context, writer storage.Writer, truncState *roachpb.RaftTruncatedState, ) error { if (*truncState == roachpb.RaftTruncatedState{}) { return errors.New("cannot persist empty RaftTruncatedState") } // "Blind" because ms == nil and timestamp == hlc.Timestamp{}. - return engine.MVCCBlindPutProto( + return storage.MVCCBlindPutProto( ctx, writer, nil, /* ms */ @@ -551,11 +553,11 @@ func (rsl StateLoader) SetRaftTruncatedState( // LoadHardState loads the HardState. func (rsl StateLoader) LoadHardState( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (raftpb.HardState, error) { var hs raftpb.HardState - found, err := engine.MVCCGetProto(ctx, reader, rsl.RaftHardStateKey(), - hlc.Timestamp{}, &hs, engine.MVCCGetOptions{}) + found, err := storage.MVCCGetProto(ctx, reader, rsl.RaftHardStateKey(), + hlc.Timestamp{}, &hs, storage.MVCCGetOptions{}) if !found || err != nil { return raftpb.HardState{}, err @@ -565,10 +567,10 @@ func (rsl StateLoader) LoadHardState( // SetHardState overwrites the HardState. func (rsl StateLoader) SetHardState( - ctx context.Context, writer engine.Writer, hs raftpb.HardState, + ctx context.Context, writer storage.Writer, hs raftpb.HardState, ) error { // "Blind" because ms == nil and timestamp == hlc.Timestamp{}. - return engine.MVCCBlindPutProto( + return storage.MVCCBlindPutProto( ctx, writer, nil, /* ms */ @@ -584,7 +586,7 @@ func (rsl StateLoader) SetHardState( // writeInitialReplicaState and, on a split, perhaps the activity of an // uninitialized Raft group) func (rsl StateLoader) SynthesizeRaftState( - ctx context.Context, readWriter engine.ReadWriter, + ctx context.Context, readWriter storage.ReadWriter, ) error { hs, err := rsl.LoadHardState(ctx, readWriter) if err != nil { @@ -605,7 +607,7 @@ func (rsl StateLoader) SynthesizeRaftState( // taking care that a HardState compatible with the existing data is written. func (rsl StateLoader) SynthesizeHardState( ctx context.Context, - readWriter engine.ReadWriter, + readWriter storage.ReadWriter, oldHS raftpb.HardState, truncState roachpb.RaftTruncatedState, raftAppliedIndex uint64, diff --git a/pkg/kv/kvserver/stats_test.go b/pkg/kv/kvserver/stats_test.go index 1e8ffb978c4c..e49b2b8abef2 100644 --- a/pkg/kv/kvserver/stats_test.go +++ b/pkg/kv/kvserver/stats_test.go @@ -15,8 +15,8 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/kr/pretty" diff --git a/pkg/kv/kvserver/storagepb/proposer_kv.pb.go b/pkg/kv/kvserver/storagepb/proposer_kv.pb.go index 3fd069f699cf..8e6a364f4eec 100644 --- a/pkg/kv/kvserver/storagepb/proposer_kv.pb.go +++ b/pkg/kv/kvserver/storagepb/proposer_kv.pb.go @@ -9,9 +9,9 @@ import ( io "io" math "math" - enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" github_com_cockroachdb_cockroach_pkg_roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" + enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" hlc "github.com/cockroachdb/cockroach/pkg/util/hlc" github_com_cockroachdb_cockroach_pkg_util_uuid "github.com/cockroachdb/cockroach/pkg/util/uuid" proto "github.com/gogo/protobuf/proto" diff --git a/pkg/kv/kvserver/storagepb/state.pb.go b/pkg/kv/kvserver/storagepb/state.pb.go index 8f7fef270f2c..55736251f070 100644 --- a/pkg/kv/kvserver/storagepb/state.pb.go +++ b/pkg/kv/kvserver/storagepb/state.pb.go @@ -8,9 +8,9 @@ import ( io "io" math "math" - enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" github_com_cockroachdb_cockroach_pkg_roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" + enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" hlc "github.com/cockroachdb/cockroach/pkg/util/hlc" proto "github.com/gogo/protobuf/proto" ) diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index 726239f17036..75655168cc57 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -29,8 +29,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -53,6 +51,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -366,7 +366,7 @@ type Store struct { Ident *roachpb.StoreIdent // pointer to catch access before Start() is called cfg StoreConfig db *client.DB - engine engine.Engine // The underlying key-value store + engine storage.Engine // The underlying key-value store compactor *compactor.Compactor // Schedules compaction of the engine tsCache tscache.Cache // Most recent timestamps for keys / key ranges allocator Allocator // Makes allocation decisions @@ -765,7 +765,7 @@ func (sc *StoreConfig) LeaseExpiration() int64 { // NewStore returns a new instance of a store. func NewStore( - ctx context.Context, cfg StoreConfig, eng engine.Engine, nodeDesc *roachpb.NodeDescriptor, + ctx context.Context, cfg StoreConfig, eng storage.Engine, nodeDesc *roachpb.NodeDescriptor, ) *Store { // TODO(tschottdorf): find better place to set these defaults. cfg.SetDefaults() @@ -1131,20 +1131,20 @@ func (s *Store) IsStarted() bool { // Iteration stops on the first error (and will pass through that error). func IterateIDPrefixKeys( ctx context.Context, - reader engine.Reader, + reader storage.Reader, keyFn func(roachpb.RangeID) roachpb.Key, msg protoutil.Message, f func(_ roachpb.RangeID) (more bool, _ error), ) error { rangeID := roachpb.RangeID(1) - iter := reader.NewIterator(engine.IterOptions{ + iter := reader.NewIterator(storage.IterOptions{ UpperBound: keys.LocalRangeIDPrefix.PrefixEnd().AsRawKey(), }) defer iter.Close() for { bumped := false - mvccKey := engine.MakeMVCCMetadataKey(keyFn(rangeID)) + mvccKey := storage.MakeMVCCMetadataKey(keyFn(rangeID)) iter.SeekGE(mvccKey) if ok, err := iter.Valid(); !ok { @@ -1169,7 +1169,7 @@ func IterateIDPrefixKeys( rangeID = curRangeID bumped = true } - mvccKey = engine.MakeMVCCMetadataKey(keyFn(rangeID)) + mvccKey = storage.MakeMVCCMetadataKey(keyFn(rangeID)) } if !unsafeKey.Key.Equal(mvccKey.Key) { @@ -1182,8 +1182,8 @@ func IterateIDPrefixKeys( continue } - ok, err := engine.MVCCGetProto( - ctx, reader, unsafeKey.Key, hlc.Timestamp{}, msg, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto( + ctx, reader, unsafeKey.Key, hlc.Timestamp{}, msg, storage.MVCCGetOptions{}) if err != nil { return err } @@ -1204,7 +1204,7 @@ func IterateIDPrefixKeys( // semantics similar to engine.MVCCIterate. func IterateRangeDescriptors( ctx context.Context, - reader engine.Reader, + reader storage.Reader, fn func(desc roachpb.RangeDescriptor) (done bool, err error), ) error { log.Event(ctx, "beginning range descriptor iteration") @@ -1234,8 +1234,8 @@ func IterateRangeDescriptors( return fn(desc) } - _, err := engine.MVCCIterate(ctx, reader, start, end, hlc.MaxTimestamp, - engine.MVCCScanOptions{Inconsistent: true}, kvToDesc) + _, err := storage.MVCCIterate(ctx, reader, start, end, hlc.MaxTimestamp, + storage.MVCCScanOptions{Inconsistent: true}, kvToDesc) log.Eventf(ctx, "iterated over %d keys to find %d range descriptors (by suffix: %v)", allCount, matchCount, bySuffix) return err @@ -1244,10 +1244,10 @@ func IterateRangeDescriptors( // ReadStoreIdent reads the StoreIdent from the store. // It returns *NotBootstrappedError if the ident is missing (meaning that the // store needs to be bootstrapped). -func ReadStoreIdent(ctx context.Context, eng engine.Engine) (roachpb.StoreIdent, error) { +func ReadStoreIdent(ctx context.Context, eng storage.Engine) (roachpb.StoreIdent, error) { var ident roachpb.StoreIdent - ok, err := engine.MVCCGetProto( - ctx, eng, keys.StoreIdentKey(), hlc.Timestamp{}, &ident, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto( + ctx, eng, keys.StoreIdentKey(), hlc.Timestamp{}, &ident, storage.MVCCGetOptions{}) if err != nil { return roachpb.StoreIdent{}, err } else if !ok { @@ -1854,7 +1854,7 @@ func (s *Store) VisitReplicas(visitor func(*Replica) bool) { // determine the approximate time that it stopped. func (s *Store) WriteLastUpTimestamp(ctx context.Context, time hlc.Timestamp) error { ctx = s.AnnotateCtx(ctx) - return engine.MVCCPutProto( + return storage.MVCCPutProto( ctx, s.engine, nil, @@ -1872,8 +1872,8 @@ func (s *Store) WriteLastUpTimestamp(ctx context.Context, time hlc.Timestamp) er // timestamp is returned instead. func (s *Store) ReadLastUpTimestamp(ctx context.Context) (hlc.Timestamp, error) { var timestamp hlc.Timestamp - ok, err := engine.MVCCGetProto(ctx, s.Engine(), keys.StoreLastUpKey(), hlc.Timestamp{}, - ×tamp, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, s.Engine(), keys.StoreLastUpKey(), hlc.Timestamp{}, + ×tamp, storage.MVCCGetOptions{}) if err != nil { return hlc.Timestamp{}, err } else if !ok { @@ -1889,7 +1889,7 @@ func (s *Store) WriteHLCUpperBound(ctx context.Context, time int64) error { batch := s.Engine().NewBatch() // Write has to sync to disk to ensure HLC monotonicity across restarts defer batch.Close() - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, batch, nil, @@ -1909,10 +1909,10 @@ func (s *Store) WriteHLCUpperBound(ctx context.Context, time int64) error { // ReadHLCUpperBound returns the upper bound to the wall time of the HLC // If this value does not exist 0 is returned -func ReadHLCUpperBound(ctx context.Context, e engine.Engine) (int64, error) { +func ReadHLCUpperBound(ctx context.Context, e storage.Engine) (int64, error) { var timestamp hlc.Timestamp - ok, err := engine.MVCCGetProto(ctx, e, keys.StoreHLCUpperBoundKey(), hlc.Timestamp{}, - ×tamp, engine.MVCCGetOptions{}) + ok, err := storage.MVCCGetProto(ctx, e, keys.StoreHLCUpperBoundKey(), hlc.Timestamp{}, + ×tamp, storage.MVCCGetOptions{}) if err != nil { return 0, err } else if !ok { @@ -1926,7 +1926,7 @@ func ReadHLCUpperBound(ctx context.Context, e engine.Engine) (int64, error) { // it is guaranteed to be higher than any wall time used by the HLC. If this // value is persisted, HLC wall clock monotonicity is guaranteed across server // restarts -func ReadMaxHLCUpperBound(ctx context.Context, engines []engine.Engine) (int64, error) { +func ReadMaxHLCUpperBound(ctx context.Context, engines []storage.Engine) (int64, error) { var hlcUpperBound int64 for _, e := range engines { engineHLCUpperBound, err := ReadHLCUpperBound(ctx, e) @@ -1940,8 +1940,8 @@ func ReadMaxHLCUpperBound(ctx context.Context, engines []engine.Engine) (int64, return hlcUpperBound, nil } -func checkEngineEmpty(ctx context.Context, eng engine.Engine) error { - kvs, err := engine.Scan(eng, roachpb.KeyMin, roachpb.KeyMax, 10) +func checkEngineEmpty(ctx context.Context, eng storage.Engine) error { + kvs, err := storage.Scan(eng, roachpb.KeyMin, roachpb.KeyMax, 10) if err != nil { return err } @@ -2044,7 +2044,7 @@ func (s *Store) StoreID() roachpb.StoreID { return s.Ident.StoreID } func (s *Store) Clock() *hlc.Clock { return s.cfg.Clock } // Engine accessor. -func (s *Store) Engine() engine.Engine { return s.engine } +func (s *Store) Engine() storage.Engine { return s.engine } // DB accessor. func (s *Store) DB() *client.DB { return s.cfg.DB } @@ -2527,18 +2527,18 @@ func (s *Store) GetClusterVersion(ctx context.Context) (clusterversion.ClusterVe // WriteClusterVersion writes the given cluster version to the store-local cluster version key. func WriteClusterVersion( - ctx context.Context, writer engine.ReadWriter, cv clusterversion.ClusterVersion, + ctx context.Context, writer storage.ReadWriter, cv clusterversion.ClusterVersion, ) error { - return engine.MVCCPutProto(ctx, writer, nil, keys.StoreClusterVersionKey(), hlc.Timestamp{}, nil, &cv) + return storage.MVCCPutProto(ctx, writer, nil, keys.StoreClusterVersionKey(), hlc.Timestamp{}, nil, &cv) } // ReadClusterVersion reads the the cluster version from the store-local version key. func ReadClusterVersion( - ctx context.Context, reader engine.Reader, + ctx context.Context, reader storage.Reader, ) (clusterversion.ClusterVersion, error) { var cv clusterversion.ClusterVersion - _, err := engine.MVCCGetProto(ctx, reader, keys.StoreClusterVersionKey(), hlc.Timestamp{}, - &cv, engine.MVCCGetOptions{}) + _, err := storage.MVCCGetProto(ctx, reader, keys.StoreClusterVersionKey(), hlc.Timestamp{}, + &cv, storage.MVCCGetOptions{}) return cv, err } diff --git a/pkg/kv/kvserver/store_bootstrap.go b/pkg/kv/kvserver/store_bootstrap.go index 38767992e996..593685861e0c 100644 --- a/pkg/kv/kvserver/store_bootstrap.go +++ b/pkg/kv/kvserver/store_bootstrap.go @@ -14,12 +14,12 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/gogo/protobuf/proto" @@ -33,7 +33,7 @@ import ( // non-empty engine. func InitEngine( ctx context.Context, - eng engine.Engine, + eng storage.Engine, ident roachpb.StoreIdent, cv clusterversion.ClusterVersion, ) error { @@ -50,7 +50,7 @@ func InitEngine( } batch := eng.NewBatch() - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, batch, nil, @@ -87,7 +87,7 @@ func InitEngine( // nowNanos: the timestamp at which to write the initial engine data. func WriteInitialClusterData( ctx context.Context, - eng engine.Engine, + eng storage.Engine, initialValues []roachpb.KeyValue, bootstrapVersion roachpb.Version, numStores int, @@ -181,7 +181,7 @@ func WriteInitialClusterData( // we write everything and then compute the stats over the whole range. // Range descriptor. - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, batch, nil /* ms */, keys.RangeDescriptorKey(desc.StartKey), now, nil /* txn */, desc, ); err != nil { @@ -189,7 +189,7 @@ func WriteInitialClusterData( } // Replica GC timestamp. - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, batch, nil /* ms */, keys.RangeLastReplicaGCTimestampKey(desc.RangeID), hlc.Timestamp{}, nil /* txn */, &now, ); err != nil { @@ -197,7 +197,7 @@ func WriteInitialClusterData( } // Range addressing for meta2. meta2Key := keys.RangeMetaKey(endKey) - if err := engine.MVCCPutProto(ctx, batch, firstRangeMS, meta2Key.AsRawKey(), + if err := storage.MVCCPutProto(ctx, batch, firstRangeMS, meta2Key.AsRawKey(), now, nil /* txn */, desc, ); err != nil { return err @@ -207,7 +207,7 @@ func WriteInitialClusterData( if startKey.Equal(roachpb.RKeyMin) { // Range addressing for meta1. meta1Key := keys.RangeMetaKey(keys.RangeMetaKey(roachpb.RKeyMax)) - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, batch, nil /* ms */, meta1Key.AsRawKey(), now, nil /* txn */, desc, ); err != nil { return err @@ -218,7 +218,7 @@ func WriteInitialClusterData( for _, kv := range rangeInitialValues { // Initialize the checksums. kv.Value.InitChecksum(kv.Key) - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, batch, nil /* ms */, kv.Key, now, kv.Value, nil, /* txn */ ); err != nil { return err diff --git a/pkg/kv/kvserver/store_create_replica.go b/pkg/kv/kvserver/store_create_replica.go index ea0a7fce70db..149bb3f86499 100644 --- a/pkg/kv/kvserver/store_create_replica.go +++ b/pkg/kv/kvserver/store_create_replica.go @@ -15,9 +15,9 @@ import ( "time" "unsafe" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/retry" @@ -144,8 +144,8 @@ func (s *Store) tryGetOrCreateReplica( // work when we know the Replica should not be created ahead of time. tombstoneKey := keys.RangeTombstoneKey(rangeID) var tombstone roachpb.RangeTombstone - if ok, err := engine.MVCCGetProto( - ctx, s.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, s.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ); err != nil { return nil, false, err } else if ok && replicaID != 0 && replicaID < tombstone.NextReplicaID { @@ -204,8 +204,8 @@ func (s *Store) tryGetOrCreateReplica( // tombstone check and the Range map linearization point. By checking // again now, we make sure to synchronize with any goroutine that wrote // a tombstone and then removed an old replica from the Range map. - if ok, err := engine.MVCCGetProto( - ctx, s.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + ctx, s.Engine(), tombstoneKey, hlc.Timestamp{}, &tombstone, storage.MVCCGetOptions{}, ); err != nil { return err } else if ok && replicaID < tombstone.NextReplicaID { diff --git a/pkg/kv/kvserver/store_pool_test.go b/pkg/kv/kvserver/store_pool_test.go index 6af0b957b1d3..facec462d609 100644 --- a/pkg/kv/kvserver/store_pool_test.go +++ b/pkg/kv/kvserver/store_pool_test.go @@ -21,13 +21,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils/gossiputil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -577,7 +577,7 @@ func TestStorePoolUpdateLocalStoreBeforeGossip(t *testing.T) { // Create store. node := roachpb.NodeDescriptor{NodeID: roachpb.NodeID(1)} - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) cfg := TestStoreConfig(clock) cfg.Transport = NewDummyRaftTransport(cfg.Settings) diff --git a/pkg/kv/kvserver/store_rebalancer_test.go b/pkg/kv/kvserver/store_rebalancer_test.go index 3f9b9f9fe99d..d63076b78ad3 100644 --- a/pkg/kv/kvserver/store_rebalancer_test.go +++ b/pkg/kv/kvserver/store_rebalancer_test.go @@ -16,8 +16,8 @@ import ( "sort" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils/gossiputil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index cc19ff29208f..9401fb9ae645 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -17,11 +17,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -100,7 +100,7 @@ type kvBatchSnapshotStrategy struct { // Limiter for sending KV batches. Only used on the sender side. limiter *rate.Limiter // Only used on the sender side. - newBatch func() engine.Batch + newBatch func() storage.Batch // The approximate size of the SST chunk to buffer in memory on the receiver // before flushing to disk. Only used on the receiver side. @@ -114,7 +114,7 @@ type kvBatchSnapshotStrategy struct { // disk. type multiSSTWriter struct { scratch *SSTSnapshotStorageScratch - currSST engine.SSTWriter + currSST storage.SSTWriter keyRanges []rditer.KeyRange currRange int // The approximate size of the SST chunk to buffer in memory on the receiver @@ -144,7 +144,7 @@ func (msstw *multiSSTWriter) initSST(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to create new sst file") } - newSST := engine.MakeIngestionSSTWriter(newSSTFile) + newSST := storage.MakeIngestionSSTWriter(newSSTFile) msstw.currSST = newSST if err := msstw.currSST.ClearRange(msstw.keyRanges[msstw.currRange].Start, msstw.keyRanges[msstw.currRange].End); err != nil { msstw.currSST.Close() @@ -163,7 +163,7 @@ func (msstw *multiSSTWriter) finalizeSST(ctx context.Context) error { return nil } -func (msstw *multiSSTWriter) Put(ctx context.Context, key engine.MVCCKey, value []byte) error { +func (msstw *multiSSTWriter) Put(ctx context.Context, key storage.MVCCKey, value []byte) error { for msstw.keyRanges[msstw.currRange].End.Key.Compare(key.Key) <= 0 { // Finish the current SST, write to the file, and move to the next key // range. @@ -238,14 +238,14 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( } if req.KVBatch != nil { - batchReader, err := engine.NewRocksDBBatchReader(req.KVBatch) + batchReader, err := storage.NewRocksDBBatchReader(req.KVBatch) if err != nil { return noSnap, errors.Wrap(err, "failed to decode batch") } // All operations in the batch are guaranteed to be puts. for batchReader.Next() { - if batchReader.BatchType() != engine.BatchTypeValue { - return noSnap, crdberrors.AssertionFailedf("expected type %d, found type %d", engine.BatchTypeValue, batchReader.BatchType()) + if batchReader.BatchType() != storage.BatchTypeValue { + return noSnap, crdberrors.AssertionFailedf("expected type %d, found type %d", storage.BatchTypeValue, batchReader.BatchType()) } key, err := batchReader.MVCCKey() if err != nil { @@ -317,7 +317,7 @@ func (kvSS *kvBatchSnapshotStrategy) Send( // Iterate over all keys using the provided iterator and stream out batches // of key-values. n := 0 - var b engine.Batch + var b storage.Batch for iter := snap.Iter; ; iter.Next() { if ok, err := iter.Valid(); err != nil { return err @@ -463,7 +463,7 @@ func (kvSS *kvBatchSnapshotStrategy) Send( } func (kvSS *kvBatchSnapshotStrategy) sendBatch( - ctx context.Context, stream outgoingSnapshotStream, batch engine.Batch, + ctx context.Context, stream outgoingSnapshotStream, batch storage.Batch, ) error { if err := kvSS.limiter.WaitN(ctx, 1); err != nil { return err @@ -887,7 +887,7 @@ func sendSnapshot( storePool SnapshotStorePool, header SnapshotRequest_Header, snap *OutgoingSnapshot, - newBatch func() engine.Batch, + newBatch func() storage.Batch, sent func(), ) error { start := timeutil.Now() diff --git a/pkg/kv/kvserver/store_split.go b/pkg/kv/kvserver/store_split.go index 03a6bfc74faf..360b69e85af0 100644 --- a/pkg/kv/kvserver/store_split.go +++ b/pkg/kv/kvserver/store_split.go @@ -14,10 +14,10 @@ import ( "bytes" "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" "go.etcd.io/etcd/raft" @@ -28,7 +28,7 @@ import ( // changes to the given ReadWriter will be written atomically with the // split commit. func splitPreApply( - ctx context.Context, readWriter engine.ReadWriter, split roachpb.SplitTrigger, r *Replica, + ctx context.Context, readWriter storage.ReadWriter, split roachpb.SplitTrigger, r *Replica, ) { // Sanity check that the store is in the split. // diff --git a/pkg/kv/kvserver/store_test.go b/pkg/kv/kvserver/store_test.go index 0057067fbc83..ae92860bc27f 100644 --- a/pkg/kv/kvserver/store_test.go +++ b/pkg/kv/kvserver/store_test.go @@ -27,8 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -42,6 +40,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -226,7 +226,7 @@ func createTestStoreWithoutStart( // and merge queues separately to cover event-driven splits and merges. cfg.TestingKnobs.DisableSplitQueue = true cfg.TestingKnobs.DisableMergeQueue = true - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) cfg.Transport = NewDummyRaftTransport(cfg.Settings) factory := &testSenderFactory{} @@ -297,7 +297,7 @@ func TestIterateIDPrefixKeys(t *testing.T) { stopper := stop.NewStopper() defer stopper.Stop(ctx) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) seed := randutil.NewPseudoSeed() @@ -331,7 +331,7 @@ func TestIterateIDPrefixKeys(t *testing.T) { for _, opIdx := range rng.Perm(len(ops))[:rng.Intn(1+len(ops))] { key := ops[opIdx](rangeID) t.Logf("writing op=%d rangeID=%d", opIdx, rangeID) - if err := engine.MVCCPut( + if err := storage.MVCCPut( ctx, eng, nil, /* ms */ @@ -369,7 +369,7 @@ func TestIterateIDPrefixKeys(t *testing.T) { wanted = append(wanted, seenT{rangeID: rangeID, tombstone: tombstone}) t.Logf("writing tombstone at rangeID=%d", rangeID) - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, eng, nil /* ms */, keys.RangeTombstoneKey(rangeID), hlc.Timestamp{}, nil /* txn */, &tombstone, ); err != nil { t.Fatal(err) @@ -426,7 +426,7 @@ func TestStoreInitAndBootstrap(t *testing.T) { stopper := stop.NewStopper() ctx := context.TODO() defer stopper.Stop(ctx) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) cfg.Transport = NewDummyRaftTransport(cfg.Settings) factory := &testSenderFactory{} @@ -499,11 +499,11 @@ func TestBootstrapOfNonEmptyStore(t *testing.T) { stopper := stop.NewStopper() ctx := context.TODO() defer stopper.Stop(ctx) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) // Put some random garbage into the engine. - if err := eng.Put(engine.MakeMVCCMetadataKey(roachpb.Key("foo")), []byte("bar")); err != nil { + if err := eng.Put(storage.MakeMVCCMetadataKey(roachpb.Key("foo")), []byte("bar")); err != nil { t.Errorf("failure putting key foo into engine: %+v", err) } cfg := TestStoreConfig(nil) @@ -1553,8 +1553,8 @@ func TestStoreResolveWriteIntent(t *testing.T) { } txnKey := keys.TransactionKey(pushee.Key, pushee.ID) var txn roachpb.Transaction - if ok, err := engine.MVCCGetProto( - context.Background(), store.Engine(), txnKey, hlc.Timestamp{}, &txn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + context.Background(), store.Engine(), txnKey, hlc.Timestamp{}, &txn, storage.MVCCGetOptions{}, ); err != nil { t.Fatal(err) } else if ok { @@ -1856,8 +1856,8 @@ func TestStoreResolveWriteIntentNoTxn(t *testing.T) { // Read pushee's txn. txnKey := keys.TransactionKey(pushee.Key, pushee.ID) var txn roachpb.Transaction - if ok, err := engine.MVCCGetProto( - context.Background(), store.Engine(), txnKey, hlc.Timestamp{}, &txn, engine.MVCCGetOptions{}, + if ok, err := storage.MVCCGetProto( + context.Background(), store.Engine(), txnKey, hlc.Timestamp{}, &txn, storage.MVCCGetOptions{}, ); !ok || err != nil { t.Fatalf("not found or err: %+v", err) } @@ -2905,7 +2905,7 @@ func (sp *fakeStorePool) throttle(reason throttleReason, why string, toStoreID r // various exceptional conditions and new capacity estimates. func TestSendSnapshotThrottling(t *testing.T) { defer leaktest.AfterTest(t)() - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() ctx := context.Background() @@ -3243,10 +3243,10 @@ func TestPreemptiveSnapshotsAreRemoved(t *testing.T) { const colID = 1 prefix := tablePrefix[0:len(tablePrefix):len(tablePrefix)] k := roachpb.Key(encoding.EncodeIntValue(prefix, colID, int64(i))) - require.NoError(t, engine.MVCCBlindPutProto(ctx, b, state.Stats, + require.NoError(t, storage.MVCCBlindPutProto(ctx, b, state.Stats, k, s.Clock().Now(), desc, nil)) } - require.NoError(t, engine.MVCCBlindPutProto(ctx, b, state.Stats, + require.NoError(t, storage.MVCCBlindPutProto(ctx, b, state.Stats, keys.RangeDescriptorKey(desc.StartKey), hlc.Timestamp{}, desc, nil)) _, err := stl.Save(ctx, b, state, stateloader.TruncatedStateUnreplicated) require.NoError(t, err) @@ -3271,7 +3271,7 @@ func TestPreemptiveSnapshotsAreRemoved(t *testing.T) { break } // There should not be any data other than the tombstone key. - if k := it.UnsafeKey(); k.Equal(engine.MVCCKey{Key: tombstoneKey}) { + if k := it.UnsafeKey(); k.Equal(storage.MVCCKey{Key: tombstoneKey}) { foundTombstoneKey = true } else { t.Fatalf("found data in the range which should have been removed: %v", k) diff --git a/pkg/kv/kvserver/stores.go b/pkg/kv/kvserver/stores.go index d45153dc9feb..0057de4be580 100644 --- a/pkg/kv/kvserver/stores.go +++ b/pkg/kv/kvserver/stores.go @@ -16,11 +16,11 @@ import ( "unsafe" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -217,8 +217,8 @@ func (ls *Stores) ReadBootstrapInfo(bi *gossip.BootstrapInfo) error { s := (*Store)(v) var storeBI gossip.BootstrapInfo var ok bool - ok, err = engine.MVCCGetProto(ctx, s.engine, keys.StoreGossipKey(), hlc.Timestamp{}, &storeBI, - engine.MVCCGetOptions{}) + ok, err = storage.MVCCGetProto(ctx, s.engine, keys.StoreGossipKey(), hlc.Timestamp{}, &storeBI, + storage.MVCCGetOptions{}) if err != nil { return false } @@ -266,7 +266,7 @@ func (ls *Stores) updateBootstrapInfoLocked(bi *gossip.BootstrapInfo) error { var err error ls.storeMap.Range(func(k int64, v unsafe.Pointer) bool { s := (*Store)(v) - err = engine.MVCCPutProto(ctx, s.engine, nil, keys.StoreGossipKey(), hlc.Timestamp{}, nil, bi) + err = storage.MVCCPutProto(ctx, s.engine, nil, keys.StoreGossipKey(), hlc.Timestamp{}, nil, bi) return err == nil }) return err @@ -275,7 +275,7 @@ func (ls *Stores) updateBootstrapInfoLocked(bi *gossip.BootstrapInfo) error { // ReadVersionFromEngineOrZero reads the persisted cluster version from the // engine, falling back to the zero value. func ReadVersionFromEngineOrZero( - ctx context.Context, e engine.Engine, + ctx context.Context, e storage.Engine, ) (clusterversion.ClusterVersion, error) { var cv clusterversion.ClusterVersion cv, err := ReadClusterVersion(ctx, e) @@ -288,7 +288,7 @@ func ReadVersionFromEngineOrZero( // WriteClusterVersionToEngines writes the given version to the given engines, // without any sanity checks. func WriteClusterVersionToEngines( - ctx context.Context, engines []engine.Engine, cv clusterversion.ClusterVersion, + ctx context.Context, engines []storage.Engine, cv clusterversion.ClusterVersion, ) error { for _, eng := range engines { if err := WriteClusterVersion(ctx, eng, cv); err != nil { @@ -310,7 +310,7 @@ func WriteClusterVersionToEngines( // any engine has a higher version. func SynthesizeClusterVersionFromEngines( ctx context.Context, - engines []engine.Engine, + engines []storage.Engine, binaryVersion, binaryMinSupportedVersion roachpb.Version, ) (clusterversion.ClusterVersion, error) { // Find the most recent bootstrap info. @@ -405,7 +405,7 @@ func SynthesizeClusterVersionFromEngines( func (ls *Stores) SynthesizeClusterVersion( ctx context.Context, ) (clusterversion.ClusterVersion, error) { - var engines []engine.Engine + var engines []storage.Engine ls.storeMap.Range(func(_ int64, v unsafe.Pointer) bool { engines = append(engines, (*Store)(v).engine) return true // want more @@ -432,8 +432,8 @@ func (ls *Stores) WriteClusterVersion(ctx context.Context, cv clusterversion.Clu return WriteClusterVersionToEngines(ctx, engines, cv) } -func (ls *Stores) engines() []engine.Engine { - var engines []engine.Engine +func (ls *Stores) engines() []storage.Engine { + var engines []storage.Engine ls.storeMap.Range(func(_ int64, v unsafe.Pointer) bool { engines = append(engines, (*Store)(v).Engine()) return true // want more @@ -454,7 +454,7 @@ func (ls *Stores) OnClusterVersionChange( // We're going to read the cluster version from any engine - all the engines // are always kept in sync so it doesn't matter which one we read from. - var someEngine engine.Engine + var someEngine storage.Engine ls.storeMap.Range(func(_ int64, v unsafe.Pointer) bool { someEngine = (*Store)(v).engine return false // don't iterate any more diff --git a/pkg/kv/kvserver/stores_server.go b/pkg/kv/kvserver/stores_server.go index e49249954608..d01807135ef8 100644 --- a/pkg/kv/kvserver/stores_server.go +++ b/pkg/kv/kvserver/stores_server.go @@ -15,8 +15,8 @@ import ( "context" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/retry" ) @@ -110,7 +110,7 @@ func (is Server) WaitForApplication( // everything up to this point to disk. // // https://github.com/cockroachdb/cockroach/issues/33120 - return engine.WriteSyncNoop(ctx, s.engine) + return storage.WriteSyncNoop(ctx, s.engine) } } if ctx.Err() == nil { diff --git a/pkg/kv/kvserver/stores_test.go b/pkg/kv/kvserver/stores_test.go index b1d9639f1f9c..08c066da72c9 100644 --- a/pkg/kv/kvserver/stores_test.go +++ b/pkg/kv/kvserver/stores_test.go @@ -18,9 +18,9 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -123,7 +123,7 @@ func TestStoresGetReplicaForRangeID(t *testing.T) { rangeID := roachpb.RangeID(i) replicaID := roachpb.ReplicaID(1) - memEngine := engine.NewDefaultInMem() + memEngine := storage.NewDefaultInMem() stopper.AddCloser(memEngine) cfg := TestStoreConfig(clock) @@ -218,7 +218,7 @@ func createStores(count int, t *testing.T) (*hlc.ManualClock, []*Store, *Stores, stores := []*Store{} for i := 0; i < count; i++ { cfg.Transport = NewDummyRaftTransport(cfg.Settings) - eng := engine.NewDefaultInMem() + eng := storage.NewDefaultInMem() stopper.AddCloser(eng) s := NewStore(context.TODO(), cfg, eng, &roachpb.NodeDescriptor{NodeID: 1}) storeIDAlloc++ diff --git a/pkg/kv/kvserver/syncing_write.go b/pkg/kv/kvserver/syncing_write.go index 5040a9ff7fe1..aac3fce14e90 100644 --- a/pkg/kv/kvserver/syncing_write.go +++ b/pkg/kv/kvserver/syncing_write.go @@ -17,9 +17,9 @@ import ( "strings" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "golang.org/x/time/rate" @@ -72,7 +72,7 @@ func writeFileSyncing( ctx context.Context, filename string, data []byte, - eng engine.Engine, + eng storage.Engine, perm os.FileMode, settings *cluster.Settings, limiter *rate.Limiter, diff --git a/pkg/kv/kvserver/track_raft_protos.go b/pkg/kv/kvserver/track_raft_protos.go index d14c6947c199..c4af188293ac 100644 --- a/pkg/kv/kvserver/track_raft_protos.go +++ b/pkg/kv/kvserver/track_raft_protos.go @@ -16,12 +16,12 @@ import ( "runtime" "strings" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/compactor" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/syncutil" ) diff --git a/pkg/kv/kvserver/ts_maintenance_queue.go b/pkg/kv/kvserver/ts_maintenance_queue.go index 6a976fbd71f6..02d9b192aba5 100644 --- a/pkg/kv/kvserver/ts_maintenance_queue.go +++ b/pkg/kv/kvserver/ts_maintenance_queue.go @@ -15,10 +15,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/mon" @@ -42,7 +42,7 @@ type TimeSeriesDataStore interface { ContainsTimeSeries(roachpb.RKey, roachpb.RKey) bool MaintainTimeSeries( context.Context, - engine.Reader, + storage.Reader, roachpb.RKey, roachpb.RKey, *client.DB, diff --git a/pkg/kv/kvserver/ts_maintenance_queue_test.go b/pkg/kv/kvserver/ts_maintenance_queue_test.go index 3d2b186d2b1c..008f5c1d9b72 100644 --- a/pkg/kv/kvserver/ts_maintenance_queue_test.go +++ b/pkg/kv/kvserver/ts_maintenance_queue_test.go @@ -20,12 +20,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/ts" @@ -63,7 +63,7 @@ func (m *modelTimeSeriesDataStore) ContainsTimeSeries(start, end roachpb.RKey) b func (m *modelTimeSeriesDataStore) MaintainTimeSeries( ctx context.Context, - snapshot engine.Reader, + snapshot storage.Reader, start, end roachpb.RKey, db *client.DB, _ *mon.BytesMonitor, diff --git a/pkg/kv/kvserver/txn_wait_queue_test.go b/pkg/kv/kvserver/txn_wait_queue_test.go index bb340c42edcb..b682bb3b675f 100644 --- a/pkg/kv/kvserver/txn_wait_queue_test.go +++ b/pkg/kv/kvserver/txn_wait_queue_test.go @@ -20,12 +20,12 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -38,7 +38,7 @@ import ( func writeTxnRecord(ctx context.Context, tc *testContext, txn *roachpb.Transaction) error { key := keys.TransactionKey(txn.Key, txn.ID) - return engine.MVCCPutProto(ctx, tc.store.Engine(), nil, key, hlc.Timestamp{}, nil, txn) + return storage.MVCCPutProto(ctx, tc.store.Engine(), nil, key, hlc.Timestamp{}, nil, txn) } // createTxnForPushQueue creates a txn struct and writes a "fake" @@ -347,7 +347,7 @@ func TestTxnWaitQueueTxnSilentlyCompletes(t *testing.T) { // value mismatches, whether transient or permanent, skip this test if the // teeing engine is being used. See // https://github.com/cockroachdb/cockroach/issues/42656 for more context. - if engine.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { + if storage.DefaultStorageEngine == enginepb.EngineTypeTeePebbleRocksDB { t.Skip("disabled on teeing engine") } tc := testContext{} diff --git a/pkg/kv/kvserver/txnwait/queue.go b/pkg/kv/kvserver/txnwait/queue.go index 17a6268e033a..8e997ea50c99 100644 --- a/pkg/kv/kvserver/txnwait/queue.go +++ b/pkg/kv/kvserver/txnwait/queue.go @@ -17,10 +17,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/kvserver/txnwait/queue_test.go b/pkg/kv/kvserver/txnwait/queue_test.go index c3425ce3118c..f8a85ed4b582 100644 --- a/pkg/kv/kvserver/txnwait/queue_test.go +++ b/pkg/kv/kvserver/txnwait/queue_test.go @@ -17,9 +17,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/split_test.go b/pkg/kv/split_test.go index 73b807554542..f029a83c02fc 100644 --- a/pkg/kv/split_test.go +++ b/pkg/kv/split_test.go @@ -20,11 +20,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -115,7 +115,7 @@ func TestRangeSplitMeta(t *testing.T) { } testutils.SucceedsSoon(t, func() error { - if _, err := engine.MVCCScan(ctx, s.Eng, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, engine.MVCCScanOptions{}); err != nil { + if _, err := storage.MVCCScan(ctx, s.Eng, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, storage.MVCCScanOptions{}); err != nil { return errors.Errorf("failed to verify no dangling intents: %s", err) } return nil @@ -225,7 +225,7 @@ func TestRangeSplitsWithWritePressure(t *testing.T) { // for timing of finishing the test writer and a possibly-ongoing // asynchronous split. testutils.SucceedsSoon(t, func() error { - if _, err := engine.MVCCScan(ctx, s.Eng, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, engine.MVCCScanOptions{}); err != nil { + if _, err := storage.MVCCScan(ctx, s.Eng, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, storage.MVCCScanOptions{}); err != nil { return errors.Errorf("failed to verify no dangling intents: %s", err) } return nil diff --git a/pkg/kv/truncate_test.go b/pkg/kv/truncate_test.go index 8040d3b99b05..09790d5f38a5 100644 --- a/pkg/kv/truncate_test.go +++ b/pkg/kv/truncate_test.go @@ -15,9 +15,9 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/uuid" diff --git a/pkg/kv/txn_coord_sender.go b/pkg/kv/txn_coord_sender.go index cb89af4bcf1a..f8a846c55b86 100644 --- a/pkg/kv/txn_coord_sender.go +++ b/pkg/kv/txn_coord_sender.go @@ -17,10 +17,10 @@ import ( "runtime/debug" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/duration" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/kv/txn_coord_sender_savepoints.go b/pkg/kv/txn_coord_sender_savepoints.go index 85ebdeb3dba0..3c803a12b7fb 100644 --- a/pkg/kv/txn_coord_sender_savepoints.go +++ b/pkg/kv/txn_coord_sender_savepoints.go @@ -13,9 +13,9 @@ package kv import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" diff --git a/pkg/kv/txn_coord_sender_test.go b/pkg/kv/txn_coord_sender_test.go index 7827e0a799c8..49cd92589e71 100644 --- a/pkg/kv/txn_coord_sender_test.go +++ b/pkg/kv/txn_coord_sender_test.go @@ -20,12 +20,12 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -410,7 +410,7 @@ func getTxn(ctx context.Context, txn *client.Txn) (*roachpb.Transaction, *roachp return &br.Responses[0].GetInner().(*roachpb.QueryTxnResponse).QueriedTxn, nil } -func verifyCleanup(key roachpb.Key, eng engine.Engine, t *testing.T, coords ...*TxnCoordSender) { +func verifyCleanup(key roachpb.Key, eng storage.Engine, t *testing.T, coords ...*TxnCoordSender) { testutils.SucceedsSoon(t, func() error { for _, coord := range coords { if coord.IsTracking() { @@ -419,7 +419,7 @@ func verifyCleanup(key roachpb.Key, eng engine.Engine, t *testing.T, coords ...* } meta := &enginepb.MVCCMetadata{} //lint:ignore SA1019 historical usage of deprecated eng.GetProto is OK - ok, _, _, err := eng.GetProto(engine.MakeMVCCMetadataKey(key), meta) + ok, _, _, err := eng.GetProto(storage.MakeMVCCMetadataKey(key), meta) if err != nil { return fmt.Errorf("error getting MVCC metadata: %s", err) } diff --git a/pkg/kv/txn_correctness_test.go b/pkg/kv/txn_correctness_test.go index 9ae22cc16dee..07cad68c92a8 100644 --- a/pkg/kv/txn_correctness_test.go +++ b/pkg/kv/txn_correctness_test.go @@ -23,10 +23,10 @@ import ( "sync" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/kv/txn_interceptor_pipeliner.go b/pkg/kv/txn_interceptor_pipeliner.go index 9fa08768df7a..2ffc551f5a3a 100644 --- a/pkg/kv/txn_interceptor_pipeliner.go +++ b/pkg/kv/txn_interceptor_pipeliner.go @@ -15,11 +15,11 @@ import ( "fmt" "sort" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/google/btree" ) diff --git a/pkg/kv/txn_interceptor_pipeliner_test.go b/pkg/kv/txn_interceptor_pipeliner_test.go index f6f455b20383..eb03cad0d786 100644 --- a/pkg/kv/txn_interceptor_pipeliner_test.go +++ b/pkg/kv/txn_interceptor_pipeliner_test.go @@ -17,9 +17,9 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" diff --git a/pkg/kv/txn_interceptor_seq_num_allocator.go b/pkg/kv/txn_interceptor_seq_num_allocator.go index 3d529a3d9051..283b50620582 100644 --- a/pkg/kv/txn_interceptor_seq_num_allocator.go +++ b/pkg/kv/txn_interceptor_seq_num_allocator.go @@ -13,9 +13,9 @@ package kv import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/errors" ) diff --git a/pkg/kv/txn_interceptor_seq_num_allocator_test.go b/pkg/kv/txn_interceptor_seq_num_allocator_test.go index d56725fdc439..07e84a73b4ce 100644 --- a/pkg/kv/txn_interceptor_seq_num_allocator_test.go +++ b/pkg/kv/txn_interceptor_seq_num_allocator_test.go @@ -15,8 +15,8 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" ) diff --git a/pkg/kv/txn_test.go b/pkg/kv/txn_test.go index feade4b199d7..cf12d1398720 100644 --- a/pkg/kv/txn_test.go +++ b/pkg/kv/txn_test.go @@ -17,12 +17,12 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/tscache" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" diff --git a/pkg/roachpb/api.pb.go b/pkg/roachpb/api.pb.go index 9e658e5bcfbe..0eb7a5eaf35d 100644 --- a/pkg/roachpb/api.pb.go +++ b/pkg/roachpb/api.pb.go @@ -11,7 +11,7 @@ import ( io "io" math "math" - github_com_cockroachdb_cockroach_pkg_storage_engine_enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + github_com_cockroachdb_cockroach_pkg_storage_engine_enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" hlc "github.com/cockroachdb/cockroach/pkg/util/hlc" tracing "github.com/cockroachdb/cockroach/pkg/util/tracing" github_com_cockroachdb_cockroach_pkg_util_uuid "github.com/cockroachdb/cockroach/pkg/util/uuid" diff --git a/pkg/roachpb/batch.go b/pkg/roachpb/batch.go index 211e1855aa6a..ef9e49d657cf 100644 --- a/pkg/roachpb/batch.go +++ b/pkg/roachpb/batch.go @@ -16,7 +16,7 @@ import ( "fmt" "strings" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/errors" ) diff --git a/pkg/roachpb/batch_test.go b/pkg/roachpb/batch_test.go index 4a76b85d2a9a..d05f052bc8b4 100644 --- a/pkg/roachpb/batch_test.go +++ b/pkg/roachpb/batch_test.go @@ -14,7 +14,7 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/kr/pretty" "github.com/stretchr/testify/require" diff --git a/pkg/roachpb/data.go b/pkg/roachpb/data.go index 020abd1e6c08..5bbe73ab6aac 100644 --- a/pkg/roachpb/data.go +++ b/pkg/roachpb/data.go @@ -27,8 +27,8 @@ import ( "time" "github.com/cockroachdb/apd" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/bitarray" "github.com/cockroachdb/cockroach/pkg/util/duration" diff --git a/pkg/roachpb/data.pb.go b/pkg/roachpb/data.pb.go index 9226c4444084..0ee0864ccee8 100644 --- a/pkg/roachpb/data.pb.go +++ b/pkg/roachpb/data.pb.go @@ -9,8 +9,8 @@ import ( io "io" math "math" - github_com_cockroachdb_cockroach_pkg_storage_engine_enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" lock "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" + github_com_cockroachdb_cockroach_pkg_storage_engine_enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" hlc "github.com/cockroachdb/cockroach/pkg/util/hlc" github_com_cockroachdb_cockroach_pkg_util_uuid "github.com/cockroachdb/cockroach/pkg/util/uuid" proto "github.com/gogo/protobuf/proto" diff --git a/pkg/roachpb/data_test.go b/pkg/roachpb/data_test.go index 66e74c3831c9..52bc510bd160 100644 --- a/pkg/roachpb/data_test.go +++ b/pkg/roachpb/data_test.go @@ -21,8 +21,8 @@ import ( "time" "github.com/cockroachdb/apd" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils/zerofields" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/bitarray" diff --git a/pkg/roachpb/ignored_seqnums.go b/pkg/roachpb/ignored_seqnums.go index dd5386707166..f28df651eeea 100644 --- a/pkg/roachpb/ignored_seqnums.go +++ b/pkg/roachpb/ignored_seqnums.go @@ -10,7 +10,7 @@ package roachpb -import "github.com/cockroachdb/cockroach/pkg/engine/enginepb" +import "github.com/cockroachdb/cockroach/pkg/storage/enginepb" // AddIgnoredSeqNumRange adds the given range to the given list of // ignored seqnum ranges. diff --git a/pkg/roachpb/ignored_seqnums_test.go b/pkg/roachpb/ignored_seqnums_test.go index c0b86e4e56ac..90c389f87bac 100644 --- a/pkg/roachpb/ignored_seqnums_test.go +++ b/pkg/roachpb/ignored_seqnums_test.go @@ -13,7 +13,7 @@ package roachpb import ( "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/stretchr/testify/require" ) diff --git a/pkg/roachpb/string_test.go b/pkg/roachpb/string_test.go index 29dd0babbb99..6bbaab66c769 100644 --- a/pkg/roachpb/string_test.go +++ b/pkg/roachpb/string_test.go @@ -14,10 +14,10 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" // Hook up the pretty printer. _ "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) diff --git a/pkg/server/config.go b/pkg/server/config.go index f4860f986e3c..96d5c0d05997 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -22,13 +22,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip/resolver" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server/status" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/ts" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -356,7 +356,7 @@ func MakeConfig(ctx context.Context, st *cluster.Settings) Config { Stores: base.StoreSpecList{ Specs: []base.StoreSpec{storeSpec}, }, - StorageEngine: engine.DefaultStorageEngine, + StorageEngine: storage.DefaultStorageEngine, TempStorageConfig: base.TempStorageConfigFromEnv( ctx, st, storeSpec, "" /* parentDir */, base.DefaultTempStorageMaxSizeBytes, 0), } @@ -401,7 +401,7 @@ func (cfg *Config) Report(ctx context.Context) { } // Engines is a container of engines, allowing convenient closing. -type Engines []engine.Engine +type Engines []storage.Engine // Close closes all the Engines. // This method has a pointer receiver so that the following pattern works: @@ -431,7 +431,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { var details []string - var cache engine.RocksDBCache + var cache storage.RocksDBCache var pebbleCache *pebble.Cache if cfg.StorageEngine == enginepb.EngineTypePebble || cfg.StorageEngine == enginepb.EngineTypeTeePebbleRocksDB { details = append(details, fmt.Sprintf("Pebble cache size: %s", humanizeutil.IBytes(cfg.CacheSize))) @@ -441,7 +441,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { if cfg.StorageEngine == enginepb.EngineTypeDefault || cfg.StorageEngine == enginepb.EngineTypeRocksDB || cfg.StorageEngine == enginepb.EngineTypeTeePebbleRocksDB { details = append(details, fmt.Sprintf("RocksDB cache size: %s", humanizeutil.IBytes(cfg.CacheSize))) - cache = engine.NewRocksDBCache(cfg.CacheSize) + cache = storage.NewRocksDBCache(cfg.CacheSize) defer cache.Release() } @@ -486,7 +486,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { } engines = append(engines, e) } else { - engines = append(engines, engine.NewInMem(ctx, cfg.StorageEngine, spec.Attributes, sizeInBytes)) + engines = append(engines, storage.NewInMem(ctx, cfg.StorageEngine, spec.Attributes, sizeInBytes)) } } else { if spec.Size.Percent > 0 { @@ -504,7 +504,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { details = append(details, fmt.Sprintf("store %d: RocksDB, max size %s, max open file limit %d", i, humanizeutil.IBytes(sizeInBytes), openFileLimitPerStore)) - var eng engine.Engine + var eng storage.Engine var err error storageConfig := base.StorageConfig{ Attrs: spec.Attributes, @@ -517,37 +517,37 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { if cfg.StorageEngine == enginepb.EngineTypePebble { // TODO(itsbilal): Tune these options, and allow them to be overridden // in the spec (similar to the existing spec.RocksDBOptions and others). - pebbleConfig := engine.PebbleConfig{ + pebbleConfig := storage.PebbleConfig{ StorageConfig: storageConfig, - Opts: engine.DefaultPebbleOptions(), + Opts: storage.DefaultPebbleOptions(), } pebbleConfig.Opts.Cache = pebbleCache pebbleConfig.Opts.MaxOpenFiles = int(openFileLimitPerStore) - eng, err = engine.NewPebble(ctx, pebbleConfig) + eng, err = storage.NewPebble(ctx, pebbleConfig) } else if cfg.StorageEngine == enginepb.EngineTypeRocksDB || cfg.StorageEngine == enginepb.EngineTypeDefault { - rocksDBConfig := engine.RocksDBConfig{ + rocksDBConfig := storage.RocksDBConfig{ StorageConfig: storageConfig, MaxOpenFiles: openFileLimitPerStore, WarnLargeBatchThreshold: 500 * time.Millisecond, RocksDBOptions: spec.RocksDBOptions, } - eng, err = engine.NewRocksDB(rocksDBConfig, cache) + eng, err = storage.NewRocksDB(rocksDBConfig, cache) } else { // cfg.StorageEngine == enginepb.EngineTypeTeePebbleRocksDB - pebbleConfig := engine.PebbleConfig{ + pebbleConfig := storage.PebbleConfig{ StorageConfig: storageConfig, - Opts: engine.DefaultPebbleOptions(), + Opts: storage.DefaultPebbleOptions(), } pebbleConfig.Dir = filepath.Join(pebbleConfig.Dir, "pebble") pebbleConfig.Opts.Cache = pebbleCache pebbleConfig.Opts.MaxOpenFiles = int(openFileLimitPerStore) - pebbleEng, err := engine.NewPebble(ctx, pebbleConfig) + pebbleEng, err := storage.NewPebble(ctx, pebbleConfig) if err != nil { return nil, err } - rocksDBConfig := engine.RocksDBConfig{ + rocksDBConfig := storage.RocksDBConfig{ StorageConfig: storageConfig, MaxOpenFiles: openFileLimitPerStore, WarnLargeBatchThreshold: 500 * time.Millisecond, @@ -555,12 +555,12 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { } rocksDBConfig.Dir = filepath.Join(rocksDBConfig.Dir, "rocksdb") - rocksdbEng, err := engine.NewRocksDB(rocksDBConfig, cache) + rocksdbEng, err := storage.NewRocksDB(rocksDBConfig, cache) if err != nil { return nil, err } - eng = engine.NewTee(ctx, rocksdbEng, pebbleEng) + eng = storage.NewTee(ctx, rocksdbEng, pebbleEng) } if err != nil { return Engines{}, err diff --git a/pkg/server/config_unix.go b/pkg/server/config_unix.go index 22a84c72b150..5323c0df2d4d 100644 --- a/pkg/server/config_unix.go +++ b/pkg/server/config_unix.go @@ -16,7 +16,7 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" ) @@ -29,15 +29,15 @@ type rlimit struct { } func setOpenFileLimitInner(physicalStoreCount int) (uint64, error) { - minimumOpenFileLimit := uint64(physicalStoreCount*engine.MinimumMaxOpenFiles + minimumNetworkFileDescriptors) - networkConstrainedFileLimit := uint64(physicalStoreCount*engine.RecommendedMaxOpenFiles + minimumNetworkFileDescriptors) - recommendedOpenFileLimit := uint64(physicalStoreCount*engine.RecommendedMaxOpenFiles + recommendedNetworkFileDescriptors) + minimumOpenFileLimit := uint64(physicalStoreCount*storage.MinimumMaxOpenFiles + minimumNetworkFileDescriptors) + networkConstrainedFileLimit := uint64(physicalStoreCount*storage.RecommendedMaxOpenFiles + minimumNetworkFileDescriptors) + recommendedOpenFileLimit := uint64(physicalStoreCount*storage.RecommendedMaxOpenFiles + recommendedNetworkFileDescriptors) var rLimit rlimit if err := getRlimitNoFile(&rLimit); err != nil { if log.V(1) { - log.Infof(context.TODO(), "could not get rlimit; setting maxOpenFiles to the recommended value %d - %s", engine.RecommendedMaxOpenFiles, err) + log.Infof(context.TODO(), "could not get rlimit; setting maxOpenFiles to the recommended value %d - %s", storage.RecommendedMaxOpenFiles, err) } - return engine.RecommendedMaxOpenFiles, nil + return storage.RecommendedMaxOpenFiles, nil } // The max open file descriptor limit is too low. @@ -131,7 +131,7 @@ func setOpenFileLimitInner(physicalStoreCount int) (uint64, error) { // for each store, than only constrain the network ones by giving the stores // their full recommended number. if rLimit.Cur >= networkConstrainedFileLimit { - return engine.RecommendedMaxOpenFiles, nil + return storage.RecommendedMaxOpenFiles, nil } // Always sacrifice all but the minimum needed network descriptors to be diff --git a/pkg/server/config_windows.go b/pkg/server/config_windows.go index d6dec09e1128..83434684f18a 100644 --- a/pkg/server/config_windows.go +++ b/pkg/server/config_windows.go @@ -10,8 +10,8 @@ package server -import "github.com/cockroachdb/cockroach/pkg/engine" +import "github.com/cockroachdb/cockroach/pkg/storage" func setOpenFileLimitInner(physicalStoreCount int) (uint64, error) { - return engine.RecommendedMaxOpenFiles, nil + return storage.RecommendedMaxOpenFiles, nil } diff --git a/pkg/server/node.go b/pkg/server/node.go index 91716a5c7846..cfd1a5ac3698 100644 --- a/pkg/server/node.go +++ b/pkg/server/node.go @@ -22,7 +22,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -34,6 +33,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/growstack" "github.com/cockroachdb/cockroach/pkg/util/grpcutil" @@ -203,7 +203,7 @@ func GetBootstrapSchema( // engines are initialized with their StoreIdent. func bootstrapCluster( ctx context.Context, - engines []engine.Engine, + engines []storage.Engine, bootstrapVersion clusterversion.ClusterVersion, defaultZoneConfig *zonepb.ZoneConfig, defaultSystemZoneConfig *zonepb.ZoneConfig, @@ -306,7 +306,7 @@ func (n *Node) AnnotateCtxWithSpan( func (n *Node) bootstrapCluster( ctx context.Context, - engines []engine.Engine, + engines []storage.Engine, bootstrapVersion clusterversion.ClusterVersion, defaultZoneConfig *zonepb.ZoneConfig, defaultSystemZoneConfig *zonepb.ZoneConfig, @@ -339,7 +339,7 @@ func (n *Node) onClusterVersionChange(ctx context.Context, cv clusterversion.Clu func (n *Node) start( ctx context.Context, addr, sqlAddr net.Addr, - initializedEngines, emptyEngines []engine.Engine, + initializedEngines, emptyEngines []storage.Engine, clusterName string, attrs roachpb.Attributes, locality roachpb.Locality, @@ -507,7 +507,7 @@ func (n *Node) start( // bumped immediately, which would be possible if gossip got started earlier). n.startGossip(ctx, n.stopper) - allEngines := append([]engine.Engine(nil), initializedEngines...) + allEngines := append([]storage.Engine(nil), initializedEngines...) allEngines = append(allEngines, emptyEngines...) log.Infof(ctx, "%s: started with %v engine(s) and attributes %v", n, allEngines, attrs.Attrs) return nil @@ -573,7 +573,7 @@ func (n *Node) validateStores(ctx context.Context) error { // allocated via a sequence id generator stored at a system key per // node. The new stores are added to n.stores. func (n *Node) bootstrapStores( - ctx context.Context, emptyEngines []engine.Engine, stopper *stop.Stopper, + ctx context.Context, emptyEngines []storage.Engine, stopper *stop.Stopper, ) error { if n.clusterID.Get() == uuid.Nil { return errors.New("ClusterID missing during store bootstrap of auxiliary store") diff --git a/pkg/server/node_engine_health.go b/pkg/server/node_engine_health.go index a06843ad5efd..f8c7d2baccfc 100644 --- a/pkg/server/node_engine_health.go +++ b/pkg/server/node_engine_health.go @@ -15,7 +15,7 @@ import ( "fmt" "time" - "github.com/cockroachdb/cockroach/pkg/engine" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -34,7 +34,7 @@ var maxSyncDurationFatalOnExceeded = envutil.EnvOrDefaultBool("COCKROACH_ENGINE_ // startAssertEngineHealth starts a goroutine that periodically verifies that // syncing the engines is possible within maxSyncDuration. If not, // the process is terminated (with an attempt at a descriptive message). -func (n *Node) startAssertEngineHealth(ctx context.Context, engines []engine.Engine) { +func (n *Node) startAssertEngineHealth(ctx context.Context, engines []storage.Engine) { n.stopper.RunWorker(ctx, func(ctx context.Context) { t := timeutil.NewTimer() t.Reset(0) @@ -58,7 +58,7 @@ func guaranteedExitFatal(ctx context.Context, msg string, args ...interface{}) { } func (n *Node) assertEngineHealth( - ctx context.Context, engines []engine.Engine, maxDuration time.Duration, + ctx context.Context, engines []storage.Engine, maxDuration time.Duration, ) { for _, eng := range engines { func() { @@ -75,7 +75,7 @@ func (n *Node) assertEngineHealth( ) }) defer t.Stop() - if err := engine.WriteSyncNoop(ctx, eng); err != nil { + if err := storage.WriteSyncNoop(ctx, eng); err != nil { log.Fatal(ctx, err) } }() diff --git a/pkg/server/node_test.go b/pkg/server/node_test.go index 45b0814cdb07..514cf279e5d2 100644 --- a/pkg/server/node_test.go +++ b/pkg/server/node_test.go @@ -24,7 +24,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/gossip/resolver" "github.com/cockroachdb/cockroach/pkg/internal/client" @@ -36,6 +35,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/server/status" "github.com/cockroachdb/cockroach/pkg/server/status/statuspb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util" @@ -55,7 +55,7 @@ import ( // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode( - addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T, + addr net.Addr, engines []storage.Engine, gossipBS net.Addr, t *testing.T, ) (*grpc.Server, net.Addr, kvserver.StoreConfig, *Node, *stop.Stopper) { cfg := kvserver.TestStoreConfig(nil /* clock */) st := cfg.Settings @@ -154,7 +154,7 @@ func createTestNode( func createAndStartTestNode( ctx context.Context, addr net.Addr, - engines []engine.Engine, + engines []storage.Engine, gossipBS net.Addr, locality roachpb.Locality, t *testing.T, @@ -206,16 +206,16 @@ func (s keySlice) Less(i, j int) bool { return bytes.Compare(s[i], s[j]) < 0 } func TestBootstrapCluster(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() if _, err := bootstrapCluster( - ctx, []engine.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), + ctx, []storage.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), ); err != nil { t.Fatal(err) } // Scan the complete contents of the local database directly from the engine. - res, err := engine.MVCCScan(ctx, e, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, engine.MVCCScanOptions{}) + res, err := storage.MVCCScan(ctx, e, keys.LocalMax, roachpb.KeyMax, hlc.MaxTimestamp, storage.MVCCScanOptions{}) if err != nil { t.Fatal(err) } @@ -262,18 +262,18 @@ func TestBootstrapCluster(t *testing.T) { func TestBootstrapNewStore(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() if _, err := bootstrapCluster( - ctx, []engine.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), + ctx, []storage.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), ); err != nil { t.Fatal(err) } // Start a new node with two new stores which will require bootstrapping. - engines := Engines([]engine.Engine{ + engines := Engines([]storage.Engine{ e, - engine.NewDefaultInMem(), - engine.NewDefaultInMem(), + storage.NewDefaultInMem(), + storage.NewDefaultInMem(), }) defer engines.Close() _, _, node, stopper := createAndStartTestNode( @@ -315,17 +315,17 @@ func TestNodeJoin(t *testing.T) { ctx := context.Background() engineStopper := stop.NewStopper() defer engineStopper.Stop(ctx) - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() engineStopper.AddCloser(e) if _, err := bootstrapCluster( - ctx, []engine.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), + ctx, []storage.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), ); err != nil { t.Fatal(err) } // Start the bootstrap node. - engines1 := []engine.Engine{e} + engines1 := []storage.Engine{e} _, server1Addr, node1, stopper1 := createAndStartTestNode( ctx, util.TestAddr, @@ -337,9 +337,9 @@ func TestNodeJoin(t *testing.T) { defer stopper1.Stop(ctx) // Create a new node. - e2 := engine.NewDefaultInMem() + e2 := storage.NewDefaultInMem() engineStopper.AddCloser(e2) - engines2 := []engine.Engine{e2} + engines2 := []storage.Engine{e2} _, server2Addr, node2, stopper2 := createAndStartTestNode( ctx, util.TestAddr, @@ -386,11 +386,11 @@ func TestCorruptedClusterID(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() if _, err := bootstrapCluster( - ctx, []engine.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), + ctx, []storage.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), ); err != nil { t.Fatal(err) } @@ -401,13 +401,13 @@ func TestCorruptedClusterID(t *testing.T) { NodeID: 1, StoreID: 1, } - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( ctx, e, nil /* ms */, keys.StoreIdentKey(), hlc.Timestamp{}, nil /* txn */, &sIdent, ); err != nil { t.Fatal(err) } - engines := []engine.Engine{e} + engines := []storage.Engine{e} _, serverAddr, _, node, stopper := createTestNode(util.TestAddr, engines, nil, t) defer stopper.Stop(ctx) bootstrappedEngines, newEngines, cv, err := inspectEngines( @@ -733,17 +733,17 @@ func TestStartNodeWithLocality(t *testing.T) { ctx := context.Background() testLocalityWithNewNode := func(locality roachpb.Locality) { - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() if _, err := bootstrapCluster( - ctx, []engine.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), + ctx, []storage.Engine{e}, clusterversion.TestingClusterVersion, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef(), ); err != nil { t.Fatal(err) } _, _, node, stopper := createAndStartTestNode( ctx, util.TestAddr, - []engine.Engine{e}, + []storage.Engine{e}, util.TestAddr, locality, t, diff --git a/pkg/server/server.go b/pkg/server/server.go index 24a88397eed8..f4586a846e51 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -33,8 +33,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/blobs" "github.com/cockroachdb/cockroach/pkg/blobs/blobspb" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -73,6 +71,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" "github.com/cockroachdb/cockroach/pkg/sql/stats" "github.com/cockroachdb/cockroach/pkg/sqlmigrations" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/ts" "github.com/cockroachdb/cockroach/pkg/ui" "github.com/cockroachdb/cockroach/pkg/util" @@ -418,7 +418,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { // Set up the DistSQL temp engine. useStoreSpec := cfg.Stores.Specs[s.cfg.TempStorageConfig.SpecIdx] - tempEngine, tempFS, err := engine.NewTempEngine(ctx, s.cfg.StorageEngine, s.cfg.TempStorageConfig, useStoreSpec) + tempEngine, tempFS, err := storage.NewTempEngine(ctx, s.cfg.StorageEngine, s.cfg.TempStorageConfig, useStoreSpec) if err != nil { return nil, errors.Wrap(err, "could not create temp storage") } @@ -437,7 +437,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { // also remove the record after the temp directory is // removed. recordPath := filepath.Join(firstStore.Path, TempDirsRecordFilename) - err = engine.CleanupTempDirs(recordPath) + err = storage.CleanupTempDirs(recordPath) } if err != nil { log.Errorf(context.TODO(), "could not remove temporary store directory: %v", err.Error()) @@ -927,12 +927,12 @@ type ListenError struct { // or to set it if no engines have a version in them already. func inspectEngines( ctx context.Context, - engines []engine.Engine, + engines []storage.Engine, binaryVersion, binaryMinSupportedVersion roachpb.Version, clusterIDContainer *base.ClusterIDContainer, ) ( - bootstrappedEngines []engine.Engine, - emptyEngines []engine.Engine, + bootstrappedEngines []storage.Engine, + emptyEngines []storage.Engine, _ clusterversion.ClusterVersion, _ error, ) { diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index e933ba5e3e06..696d43d4ffb3 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -30,7 +30,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/build" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -38,6 +37,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/server/status/statuspb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/ui" @@ -667,9 +667,9 @@ func TestListenerFileCreation(t *testing.T) { func TestClusterIDMismatch(t *testing.T) { defer leaktest.AfterTest(t)() - engines := make([]engine.Engine, 2) + engines := make([]storage.Engine, 2) for i := range engines { - e := engine.NewDefaultInMem() + e := storage.NewDefaultInMem() defer e.Close() sIdent := roachpb.StoreIdent{ @@ -677,7 +677,7 @@ func TestClusterIDMismatch(t *testing.T) { NodeID: roachpb.NodeID(i), StoreID: roachpb.StoreID(i), } - if err := engine.MVCCPutProto( + if err := storage.MVCCPutProto( context.Background(), e, nil, keys.StoreIdentKey(), hlc.Timestamp{}, nil, &sIdent); err != nil { t.Fatal(err) diff --git a/pkg/server/serverpb/admin.pb.go b/pkg/server/serverpb/admin.pb.go index 1f2aec5ab37d..bd6cef66a749 100644 --- a/pkg/server/serverpb/admin.pb.go +++ b/pkg/server/serverpb/admin.pb.go @@ -12,11 +12,11 @@ import ( time "time" zonepb "github.com/cockroachdb/cockroach/pkg/config/zonepb" - enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" jobspb "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" storagepb "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" github_com_cockroachdb_cockroach_pkg_roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" github_com_cockroachdb_cockroach_pkg_sql_sqlbase "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" catalog "github.com/cockroachdb/cockroach/pkg/ts/catalog" metric "github.com/cockroachdb/cockroach/pkg/util/metric" proto "github.com/gogo/protobuf/proto" diff --git a/pkg/server/serverpb/status.pb.go b/pkg/server/serverpb/status.pb.go index 2b6525cce5e0..850c415459c7 100644 --- a/pkg/server/serverpb/status.pb.go +++ b/pkg/server/serverpb/status.pb.go @@ -12,7 +12,6 @@ import ( time "time" build "github.com/cockroachdb/cockroach/pkg/build" - enginepb "github.com/cockroachdb/cockroach/pkg/engine/enginepb" gossip "github.com/cockroachdb/cockroach/pkg/gossip" jobspb "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" storagepb "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" @@ -20,6 +19,7 @@ import ( roachpb "github.com/cockroachdb/cockroach/pkg/roachpb" diagnosticspb "github.com/cockroachdb/cockroach/pkg/server/diagnosticspb" statuspb "github.com/cockroachdb/cockroach/pkg/server/status/statuspb" + enginepb "github.com/cockroachdb/cockroach/pkg/storage/enginepb" util "github.com/cockroachdb/cockroach/pkg/util" log "github.com/cockroachdb/cockroach/pkg/util/log" github_com_cockroachdb_cockroach_pkg_util_uuid "github.com/cockroachdb/cockroach/pkg/util/uuid" diff --git a/pkg/server/status.go b/pkg/server/status.go index 5830aefbacf8..24ad2091b32a 100644 --- a/pkg/server/status.go +++ b/pkg/server/status.go @@ -34,7 +34,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/build" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -50,6 +49,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/httputil" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -284,7 +284,7 @@ func (s *statusServer) EngineStats( } switch e := store.Engine().(type) { - case *engine.RocksDB: + case *storage.RocksDB: tickersAndHistograms, err := e.GetTickersAndHistograms() if err != nil { return grpcstatus.Errorf(codes.Internal, err.Error()) diff --git a/pkg/server/status_test.go b/pkg/server/status_test.go index 08f9282f09b5..cde180d17d10 100644 --- a/pkg/server/status_test.go +++ b/pkg/server/status_test.go @@ -28,7 +28,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/build" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -40,6 +39,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/server/status/statuspb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" diff --git a/pkg/server/sticky_engine.go b/pkg/server/sticky_engine.go index 66c2c31916e6..f39818d46ac4 100644 --- a/pkg/server/sticky_engine.go +++ b/pkg/server/sticky_engine.go @@ -13,9 +13,9 @@ package server import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/pkg/errors" @@ -35,11 +35,11 @@ type stickyInMemEngine struct { closed bool // Engine extends the Engine interface. - engine.Engine + storage.Engine } // stickyInMemEngine implements Engine. -var _ engine.Engine = &stickyInMemEngine{} +var _ storage.Engine = &stickyInMemEngine{} // Close overwrites the default Engine interface to not close the underlying // engine if called. We mark the state as closed to reflect a correct result @@ -77,7 +77,7 @@ func getOrCreateStickyInMemEngine( engineType enginepb.EngineType, attrs roachpb.Attributes, cacheSize int64, -) (engine.Engine, error) { +) (storage.Engine, error) { stickyInMemEnginesRegistry.mu.Lock() defer stickyInMemEnginesRegistry.mu.Unlock() @@ -95,7 +95,7 @@ func getOrCreateStickyInMemEngine( engine := &stickyInMemEngine{ id: id, closed: false, - Engine: engine.NewInMem(ctx, engineType, attrs, cacheSize), + Engine: storage.NewInMem(ctx, engineType, attrs, cacheSize), } stickyInMemEnginesRegistry.entries[id] = engine return engine, nil diff --git a/pkg/server/sticky_engine_test.go b/pkg/server/sticky_engine_test.go index 3ce12523458f..30c133f3c390 100644 --- a/pkg/server/sticky_engine_test.go +++ b/pkg/server/sticky_engine_test.go @@ -14,9 +14,9 @@ import ( "context" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" ) @@ -69,7 +69,7 @@ func TestStickyEngines(t *testing.T) { // Cleaning up everything asserts everything is closed. CloseAllStickyInMemEngines() require.Len(t, stickyInMemEnginesRegistry.entries, 0) - for _, engine := range []engine.Engine{engine1, newEngine1, engine2} { + for _, engine := range []storage.Engine{engine1, newEngine1, engine2} { require.True(t, engine.Closed()) require.True(t, engine.(*stickyInMemEngine).Engine.Closed()) } diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index 5ca23adeebe4..5f5bfcb58ac0 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -42,6 +41,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sqlmigrations" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/ts" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -67,7 +67,7 @@ func makeTestConfig(st *cluster.Settings) Config { cfg.Insecure = false // Configure test storage engine. - cfg.StorageEngine = engine.DefaultStorageEngine + cfg.StorageEngine = storage.DefaultStorageEngine // Configure the default in-memory temp storage for all tests unless // otherwise configured. @@ -434,7 +434,7 @@ func (ts *TestServer) ClusterSettings() *cluster.Settings { } // Engines returns the TestServer's engines. -func (ts *TestServer) Engines() []engine.Engine { +func (ts *TestServer) Engines() []storage.Engine { return ts.engines } diff --git a/pkg/sql/colcontainer/diskqueue.go b/pkg/sql/colcontainer/diskqueue.go index d5f83fb5e894..3a277824ed6b 100644 --- a/pkg/sql/colcontainer/diskqueue.go +++ b/pkg/sql/colcontainer/diskqueue.go @@ -19,7 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/col/coldata" "github.com/cockroachdb/cockroach/pkg/col/colserde" "github.com/cockroachdb/cockroach/pkg/col/coltypes" - "github.com/cockroachdb/cockroach/pkg/engine/fs" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/golang/snappy" diff --git a/pkg/sql/colcontainer/partitionedqueue_test.go b/pkg/sql/colcontainer/partitionedqueue_test.go index 62a5efcf2dc7..c9512825f3a5 100644 --- a/pkg/sql/colcontainer/partitionedqueue_test.go +++ b/pkg/sql/colcontainer/partitionedqueue_test.go @@ -17,9 +17,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/col/coldata" "github.com/cockroachdb/cockroach/pkg/col/coltypes" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/sql/colcontainer" "github.com/cockroachdb/cockroach/pkg/sql/colexec" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils/colcontainerutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/randutil" diff --git a/pkg/sql/colflow/vectorized_flow_test.go b/pkg/sql/colflow/vectorized_flow_test.go index 2968ad04d323..12445a813607 100644 --- a/pkg/sql/colflow/vectorized_flow_test.go +++ b/pkg/sql/colflow/vectorized_flow_test.go @@ -16,7 +16,6 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/col/coltypes" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/colcontainer" @@ -27,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/flowinfra" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/stretchr/testify/require" ) @@ -237,7 +237,7 @@ func TestVectorizedFlowTempDirectory(t *testing.T) { const baseDirName = "base" - ngn := engine.NewDefaultInMem() + ngn := storage.NewDefaultInMem() defer ngn.Close() require.NoError(t, ngn.CreateDir(baseDirName)) diff --git a/pkg/sql/distsql/columnar_utils_test.go b/pkg/sql/distsql/columnar_utils_test.go index 044b7e5eb57a..d5ee271ade42 100644 --- a/pkg/sql/distsql/columnar_utils_test.go +++ b/pkg/sql/distsql/columnar_utils_test.go @@ -17,7 +17,6 @@ import ( "strconv" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/colcontainer" "github.com/cockroachdb/cockroach/pkg/sql/colexec" @@ -27,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/pkg/errors" ) @@ -65,7 +65,7 @@ func verifyColOperator(args verifyColOperatorArgs) error { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, tempFS, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, tempFS, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { return err } diff --git a/pkg/sql/execinfra/server_config.go b/pkg/sql/execinfra/server_config.go index 0f570ad78f35..47ca8bd21ff0 100644 --- a/pkg/sql/execinfra/server_config.go +++ b/pkg/sql/execinfra/server_config.go @@ -14,7 +14,6 @@ package execinfra import ( "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -28,6 +27,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/mon" "github.com/cockroachdb/cockroach/pkg/util/stop" diff --git a/pkg/sql/row/fetcher_mvcc_test.go b/pkg/sql/row/fetcher_mvcc_test.go index 299e060a8b58..7106e383aa42 100644 --- a/pkg/sql/row/fetcher_mvcc_test.go +++ b/pkg/sql/row/fetcher_mvcc_test.go @@ -16,13 +16,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/row" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -32,7 +32,7 @@ import ( "github.com/pkg/errors" ) -func slurpUserDataKVs(t testing.TB, e engine.Engine) []roachpb.KeyValue { +func slurpUserDataKVs(t testing.TB, e storage.Engine) []roachpb.KeyValue { t.Helper() // Scan meta keys directly from engine. We put this in a retry loop @@ -41,9 +41,9 @@ func slurpUserDataKVs(t testing.TB, e engine.Engine) []roachpb.KeyValue { var kvs []roachpb.KeyValue testutils.SucceedsSoon(t, func() error { kvs = nil - it := e.NewIterator(engine.IterOptions{UpperBound: roachpb.KeyMax}) + it := e.NewIterator(storage.IterOptions{UpperBound: roachpb.KeyMax}) defer it.Close() - for it.SeekGE(engine.MVCCKey{Key: keys.UserTableDataMin}); ; it.NextKey() { + for it.SeekGE(storage.MVCCKey{Key: keys.UserTableDataMin}); ; it.NextKey() { ok, err := it.Valid() if err != nil { t.Fatal(err) diff --git a/pkg/sql/row/kv_fetcher.go b/pkg/sql/row/kv_fetcher.go index dfc1aae79708..7743a40f0436 100644 --- a/pkg/sql/row/kv_fetcher.go +++ b/pkg/sql/row/kv_fetcher.go @@ -13,10 +13,10 @@ package row import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" ) // KVFetcher wraps kvBatchFetcher, providing a NextKV interface that returns the diff --git a/pkg/sql/rowcontainer/disk_row_container_test.go b/pkg/sql/rowcontainer/disk_row_container_test.go index d01928f3c338..63b9aef7ac30 100644 --- a/pkg/sql/rowcontainer/disk_row_container_test.go +++ b/pkg/sql/rowcontainer/disk_row_container_test.go @@ -18,13 +18,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/mon" @@ -63,7 +63,7 @@ func TestDiskRowContainer(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -245,7 +245,7 @@ func TestDiskRowContainerDiskFull(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -285,7 +285,7 @@ func TestDiskRowContainerFinalIterator(t *testing.T) { st := cluster.MakeTestingClusterSettings() alloc := &sqlbase.DatumAlloc{} evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -413,7 +413,7 @@ func TestDiskRowContainerUnsafeReset(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/sql/rowcontainer/hash_row_container_test.go b/pkg/sql/rowcontainer/hash_row_container_test.go index a0d9a4c480df..f7da568f105f 100644 --- a/pkg/sql/rowcontainer/hash_row_container_test.go +++ b/pkg/sql/rowcontainer/hash_row_container_test.go @@ -17,13 +17,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/mon" @@ -35,7 +35,7 @@ func TestHashDiskBackedRowContainer(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -339,7 +339,7 @@ func TestHashDiskBackedRowContainerPreservesMatchesAndMarks(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/sql/rowcontainer/row_container_test.go b/pkg/sql/rowcontainer/row_container_test.go index b232174bdc56..28224f897471 100644 --- a/pkg/sql/rowcontainer/row_container_test.go +++ b/pkg/sql/rowcontainer/row_container_test.go @@ -19,13 +19,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/mon" @@ -187,7 +187,7 @@ func TestDiskBackedRowContainer(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -379,7 +379,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -822,7 +822,7 @@ func BenchmarkDiskBackedIndexedRowContainer(b *testing.B) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() evalCtx := tree.MakeTestingEvalContext(st) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.TempStorageConfig{InMemory: true}, base.DefaultTestStoreSpec) if err != nil { b.Fatal(err) } diff --git a/pkg/sql/rowexec/hashjoiner_test.go b/pkg/sql/rowexec/hashjoiner_test.go index 8b3103579b9c..7ff00c05d7d5 100644 --- a/pkg/sql/rowexec/hashjoiner_test.go +++ b/pkg/sql/rowexec/hashjoiner_test.go @@ -19,13 +19,13 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/distsqlutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -50,7 +50,7 @@ func TestHashJoiner(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestHashJoinerError(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -514,7 +514,7 @@ func BenchmarkHashJoiner(b *testing.B) { DiskMonitor: diskMonitor, }, } - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { b.Fatal(err) } diff --git a/pkg/sql/rowexec/joinreader_test.go b/pkg/sql/rowexec/joinreader_test.go index 9d8ebafbc08e..623fd97a746d 100644 --- a/pkg/sql/rowexec/joinreader_test.go +++ b/pkg/sql/rowexec/joinreader_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" @@ -29,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils/distsqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -385,7 +385,7 @@ func TestJoinReader(t *testing.T) { }, } st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -498,7 +498,7 @@ CREATE TABLE test.t (a INT, s STRING, INDEX (a, s))`); err != nil { td := sqlbase.GetTableDescriptor(kvDB, "test", "t") st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } @@ -593,7 +593,7 @@ func TestJoinReaderDrain(t *testing.T) { td := sqlbase.GetTableDescriptor(kvDB, "test", "t") st := s.ClusterSettings() - tempEngine, _, err := engine.NewTempEngine(context.Background(), engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(context.Background(), storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/sql/rowexec/sorter_test.go b/pkg/sql/rowexec/sorter_test.go index a09ec28fe56e..5586aceebae6 100644 --- a/pkg/sql/rowexec/sorter_test.go +++ b/pkg/sql/rowexec/sorter_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" @@ -26,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/distsqlutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" @@ -293,7 +293,7 @@ func TestSorter(t *testing.T) { t.Run(name, func(t *testing.T) { ctx := context.Background() st := cluster.MakeTestingClusterSettings() - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/sql/rowexec/windower_test.go b/pkg/sql/rowexec/windower_test.go index 6aa27cd1c36b..be9481fecaf4 100644 --- a/pkg/sql/rowexec/windower_test.go +++ b/pkg/sql/rowexec/windower_test.go @@ -18,12 +18,12 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils/distsqlutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -50,7 +50,7 @@ func TestWindowerAccountingForResults(t *testing.T) { defer evalCtx.Stop(ctx) diskMonitor := execinfra.NewTestDiskMonitor(ctx, st) defer diskMonitor.Stop(ctx) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/sql/rowflow/routers_test.go b/pkg/sql/rowflow/routers_test.go index 78c78b033371..7eabda6af4f3 100644 --- a/pkg/sql/rowflow/routers_test.go +++ b/pkg/sql/rowflow/routers_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" @@ -30,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/distsqlutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" @@ -710,7 +710,7 @@ func TestRouterDiskSpill(t *testing.T) { st := cluster.MakeTestingClusterSettings() diskMonitor := execinfra.NewTestDiskMonitor(ctx, st) defer diskMonitor.Stop(ctx) - tempEngine, _, err := engine.NewTempEngine(ctx, engine.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) + tempEngine, _, err := storage.NewTempEngine(ctx, storage.DefaultStorageEngine, base.DefaultTestTempStorageConfig(st), base.DefaultTestStoreSpec) if err != nil { t.Fatal(err) } diff --git a/pkg/storage/batch.go b/pkg/storage/batch.go index b24dbc5c83c7..9d93c19cb2a7 100644 --- a/pkg/storage/batch.go +++ b/pkg/storage/batch.go @@ -8,12 +8,12 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "encoding/binary" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/pebble" "github.com/pkg/errors" diff --git a/pkg/storage/batch_test.go b/pkg/storage/batch_test.go index 7b5b3d119058..a50bed71f696 100644 --- a/pkg/storage/batch_test.go +++ b/pkg/storage/batch_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -18,8 +18,8 @@ import ( "sync/atomic" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/storage/bench_pebble_test.go b/pkg/storage/bench_pebble_test.go index f677aa226a58..b421cd152a76 100644 --- a/pkg/storage/bench_pebble_test.go +++ b/pkg/storage/bench_pebble_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" diff --git a/pkg/storage/bench_rocksdb_test.go b/pkg/storage/bench_rocksdb_test.go index 03ad40e8c23c..b5c1c8dea6fd 100644 --- a/pkg/storage/bench_rocksdb_test.go +++ b/pkg/storage/bench_rocksdb_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" diff --git a/pkg/storage/bench_test.go b/pkg/storage/bench_test.go index d6b045299e7f..21adea196f32 100644 --- a/pkg/storage/bench_test.go +++ b/pkg/storage/bench_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/fileutil" diff --git a/pkg/storage/disk_map.go b/pkg/storage/disk_map.go index c7f2f8bac335..f05922e4767e 100644 --- a/pkg/storage/disk_map.go +++ b/pkg/storage/disk_map.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/disk_map_test.go b/pkg/storage/disk_map_test.go index 097f528cda1a..fb66349d0154 100644 --- a/pkg/storage/disk_map_test.go +++ b/pkg/storage/disk_map_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/doc.go b/pkg/storage/doc.go index a0726bf3e23a..b1f696f9ced0 100644 --- a/pkg/storage/doc.go +++ b/pkg/storage/doc.go @@ -100,4 +100,4 @@ on a Put operation and are cleared from the engine on a delete. Importantly, zero-timestamped MVCC values may be merged, as is necessary for stats counters and time series data. */ -package engine +package storage diff --git a/pkg/storage/engine.go b/pkg/storage/engine.go index fdbe4d7d43c4..c7e5e19b254b 100644 --- a/pkg/storage/engine.go +++ b/pkg/storage/engine.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" @@ -17,11 +17,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" diff --git a/pkg/storage/engine_test.go b/pkg/storage/engine_test.go index 737a5d360a7d..f90a39304b9a 100644 --- a/pkg/storage/engine_test.go +++ b/pkg/storage/engine_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -27,10 +27,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/storage/enginepb/decode_test.go b/pkg/storage/enginepb/decode_test.go index f1a92aef6ad8..b4437cb35a19 100644 --- a/pkg/storage/enginepb/decode_test.go +++ b/pkg/storage/enginepb/decode_test.go @@ -14,9 +14,9 @@ import ( "encoding/binary" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -25,7 +25,7 @@ func BenchmarkScanDecodeKeyValue(b *testing.B) { ts := hlc.Timestamp{WallTime: int64(1000000)} value := []byte("foo foo foo") rep := make([]byte, 8) - keyBytes := engine.EncodeKey(engine.MVCCKey{Key: key, Timestamp: ts}) + keyBytes := storage.EncodeKey(storage.MVCCKey{Key: key, Timestamp: ts}) binary.LittleEndian.PutUint64(rep, uint64(len(keyBytes)<<32)|uint64(len(value))) rep = append(rep, keyBytes...) rep = append(rep, value...) diff --git a/pkg/storage/enginepb/mvcc_test.go b/pkg/storage/enginepb/mvcc_test.go index aa2816732811..122fbeeb46de 100644 --- a/pkg/storage/enginepb/mvcc_test.go +++ b/pkg/storage/enginepb/mvcc_test.go @@ -14,8 +14,8 @@ import ( "fmt" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/stretchr/testify/assert" diff --git a/pkg/storage/error.go b/pkg/storage/error.go index cfdca3f8d6af..cf408732e88a 100644 --- a/pkg/storage/error.go +++ b/pkg/storage/error.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "regexp" diff --git a/pkg/storage/error_test.go b/pkg/storage/error_test.go index 42bbb5764b8e..601d93e69d7d 100644 --- a/pkg/storage/error_test.go +++ b/pkg/storage/error_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "testing" diff --git a/pkg/storage/file_util.go b/pkg/storage/file_util.go index bd61ddd70538..2e546bf6e25b 100644 --- a/pkg/storage/file_util.go +++ b/pkg/storage/file_util.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/in_mem.go b/pkg/storage/in_mem.go index 7266bc38c14c..dda9617f6709 100644 --- a/pkg/storage/in_mem.go +++ b/pkg/storage/in_mem.go @@ -8,14 +8,14 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" ) // NewInMem allocates and returns a new, opened in-memory engine. The caller diff --git a/pkg/storage/main_test.go b/pkg/storage/main_test.go index 7f6d27fe9d7e..b00796c1801b 100644 --- a/pkg/storage/main_test.go +++ b/pkg/storage/main_test.go @@ -8,6 +8,6 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine_test +package storage_test //go:generate ../../util/leaktest/add-leaktest.sh *_test.go diff --git a/pkg/storage/merge.go b/pkg/storage/merge.go index fc43c9f739da..5ecfc7391e98 100644 --- a/pkg/storage/merge.go +++ b/pkg/storage/merge.go @@ -8,11 +8,11 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" ) diff --git a/pkg/storage/merge_test.go b/pkg/storage/merge_test.go index 18ea8e8c292c..55fc361f4611 100644 --- a/pkg/storage/merge_test.go +++ b/pkg/storage/merge_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "math" @@ -16,8 +16,8 @@ import ( "reflect" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/gogo/protobuf/proto" diff --git a/pkg/storage/metamorphic/generator.go b/pkg/storage/metamorphic/generator.go index f0d60b0fe549..7cfcd24b09e4 100644 --- a/pkg/storage/metamorphic/generator.go +++ b/pkg/storage/metamorphic/generator.go @@ -20,9 +20,9 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/pebble" ) @@ -36,31 +36,31 @@ func makeStorageConfig(path string) base.StorageConfig { } } -func createTestRocksDBEngine(path string) (engine.Engine, error) { - cache := engine.NewRocksDBCache(1 << 20) +func createTestRocksDBEngine(path string) (storage.Engine, error) { + cache := storage.NewRocksDBCache(1 << 20) defer cache.Release() - cfg := engine.RocksDBConfig{ + cfg := storage.RocksDBConfig{ StorageConfig: makeStorageConfig(path), ReadOnly: false, } - return engine.NewRocksDB(cfg, cache) + return storage.NewRocksDB(cfg, cache) } -func createTestPebbleEngine(path string) (engine.Engine, error) { - pebbleConfig := engine.PebbleConfig{ +func createTestPebbleEngine(path string) (storage.Engine, error) { + pebbleConfig := storage.PebbleConfig{ StorageConfig: makeStorageConfig(path), - Opts: engine.DefaultPebbleOptions(), + Opts: storage.DefaultPebbleOptions(), } pebbleConfig.Opts.Cache = pebble.NewCache(1 << 20) defer pebbleConfig.Opts.Cache.Unref() - return engine.NewPebble(context.Background(), pebbleConfig) + return storage.NewPebble(context.Background(), pebbleConfig) } type engineImpl struct { name string - create func(path string) (engine.Engine, error) + create func(path string) (storage.Engine, error) } var _ fmt.Stringer = &engineImpl{} @@ -84,7 +84,7 @@ type metaTestRunner struct { engineImpls []engineImpl curEngine int restarts bool - engine engine.Engine + engine storage.Engine tsGenerator tsGenerator opGenerators map[operandType]operandGenerator txnGenerator *txnGenerator @@ -95,7 +95,7 @@ type metaTestRunner struct { pastTSGenerator *pastTSGenerator nextTSGenerator *nextTSGenerator openIters map[iteratorID]iteratorInfo - openBatches map[readWriterID]engine.ReadWriter + openBatches map[readWriterID]storage.ReadWriter openTxns map[txnID]*roachpb.Transaction nameToGenerator map[string]*opGenerator ops []opRun @@ -127,7 +127,7 @@ func (m *metaTestRunner) init() { m.rwGenerator = &readWriterGenerator{ rng: m.rng, m: m, - batchIDMap: make(map[readWriterID]engine.Batch), + batchIDMap: make(map[readWriterID]storage.Batch), } m.iterGenerator = &iteratorGenerator{ rng: m.rng, @@ -168,7 +168,7 @@ func (m *metaTestRunner) init() { } m.ops = nil m.openIters = make(map[iteratorID]iteratorInfo) - m.openBatches = make(map[readWriterID]engine.ReadWriter) + m.openBatches = make(map[readWriterID]storage.ReadWriter) m.openTxns = make(map[txnID]*roachpb.Transaction) } @@ -199,7 +199,7 @@ func (m *metaTestRunner) closeAll() { } // TODO(itsbilal): Abort all txns. m.openIters = make(map[iteratorID]iteratorInfo) - m.openBatches = make(map[readWriterID]engine.ReadWriter) + m.openBatches = make(map[readWriterID]storage.ReadWriter) m.openTxns = make(map[txnID]*roachpb.Transaction) m.engine.Close() m.engine = nil @@ -218,7 +218,7 @@ func (m *metaTestRunner) setTxn(id txnID, txn *roachpb.Transaction) { m.openTxns[id] = txn } -func (m *metaTestRunner) getReadWriter(id readWriterID) engine.ReadWriter { +func (m *metaTestRunner) getReadWriter(id readWriterID) storage.ReadWriter { if id == "engine" { return m.engine } @@ -230,7 +230,7 @@ func (m *metaTestRunner) getReadWriter(id readWriterID) engine.ReadWriter { return batch } -func (m *metaTestRunner) setReadWriter(id readWriterID, rw engine.ReadWriter) { +func (m *metaTestRunner) setReadWriter(id readWriterID, rw storage.ReadWriter) { if id == "engine" { // no-op return diff --git a/pkg/storage/metamorphic/operands.go b/pkg/storage/metamorphic/operands.go index 4bacd74ad6c1..07fd2c50a393 100644 --- a/pkg/storage/metamorphic/operands.go +++ b/pkg/storage/metamorphic/operands.go @@ -15,8 +15,8 @@ import ( "math/rand" "strconv" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -80,7 +80,7 @@ func generateBytes(rng *rand.Rand, min int, max int) []byte { } type keyGenerator struct { - liveKeys []engine.MVCCKey + liveKeys []storage.MVCCKey rng *rand.Rand tsGenerator *tsGenerator } @@ -96,8 +96,8 @@ func (k *keyGenerator) count() int { return len(k.liveKeys) + 1 } -func (k *keyGenerator) open() engine.MVCCKey { - var key engine.MVCCKey +func (k *keyGenerator) open() storage.MVCCKey { + var key storage.MVCCKey key.Key = generateBytes(k.rng, 8, maxValueSize) key.Timestamp = k.tsGenerator.lastTS k.liveKeys = append(k.liveKeys, key) @@ -105,7 +105,7 @@ func (k *keyGenerator) open() engine.MVCCKey { return key } -func (k *keyGenerator) toString(key engine.MVCCKey) string { +func (k *keyGenerator) toString(key storage.MVCCKey) string { return fmt.Sprintf("%s/%d", key.Key, key.Timestamp.WallTime) } @@ -126,8 +126,8 @@ func (k *keyGenerator) closeAll() { // No-op. } -func (k *keyGenerator) parse(input string) engine.MVCCKey { - var key engine.MVCCKey +func (k *keyGenerator) parse(input string) storage.MVCCKey { + var key storage.MVCCKey key.Key = make([]byte, 0, maxValueSize) _, err := fmt.Sscanf(input, "%q/%d", &key.Key, &key.Timestamp.WallTime) if err != nil { @@ -320,7 +320,7 @@ type readWriterGenerator struct { rng *rand.Rand m *metaTestRunner liveBatches []readWriterID - batchIDMap map[readWriterID]engine.Batch + batchIDMap map[readWriterID]storage.Batch batchGenCounter uint64 } @@ -377,13 +377,13 @@ func (w *readWriterGenerator) closeAll() { } } w.liveBatches = w.liveBatches[:0] - w.batchIDMap = make(map[readWriterID]engine.Batch) + w.batchIDMap = make(map[readWriterID]storage.Batch) } type iteratorID string type iteratorInfo struct { id iteratorID - iter engine.Iterator + iter storage.Iterator lowerBound roachpb.Key isBatchIter bool } diff --git a/pkg/storage/metamorphic/operations.go b/pkg/storage/metamorphic/operations.go index 9bb86d65ecc8..a3711c900d04 100644 --- a/pkg/storage/metamorphic/operations.go +++ b/pkg/storage/metamorphic/operations.go @@ -18,10 +18,10 @@ import ( "path/filepath" "sort" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/uint128" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -134,7 +134,7 @@ func generateMVCCScan( } // Prints the key where an iterator is positioned, or valid = false if invalid. -func printIterState(iter engine.Iterator) string { +func printIterState(iter storage.Iterator) string { if ok, err := iter.Valid(); !ok || err != nil { if err != nil { return fmt.Sprintf("valid = %v, err = %s", ok, err.Error()) @@ -163,7 +163,7 @@ func (m mvccGetOp) run(ctx context.Context) string { // TODO(itsbilal): Specify these bools as operands instead of having a // separate operation for inconsistent cases. This increases visibility for // anyone reading the output file. - val, intent, err := engine.MVCCGet(ctx, reader, m.key, m.ts, engine.MVCCGetOptions{ + val, intent, err := storage.MVCCGet(ctx, reader, m.key, m.ts, storage.MVCCGetOptions{ Inconsistent: m.inconsistent, Tombstones: true, Txn: txn, @@ -187,7 +187,7 @@ func (m mvccPutOp) run(ctx context.Context) string { txn.Sequence++ writer := m.m.getReadWriter(m.writer) - err := engine.MVCCPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, txn) + err := storage.MVCCPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, txn) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -213,7 +213,7 @@ func (m mvccCPutOp) run(ctx context.Context) string { writer := m.m.getReadWriter(m.writer) txn.Sequence++ - err := engine.MVCCConditionalPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, &m.expVal, true, txn) + err := storage.MVCCConditionalPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, &m.expVal, true, txn) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -238,7 +238,7 @@ func (m mvccInitPutOp) run(ctx context.Context) string { writer := m.m.getReadWriter(m.writer) txn.Sequence++ - err := engine.MVCCInitPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, false, txn) + err := storage.MVCCInitPut(ctx, writer, nil, m.key, txn.WriteTimestamp, m.value, false, txn) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -263,7 +263,7 @@ func (m mvccDeleteRangeOp) run(ctx context.Context) string { writer := m.m.getReadWriter(m.writer) txn.Sequence++ - keys, _, _, err := engine.MVCCDeleteRange(ctx, writer, nil, m.key, m.endKey, 0, txn.WriteTimestamp, txn, true) + keys, _, _, err := storage.MVCCDeleteRange(ctx, writer, nil, m.key, m.endKey, 0, txn.WriteTimestamp, txn, true) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -288,7 +288,7 @@ type mvccClearTimeRangeOp struct { func (m mvccClearTimeRangeOp) run(ctx context.Context) string { writer := m.m.getReadWriter(m.writer) - span, err := engine.MVCCClearTimeRange(ctx, writer, nil, m.key, m.endKey, m.startTime, m.endTime, math.MaxInt64) + span, err := storage.MVCCClearTimeRange(ctx, writer, nil, m.key, m.endKey, m.startTime, m.endTime, math.MaxInt64) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -307,7 +307,7 @@ func (m mvccDeleteOp) run(ctx context.Context) string { writer := m.m.getReadWriter(m.writer) txn.Sequence++ - err := engine.MVCCDelete(ctx, writer, nil, m.key, txn.WriteTimestamp, txn) + err := storage.MVCCDelete(ctx, writer, nil, m.key, txn.WriteTimestamp, txn) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -327,7 +327,7 @@ type mvccFindSplitKeyOp struct { } func (m mvccFindSplitKeyOp) run(ctx context.Context) string { - splitKey, err := engine.MVCCFindSplitKey(ctx, m.m.engine, m.key, m.endKey, m.splitSize) + splitKey, err := storage.MVCCFindSplitKey(ctx, m.m.engine, m.key, m.endKey, m.splitSize) if err != nil { return fmt.Sprintf("error: %s", err) } @@ -356,7 +356,7 @@ func (m mvccScanOp) run(ctx context.Context) string { // m.engine instead of a readWriterGenerator-generated engine.Reader, otherwise // we will try MVCCScanning on batches and produce diffs between runs on // different engines that don't point to an actual issue. - result, err := engine.MVCCScan(ctx, m.m.engine, m.key, m.endKey, m.ts, engine.MVCCScanOptions{ + result, err := storage.MVCCScan(ctx, m.m.engine, m.key, m.endKey, m.ts, storage.MVCCScanOptions{ Inconsistent: m.inconsistent, Tombstones: true, Reverse: m.reverse, @@ -407,7 +407,7 @@ func (t txnCommitOp) run(ctx context.Context) string { for _, span := range txn.IntentSpans { intent := roachpb.MakeLockUpdate(txn, span) intent.Status = roachpb.COMMITTED - _, err := engine.MVCCResolveWriteIntent(context.TODO(), t.m.engine, nil, intent) + _, err := storage.MVCCResolveWriteIntent(context.TODO(), t.m.engine, nil, intent) if err != nil { panic(err) } @@ -453,7 +453,7 @@ type iterOpenOp struct { func (i iterOpenOp) run(ctx context.Context) string { rw := i.m.getReadWriter(i.rw) - iter := rw.NewIterator(engine.IterOptions{ + iter := rw.NewIterator(storage.IterOptions{ Prefix: false, LowerBound: i.key, UpperBound: i.endKey.Next(), @@ -466,12 +466,12 @@ func (i iterOpenOp) run(ctx context.Context) string { isBatchIter: i.rw != "engine", }) - if _, ok := rw.(engine.Batch); ok { + if _, ok := rw.(storage.Batch); ok { // When Next()-ing on a newly initialized batch iter without a key, // pebble's iterator stays invalid while RocksDB's finds the key after // the first key. This is a known difference. For now seek the iterator // to standardize behavior for this test. - iter.SeekGE(engine.MakeMVCCMetadataKey(i.key)) + iter.SeekGE(storage.MakeMVCCMetadataKey(i.key)) } return string(i.id) @@ -492,7 +492,7 @@ func (i iterCloseOp) run(ctx context.Context) string { type iterSeekOp struct { m *metaTestRunner iter iteratorID - key engine.MVCCKey + key storage.MVCCKey seekLT bool } @@ -578,7 +578,7 @@ type clearRangeOp struct { func (c clearRangeOp) run(ctx context.Context) string { // All ClearRange calls in Cockroach usually happen with metadata keys, so // mimic the same behavior here. - err := c.m.engine.ClearRange(engine.MakeMVCCMetadataKey(c.key), engine.MakeMVCCMetadataKey(c.endKey)) + err := c.m.engine.ClearRange(storage.MakeMVCCMetadataKey(c.key), storage.MakeMVCCMetadataKey(c.endKey)) if err != nil { return fmt.Sprintf("error: %s", err.Error()) } @@ -601,7 +601,7 @@ func (c compactOp) run(ctx context.Context) string { type ingestOp struct { m *metaTestRunner - keys []engine.MVCCKey + keys []storage.MVCCKey } func (i ingestOp) run(ctx context.Context) string { @@ -612,7 +612,7 @@ func (i ingestOp) run(ctx context.Context) string { } defer f.Close() - sstWriter := engine.MakeIngestionSSTWriter(f) + sstWriter := storage.MakeIngestionSSTWriter(f) for _, key := range i.keys { _ = sstWriter.Put(key, []byte("ingested")) } @@ -1185,7 +1185,7 @@ var opGenerators = []opGenerator{ { name: "ingest", generate: func(ctx context.Context, m *metaTestRunner, args ...string) mvccOp { - var keys []engine.MVCCKey + var keys []storage.MVCCKey for _, arg := range args { key := m.keyGenerator.parse(arg) // Don't put anything at the 0 timestamp; the MVCC code expects diff --git a/pkg/storage/multi_iterator.go b/pkg/storage/multi_iterator.go index f4c9d97a051a..9bdfbcc76b1f 100644 --- a/pkg/storage/multi_iterator.go +++ b/pkg/storage/multi_iterator.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/multi_iterator_test.go b/pkg/storage/multi_iterator_test.go index ec067dd43e27..1c0ee6a1442c 100644 --- a/pkg/storage/multi_iterator_test.go +++ b/pkg/storage/multi_iterator_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/mvcc.go b/pkg/storage/mvcc.go index 27c13e5c58ed..f93f5ffb7304 100644 --- a/pkg/storage/mvcc.go +++ b/pkg/storage/mvcc.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -21,9 +21,9 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" diff --git a/pkg/storage/mvcc_history_test.go b/pkg/storage/mvcc_history_test.go index f9cc2ed6e6cd..75b14016d29e 100644 --- a/pkg/storage/mvcc_history_test.go +++ b/pkg/storage/mvcc_history_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -19,8 +19,8 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" diff --git a/pkg/storage/mvcc_incremental_iterator.go b/pkg/storage/mvcc_incremental_iterator.go index a8f6267307c2..4e757cd594b6 100644 --- a/pkg/storage/mvcc_incremental_iterator.go +++ b/pkg/storage/mvcc_incremental_iterator.go @@ -8,11 +8,11 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/pkg/errors" diff --git a/pkg/storage/mvcc_incremental_iterator_test.go b/pkg/storage/mvcc_incremental_iterator_test.go index 0786133194e9..5e9a073a9814 100644 --- a/pkg/storage/mvcc_incremental_iterator_test.go +++ b/pkg/storage/mvcc_incremental_iterator_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -16,8 +16,8 @@ import ( "math" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/storage/mvcc_logical_ops.go b/pkg/storage/mvcc_logical_ops.go index b0e60f4cb7f9..b74afcce384e 100644 --- a/pkg/storage/mvcc_logical_ops.go +++ b/pkg/storage/mvcc_logical_ops.go @@ -8,14 +8,14 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "fmt" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/bufalloc" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) diff --git a/pkg/storage/mvcc_logical_ops_test.go b/pkg/storage/mvcc_logical_ops_test.go index aaa7269a2476..842c1f628706 100644 --- a/pkg/storage/mvcc_logical_ops_test.go +++ b/pkg/storage/mvcc_logical_ops_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" @@ -16,9 +16,9 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/kr/pretty" diff --git a/pkg/storage/mvcc_stats_test.go b/pkg/storage/mvcc_stats_test.go index 51a78a6136a0..4e9383e183b4 100644 --- a/pkg/storage/mvcc_stats_test.go +++ b/pkg/storage/mvcc_stats_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" @@ -17,9 +17,9 @@ import ( "sort" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" diff --git a/pkg/storage/mvcc_test.go b/pkg/storage/mvcc_test.go index 16edf6a09947..dc5246d05554 100644 --- a/pkg/storage/mvcc_test.go +++ b/pkg/storage/mvcc_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -23,9 +23,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/zerofields" "github.com/cockroachdb/cockroach/pkg/util/caller" diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index 34e0cfc79158..71020eda1584 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -21,10 +21,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" diff --git a/pkg/storage/pebble_batch.go b/pkg/storage/pebble_batch.go index b197d066b223..c3030451c029 100644 --- a/pkg/storage/pebble_batch.go +++ b/pkg/storage/pebble_batch.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "sync" diff --git a/pkg/storage/pebble_file_registry.go b/pkg/storage/pebble_file_registry.go index dcf4b71e7841..9dae775b0217 100644 --- a/pkg/storage/pebble_file_registry.go +++ b/pkg/storage/pebble_file_registry.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "fmt" @@ -16,7 +16,7 @@ import ( "os" "strings" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/pebble/vfs" diff --git a/pkg/storage/pebble_file_registry_test.go b/pkg/storage/pebble_file_registry_test.go index e48a22a6b2e3..fc688bbb5390 100644 --- a/pkg/storage/pebble_file_registry_test.go +++ b/pkg/storage/pebble_file_registry_test.go @@ -8,14 +8,14 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "runtime/debug" "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/pebble/vfs" "github.com/kr/pretty" diff --git a/pkg/storage/pebble_iterator.go b/pkg/storage/pebble_iterator.go index 232bbd545823..77a7d527a5cd 100644 --- a/pkg/storage/pebble_iterator.go +++ b/pkg/storage/pebble_iterator.go @@ -8,16 +8,16 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" "math" "sync" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/pebble" diff --git a/pkg/storage/pebble_merge.go b/pkg/storage/pebble_merge.go index 7912958fbf46..cde14ecc19e5 100644 --- a/pkg/storage/pebble_merge.go +++ b/pkg/storage/pebble_merge.go @@ -8,13 +8,13 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "sort" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/errors" "github.com/gogo/protobuf/proto" diff --git a/pkg/storage/pebble_mvcc_scanner.go b/pkg/storage/pebble_mvcc_scanner.go index b2c131fab2fc..1f2627efce96 100644 --- a/pkg/storage/pebble_mvcc_scanner.go +++ b/pkg/storage/pebble_mvcc_scanner.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -16,9 +16,9 @@ import ( "sort" "sync" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/errors" diff --git a/pkg/storage/pebble_test.go b/pkg/storage/pebble_test.go index 9ea0096fb962..2f72370a9dca 100644 --- a/pkg/storage/pebble_test.go +++ b/pkg/storage/pebble_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -20,7 +20,7 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/protoutil" diff --git a/pkg/storage/rocksdb.go b/pkg/storage/rocksdb.go index a6ad030692fe..3fceb1efce8f 100644 --- a/pkg/storage/rocksdb.go +++ b/pkg/storage/rocksdb.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -27,10 +27,10 @@ import ( "unsafe" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/hlc" diff --git a/pkg/storage/rocksdb_32bit.go b/pkg/storage/rocksdb_32bit.go index 261247367f22..360f8944147c 100644 --- a/pkg/storage/rocksdb_32bit.go +++ b/pkg/storage/rocksdb_32bit.go @@ -10,7 +10,7 @@ // +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc -package engine +package storage const ( // MaxArrayLen is a safe maximum length for slices on this architecture. diff --git a/pkg/storage/rocksdb_64bit.go b/pkg/storage/rocksdb_64bit.go index 7bdb55c3270f..381640f699cc 100644 --- a/pkg/storage/rocksdb_64bit.go +++ b/pkg/storage/rocksdb_64bit.go @@ -10,7 +10,7 @@ // +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 -package engine +package storage const ( // MaxArrayLen is a safe maximum length for slices on this architecture. diff --git a/pkg/storage/rocksdb_error_dict.go b/pkg/storage/rocksdb_error_dict.go index 90bb8dcbd53c..fdb87c2c6040 100644 --- a/pkg/storage/rocksdb_error_dict.go +++ b/pkg/storage/rocksdb_error_dict.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage /* Generated manually via: diff --git a/pkg/storage/rocksdb_iter_stats_test.go b/pkg/storage/rocksdb_iter_stats_test.go index cd606bdeb40c..bdc49b59c33e 100644 --- a/pkg/storage/rocksdb_iter_stats_test.go +++ b/pkg/storage/rocksdb_iter_stats_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" diff --git a/pkg/storage/rocksdb_jemalloc.go b/pkg/storage/rocksdb_jemalloc.go index d4225996cda0..f4819604c287 100644 --- a/pkg/storage/rocksdb_jemalloc.go +++ b/pkg/storage/rocksdb_jemalloc.go @@ -10,7 +10,7 @@ // +build !stdmalloc -package engine +package storage // #cgo LDFLAGS: -ljemalloc // #cgo dragonfly freebsd LDFLAGS: -lm diff --git a/pkg/storage/rocksdb_test.go b/pkg/storage/rocksdb_test.go index 640794f5a1e3..35e9d7e7900b 100644 --- a/pkg/storage/rocksdb_test.go +++ b/pkg/storage/rocksdb_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/row_counter.go b/pkg/storage/row_counter.go index 848bb99edba8..475ee2375652 100644 --- a/pkg/storage/row_counter.go +++ b/pkg/storage/row_counter.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/slice.go b/pkg/storage/slice.go index 51c25f7adef3..cd99da4a94ca 100644 --- a/pkg/storage/slice.go +++ b/pkg/storage/slice.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import "unsafe" diff --git a/pkg/storage/slice_gccgo.go b/pkg/storage/slice_gccgo.go index 9ea0ac1063d9..95b17bb5f965 100644 --- a/pkg/storage/slice_gccgo.go +++ b/pkg/storage/slice_gccgo.go @@ -10,7 +10,7 @@ // +build gccgo -package engine +package storage import "unsafe" diff --git a/pkg/storage/slice_go1.9.go b/pkg/storage/slice_go1.9.go index 1c06bdf83a09..574e3ad9bd1a 100644 --- a/pkg/storage/slice_go1.9.go +++ b/pkg/storage/slice_go1.9.go @@ -10,7 +10,7 @@ // +build gc,go1.9 -package engine +package storage import "unsafe" diff --git a/pkg/storage/slice_test.go b/pkg/storage/slice_test.go index c53cc2a93307..6026c41fa721 100644 --- a/pkg/storage/slice_test.go +++ b/pkg/storage/slice_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "testing" diff --git a/pkg/storage/sst_iterator.go b/pkg/storage/sst_iterator.go index 888d7ee13806..17ce610d67c5 100644 --- a/pkg/storage/sst_iterator.go +++ b/pkg/storage/sst_iterator.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/sst_iterator_test.go b/pkg/storage/sst_iterator_test.go index 5d343d207c67..76dd32be0028 100644 --- a/pkg/storage/sst_iterator_test.go +++ b/pkg/storage/sst_iterator_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "io/ioutil" diff --git a/pkg/storage/sst_writer.go b/pkg/storage/sst_writer.go index d2cd205ac16d..17cd1dee17bf 100644 --- a/pkg/storage/sst_writer.go +++ b/pkg/storage/sst_writer.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/sst_writer_test.go b/pkg/storage/sst_writer_test.go index 6ddc490bb6bf..ef3a24367973 100644 --- a/pkg/storage/sst_writer_test.go +++ b/pkg/storage/sst_writer_test.go @@ -8,24 +8,24 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine_test +package storage_test import ( "math/rand" "testing" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/randutil" "github.com/stretchr/testify/require" ) -func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []engine.MVCCKeyValue { +func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []storage.MVCCKeyValue { prefix := encoding.EncodeUvarintAscending(keys.MakeTablePrefix(uint32(100)), uint64(1)) - kvs := make([]engine.MVCCKeyValue, numKeys) + kvs := make([]storage.MVCCKeyValue, numKeys) r, _ := randutil.NewPseudoRand() var k int @@ -49,8 +49,8 @@ func makeIntTableKVs(numKeys, valueSize, maxRevisions int) []engine.MVCCKeyValue return kvs } -func makeRocksSST(t testing.TB, kvs []engine.MVCCKeyValue) []byte { - w, err := engine.MakeRocksDBSstFileWriter() +func makeRocksSST(t testing.TB, kvs []storage.MVCCKeyValue) []byte { + w, err := storage.MakeRocksDBSstFileWriter() require.NoError(t, err) defer w.Close() @@ -64,13 +64,13 @@ func makeRocksSST(t testing.TB, kvs []engine.MVCCKeyValue) []byte { return sst } -func makePebbleSST(t testing.TB, kvs []engine.MVCCKeyValue, ingestion bool) []byte { - f := &engine.MemFile{} - var w engine.SSTWriter +func makePebbleSST(t testing.TB, kvs []storage.MVCCKeyValue, ingestion bool) []byte { + f := &storage.MemFile{} + var w storage.SSTWriter if ingestion { - w = engine.MakeIngestionSSTWriter(f) + w = storage.MakeIngestionSSTWriter(f) } else { - w = engine.MakeBackupSSTWriter(f) + w = storage.MakeBackupSSTWriter(f) } defer w.Close() @@ -111,13 +111,13 @@ func TestPebbleWritesSameSSTs(t *testing.T) { sstRocks := makeRocksSST(t, kvs) sstPebble := makePebbleSST(t, kvs, false /* ingestion */) - itRocks, err := engine.NewMemSSTIterator(sstRocks, false) + itRocks, err := storage.NewMemSSTIterator(sstRocks, false) require.NoError(t, err) - itPebble, err := engine.NewMemSSTIterator(sstPebble, false) + itPebble, err := storage.NewMemSSTIterator(sstPebble, false) require.NoError(t, err) - itPebble.SeekGE(engine.NilKey) - for itRocks.SeekGE(engine.NilKey); ; { + itPebble.SeekGE(storage.NilKey) + for itRocks.SeekGE(storage.NilKey); ; { okRocks, err := itRocks.Valid() if err != nil { t.Fatal(err) diff --git a/pkg/storage/tee.go b/pkg/storage/tee.go index 082b31517250..618cf5da165e 100644 --- a/pkg/storage/tee.go +++ b/pkg/storage/tee.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" @@ -19,9 +19,9 @@ import ( "path/filepath" "strings" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" diff --git a/pkg/storage/temp_dir.go b/pkg/storage/temp_dir.go index 7af75d7b29cc..0f865a26df83 100644 --- a/pkg/storage/temp_dir.go +++ b/pkg/storage/temp_dir.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bufio" diff --git a/pkg/storage/temp_dir_test.go b/pkg/storage/temp_dir_test.go index d092c785c779..b0123969764d 100644 --- a/pkg/storage/temp_dir_test.go +++ b/pkg/storage/temp_dir_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "bytes" diff --git a/pkg/storage/temp_engine.go b/pkg/storage/temp_engine.go index 5fda0e37defe..66bbae614e3f 100644 --- a/pkg/storage/temp_engine.go +++ b/pkg/storage/temp_engine.go @@ -8,17 +8,17 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "context" "fmt" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/diskmap" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/pebble" "github.com/cockroachdb/pebble/vfs" diff --git a/pkg/storage/temp_engine_test.go b/pkg/storage/temp_engine_test.go index eecd73d3d40c..d97bbe373931 100644 --- a/pkg/storage/temp_engine_test.go +++ b/pkg/storage/temp_engine_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "testing" diff --git a/pkg/storage/version.go b/pkg/storage/version.go index a2a24d743c19..7ca15b43e010 100644 --- a/pkg/storage/version.go +++ b/pkg/storage/version.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "encoding/json" diff --git a/pkg/storage/version_test.go b/pkg/storage/version_test.go index 1b3262737b78..ab6e65f1b5f4 100644 --- a/pkg/storage/version_test.go +++ b/pkg/storage/version_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package engine +package storage import ( "io/ioutil" diff --git a/pkg/testutils/colcontainerutils/diskqueuecfg.go b/pkg/testutils/colcontainerutils/diskqueuecfg.go index 4f803ce641e3..6be2db9a4197 100644 --- a/pkg/testutils/colcontainerutils/diskqueuecfg.go +++ b/pkg/testutils/colcontainerutils/diskqueuecfg.go @@ -14,9 +14,9 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/engine" - "github.com/cockroachdb/cockroach/pkg/engine/fs" "github.com/cockroachdb/cockroach/pkg/sql/colcontainer" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" ) @@ -34,7 +34,7 @@ func NewTestingDiskQueueCfg(t testing.TB, inMem bool) (colcontainer.DiskQueueCfg ) if inMem { - ngn := engine.NewDefaultInMem() + ngn := storage.NewDefaultInMem() testingFS = ngn.(fs.FS) if err := testingFS.CreateDir(inMemDirName); err != nil { t.Fatal(err) @@ -44,7 +44,7 @@ func NewTestingDiskQueueCfg(t testing.TB, inMem bool) (colcontainer.DiskQueueCfg } else { tempPath, dirCleanup := testutils.TempDir(t) path = tempPath - ngn, err := engine.NewDefaultEngine(0 /* cacheSize */, base.StorageConfig{Dir: tempPath}) + ngn, err := storage.NewDefaultEngine(0 /* cacheSize */, base.StorageConfig{Dir: tempPath}) if err != nil { t.Fatal(err) } diff --git a/pkg/testutils/localtestcluster/local_test_cluster.go b/pkg/testutils/localtestcluster/local_test_cluster.go index f357375d676a..13da93a297fb 100644 --- a/pkg/testutils/localtestcluster/local_test_cluster.go +++ b/pkg/testutils/localtestcluster/local_test_cluster.go @@ -20,7 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" @@ -29,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -53,7 +53,7 @@ type LocalTestCluster struct { Manual *hlc.ManualClock Clock *hlc.Clock Gossip *gossip.Gossip - Eng engine.Engine + Eng storage.Engine Store *kvserver.Store StoreTestingKnobs *kvserver.StoreTestingKnobs DBContext *client.DBContext @@ -118,8 +118,8 @@ func (ltc *LocalTestCluster) Start(t testing.TB, baseCtx *base.Config, initFacto c := &cfg.RPCContext.ClusterID server := rpc.NewServer(cfg.RPCContext) // never started ltc.Gossip = gossip.New(ambient, c, nc, cfg.RPCContext, server, ltc.Stopper, metric.NewRegistry(), roachpb.Locality{}, zonepb.DefaultZoneConfigRef()) - ltc.Eng = engine.NewInMem(ambient.AnnotateCtx(context.Background()), - engine.DefaultStorageEngine, roachpb.Attributes{}, 50<<20) + ltc.Eng = storage.NewInMem(ambient.AnnotateCtx(context.Background()), + storage.DefaultStorageEngine, roachpb.Attributes{}, 50<<20) ltc.Stopper.AddCloser(ltc.Eng) ltc.Stores = kvserver.NewStores(ambient, ltc.Clock, clusterversion.TestingBinaryVersion, clusterversion.TestingBinaryMinSupportedVersion) @@ -150,7 +150,7 @@ func (ltc *LocalTestCluster) Start(t testing.TB, baseCtx *base.Config, initFacto cfg.AmbientCtx, cfg.Clock, cfg.DB, - []engine.Engine{ltc.Eng}, + []storage.Engine{ltc.Eng}, cfg.Gossip, active, renewal, diff --git a/pkg/ts/db_test.go b/pkg/ts/db_test.go index 1987f94ca2a8..fe3c9c9d0ced 100644 --- a/pkg/ts/db_test.go +++ b/pkg/ts/db_test.go @@ -21,11 +21,11 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster" "github.com/cockroachdb/cockroach/pkg/ts/testmodel" @@ -122,7 +122,7 @@ func (tm *testModelRunner) getActualData() map[string]roachpb.Value { // Scan over all TS Keys stored in the engine startKey := keys.TimeseriesPrefix endKey := startKey.PrefixEnd() - res, err := engine.MVCCScan(context.Background(), tm.Eng, startKey, endKey, tm.Clock.Now(), engine.MVCCScanOptions{}) + res, err := storage.MVCCScan(context.Background(), tm.Eng, startKey, endKey, tm.Clock.Now(), storage.MVCCScanOptions{}) if err != nil { tm.t.Fatalf("error scanning TS data from engine: %s", err) } diff --git a/pkg/ts/maintenance.go b/pkg/ts/maintenance.go index 10bb77293d7c..42a9622c4923 100644 --- a/pkg/ts/maintenance.go +++ b/pkg/ts/maintenance.go @@ -13,10 +13,10 @@ package ts import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/mon" ) @@ -47,7 +47,7 @@ func (tsdb *DB) ContainsTimeSeries(start, end roachpb.RKey) bool { // snapshot to quickly obtain a set of keys to be pruned with no network calls. func (tsdb *DB) MaintainTimeSeries( ctx context.Context, - snapshot engine.Reader, + snapshot storage.Reader, start, end roachpb.RKey, db *client.DB, mem *mon.BytesMonitor, diff --git a/pkg/ts/pruning.go b/pkg/ts/pruning.go index 9926d21304e0..35b8cbc78b89 100644 --- a/pkg/ts/pruning.go +++ b/pkg/ts/pruning.go @@ -13,10 +13,10 @@ package ts import ( "context" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -40,29 +40,29 @@ type timeSeriesResolutionInfo struct { // intended to be called by a storage queue which can inspect the local data for // a single range without the need for expensive network calls. func (tsdb *DB) findTimeSeries( - snapshot engine.Reader, startKey, endKey roachpb.RKey, now hlc.Timestamp, + snapshot storage.Reader, startKey, endKey roachpb.RKey, now hlc.Timestamp, ) ([]timeSeriesResolutionInfo, error) { var results []timeSeriesResolutionInfo // Set start boundary for the search, which is the lesser of the range start // key and the beginning of time series data. - start := engine.MakeMVCCMetadataKey(startKey.AsRawKey()) - next := engine.MakeMVCCMetadataKey(keys.TimeseriesPrefix) + start := storage.MakeMVCCMetadataKey(startKey.AsRawKey()) + next := storage.MakeMVCCMetadataKey(keys.TimeseriesPrefix) if next.Less(start) { next = start } // Set end boundary for the search, which is the lesser of the range end key // and the end of time series data. - end := engine.MakeMVCCMetadataKey(endKey.AsRawKey()) - lastTS := engine.MakeMVCCMetadataKey(keys.TimeseriesPrefix.PrefixEnd()) + end := storage.MakeMVCCMetadataKey(endKey.AsRawKey()) + lastTS := storage.MakeMVCCMetadataKey(keys.TimeseriesPrefix.PrefixEnd()) if lastTS.Less(end) { end = lastTS } thresholds := tsdb.computeThresholds(now.WallTime) - iter := snapshot.NewIterator(engine.IterOptions{UpperBound: endKey.AsRawKey()}) + iter := snapshot.NewIterator(storage.IterOptions{UpperBound: endKey.AsRawKey()}) defer iter.Close() for iter.SeekGE(next); ; iter.SeekGE(next) { @@ -90,7 +90,7 @@ func (tsdb *DB) findTimeSeries( // Set 'next' is initialized to the next possible time series key // which could belong to a previously undiscovered time series. - next = engine.MakeMVCCMetadataKey(makeDataKeySeriesPrefix(name, res).PrefixEnd()) + next = storage.MakeMVCCMetadataKey(makeDataKeySeriesPrefix(name, res).PrefixEnd()) } return results, nil diff --git a/pkg/ts/timeseries_test.go b/pkg/ts/timeseries_test.go index 26ff5832db5d..59bb5839158a 100644 --- a/pkg/ts/timeseries_test.go +++ b/pkg/ts/timeseries_test.go @@ -15,8 +15,8 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/engine" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/ts/tspb" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -192,7 +192,7 @@ func TestDiscardEarlierSamples(t *testing.T) { t.Fatal(err) } - out, err := engine.MergeInternalTimeSeriesData(true /* mergeIntoNil */, false /* usePartialMerge */, internal...) + out, err := storage.MergeInternalTimeSeriesData(true /* mergeIntoNil */, false /* usePartialMerge */, internal...) if err != nil { t.Fatal(err) } diff --git a/pkg/util/protoutil/clone_test.go b/pkg/util/protoutil/clone_test.go index a1c88e223092..a261f70d7ba6 100644 --- a/pkg/util/protoutil/clone_test.go +++ b/pkg/util/protoutil/clone_test.go @@ -17,11 +17,11 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/config/zonepb" - "github.com/cockroachdb/cockroach/pkg/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagepb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/gogo/protobuf/proto" )