[encryption] Manage file key by the file to encrypt itself (#16)

apache/incubator-pegasus#1575

After all encryption related patches been cherry-picked from
[tikv](https://github.com/tikv/rocksdb/commits/6.29.tikv) and
merged, now we will improve the encrytion, including:
- Fix action job `build-linux-encrypted_env-no_compression-no_openssl`
  to build binaries without openssl and compression libs correctly.
- Fix action job `build-linux-encrypted_env-openssl` to export the
  `ENCRYPTED_ENV` enviroment variable correctly.
- Don not skip tests which are skipped by TiKV.
- Refactor `AESCTRCipherStream` and `AESEncryptionProvider` to support
  manage file key by the file itself, according to the design docs in
[Data at rest
encryption](apache/incubator-pegasus#1575).
- Remove all KeyManager related codes.
- Replace KeyManager tests by AES encryption tests.
- Refactor encryption/encryption_test.cc and add more tests.
- Make it possible to construct AESEncryptionProvider object via
  `EncryptionProvider::CreateFromString()` by registering a
  factory in "encryption" library.
  It's possible to construct an object by URI: `AES`, `AES://test` or
  `AES:<instance_key>,<EncryptionMethod>`.
- `ldb` tool support to parse `--fs_uri` flags as the URI mentioned
above.
- Add tests to create AESEncryptionProvider object in
  `CreateEncryptedEnvTest.CreateEncryptedFileSystem`
- `db_bench` support to run benchmark with encryption enabled, by adding
new flags for `db_bench`, they are `encryption_method` and
`encryption_instance_key`.
- Move code from the exported header directory (i.e.
include/rocksdb/encryption.h)
to rocksdb internal (i.e. encryption/encryption.h), do not expose them
to users.
- Code format.

Review hint: #17 shows all the
code changes
from the base branch (i.e. `pegasus-kv:v8.3.2-pegasus`), you can review
it together to
make sure the request branch `acelyc111:pk_enc_new` doesn't have vice
effect on the base.

Manual test:
```
// Generate some data.
./db_bench --encryption_method=AES128CTR --encryption_instance_key=test_instance_key  --num=10000

// Dump WAL OK
./tools/ldb --fs_uri="provider=AES; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log
./tools/ldb --fs_uri="provider=AES://test; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log
./tools/ldb --fs_uri="provider=AES:test_instance_key,AES128CTR; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log

// Dump WAL failed. Pass bad provider parameters to --fs_uri, e.g.
./tools/ldb --fs_uri="provider=AES1:test_instance_key,1AES128CTR; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log
./tools/ldb --fs_uri="provider=AES:bad_test_instance_key,AES128CTR; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log
./tools/ldb --fs_uri="provider=AES:test_instance_key,AES192CTR; id=EncryptedFileSystem" dump_wal --walfile=/tmp/rocksdbtest-1000/dbbench/000004.log

// The same to other ldb tools.

```

.github/workflows/jobs-linux-run-tests.yml

@@ -85,7 +85,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3.5.0
     - uses: "./.github/actions/pre-steps"
-    - run: mkdir build && cd build && cmake -DWITH_SNAPPY=0 -DWITH_ZLIB=0 -DWITH_BZ2=0 -DWITH_LZ4=0 -DWITH_ZSTD=0 .. && make V=1 -j5 && ctest -j5 -V
+    - run: mkdir build && cd build && cmake -DWITH_OPENSSL=0 -DWITH_SNAPPY=0 -DWITH_ZLIB=0 -DWITH_BZ2=0 -DWITH_LZ4=0 -DWITH_ZSTD=0 .. && make V=1 -j5 && ctest -j5 -V
     - run: "cd build/tools && ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression'"
     - uses: "./.github/actions/post-steps"
   build-linux-encrypted_env-openssl:
@@ -95,5 +95,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3.5.0
     - uses: "./.github/actions/pre-steps"
-    - run: mkdir build && cd build && cmake -DWITH_OPENSSL=1 -DENCRYPTED_ENV=1 .. && make V=1 -j5 && ctest -j5 -V
+    - run: |
+       export ENCRYPTED_ENV=AES
+       mkdir build && cd build && cmake -DWITH_OPENSSL=1 .. && make V=1 -j5 && ctest -j5 -V
     - uses: "./.github/actions/post-steps"

db/db_test.cc

@@ -2414,10 +2414,6 @@ TEST_F(DBTest, DestroyDBMetaDatabase) {
 }
 
 TEST_F(DBTest, SnapshotFiles) {
-  if (getenv("ENCRYPTED_ENV")) {
-    // File copy does not carry encryption key.
-    return;
-  }
   do {
     Options options = CurrentOptions();
     options.write_buffer_size = 100000000;  // Large write buffer

db/db_test_util.cc

@@ -71,14 +71,15 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
     mem_env_ = MockEnv::Create(base_env, base_env->GetSystemClock());
   }
   if (getenv("ENCRYPTED_ENV")) {
-#ifdef OPENSSL
-    std::shared_ptr<encryption::KeyManager> key_manager(
-        new test::TestKeyManager);
-    encrypted_env_ = NewKeyManagedEncryptedEnv(Env::Default(), key_manager);
-#else
-    fprintf(stderr, "EncryptedEnv is not available without OpenSSL.");
-    assert(false);
-#endif
+    std::shared_ptr<EncryptionProvider> provider;
+    std::string provider_id = getenv("ENCRYPTED_ENV");
+    if (provider_id.find("=") == std::string::npos &&
+        !EndsWith(provider_id, "://test")) {
+      provider_id = provider_id + "://test";
+    }
+    EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
+                                                   &provider));
+    encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
   }
   env_ = new SpecialEnv(encrypted_env_ ? encrypted_env_
                                        : (mem_env_ ? mem_env_ : base_env));

db/db_test_util.h

@@ -29,7 +29,6 @@
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
-#include "rocksdb/encryption.h"
 #include "rocksdb/env.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/filter_policy.h"