Merge pull request #187 from turt2live/travis/rust-sdk

Use rust-sdk for crypto
turt2live · Dec 29, 2021 · e009414 · e009414
2 parents 7a5f9ad + 78aea9d
commit e009414
Show file tree

Hide file tree

Showing 31 changed files with 2,513 additions and 10,702 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -5,40 +5,40 @@ jobs:
   # Global
   # ================================================
 
-  eslint-14:
-    name: 'ESLint Node 14'
+  eslint-16:
+    name: 'ESLint Node 16'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '16'
       - run: yarn install
       - run: yarn lint
 
-  # Node 12
+  # Node 16
   # ================================================
 
-  build-12:
-    name: 'Build Node 12'
+  build-16:
+    name: 'Build Node 16'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '16'
       - run: yarn install
       - run: yarn build
       - run: yarn build:examples
 
-  tests-12:
-    name: 'Mocha Tests Node 12'
+  tests-16:
+    name: 'Mocha Tests Node 16'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '16'
       - run: yarn install
       - uses: nick-invision/retry@v2
         with:

diff --git a/docs/tutorials/encryption-appservices.md b/docs/tutorials/encryption-appservices.md
@@ -1,102 +1,29 @@
 Encryption for appservices is just about as easy as bots, though involves using a storage mechanism which is capable of
-handling the higher traffic. The bot-sdk provides a PostgreSQL implementation of {@link ICryptoStorageProvider}, though
-you're welcome to write your own. Note that `storageForUser()` **must** be implemented if writing your own provider.
-
-## {@link NamespacingPostgresCryptoStorageProvider}
-
-This provider requires an {@link ICryptoSecureStorageProvider}. The bot-sdk provides one using 
-[Cryptex](https://github.com/TomFrost/Cryptex), though you're welcome to write your own.
-
-Upon instantiation, the class will automatically prepare the database for you.
-
-⚠ It is important not to lose the PostgreSQL database or the Cryptex key (eg, in AWS KMS). Losing either will make for
-a difficult recovery of the encryption state.
+handling the higher traffic. Eventually the SDK will support custom stores, however for now the crypto store must be
+a {@link RustSdkAppserviceCryptoStorageProvider}.
 
 ```typescript
 const storage = new SimpleFsStorageProvider("./path/to/appservice.json"); // or any other {@link IStorageProvider}
-const secureStorage = new CryptexCryptoSecureStorageProvider(); // instance of an {@link ICryptoSecureStorageProvider}
-const psqlStorage = new NamespacingPostgresCryptoStorageProvider("postgresql://user:pass@domain/database", secureStorage);
+const cryptoStorage = new RustSdkAppserviceCryptoStorageProvider("./path/to/directory");
+
+// ⚠⚠ Be sure to back up both `./path/to/appservice.json` and `./path/to/directory` when using this setup
 
 const registration: IAppserviceRegistration = {
-    /* ... */
-    "de.sorunome.msc2409.push_ephemeral": true,
+   /* ... */
+   "de.sorunome.msc2409.push_ephemeral": true,
 };
 const options: IAppserviceOptions = {
-    /* ... */
-    storage: storage,
-    cryptoStorage: psqlStorage,
-    intentOptions: {
-        // Enable encryption on all appservice users, including the `sender_localpart` user
-        encryption: true,
-    },
+   /* ... */
+   storage: storage,
+   cryptoStorage: cryptoStorage,
+   intentOptions: {
+      // Enable encryption on all appservice users, including the `sender_localpart` user
+      encryption: true,
+   },
 }
 const appservice = new Appservice(options);
 ```
 
-## {@link CryptexCryptoSecureStorageProvider}
-
-This is an implementation of {@link ICryptoSecureStorageProvider} using [Cryptex](https://github.com/TomFrost/Cryptex).
-
-The configuration methods supported by the Cryptex library are all supported here. The recommendation for appservices is
-to add to their own config a section which mirrors the Cryptex configuration to pass it through to the provider.
-
-⚠ It is important not to lose the Cryptex key (eg, in AWS KMS). Losing it will make for  a difficult recovery of the 
-encryption state.
-
-For example:
-
-```yaml
-# Your appservice config
-
-crypto:
-  algorithm: "aes256"
-  keySource: "kms"
-  keySourceOpts:
-    dataKey: "... from KMS ..."
-    region: "us-east-2"
-  secrets:
-    # `pickle_key` is the only secret the provider cares about (currently)
-    pickle_key: "... encrypted with KMS key ..."
-```
-
-Then, after parsing, the provider instance can be created as:
-
-```typescript
-const secureStorage = new CryptexCryptoSecureStorageProvider({
-    config: config.get("crypto"),
-});
-```
-
-### Using AWS [KMS](https://aws.amazon.com/kms/)
-
-1. Create a new symmetric key in the [KMS](https://aws.amazon.com/kms/) Console.
-2. Give access to a user which has access to the [AWS CLI tool](http://docs.aws.amazon.com/cli/latest/userguide/installing.html).
-3. Run the following to get the `dataKey` for `keySourceOpts` above:
-    ```bash
-    aws kms generate-data-key-without-plaintext \
-        --key-id alias/<YOUR_KEY_ALIAS> \
-        --key-spec AES_256 \
-        --output text \
-        --query CiphertextBlob
-    ```
-4. In an empty directory somewhere, create a `cryptex.json` file which looks similar to the following:
-    ```json
-    {
-      "production": {
-        "algorithm": "aes256",
-        "keySource": "kms",
-        "keySourceOpts": {
-          "dataKey": "... from KMS ...",
-          "region": "us-east-2"
-        },
-        "secrets": {}
-      }
-    }
-    ```
-5. Run `cryptex encrypt $YOUR_PICKLE_KEY` in that directory - the pickle key should be sufficiently complex.
-6. Copy the output of that command into the `pickle_key` value for the appservice config.
-7. (Optional) Delete the directory used to encrypt the pickle key.
-
 ## Advanced usage
 
 To monitor the encryption/decryption process, add the following listeners:

diff --git a/docs/tutorials/encryption-bots.md b/docs/tutorials/encryption-bots.md
@@ -3,9 +3,9 @@ providers and it'll start working behind the scenes.
 
 ```typescript
 const storageProvider = new SimpleFsStorageProvider("./path/to/bot.json"); // or any other {@link IStorageProvider}
-const cryptoProvider = new NamespacingSqliteCryptoStorageProvider("./path/to/bot.db"); // or any other {@link ICryptoStorageProvider}
+const cryptoProvider = new RustSdkCryptoStorageProvider("./path/to/directory");
 
-// ⚠⚠ Be sure to back up both `bot.json` and `bot.db` when using this setup
+// ⚠⚠ Be sure to back up both `./path/to/bot.json` and `./path/to/directory` when using this setup
 
 const homeserverUrl = "https://example.org"; // where the bot can reach the homeserver at
 const accessToken = "..."; // acquired from login or registration.
@@ -19,7 +19,7 @@ const client = new MatrixClient(homeserverUrl, accessToken, storageProvider, cry
 // set up your listeners here
 client.on("room.message", (roomId: string, event: any) => {
     if (!event['content']?.['msgtype']) return;
-    
+
     // handle message here. It'll be decrypted already.
 });
 

diff --git a/docs/tutorials/encryption.md b/docs/tutorials/encryption.md
@@ -9,21 +9,5 @@ The following guides go into detail on how to enable encryption for different us
 
 ## General principles
 
-For both bots and appservices, an {@link ICryptoStorageProvider} will be needed to actually enable encryption. This can
-be your own implementation of the interface (including `storageForUser` in the case of appservices), or it can be one of
-the built in types.
-
-For bots, {@link NamespacingSqliteCryptoStorageProvider} is typically best. There is also an 
-{@link NamespacingPostgresCryptoStorageProvider} intended for appservices, though works fine with bots too. The PostgreSQL
-implementation takes a {@link ICryptoSecureStorageProvider} which can also be implemented on your own, or using something
-like {@link CryptexCryptoSecureStorageProvider} instead.
-
-```typescript
-// Sqlite is easiest for most bots
-const sqliteStorage = new NamespacingSqliteCryptoStorageProvider("./path/to/bot.db");
-
-// PostgreSQL is easier for appservices, but just as available to bots. It requires a secret management provider.
-// See the appservices tutorial for more information: {@tutorial encryption-appservices}
-const cryptexInstance = new CryptexCryptoSecureStorageProvider(); // read from default locations
-const psqlStorage = new NamespacingPostgresCryptoStorageProvider("postgresql://user:pass@domain/database", cryptexInstance);
-```
+For both bots and appservices, an {@link ICryptoStorageProvider} will be needed to actually enable encryption. Eventually
+this will be able to be your own implementation, but for now must be a {@link RustSdkCryptoStorageProvider} or derivative.
diff --git a/examples/encryption_appservice.ts b/examples/encryption_appservice.ts
@@ -8,13 +8,11 @@ import {
     LogService,
     MessageEvent,
     RichConsoleLogger,
+    RustSdkAppserviceCryptoStorageProvider,
     SimpleFsStorageProvider,
-    SimpleRetryJoinStrategy
+    SimpleRetryJoinStrategy,
 } from "../src";
 import * as fs from "fs";
-import { NamespacingSqliteCryptoStorageProvider } from "../src/storage/NamespacingSqliteCryptoStorageProvider";
-import { NamespacingPostgresCryptoStorageProvider } from "../src/storage/NamespacingPostgresCryptoStorageProvider";
-import { CryptexCryptoSecureStorageProvider } from "../src/storage/CryptexCryptoSecureStorageProvider";
 
 LogService.setLogger(new RichConsoleLogger());
 LogService.setLevel(LogLevel.TRACE);
@@ -31,24 +29,13 @@ try {
 const dmTarget = creds?.['dmTarget'] ?? "@admin:localhost";
 const homeserverUrl = creds?.['homeserverUrl'] ?? "http://localhost:8008";
 const storage = new SimpleFsStorageProvider("./examples/storage/encryption_appservice.json");
-const cryptexStore = new CryptexCryptoSecureStorageProvider({
-    config: {
-        keySource: "kms",
-        keySourceOpts: {
-            dataKey: creds?.['kmsKey'] || 'NOT_SET',
-            region: "us-east-2",
-        },
-        secrets: creds?.['secrets'] || {},
-        algorithm: "aes256",
-    },
-});
-const crypto = new NamespacingPostgresCryptoStorageProvider(creds?.["psql"] ?? "postgresql://localhost/encrypted_appservice", cryptexStore);
+const crypto = new RustSdkAppserviceCryptoStorageProvider("./examples/storage/encryption_appservice_sled");
 const worksImage = fs.readFileSync("./examples/static/it-works.png");
 
 const registration: IAppserviceRegistration = {
     as_token: creds?.['asToken'] ?? "change_me",
     hs_token: creds?.['hsToken'] ?? "change_me",
-    sender_localpart: "crypto_test_appservice_bot2",
+    sender_localpart: "crypto_test_appservice_rust3",
     namespaces: {
         users: [{
             regex: "@crypto.*:localhost",
@@ -77,7 +64,8 @@ const options: IAppserviceOptions = {
 };
 
 const appservice = new Appservice(options);
-const bot = appservice.botIntent;
+// const bot = appservice.botIntent;
+const bot = appservice.getIntentForUserId("@crypto_nondefault_test2:localhost");
 
 (async function() {
     await bot.enableEncryption();

diff --git a/examples/encryption_bot.ts b/examples/encryption_bot.ts
@@ -3,12 +3,13 @@ import {
     FileMessageEventContent,
     LogLevel,
     LogService,
-    MatrixClient, MessageEvent,
+    MatrixClient,
+    MessageEvent,
     RichConsoleLogger,
+    RustSdkCryptoStorageProvider,
     SimpleFsStorageProvider,
 } from "../src";
 import * as fs from "fs";
-import { NamespacingSqliteCryptoStorageProvider } from "../src/storage/NamespacingSqliteCryptoStorageProvider";
 
 LogService.setLogger(new RichConsoleLogger());
 LogService.setLevel(LogLevel.TRACE);
@@ -26,7 +27,7 @@ const dmTarget = creds?.['dmTarget'] ?? "@admin:localhost";
 const homeserverUrl = creds?.['homeserverUrl'] ?? "http://localhost:8008";
 const accessToken = creds?.['accessToken'] ?? 'YOUR_TOKEN';
 const storage = new SimpleFsStorageProvider("./examples/storage/encryption_bot.json");
-const crypto = new NamespacingSqliteCryptoStorageProvider("./examples/storage/encryption_bot.db");
+const crypto = new RustSdkCryptoStorageProvider("./examples/storage/encryption_bot_sled");
 const worksImage = fs.readFileSync("./examples/static/it-works.png");
 
 const client = new MatrixClient(homeserverUrl, accessToken, storage, crypto);

diff --git a/package.json b/package.json
@@ -27,7 +27,7 @@
   "main": "./lib/index.js",
   "typings": "./lib/index.d.ts",
   "engines": {
-    "node": ">=10.0.0"
+    "node": ">=14.0.0"
   },
   "keywords": [
     "matrix",
@@ -49,13 +49,9 @@
     "scripts/*",
     "tsconfig.json"
   ],
-  "optionalDependencies": {
-    "better-sqlite3": "^7.4.3",
-    "cryptex": "^1.0.1",
-    "pg-promise": "^10.11.1"
-  },
+  "optionalDependencies": {},
   "dependencies": {
-    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
+    "@turt2live/matrix-sdk-crypto-nodejs": "^0.1.0-beta.7",
     "@types/express": "^4.17.13",
     "another-json": "^0.2.0",
     "chalk": "^4.1.0",
@@ -93,6 +89,6 @@
     "tmp": "^0.2.1",
     "ts-mocha": "^7.0.0",
     "tslint": "^6.1.3",
-    "typescript": "^4.3.5"
+    "typescript": "^4.5.4"
   }
 }