From 2d89fd42f2d6f613cceeb32295f11d62cc13b411 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Wed, 16 Feb 2022 18:27:46 +1100 Subject: [PATCH] WIP - #305 --- src/PolykeyAgent.ts | 1 - src/acl/types.ts | 2 +- src/agent/service/vaultsGitInfoGet.ts | 54 +- src/agent/service/vaultsGitPackGet.ts | 46 +- src/agent/service/vaultsScan.ts | 2 +- src/bootstrap/utils.ts | 1 - src/client/service/vaultsClone.ts | 2 +- src/client/service/vaultsShare.ts | 2 +- src/proto/js/polykey/v1/vaults/vaults_pb.d.ts | 6 - src/proto/js/polykey/v1/vaults/vaults_pb.js | 51 - .../schemas/polykey/v1/vaults/vaults.proto | 1 - src/sigchain/Sigchain.ts | 2 +- src/vaults/VaultInternal.ts | 467 +++- src/vaults/VaultManager.ts | 697 +++--- src/vaults/VaultOps.ts | 5 + src/vaults/errors.ts | 13 +- src/vaults/types.ts | 3 +- src/vaults/utils.ts | 244 +- tests/acl/ACL.test.ts | 1 - tests/agent/GRPCClientAgent.test.ts | 1 - tests/nodes/NodeConnection.test.ts | 1 - tests/notifications/utils.test.ts | 3 +- tests/vaults/VaultInternal.test.ts | 545 +++-- tests/vaults/VaultManager.test.ts | 2092 +++++++++++------ tests/vaults/VaultOps.test.ts | 46 +- tests/vaults/utils.test.ts | 4 - 26 files changed, 2784 insertions(+), 1508 deletions(-) diff --git a/src/PolykeyAgent.ts b/src/PolykeyAgent.ts index b22c2e7955..82c3402be0 100644 --- a/src/PolykeyAgent.ts +++ b/src/PolykeyAgent.ts @@ -327,7 +327,6 @@ class PolykeyAgent { vaultsPath, keyManager, nodeConnectionManager, - nodeManager, notificationsManager, gestaltGraph, acl, diff --git a/src/acl/types.ts b/src/acl/types.ts index 92ae07d137..7770edd7d1 100644 --- a/src/acl/types.ts +++ b/src/acl/types.ts @@ -8,7 +8,7 @@ type PermissionIdString = Opaque<'PermissionIdString', string>; type Permission = { gestalt: GestaltActions; - vaults: Record; // FIXME: the string union on VaultId is to prevent some false errors. + vaults: Record; }; type GestaltActions = Partial>; diff --git a/src/agent/service/vaultsGitInfoGet.ts b/src/agent/service/vaultsGitInfoGet.ts index f5a9d8f41f..7b9461f106 100644 --- a/src/agent/service/vaultsGitInfoGet.ts +++ b/src/agent/service/vaultsGitInfoGet.ts @@ -1,19 +1,23 @@ import type { VaultName } from '../../vaults/types'; import type { VaultManager } from '../../vaults'; import type { ACL } from '../../acl'; +import type { ConnectionInfoGetter } from '../../agent/types'; import * as grpc from '@grpc/grpc-js'; -import { utils as idUtils } from '@matrixai/id'; import { utils as grpcUtils } from '../../grpc'; import { utils as vaultsUtils, errors as vaultsErrors } from '../../vaults'; import * as vaultsPB from '../../proto/js/polykey/v1/vaults/vaults_pb'; import * as validationUtils from '../../validation/utils'; +import * as nodesUtils from '../../nodes/utils'; +import { never } from '../../utils/utils'; function vaultsGitInfoGet({ vaultManager, acl, + connectionInfoGetter, }: { vaultManager: VaultManager; acl: ACL; + connectionInfoGetter: ConnectionInfoGetter; }) { return async ( call: grpc.ServerWritableStream, @@ -25,11 +29,6 @@ function vaultsGitInfoGet({ await genWritable.throw({ code: grpc.status.NOT_FOUND }); return; } - const nodeMessage = request.getNode(); - if (nodeMessage == null) { - await genWritable.throw({ code: grpc.status.NOT_FOUND }); - return; - } let vaultName; const vaultNameOrId = vaultMessage.getNameOrId(); let vaultId = await vaultManager.getVaultId(vaultNameOrId as VaultName); @@ -37,30 +36,39 @@ function vaultsGitInfoGet({ if (!vaultId) { try { vaultId = validationUtils.parseVaultId(vaultNameOrId); - vaultName = (await vaultManager.getVaultMeta(vaultId)).name; + vaultName = (await vaultManager.getVaultMeta(vaultId))?.vaultName; } catch (err) { await genWritable.throw(new vaultsErrors.ErrorVaultsVaultUndefined()); return; } } - const nodeId = validationUtils.parseNodeId(nodeMessage.getNodeId()); - const actionType = request.getAction(); - const perms = await acl.getNodePerm(nodeId); - if (!perms) { - await genWritable.throw(new vaultsErrors.ErrorVaultsPermissionDenied()); + // Getting the NodeId from the ReverseProxy connection info + const connectionInfo = connectionInfoGetter(call.getPeer()); + // If this is getting run the connection exists + // It SHOULD exist here + if (connectionInfo == null) never(); + const nodeId = connectionInfo.nodeId; + const nodeIdEncoded = nodesUtils.encodeNodeId(nodeId); + const actionType = validationUtils.parseVaultAction(request.getAction()); + const permissions = await acl.getNodePerm(nodeId); + if (permissions == null) { + await genWritable.throw( + new vaultsErrors.ErrorVaultsPermissionDenied( + `No permissions found for ${nodeIdEncoded}`, + ), + ); return; } - const vaultPerms = perms.vaults[idUtils.toString(vaultId)]; - try { - if (vaultPerms[actionType] !== null) { - await genWritable.throw(new vaultsErrors.ErrorVaultsPermissionDenied()); - return; - } - } catch (err) { - if (err instanceof TypeError) { - await genWritable.throw(new vaultsErrors.ErrorVaultsPermissionDenied()); - return; - } + const vaultPerms = permissions.vaults[vaultId]; + if (vaultPerms[actionType] !== null) { + await genWritable.throw( + new vaultsErrors.ErrorVaultsPermissionDenied( + `${nodeIdEncoded} does not have permission to ${actionType} from vault ${vaultsUtils.encodeVaultId( + vaultId, + )}`, + ), + ); + return; } const meta = new grpc.Metadata(); meta.set('vaultName', vaultName); diff --git a/src/agent/service/vaultsGitPackGet.ts b/src/agent/service/vaultsGitPackGet.ts index 72e158fae1..3ae0b70fac 100644 --- a/src/agent/service/vaultsGitPackGet.ts +++ b/src/agent/service/vaultsGitPackGet.ts @@ -1,11 +1,24 @@ import type * as grpc from '@grpc/grpc-js'; import type { VaultName } from '../../vaults/types'; import type { VaultManager } from '../../vaults'; +import type { ConnectionInfoGetter } from '../../agent/types'; +import type ACL from '../../acl/ACL'; +import { never } from '../../utils/utils'; +import * as nodesUtils from '../../nodes/utils'; import { errors as grpcErrors, utils as grpcUtils } from '../../grpc'; import { utils as vaultsUtils, errors as vaultsErrors } from '../../vaults'; import * as vaultsPB from '../../proto/js/polykey/v1/vaults/vaults_pb'; +import * as validationUtils from '../../validation/utils'; -function vaultsGitPackGet({ vaultManager }: { vaultManager: VaultManager }) { +function vaultsGitPackGet({ + vaultManager, + acl, + connectionInfoGetter, +}: { + vaultManager: VaultManager; + acl: ACL; + connectionInfoGetter: ConnectionInfoGetter; +}) { return async ( call: grpc.ServerDuplexStream, ) => { @@ -15,6 +28,14 @@ function vaultsGitPackGet({ vaultManager }: { vaultManager: VaultManager }) { clientBodyBuffers.push(clientRequest!.getChunk_asU8()); const body = Buffer.concat(clientBodyBuffers); const meta = call.metadata; + // Getting the NodeId from the ReverseProxy connection info + const connectionInfo = connectionInfoGetter(call.getPeer()); + // If this is getting run the connection exists + // It SHOULD exist here + if (connectionInfo == null) never(); + const nodeId = connectionInfo.nodeId; + const nodeIdEncoded = nodesUtils.encodeNodeId(nodeId); + // Getting vaultId const vaultNameOrId = meta.get('vaultNameOrId').pop()!.toString(); if (vaultNameOrId == null) { throw new grpcErrors.ErrorGRPC('vault-name not in metadata'); @@ -22,7 +43,28 @@ function vaultsGitPackGet({ vaultManager }: { vaultManager: VaultManager }) { let vaultId = await vaultManager.getVaultId(vaultNameOrId as VaultName); vaultId = vaultId ?? vaultsUtils.decodeVaultId(vaultNameOrId); if (vaultId == null) { - await genDuplex.throw(new vaultsErrors.ErrorVaultsVaultUndefined()); + await genDuplex.throw( + // Throwing permission error to hide information about vaults existence + new vaultsErrors.ErrorVaultsPermissionDenied( + `No permissions found for ${nodeIdEncoded}`, + ), + ); + return; + } + // Checking permissions + const permissions = await acl.getNodePerm(nodeId); + const vaultPerms = permissions?.vaults[vaultId]; + const actionType = validationUtils.parseVaultAction( + meta.get('vaultAction').pop(), + ); + if (vaultPerms?.[actionType] !== null) { + await genDuplex.throw( + new vaultsErrors.ErrorVaultsPermissionDenied( + `${nodeIdEncoded} does not have permission to ${actionType} from vault ${vaultsUtils.encodeVaultId( + vaultId, + )}`, + ), + ); return; } const response = new vaultsPB.PackChunk(); diff --git a/src/agent/service/vaultsScan.ts b/src/agent/service/vaultsScan.ts index ee473872df..682863af05 100644 --- a/src/agent/service/vaultsScan.ts +++ b/src/agent/service/vaultsScan.ts @@ -2,7 +2,7 @@ import type * as grpc from '@grpc/grpc-js'; import type { GestaltGraph } from '../../gestalts'; import type { VaultManager } from '../../vaults'; import type * as nodesPB from '../../proto/js/polykey/v1/nodes/nodes_pb'; -import * as validationUtils from '@/validation/utils'; +import * as validationUtils from '../../validation/utils'; import * as vaultsPB from '../../proto/js/polykey/v1/vaults/vaults_pb'; import { utils as vaultsUtils, errors as vaultsErrors } from '../../vaults'; import { utils as grpcUtils } from '../../grpc'; diff --git a/src/bootstrap/utils.ts b/src/bootstrap/utils.ts index 73deb176e8..fc855bb02d 100644 --- a/src/bootstrap/utils.ts +++ b/src/bootstrap/utils.ts @@ -177,7 +177,6 @@ async function bootstrapState({ keyManager, nodeConnectionManager, vaultsPath, - nodeManager, notificationsManager, logger: logger.getChild(VaultManager.name), fresh, diff --git a/src/client/service/vaultsClone.ts b/src/client/service/vaultsClone.ts index c7338b9a28..5b4e516f70 100644 --- a/src/client/service/vaultsClone.ts +++ b/src/client/service/vaultsClone.ts @@ -37,7 +37,7 @@ function vaultsClone({ // Vault id let vaultId; const vaultNameOrId = vaultMessage.getNameOrId(); - vaultId = vaultManager.getVaultId(vaultNameOrId) + vaultId = vaultManager.getVaultId(vaultNameOrId); vaultId = vaultId ?? vaultsUtils.decodeVaultId(vaultNameOrId); if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined(); // Node id diff --git a/src/client/service/vaultsShare.ts b/src/client/service/vaultsShare.ts index 54e2ef4bce..9d2bbf85e2 100644 --- a/src/client/service/vaultsShare.ts +++ b/src/client/service/vaultsShare.ts @@ -4,7 +4,7 @@ import type { VaultId, VaultName } from '../../vaults/types'; import type * as vaultsPB from '../../proto/js/polykey/v1/vaults/vaults_pb'; import * as grpc from '@grpc/grpc-js'; import { utils as idUtils } from '@matrixai/id'; -import * as validationUtils from '@/validation/utils'; +import * as validationUtils from '../../validation/utils'; import { errors as vaultsErrors } from '../../vaults'; import { utils as grpcUtils } from '../../grpc'; import * as utilsPB from '../../proto/js/polykey/v1/utils/utils_pb'; diff --git a/src/proto/js/polykey/v1/vaults/vaults_pb.d.ts b/src/proto/js/polykey/v1/vaults/vaults_pb.d.ts index 9e1a08b0b1..4b0ab311c0 100644 --- a/src/proto/js/polykey/v1/vaults/vaults_pb.d.ts +++ b/src/proto/js/polykey/v1/vaults/vaults_pb.d.ts @@ -409,11 +409,6 @@ export class InfoRequest extends jspb.Message { clearVault(): void; getVault(): Vault | undefined; setVault(value?: Vault): InfoRequest; - - hasNode(): boolean; - clearNode(): void; - getNode(): polykey_v1_nodes_nodes_pb.Node | undefined; - setNode(value?: polykey_v1_nodes_nodes_pb.Node): InfoRequest; getAction(): string; setAction(value: string): InfoRequest; @@ -430,7 +425,6 @@ export class InfoRequest extends jspb.Message { export namespace InfoRequest { export type AsObject = { vault?: Vault.AsObject, - node?: polykey_v1_nodes_nodes_pb.Node.AsObject, action: string, } } diff --git a/src/proto/js/polykey/v1/vaults/vaults_pb.js b/src/proto/js/polykey/v1/vaults/vaults_pb.js index 2a78b7d18a..aebbcbb332 100644 --- a/src/proto/js/polykey/v1/vaults/vaults_pb.js +++ b/src/proto/js/polykey/v1/vaults/vaults_pb.js @@ -3264,7 +3264,6 @@ proto.polykey.v1.vaults.InfoRequest.prototype.toObject = function(opt_includeIns proto.polykey.v1.vaults.InfoRequest.toObject = function(includeInstance, msg) { var f, obj = { vault: (f = msg.getVault()) && proto.polykey.v1.vaults.Vault.toObject(includeInstance, f), - node: (f = msg.getNode()) && polykey_v1_nodes_nodes_pb.Node.toObject(includeInstance, f), action: jspb.Message.getFieldWithDefault(msg, 3, "") }; @@ -3307,11 +3306,6 @@ proto.polykey.v1.vaults.InfoRequest.deserializeBinaryFromReader = function(msg, reader.readMessage(value,proto.polykey.v1.vaults.Vault.deserializeBinaryFromReader); msg.setVault(value); break; - case 2: - var value = new polykey_v1_nodes_nodes_pb.Node; - reader.readMessage(value,polykey_v1_nodes_nodes_pb.Node.deserializeBinaryFromReader); - msg.setNode(value); - break; case 3: var value = /** @type {string} */ (reader.readString()); msg.setAction(value); @@ -3353,14 +3347,6 @@ proto.polykey.v1.vaults.InfoRequest.serializeBinaryToWriter = function(message, proto.polykey.v1.vaults.Vault.serializeBinaryToWriter ); } - f = message.getNode(); - if (f != null) { - writer.writeMessage( - 2, - f, - polykey_v1_nodes_nodes_pb.Node.serializeBinaryToWriter - ); - } f = message.getAction(); if (f.length > 0) { writer.writeString( @@ -3408,43 +3394,6 @@ proto.polykey.v1.vaults.InfoRequest.prototype.hasVault = function() { }; -/** - * optional polykey.v1.nodes.Node node = 2; - * @return {?proto.polykey.v1.nodes.Node} - */ -proto.polykey.v1.vaults.InfoRequest.prototype.getNode = function() { - return /** @type{?proto.polykey.v1.nodes.Node} */ ( - jspb.Message.getWrapperField(this, polykey_v1_nodes_nodes_pb.Node, 2)); -}; - - -/** - * @param {?proto.polykey.v1.nodes.Node|undefined} value - * @return {!proto.polykey.v1.vaults.InfoRequest} returns this -*/ -proto.polykey.v1.vaults.InfoRequest.prototype.setNode = function(value) { - return jspb.Message.setWrapperField(this, 2, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.polykey.v1.vaults.InfoRequest} returns this - */ -proto.polykey.v1.vaults.InfoRequest.prototype.clearNode = function() { - return this.setNode(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.polykey.v1.vaults.InfoRequest.prototype.hasNode = function() { - return jspb.Message.getField(this, 2) != null; -}; - - /** * optional string action = 3; * @return {string} diff --git a/src/proto/schemas/polykey/v1/vaults/vaults.proto b/src/proto/schemas/polykey/v1/vaults/vaults.proto index 478506e1f2..0c8ca81431 100644 --- a/src/proto/schemas/polykey/v1/vaults/vaults.proto +++ b/src/proto/schemas/polykey/v1/vaults/vaults.proto @@ -89,7 +89,6 @@ message LogEntry { // Agent specific. message InfoRequest { Vault vault = 1; - polykey.v1.nodes.Node node = 2; string action = 3; } diff --git a/src/sigchain/Sigchain.ts b/src/sigchain/Sigchain.ts index a6d2e6f193..fdbcd29402 100644 --- a/src/sigchain/Sigchain.ts +++ b/src/sigchain/Sigchain.ts @@ -440,7 +440,7 @@ class Sigchain { @ready(new sigchainErrors.ErrorSigchainNotRunning()) public async clearDB() { - this.sigchainDb.clear(); + await this.sigchainDb.clear(); await this._transaction(async () => { await this.db.put( diff --git a/src/vaults/VaultInternal.ts b/src/vaults/VaultInternal.ts index 45db2d7e3e..dfba2b68e8 100644 --- a/src/vaults/VaultInternal.ts +++ b/src/vaults/VaultInternal.ts @@ -2,27 +2,44 @@ import type { ReadCommitResult } from 'isomorphic-git'; import type { EncryptedFS } from 'encryptedfs'; import type { DB, DBDomain, DBLevel } from '@matrixai/db'; import type { - VaultId, - VaultRef, CommitId, CommitLog, FileSystemReadable, FileSystemWritable, + VaultAction, + VaultId, + VaultIdEncoded, + VaultName, + VaultRef, } from './types'; -import type { KeyManager } from '../keys'; -import type { NodeId } from '../nodes/types'; -import type { ResourceAcquire } from '../utils'; +import type KeyManager from '../keys/KeyManager'; +import type { NodeId, NodeIdEncoded } from '../nodes/types'; +import type NodeConnectionManager from '../nodes/NodeConnectionManager'; +import type { ResourceAcquire } from '../utils/context'; +import type GRPCClientAgent from '../agent/GRPCClientAgent'; +import type { POJO } from '../types'; import path from 'path'; import git from 'isomorphic-git'; -import { Mutex } from 'async-mutex'; +import * as grpc from '@grpc/grpc-js'; import Logger from '@matrixai/logger'; import { CreateDestroyStartStop, ready, } from '@matrixai/async-init/dist/CreateDestroyStartStop'; -import * as vaultsUtils from './utils'; import * as vaultsErrors from './errors'; -import { withF, withG } from '../utils'; +import * as vaultsUtils from './utils'; +import * as nodesUtils from '../nodes/utils'; +import * as validationUtils from '../validation/utils'; +import { withF, withG } from '../utils/context'; +import { RWLock } from '../utils/locks'; +import * as vaultsPB from '../proto/js/polykey/v1/vaults/vaults_pb'; +import { never } from '../utils/utils'; + +// TODO: this might be temp? +export type RemoteInfo = { + remoteNode: NodeIdEncoded; + remoteVault: VaultIdEncoded; +}; interface VaultInternal extends CreateDestroyStartStop {} @CreateDestroyStartStop( @@ -32,22 +49,22 @@ interface VaultInternal extends CreateDestroyStartStop {} class VaultInternal { public static async createVaultInternal({ vaultId, + vaultName, db, vaultsDb, vaultsDbDomain, keyManager, efs, - remote = false, logger = new Logger(this.name), fresh = false, }: { vaultId: VaultId; + vaultName?: VaultName; db: DB; vaultsDb: DBLevel; vaultsDbDomain: DBDomain; keyManager: KeyManager; efs: EncryptedFS; - remote?: boolean; logger?: Logger; fresh?: boolean; }): Promise { @@ -62,34 +79,36 @@ class VaultInternal { efs, logger, }); - await vault.start({ fresh }); + await vault.start({ fresh, vaultName }); logger.info(`Created ${this.name} - ${vaultIdEncoded}`); return vault; } public static async cloneVaultInternal({ + targetNodeId, + targetVaultNameOrId, vaultId, db, vaultsDb, vaultsDbDomain, keyManager, + nodeConnectionManager, efs, logger = new Logger(this.name), }: { + targetNodeId: NodeId; + targetVaultNameOrId: VaultId | VaultName; vaultId: VaultId; db: DB; vaultsDb: DBLevel; vaultsDbDomain: DBDomain; efs: EncryptedFS; keyManager: KeyManager; - remote?: boolean; + nodeConnectionManager: NodeConnectionManager; logger?: Logger; }): Promise { const vaultIdEncoded = vaultsUtils.encodeVaultId(vaultId); logger.info(`Cloning ${this.name} - ${vaultIdEncoded}`); - // TODO: - // Perform the cloning operation to preseed state - // and also seed the remote state const vault = new VaultInternal({ vaultId, db, @@ -99,11 +118,61 @@ class VaultInternal { efs, logger, }); - await vault.start(); + // This error flag will contain the error returned by the cloning grpc stream + let error; + // Make the directory where the .git files will be auto generated and + // where the contents will be cloned to ('contents' file) + await efs.mkdir(vault.vaultDataDir, { recursive: true }); + let vaultName: VaultName, remoteVaultId: VaultId, remote: RemoteInfo; + try { + [vaultName, remoteVaultId] = await nodeConnectionManager.withConnF( + targetNodeId, + async (connection) => { + const client = connection.getClient(); + const [request, vaultName, remoteVaultId] = await vault.request( + client, + targetVaultNameOrId, + 'clone', + ); + await git.clone({ + fs: efs, + http: { request }, + dir: vault.vaultDataDir, + gitdir: vault.vaultGitDir, + url: 'http://', + singleBranch: true, + }); + return [vaultName, remoteVaultId]; + }, + ); + remote = { + remoteNode: nodesUtils.encodeNodeId(targetNodeId), + remoteVault: vaultsUtils.encodeVaultId(remoteVaultId), + }; + } catch (err) { + // If the error flag set and we have the generalised SmartHttpError from + // isomorphic git then we need to throw the polykey error + if (err instanceof git.Errors.SmartHttpError && error) { + throw error; + } + throw err; + } + + await vault.start({ vaultName }); + // Setting the remote in the metadata + await vault.db.put( + vault.vaultMetadataDbDomain, + VaultInternal.remoteKey, + remote, + ); logger.info(`Cloned ${this.name} - ${vaultIdEncoded}`); return vault; } + static dirtyKey = 'dirty'; + static remoteKey = 'remote'; + static nameKey = 'key'; + public readonly vaultId: VaultId; public readonly vaultIdEncoded: string; public readonly vaultDataDir: string; @@ -113,17 +182,22 @@ class VaultInternal { protected db: DB; protected vaultsDbDomain: DBDomain; protected vaultsDb: DBLevel; - protected vaultDbDomain: DBDomain; - protected vaultDb: DBLevel; + protected vaultMetadataDbDomain: DBDomain; + protected vaultMetadataDb: DBLevel; protected keyManager: KeyManager; + protected vaultsNamesDomain: DBDomain; protected efs: EncryptedFS; protected efsVault: EncryptedFS; - protected remote: boolean; - protected _lock: Mutex = new Mutex(); + protected lock: RWLock = new RWLock(); - public lock: ResourceAcquire = async () => { - const release = await this._lock.acquire(); - return [async () => release(), this._lock]; + public readLock: ResourceAcquire = async () => { + const release = await this.lock.acquireRead(); + return [async () => release()]; + }; + + public writeLock: ResourceAcquire = async () => { + const release = await this.lock.acquireWrite(); + return [async () => release()]; }; constructor({ @@ -156,18 +230,31 @@ class VaultInternal { this.efs = efs; } + /** + * + * @param fresh Clears all state before starting + * @param vaultName Name of the vault, Only used when creating a new vault + */ public async start({ fresh = false, + vaultName, }: { fresh?: boolean; + vaultName?: VaultName; } = {}): Promise { this.logger.info( `Starting ${this.constructor.name} - ${this.vaultIdEncoded}`, ); - const vaultDbDomain = [...this.vaultsDbDomain, this.vaultIdEncoded]; - const vaultDb = await this.db.level(this.vaultIdEncoded, this.vaultsDb); + this.vaultMetadataDbDomain = [...this.vaultsDbDomain, this.vaultIdEncoded]; + this.vaultsNamesDomain = [...this.vaultsDbDomain, 'names']; + this.vaultMetadataDb = await this.db.level( + this.vaultIdEncoded, + this.vaultsDb, + ); + // Let's backup any metadata. + if (fresh) { - await vaultDb.clear(); + await this.vaultMetadataDb.clear(); try { await this.efs.rmdir(this.vaultIdEncoded, { recursive: true, @@ -178,20 +265,27 @@ class VaultInternal { } } } - await this.efs.mkdir(this.vaultIdEncoded, { recursive: true }); - await this.efs.mkdir(this.vaultDataDir, { recursive: true }); - await this.efs.mkdir(this.vaultGitDir, { recursive: true }); - await this.setupMeta(); + await this.mkdirExists(this.vaultIdEncoded); + await this.mkdirExists(this.vaultDataDir); + await this.mkdirExists(this.vaultGitDir); + await this.setupMeta({ vaultName }); await this.setupGit(); - const efsVault = await this.efs.chroot(this.vaultDataDir); - this.vaultDbDomain = vaultDbDomain; - this.vaultDb = vaultDb; - this.efsVault = efsVault; + this.efsVault = await this.efs.chroot(this.vaultDataDir); this.logger.info( `Started ${this.constructor.name} - ${this.vaultIdEncoded}`, ); } + private async mkdirExists(directory: string) { + try { + await this.efs.mkdir(directory, { recursive: true }); + } catch (e) { + if (e.code !== 'EEXIST') { + throw e; + } + } + } + public async stop(): Promise { this.logger.info( `Stopping ${this.constructor.name} - ${this.vaultIdEncoded}`, @@ -207,9 +301,14 @@ class VaultInternal { ); const vaultDb = await this.db.level(this.vaultIdEncoded, this.vaultsDb); await vaultDb.clear(); - await this.efs.rmdir(this.vaultIdEncoded, { - recursive: true, - }); + try { + await this.efs.rmdir(this.vaultIdEncoded, { + recursive: true, + }); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + // Otherwise ignore + } this.logger.info( `Destroyed ${this.constructor.name} - ${this.vaultIdEncoded}`, ); @@ -291,7 +390,7 @@ class VaultInternal { @ready(new vaultsErrors.ErrorVaultNotRunning()) public async readF(f: (fs: FileSystemReadable) => Promise): Promise { - return withF([this.lock], async () => { + return withF([this.readLock], async () => { return await f(this.efsVault); }); } @@ -300,8 +399,9 @@ class VaultInternal { public readG( g: (fs: FileSystemReadable) => AsyncGenerator, ): AsyncGenerator { - return withG([this.lock], async function* () { - return yield* g(this.efsVault); + const efsVault = this.efsVault; + return withG([this.readLock], async function* () { + return yield* g(efsVault); }); } @@ -309,29 +409,40 @@ class VaultInternal { public async writeF( f: (fs: FileSystemWritable) => Promise, ): Promise { - return withF([this.lock], async () => { - await this.db.put(this.vaultsDbDomain, 'dirty', true); - // This should really be an internal property - // get whether this is remote, and the remote address - // if it is, we consider this repo an "attached repo" - // this vault is a "mirrored" vault - if (this.remote) { - // Mirrored vaults are immutable - throw new vaultsErrors.ErrorVaultImmutable(); - } + // This should really be an internal property + // get whether this is remote, and the remote address + // if it is, we consider this repo an "attached repo" + // this vault is a "mirrored" vault + if ( + (await this.db.get( + this.vaultMetadataDbDomain, + VaultInternal.remoteKey, + )) != null + ) { + // Mirrored vaults are immutable + throw new vaultsErrors.ErrorVaultRemoteDefined(); + } + return withF([this.writeLock], async () => { + await this.db.put( + this.vaultMetadataDbDomain, + VaultInternal.dirtyKey, + true, + ); // We have to chroot it // and then remove it - // but this is done byitself? - + // but this is done by itself? await f(this.efsVault); - - await this.db.put(this.vaultsDbDomain, 'dirty', false); + await this.db.put( + this.vaultMetadataDbDomain, + VaultInternal.dirtyKey, + false, + ); }); // Const message: string[] = []; // try { - + // // // If the version of the vault has been changed, checkout the working // // directory to this point in history and discard any unlinked commits // await git.checkout({ @@ -480,40 +591,170 @@ class VaultInternal { public writeG( g: (fs: FileSystemWritable) => AsyncGenerator, ): AsyncGenerator { - return withG([this.lock], async function* () { - const result = yield* g(this.efsVault); - // At the end of the geneartor + const efsVault = this.efsVault; + const db = this.db; + const vaultDbDomain = this.vaultMetadataDbDomain; + return withG([this.writeLock], async function* () { + if ((await db.get(vaultDbDomain, VaultInternal.remoteKey)) != null) { + // Mirrored vaults are immutable + throw new vaultsErrors.ErrorVaultRemoteDefined(); + } + await db.put(vaultDbDomain, VaultInternal.dirtyKey, true); + const result = yield* g(efsVault); + // At the end of the generator // you need to do this // but just before // you need to finish it up // DO what you need to do here, create the commit + await db.put(vaultDbDomain, VaultInternal.dirtyKey, false); return result; }); } + // TODO: this needs to respect the write lock since we are writing to the EFS + @ready(new vaultsErrors.ErrorVaultNotRunning()) + public async pullVault({ + nodeConnectionManager, + pullNodeId, + pullVaultNameOrId, + }: { + nodeConnectionManager: NodeConnectionManager; + pullNodeId?: NodeId; + pullVaultNameOrId?: VaultId | VaultName; + }) { + // This error flag will contain the error returned by the cloning grpc stream + let error; + // Keeps track of whether the metadata needs changing to avoid unnecessary db ops + // 0 = no change, 1 = change with vault Id, 2 = change with vault name + let metaChange = 0; + const remoteInfo = await this.db.get( + this.vaultMetadataDbDomain, + VaultInternal.remoteKey, + ); + if (remoteInfo == null) throw new vaultsErrors.ErrorVaultRemoteUndefined(); + + if (pullNodeId == null) { + pullNodeId = nodesUtils.decodeNodeId(remoteInfo.remoteNode)!; + } else { + metaChange = 1; + remoteInfo.remoteNode = nodesUtils.encodeNodeId(pullNodeId); + } + if (pullVaultNameOrId == null) { + pullVaultNameOrId = vaultsUtils.decodeVaultId(remoteInfo.remoteVault!); + } else { + metaChange = 1; + if (typeof pullVaultNameOrId === 'string') { + metaChange = 2; + } else { + remoteInfo.remoteVault = vaultsUtils.encodeVaultId(pullVaultNameOrId); + } + } + this.logger.info( + `Pulling Vault ${vaultsUtils.encodeVaultId( + this.vaultId, + )} from Node ${pullNodeId}`, + ); + let remoteVaultId: VaultIdEncoded; + try { + remoteVaultId = await nodeConnectionManager.withConnF( + pullNodeId!, + async (connection) => { + const client = connection.getClient(); + const [request, , remoteVaultId] = await this.request( + client, + pullVaultNameOrId!, + 'pull', + ); + await withF([this.writeLock], async () => { + await git.pull({ + fs: this.efs, + http: { request }, + dir: this.vaultDataDir, + gitdir: this.vaultGitDir, + url: `http://`, + ref: 'HEAD', + singleBranch: true, + author: { + name: nodesUtils.encodeNodeId(pullNodeId!), + }, + }); + }); + return remoteVaultId; + }, + ); + } catch (err) { + // If the error flag set and we have the generalised SmartHttpError from + // isomorphic git then we need to throw the polykey error + if (err instanceof git.Errors.SmartHttpError && error) { + throw error; + } else if (err instanceof git.Errors.MergeNotSupportedError) { + throw new vaultsErrors.ErrorVaultsMergeConflict(); + } + throw err; + } + if (metaChange !== 0) { + if (metaChange === 2) remoteInfo.remoteVault = remoteVaultId; + await this.db.put( + this.vaultMetadataDbDomain, + VaultInternal.remoteKey, + remoteInfo, + ); + } + this.logger.info( + `Pulled Vault ${vaultsUtils.encodeVaultId( + this.vaultId, + )} from Node ${pullNodeId}`, + ); + } + /** * Setup the vault metadata */ - protected async setupMeta(): Promise { + protected async setupMeta({ + vaultName, + }: { + vaultName?: VaultName; + }): Promise { // Setup the vault metadata - // setup metadata // and you need to make certain preparations // the meta gets created first // if the SoT is the database - // are we suposed to check this? - - if ((await this.db.get(this.vaultDbDomain, 'remote')) == null) { - await this.db.put(this.vaultDbDomain, 'remote', true); - } + // are we supposed to check this? // If this is not existing // setup default vaults db - await this.db.get(this.vaultsDbDomain, 'dirty'); + if ( + (await this.db.get( + this.vaultMetadataDbDomain, + VaultInternal.dirtyKey, + )) == null + ) { + await this.db.put( + this.vaultMetadataDbDomain, + VaultInternal.dirtyKey, + true, + ); + } + + // Set up vault Name + if ( + (await this.db.get( + this.vaultMetadataDbDomain, + VaultInternal.nameKey, + )) == null && + vaultName != null + ) { + await this.db.put( + this.vaultMetadataDbDomain, + VaultInternal.nameKey, + vaultName, + ); + } // Remote: [NodeId, VaultId] | undefined // dirty: boolean - // name: string + // name: string | undefined } /** @@ -568,6 +809,98 @@ class VaultInternal { } return commitIdLatest; } + + protected async request( + client: GRPCClientAgent, + vaultNameOrId: VaultId | VaultName, + vaultAction: VaultAction, + ): Promise { + const requestMessage = new vaultsPB.InfoRequest(); + const vaultMessage = new vaultsPB.Vault(); + requestMessage.setAction(vaultAction); + if (typeof vaultNameOrId === 'string') { + vaultMessage.setNameOrId(vaultNameOrId); + } else { + // To have consistency between GET and POST, send the user + // readable form of the vault Id + vaultMessage.setNameOrId(vaultsUtils.encodeVaultId(vaultNameOrId)); + } + requestMessage.setVault(vaultMessage); + const response = client.vaultsGitInfoGet(requestMessage); + let vaultName, remoteVaultId; + response.stream.on('metadata', async (meta) => { + // Receive the Id of the remote vault + vaultName = meta.get('vaultName').pop(); + if (vaultName) vaultName = vaultName.toString(); + const vId = meta.get('vaultId').pop(); + if (vId) remoteVaultId = validationUtils.parseVaultId(vId.toString()); + }); + // Collect the response buffers from the GET request + const infoResponse: Uint8Array[] = []; + for await (const resp of response) { + infoResponse.push(resp.getChunk_asU8()); + } + const metadata = new grpc.Metadata(); + metadata.set('vaultAction', vaultAction); + if (typeof vaultNameOrId === 'string') { + metadata.set('vaultNameOrId', vaultNameOrId); + } else { + // Metadata only accepts the user readable form of the vault Id + // as the string form has illegal characters + metadata.set('vaultNameOrId', vaultsUtils.encodeVaultId(vaultNameOrId)); + } + return [ + async function ({ + url, + method = 'GET', + headers = {}, + body = [Buffer.from('')], + }: { + url: string; + method: string; + headers: POJO; + body: Buffer[]; + }) { + if (method === 'GET') { + // Send back the GET request info response + return { + url: url, + method: method, + body: infoResponse, + headers: headers, + statusCode: 200, + statusMessage: 'OK', + }; + } else if (method === 'POST') { + const responseBuffers: Array = []; + const stream = client.vaultsGitPackGet(metadata); + const chunk = new vaultsPB.PackChunk(); + // Body is usually an async generator but in the cases we are using, + // only the first value is used + chunk.setChunk(body[0]); + // Tell the server what commit we need + await stream.write(chunk); + let packResponse = (await stream.read()).value; + while (packResponse != null) { + responseBuffers.push(packResponse.getChunk_asU8()); + packResponse = (await stream.read()).value; + } + return { + url: url, + method: method, + body: responseBuffers, + headers: headers, + statusCode: 200, + statusMessage: 'OK', + }; + } else { + never(); + } + }, + vaultName, + remoteVaultId, + ]; + } } export default VaultInternal; diff --git a/src/vaults/VaultManager.ts b/src/vaults/VaultManager.ts index 596863a133..eff016ca46 100644 --- a/src/vaults/VaultManager.ts +++ b/src/vaults/VaultManager.ts @@ -1,6 +1,5 @@ -import type { MutexInterface } from 'async-mutex'; import type { DB, DBDomain, DBLevel } from '@matrixai/db'; -import type { VaultId, VaultName, VaultActions } from './types'; +import type { VaultId, VaultName, VaultActions, VaultIdString } from './types'; import type { Vault } from './Vault'; import type { FileSystem } from '../types'; @@ -8,15 +7,15 @@ import type { PolykeyWorkerManagerInterface } from '../workers/types'; import type { NodeId } from '../nodes/types'; import type { KeyManager } from '../keys'; -import type { NodeConnectionManager, NodeManager } from '../nodes'; +import type { NodeConnectionManager } from '../nodes'; import type { GestaltGraph } from '../gestalts'; import type { ACL } from '../acl'; import type { NotificationsManager } from '../notifications'; +import type { RemoteInfo } from './VaultInternal'; +import type { ResourceAcquire } from '../utils'; import path from 'path'; import { PassThrough } from 'readable-stream'; -import { Mutex } from 'async-mutex'; -import git from 'isomorphic-git'; import { EncryptedFS, errors as encryptedfsErrors } from 'encryptedfs'; import Logger from '@matrixai/logger'; import { @@ -33,30 +32,33 @@ import { utils as nodesUtils } from '../nodes'; import { utils as keysUtils } from '../keys'; import * as validationUtils from '../validation/utils'; import config from '../config'; -import { mkdirExists } from '../utils'; +import { mkdirExists, RWLock, withF, withG } from '../utils'; import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb'; /** * Object map pattern for each vault */ type VaultMap = Map< - VaultId, + VaultIdString, { vault?: VaultInternal; - lock: MutexInterface; + lock: RWLock; } >; type VaultList = Map; - -// FIXME: this will be removed when moved into VaultInternal. type VaultMetadata = { - name: VaultName; - workingDirectoryIndex: string; - remoteNode?: NodeId; - remoteVault?: string; + dirty: boolean; + vaultName: VaultName; + remoteInfo?: RemoteInfo; }; +// TODO: We need a way of tracking vaultName -> vaultId mapping. +// This may be the only metadata we need in the VaultManager. +// TODO: We may need a way to peek into the vaultInternal +// Metadata without needing to create a VaultInternal instance. +// I think this can be readOnly. + interface VaultManager extends CreateDestroyStartStop {} @CreateDestroyStartStop( new vaultsErrors.ErrorVaultManagerRunning(), @@ -69,7 +71,6 @@ class VaultManager { acl, keyManager, nodeConnectionManager, - nodeManager, gestaltGraph, notificationsManager, keyBits = 256, @@ -82,7 +83,6 @@ class VaultManager { acl: ACL; keyManager: KeyManager; nodeConnectionManager: NodeConnectionManager; - nodeManager: NodeManager; gestaltGraph: GestaltGraph; notificationsManager: NotificationsManager; keyBits?: 128 | 192 | 256; @@ -98,7 +98,6 @@ class VaultManager { acl, keyManager, nodeConnectionManager, - nodeManager, gestaltGraph, notificationsManager, keyBits, @@ -119,16 +118,16 @@ class VaultManager { protected db: DB; protected acl: ACL; protected keyManager: KeyManager; - // FIXME, add this to create and constructor protected nodeConnectionManager: NodeConnectionManager; - protected nodeManager: NodeManager; protected gestaltGraph: GestaltGraph; protected notificationsManager: NotificationsManager; protected vaultsDbDomain: DBDomain = [this.constructor.name]; protected vaultsDb: DBLevel; + protected vaultsNamesDbDomain: DBDomain = [...this.vaultsDbDomain, 'names']; + protected vaultsNamesDb: DBLevel; + protected vaultsNamesLock: RWLock = new RWLock(); // VaultId -> VaultMetadata - protected vaultsMetaDbDomain: DBDomain = [this.vaultsDbDomain[0], 'meta']; - protected vaultsMetaDb: DBLevel; + // FIXME: vaultMap should use VaultIdString as the key. protected vaultMap: VaultMap = new Map(); protected vaultKey: Buffer; protected efs: EncryptedFS; @@ -139,7 +138,6 @@ class VaultManager { acl, keyManager, nodeConnectionManager, - nodeManager, gestaltGraph, notificationsManager, keyBits, @@ -151,7 +149,6 @@ class VaultManager { acl: ACL; keyManager: KeyManager; nodeConnectionManager: NodeConnectionManager; - nodeManager: NodeManager; gestaltGraph: GestaltGraph; notificationsManager: NotificationsManager; keyBits: 128 | 192 | 256; @@ -165,7 +162,6 @@ class VaultManager { this.acl = acl; this.keyManager = keyManager; this.nodeConnectionManager = nodeConnectionManager; - this.nodeManager = nodeManager; this.gestaltGraph = gestaltGraph; this.notificationsManager = notificationsManager; this.keyBits = keyBits; @@ -180,12 +176,11 @@ class VaultManager { try { this.logger.info(`Starting ${this.constructor.name}`); const vaultsDb = await this.db.level(this.vaultsDbDomain[0]); - const vaultsMetaDb = await this.db.level( - this.vaultsMetaDbDomain[1], - this.vaultsDb, + const vaultsNamesDb = await this.db.level( + this.vaultsNamesDbDomain[1], + vaultsDb, ); if (fresh) { - await vaultsMetaDb.clear(); await vaultsDb.clear(); await this.fs.promises.rm(this.vaultsPath, { force: true, @@ -213,7 +208,7 @@ class VaultManager { }); } this.vaultsDb = vaultsDb; - this.vaultsMetaDb = vaultsMetaDb; + this.vaultsNamesDb = vaultsNamesDb; this.vaultKey = vaultKey; this.efs = efs; this.logger.info(`Started ${this.constructor.name}`); @@ -230,7 +225,7 @@ class VaultManager { // Iterate over vaults in memory and destroy them, ensuring that // the working directory commit state is saved - for (const [vaultId, vaultAndLock] of this.vaultMap) { + for (const [vaultIdString, vaultAndLock] of this.vaultMap) { // This is locking each vault... before it tries to do this // but if we are calling stop now // we will have blocked all the other methods @@ -240,13 +235,11 @@ class VaultManager { // this this applies already just be calling stop // in that it waits for stop to finish - await this.transact(async () => { - // Think about it, maybe we should use stop instead - // it will be clearer!! - // await vaultAndLock.vault?.stop(); - - await vaultAndLock.vault?.destroy(); - }, [vaultId]); + const vaultId = IdInternal.fromString(vaultIdString); + await withF([this.getWriteLock(vaultId)], async () => { + await vaultAndLock.vault?.stop(); + }); + this.vaultMap.delete(vaultIdString); } // Need to figure out if this id thing is a good idea @@ -264,7 +257,9 @@ class VaultManager { // If the DB was stopped, the existing sublevel `this.vaultsDb` will not be valid // Therefore we recreate the sublevel here const vaultsDb = await this.db.level(this.vaultsDbDomain[0]); + // Clearing all vaults db data await vaultsDb.clear(); + // Is it necessary to remove the vaults domain? await this.fs.promises.rm(this.vaultsPath, { force: true, recursive: true, @@ -280,50 +275,40 @@ class VaultManager { this.efs.unsetWorkerManager(); } - // The with locks thing - // can be generalised a bit - // we can address the with locking mechanism in general - // with withF and withG - // this will become our generic of way locking anything - - // REPLACE THE FOLLOWING 3 functions - // replace this transact with our new withF and withG mechanisms - // all we need to do is create `ResourceAcquire` types in this domain - - /** - * By default will not lock anything - */ - public async transact(f: () => Promise, vaults: Array = []) { - // Will lock nothing by default - return await this.withLocks(f, vaults.map(this.getLock.bind(this))); + // TODO: + // The with locks thing + // can be generalised a bit + // we can address the with locking mechanism in general + // with withF and withG + // this will become our generic of way locking anything + // ... + // REPLACE THE FOLLOWING 3 functions + // replace this transact with our new withF and withG mechanisms + // all we need to do is create `ResourceAcquire` types in this domain + + protected getLock(vaultId: VaultId): RWLock { + const vaultIdString = vaultId.toString() as VaultIdString; + const vaultAndLock = this.vaultMap.get(vaultIdString); + if (vaultAndLock != null) return vaultAndLock.lock; + const lock = new RWLock(); + this.vaultMap.set(vaultIdString, { lock }); + return lock; } - protected async withLocks( - f: () => Promise, - locks: Array = [], - ): Promise { - const releases: Array = []; - for (const lock of locks) { - // Take the lock for each vault in memory and acquire it - releases.push(await lock.acquire()); - } - try { - return await f(); - } finally { - // Release the vault locks in the opposite order - releases.reverse(); - for (const r of releases) { - r(); - } - } + protected getReadLock(vaultId: VaultId): ResourceAcquire { + const lock = this.getLock(vaultId); + return async () => { + const release = await lock.acquireRead(); + return [async () => release()]; + }; } - protected getLock(vaultId: VaultId): MutexInterface { - const vaultAndLock = this.vaultMap.get(vaultId); - if (vaultAndLock != null) return vaultAndLock.lock; - const lock = new Mutex(); - this.vaultMap.set(vaultId, { lock }); - return lock; + protected getWriteLock(vaultId: VaultId): ResourceAcquire { + const lock = this.getLock(vaultId); + return async () => { + const release = await lock.acquireWrite(); + return [async () => release()]; + }; } /** @@ -335,26 +320,45 @@ class VaultManager { @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) public async createVault(vaultName: VaultName): Promise { + // Adding vault to name map const vaultId = await this.generateVaultId(); - const lock = new Mutex(); - this.vaultMap.set(vaultId, { lock }); - return await this.transact(async () => { - this.logger.info( - `Storing metadata for Vault ${vaultsUtils.encodeVaultId(vaultId)}`, + await this.vaultsNamesLock.withWrite(async () => { + const vaultIdBuffer = await this.db.get( + this.vaultsNamesDbDomain, + vaultName, + true, ); - await this.db.put(this.vaultsMetaDbDomain, idUtils.toBuffer(vaultId), { - name: vaultName, - }); - const vault = await VaultInternal.create({ + // Check if the vault name already exists; + if (vaultIdBuffer != null) { + throw new vaultsErrors.ErrorVaultsVaultDefined(); + } + await this.db.put( + this.vaultsNamesDbDomain, + vaultName, + vaultId.toBuffer(), + true, + ); + }); + const lock = new RWLock(); + const vaultIdString = vaultId.toString() as VaultIdString; + this.vaultMap.set(vaultIdString, { lock }); + return await withF([this.getWriteLock(vaultId)], async () => { + // Creating vault + const vault = await VaultInternal.createVaultInternal({ vaultId, + vaultName, keyManager: this.keyManager, efs: this.efs, logger: this.logger.getChild(VaultInternal.name), + db: this.db, + vaultsDb: this.vaultsDb, + vaultsDbDomain: this.vaultsDbDomain, fresh: true, }); - this.vaultMap.set(vaultId, { lock, vault }); + // Adding vault to object map + this.vaultMap.set(vaultIdString, { lock, vault }); return vault.vaultId; - }, [vaultId]); + }); } /** @@ -362,13 +366,33 @@ class VaultManager { * and parses it to return the associated vault name */ @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) - public async getVaultMeta(vaultId: VaultId): Promise { - const vaultMeta = await this.db.get( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), + public async getVaultMeta( + vaultId: VaultId, + ): Promise { + // First check if the metadata exists + const vaultIdEncoded = vaultsUtils.encodeVaultId(vaultId); + const vaultDbDomain = [...this.vaultsDbDomain, vaultIdEncoded]; + const vaultDb = await this.db.level(vaultIdEncoded, this.vaultsDb); + // Return if metadata has no data + if ((await this.db.count(vaultDb)) === 0) return; + // Obtain the metadata; + const dirty = (await this.db.get( + vaultDbDomain, + VaultInternal.dirtyKey, + ))!; + const vaultName = (await this.db.get( + vaultDbDomain, + VaultInternal.nameKey, + ))!; + const remoteInfo = await this.db.get( + vaultDbDomain, + VaultInternal.remoteKey, ); - if (vaultMeta == null) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - return vaultMeta; + return { + dirty, + vaultName, + remoteInfo, + }; } /** @@ -377,68 +401,54 @@ class VaultManager { */ @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) public async destroyVault(vaultId: VaultId) { + const vaultMeta = await this.getVaultMeta(vaultId); + if (vaultMeta == null) return; + const vaultName = vaultMeta.vaultName; this.logger.info(`Destroying Vault ${vaultsUtils.encodeVaultId(vaultId)}`); - await this.transact(async () => { - const vaultMeta = await this.getVaultMeta(vaultId); - if (!vaultMeta) return; - await this.db.del(this.vaultsMetaDbDomain, idUtils.toBuffer(vaultId)); - this.vaultMap.delete(vaultId); - await this.efs.rmdir(vaultsUtils.encodeVaultId(vaultId), { - recursive: true, + const vaultIdString = vaultId.toString() as VaultIdString; + await withF([this.getWriteLock(vaultId)], async () => { + const vault = await this.getVault(vaultId); + // Destroying vault state and metadata + await vault.stop(); + await vault.destroy(); + // Removing from map + this.vaultMap.delete(vaultIdString); + // Removing name->id mapping + await this.vaultsNamesLock.withWrite(async () => { + await this.db.del(this.vaultsNamesDbDomain, vaultName); }); - }, [vaultId]); + }); + this.logger.info(`Destroyed Vault ${vaultsUtils.encodeVaultId(vaultId)}`); } - // /** - // * Constructs or returns the in-memory instance of a vault - // * from metadata using a given vault Id - // */ - // @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) - // private async openVault(vaultId: VaultId): Promise { - // const vaultMeta = await this.getVaultMeta(vaultId); - // if (!vaultMeta) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - // return await this.getVault(vaultId); - // } - /** - * Writes the working directory commit state of a vault Id - * and removes the vault from memory + * Removes vault from the vault map */ @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) public async closeVault(vaultId: VaultId) { - const vaultMeta = await this.getVaultMeta(vaultId); - if (!vaultMeta) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - const vault = await this.getVault(vaultId); - // Updating workingDirectoryIndex in the vault metadata. - vaultMeta.workingDirectoryIndex = vault.getworkingDirIndex(); - await this.db.put( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - vaultMeta, - ); - await vault.destroy(); - this.vaultMap.delete(vaultId); + if ((await this.getVaultName(vaultId)) == null) { + throw new vaultsErrors.ErrorVaultsVaultUndefined(); + } + const vaultIdString = vaultId.toString() as VaultIdString; + await withF([this.getWriteLock(vaultId)], async () => { + const vault = await this.getVault(vaultId); + await vault.stop(); + this.vaultMap.delete(vaultIdString); + }); } /** * Lists the vault name and associated vault Id of all * the vaults stored */ - // FIXME: this will have to peek into the vaults metadata. - // This will be inside the vaultInternal now. Need to work this out. @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) public async listVaults(): Promise { const vaults: VaultList = new Map(); - // Stream all the vault Id and associated metadata values - for await (const o of this.vaultsMetaDb.createReadStream({})) { - const dbMeta = (o as any).value; - const dbId = (o as any).key; - // Manually decrypt the vault metadata - const vaultMeta = await this.db.deserializeDecrypt( - dbMeta, - false, - ); - vaults.set(vaultMeta.name, IdInternal.fromBuffer(dbId)); + // Stream of vaultName VaultId key value pairs + for await (const vaultNameBuffer of this.vaultsNamesDb.createKeyStream()) { + const vaultName = vaultNameBuffer.toString() as VaultName; + const vaultId = (await this.getVaultId(vaultName))!; + vaults.set(vaultName, vaultId); } return vaults; } @@ -451,20 +461,35 @@ class VaultManager { vaultId: VaultId, newVaultName: VaultName, ): Promise { - this.logger.info(`Renaming Vault ${vaultsUtils.encodeVaultId(vaultId)}`); - await this.transact(async () => { - const meta = await this.db.get( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - ); - if (!meta) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - meta.name = newVaultName; - await this.db.put( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - meta, - ); - }, [vaultId]); + await withF([this.getWriteLock(vaultId)], async () => { + this.logger.info(`Renaming Vault ${vaultsUtils.encodeVaultId(vaultId)}`); + // Checking if new name exists + if (await this.getVaultId(newVaultName)) { + throw new vaultsErrors.ErrorVaultsVaultDefined(); + } + // Checking if vault exists + const vaultMetadata = await this.getVaultMeta(vaultId); + if (vaultMetadata == null) { + throw new vaultsErrors.ErrorVaultsVaultUndefined(); + } + const oldVaultName = vaultMetadata.vaultName; + // Updating metadata with new name; + const vaultDbDomain = [ + ...this.vaultsDbDomain, + vaultsUtils.encodeVaultId(vaultId), + ]; + await this.db.put(vaultDbDomain, VaultInternal.nameKey, newVaultName); + // Updating name->id map + await this.vaultsNamesLock.withWrite(async () => { + await this.db.del(this.vaultsNamesDbDomain, oldVaultName); + await this.db.put( + this.vaultsNamesDbDomain, + newVaultName, + vaultId.toBuffer(), + true, + ); + }); + }); } /** @@ -472,20 +497,24 @@ class VaultManager { */ @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) public async getVaultId(vaultName: VaultName): Promise { - // Stream all the metadata and associated vault Id values - for await (const o of this.vaultsMetaDb.createReadStream({})) { - const dbMeta = (o as any).value; - const dbId = (o as any).key; - // Manually decrypt the vault metadata - const vaultMeta = await this.db.deserializeDecrypt( - dbMeta, - false, + return await this.vaultsNamesLock.withWrite(async () => { + const vaultIdBuffer = await this.db.get( + this.vaultsNamesDbDomain, + vaultName, + true, ); - // If the name metadata matches the given name, return the associated vault Id - if (vaultName === vaultMeta.name) { - return IdInternal.fromBuffer(dbId); - } - } + if (vaultIdBuffer == null) return; + return IdInternal.fromBuffer(vaultIdBuffer); + }); + } + + /** + * Retreives the vault name associated with a vault Id + */ + @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) + public async getVaultName(vaultId: VaultId): Promise { + const metadata = await this.getVaultMeta(vaultId); + return metadata?.vaultName; } /** @@ -513,7 +542,9 @@ class VaultManager { public async shareVault(vaultId: VaultId, nodeId: NodeId): Promise { const vaultMeta = await this.getVaultMeta(vaultId); if (!vaultMeta) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - await this.transact(async () => { + // FIXME: does this need locking? + // We don't mutate the vault and the domains have their own locking + await withF([this.getWriteLock(vaultId)], async () => { await this.gestaltGraph._transaction(async () => { await this.acl._transaction(async () => { // Node Id permissions translated to other nodes in @@ -524,7 +555,7 @@ class VaultManager { await this.notificationsManager.sendNotification(nodeId, { type: 'VaultShare', vaultId: vaultId.toString(), - vaultName: vaultMeta.name, + vaultName: vaultMeta.vaultName, actions: { clone: null, pull: null, @@ -532,7 +563,7 @@ class VaultManager { }); }); }); - }, [vaultId]); + }); } /** @@ -561,75 +592,33 @@ class VaultManager { nodeId: NodeId, vaultNameOrId: VaultId | VaultName, ): Promise { - // This error flag will contain the error returned by the cloning grpc stream - let error; - // Let vaultName, remoteVaultId; - const thisNodeId = this.keyManager.getNodeId(); - const nodeConnection = await this.nodeManager.getConnectionToNode(nodeId); - const client = nodeConnection.getClient(); const vaultId = await this.generateVaultId(); - const lock = new Mutex(); - this.vaultMap.set(vaultId, { lock }); + const lock = new RWLock(); + const vaultIdString = vaultId.toString() as VaultIdString; + this.vaultMap.set(vaultIdString, { lock }); this.logger.info( `Cloning Vault ${vaultsUtils.encodeVaultId(vaultId)} on Node ${nodeId}`, ); - return await this.transact(async () => { - // Make the directory where the .git files will be auto generated and - // where the contents will be cloned to ('contents' file) - await this.efs.mkdir( - path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - { recursive: true }, - ); - const [request, vaultName, remoteVaultId] = await vaultsUtils.request( - client, - thisNodeId, - vaultNameOrId, - ); - try { - await git.clone({ - fs: this.efs, - http: { request }, - dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), - url: 'http://', - singleBranch: true, - }); - } catch (err) { - // If the error flag set and we have the generalised SmartHttpError from - // isomorphic git then we need to throw the polykey error - if (err instanceof git.Errors.SmartHttpError && error) { - throw error; - } - throw err; - } - const workingDirIndex = ( - await git.log({ - fs: this.efs, - dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), - depth: 1, - }) - ).pop()!; - // Store the node and vault Id to be used as default remote values when pulling - await this.db.put(this.vaultsMetaDbDomain, idUtils.toBuffer(vaultId), { - name: vaultName, - workingDirectoryIndex: workingDirIndex.oid, - remoteNode: nodeId, - remoteVault: remoteVaultId.toString(), - } as VaultMetadata); - const vault = await VaultInternal.create({ + return await withF([this.getWriteLock(vaultId)], async () => { + const vault = await VaultInternal.cloneVaultInternal({ + targetNodeId: nodeId, + targetVaultNameOrId: vaultNameOrId, vaultId, + db: this.db, + nodeConnectionManager: this.nodeConnectionManager, + vaultsDb: this.vaultsDb, + vaultsDbDomain: this.vaultsDbDomain, keyManager: this.keyManager, efs: this.efs, logger: this.logger.getChild(VaultInternal.name), - remote: true, }); - this.vaultMap.set(vaultId, { lock, vault }); + // TODO: We need to add the cloned vaultName to the name->id mapping + this.vaultMap.set(vaultIdString, { lock, vault }); this.logger.info( `Cloned Vault ${vaultsUtils.encodeVaultId(vaultId)} on Node ${nodeId}`, ); return vault.vaultId; - }, [vaultId]); + }); } /** @@ -644,93 +633,16 @@ class VaultManager { vaultId: VaultId; pullNodeId?: NodeId; pullVaultNameOrId?: VaultId | VaultName; - }): Promise { - return await this.transact(async () => { - // This error flag will contain the error returned by the cloning grpc stream - let error; - // Keeps track of whether the metadata needs changing to avoid unnecessary db ops - // 0 = no change, 1 = change with vault Id, 2 = change with vault name - let metaChange = 0; - const thisNodeId = this.keyManager.getNodeId(); - const vaultMeta = await this.db.get( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - ); - if (!vaultMeta) throw new vaultsErrors.ErrorVaultsVaultUnlinked(); - if (pullNodeId == null) { - pullNodeId = vaultMeta.remoteNode; - } else { - metaChange = 1; - vaultMeta.remoteNode = pullNodeId; - } - if (pullVaultNameOrId == null) { - pullVaultNameOrId = IdInternal.fromString( - vaultMeta.remoteVault!, - ); - } else { - metaChange = 1; - if (typeof pullVaultNameOrId === 'string') { - metaChange = 2; - } else { - vaultMeta.remoteVault = pullVaultNameOrId.toString(); - } - } - this.logger.info( - `Pulling Vault ${vaultsUtils.encodeVaultId( - vaultId, - )} from Node ${pullNodeId}`, - ); - const nodeConnection = await this.nodeManager.getConnectionToNode( - pullNodeId!, - ); - const client = nodeConnection.getClient(); - const [request,, remoteVaultId] = await vaultsUtils.request( - client, - thisNodeId, - pullVaultNameOrId!, - ); - try { - await git.pull({ - fs: this.efs, - http: { request }, - dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), - url: `http://`, - ref: 'HEAD', - singleBranch: true, - author: { - name: nodesUtils.encodeNodeId(pullNodeId!), - }, - }); - } catch (err) { - // If the error flag set and we have the generalised SmartHttpError from - // isomorphic git then we need to throw the polykey error - if (err instanceof git.Errors.SmartHttpError && error) { - throw error; - } else if (err instanceof git.Errors.MergeNotSupportedError) { - throw new vaultsErrors.ErrorVaultsMergeConflict( - 'Merge Conflicts are not supported yet', - ); - } - throw err; - } - if (metaChange !== 0) { - if (metaChange === 2) vaultMeta.remoteVault = remoteVaultId; - await this.db.put( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - vaultMeta, - ); - } + }): Promise { + if ((await this.getVaultName(vaultId)) == null) return; + await withF([this.getWriteLock(vaultId)], async () => { const vault = await this.getVault(vaultId); - // Store the working directory commit state in the '.git' directory - this.logger.info( - `Pulled Vault ${vaultsUtils.encodeVaultId( - vaultId, - )} from Node ${pullNodeId}`, - ); - return vault.vaultId; - }, [vaultId]); + await vault.pullVault({ + nodeConnectionManager: this.nodeConnectionManager, + pullNodeId, + pullVaultNameOrId, + }); + }); } /** @@ -738,24 +650,29 @@ class VaultManager { * cloned or pulled from */ @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) - public async *handleInfoRequest( - vaultId: VaultId, - ): AsyncGenerator { - // Adehrance to git protocol - yield Buffer.from( - gitUtils.createGitPacketLine('# service=git-upload-pack\n'), + public async *handleInfoRequest(vaultId: VaultId): AsyncGenerator { + const efs = this.efs; + const vault = await this.getVault(vaultId); + return yield* withG( + [this.getReadLock(vaultId), vault.readLock], + async function* (): AsyncGenerator { + // Adherence to git protocol + yield Buffer.from( + gitUtils.createGitPacketLine('# service=git-upload-pack\n'), + ); + yield Buffer.from('0000'); + // Read the commit state of the vault + const uploadPack = await gitUtils.uploadPack({ + fs: efs, + dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), + gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), + advertiseRefs: true, + }); + for (const buffer of uploadPack) { + yield buffer; + } + }, ); - yield Buffer.from('0000'); - // Read the commit state of the vault - const uploadPack = await gitUtils.uploadPack({ - fs: this.efs, - dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), - advertiseRefs: true, - }); - for (const buffer of uploadPack) { - yield buffer; - } } /** @@ -767,32 +684,38 @@ class VaultManager { vaultId: VaultId, body: Buffer, ): Promise<[PassThrough, PassThrough]> { - if (body.toString().slice(4, 8) === 'want') { - // Parse the request to get the wanted git object - const wantedObjectId = body.toString().slice(9, 49); - const packResult = await gitUtils.packObjects({ - fs: this.efs, - dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), - gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), - refs: [wantedObjectId], - }); - // Generate a contents and progress stream - const readable = new PassThrough(); - const progressStream = new PassThrough(); - const sideBand = gitUtils.mux( - 'side-band-64', - readable, - packResult.packstream, - progressStream, - ); - return [sideBand, progressStream]; - } else { - throw new gitErrors.ErrorGitUnimplementedMethod( - `Request of type '${body - .toString() - .slice(4, 8)}' not valid, expected 'want'`, - ); - } + const vault = await this.getVault(vaultId); + return await withF( + [this.getReadLock(vaultId), vault.readLock], + async () => { + if (body.toString().slice(4, 8) === 'want') { + // Parse the request to get the wanted git object + const wantedObjectId = body.toString().slice(9, 49); + const packResult = await gitUtils.packObjects({ + fs: this.efs, + dir: path.join(vaultsUtils.encodeVaultId(vaultId), 'contents'), + gitdir: path.join(vaultsUtils.encodeVaultId(vaultId), '.git'), + refs: [wantedObjectId], + }); + // Generate a contents and progress stream + const readable = new PassThrough(); + const progressStream = new PassThrough(); + const sideBand = gitUtils.mux( + 'side-band-64', + readable, + packResult.packstream, + progressStream, + ); + return [sideBand, progressStream]; + } else { + throw new gitErrors.ErrorGitUnimplementedMethod( + `Request of type '${body + .toString() + .slice(4, 8)}' not valid, expected 'want'`, + ); + } + }, + ); } /** @@ -842,8 +765,9 @@ class VaultManager { @ready(new vaultsErrors.ErrorVaultManagerNotRunning()) protected async getVault(vaultId: VaultId): Promise { let vault: VaultInternal | undefined; - let lock: MutexInterface; - let vaultAndLock = this.vaultMap.get(vaultId); + let lock: RWLock; + const vaultIdString = vaultId.toString() as VaultIdString; + let vaultAndLock = this.vaultMap.get(vaultIdString); if (vaultAndLock != null) { ({ vault, lock } = vaultAndLock); // Lock and vault exist @@ -853,62 +777,57 @@ class VaultManager { // Only lock exists let release; try { - release = await lock.acquire(); - ({ vault, lock } = vaultAndLock); + release = await lock.acquireWrite(); + ({ vault } = vaultAndLock); if (vault != null) { return vault; } - const vaultMeta = await this.db.get( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - ); - let remote; - if (vaultMeta) { - if (vaultMeta.remoteVault || vaultMeta.remoteNode) { - remote = true; - } + // Only create if the vault state already exists + if ((await this.getVaultMeta(vaultId)) == null) { + throw new vaultsErrors.ErrorVaultsVaultUndefined( + `Vault ${vaultsUtils.encodeVaultId(vaultId)} doesn't exist`, + ); } - vault = await VaultInternal.create({ + vault = await VaultInternal.createVaultInternal({ vaultId, keyManager: this.keyManager, efs: this.efs, logger: this.logger.getChild(VaultInternal.name), - remote, + db: this.db, + vaultsDb: this.vaultsDb, + vaultsDbDomain: this.vaultsDbDomain, }); vaultAndLock.vault = vault; - this.vaultMap.set(vaultId, vaultAndLock); + this.vaultMap.set(vaultIdString, vaultAndLock); return vault; } finally { release(); } } else { // Neither vault nor lock exists - lock = new Mutex(); + lock = new RWLock(); vaultAndLock = { lock }; - this.vaultMap.set(vaultId, vaultAndLock); + this.vaultMap.set(vaultIdString, vaultAndLock); let release; try { - release = await lock.acquire(); - const vaultMeta = await this.db.get( - this.vaultsMetaDbDomain, - idUtils.toBuffer(vaultId), - ); - let remote; - if (vaultMeta) { - if (vaultMeta.remoteVault || vaultMeta.remoteNode) { - remote = true; - } + release = await lock.acquireWrite(); + // Only create if the vault state already exists + if ((await this.getVaultMeta(vaultId)) == null) { + throw new vaultsErrors.ErrorVaultsVaultUndefined( + `Vault ${vaultsUtils.encodeVaultId(vaultId)} doesn't exist`, + ); } - vault = await VaultInternal.create({ + vault = await VaultInternal.createVaultInternal({ vaultId, keyManager: this.keyManager, efs: this.efs, - workingDirIndex: vaultMeta?.workingDirectoryIndex, + db: this.db, + vaultsDb: this.vaultsDb, + vaultsDbDomain: this.vaultsDbDomain, logger: this.logger.getChild(VaultInternal.name), - remote, }); vaultAndLock.vault = vault; - this.vaultMap.set(vaultId, vaultAndLock); + this.vaultMap.set(vaultIdString, vaultAndLock); return vault; } finally { release(); @@ -942,13 +861,13 @@ class VaultManager { // Obtaining locks. const vaultLocks = vaultIds.map((vaultId) => { - return this.getLock(vaultId); + return this.getReadLock(vaultId); }); // Running the function with locking. - return await this.withLocks(() => { + return await withF(vaultLocks, () => { return f(...vaults); - }, vaultLocks); + }); } protected async setupKey(bits: 128 | 192 | 256): Promise { diff --git a/src/vaults/VaultOps.ts b/src/vaults/VaultOps.ts index 9059471435..f9bef210a3 100644 --- a/src/vaults/VaultOps.ts +++ b/src/vaults/VaultOps.ts @@ -7,6 +7,11 @@ import path from 'path'; import * as vaultsErrors from './errors'; import * as vaultsUtils from './utils'; +// TODO: remove? +type FileOptions = { + recursive?: boolean; +}; + // TODO: tests // - add succeeded // - secret exists diff --git a/src/vaults/errors.ts b/src/vaults/errors.ts index 3bd7c17aaa..88454722ec 100644 --- a/src/vaults/errors.ts +++ b/src/vaults/errors.ts @@ -54,11 +54,13 @@ class ErrorVaultReferenceMissing extends ErrorVault { exitCode = sysexits.USAGE; } -// Yes it is immutable -// But this is because you don't own the vault right now +class ErrorVaultRemoteDefined extends ErrorVaults { + description = 'Vault is a clone of a remote vault and can not be mutated'; + exitCode = sysexits.USAGE; +} -class ErrorVaultImmutable extends ErrorVaults { - description = 'Vault cannot be mutated'; +class ErrorVaultRemoteUndefined extends ErrorVaults { + description = 'Vault has no remote set and can not be pulled'; exitCode = sysexits.USAGE; } @@ -102,7 +104,8 @@ export { ErrorVaultDestroyed, ErrorVaultReferenceInvalid, ErrorVaultReferenceMissing, - ErrorVaultImmutable, + ErrorVaultRemoteDefined, + ErrorVaultRemoteUndefined, ErrorVaultsVaultUndefined, ErrorVaultsVaultDefined, ErrorVaultsRecursive, diff --git a/src/vaults/types.ts b/src/vaults/types.ts index 66e053ebfb..8635f526a3 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -21,8 +21,8 @@ const tagLast = 'last'; const refs = ['HEAD', tagLast] as const; type VaultId = Opaque<'VaultId', Id>; - type VaultIdEncoded = Opaque<'VaultIdEncoded', string>; +type VaultIdString = Opaque<'VaultIdString', string>; type VaultRef = typeof refs[number]; @@ -161,6 +161,7 @@ export { vaultActions }; export type { VaultId, VaultIdEncoded, + VaultIdString, VaultRef, VaultAction, CommitId, diff --git a/src/vaults/utils.ts b/src/vaults/utils.ts index d712691f45..aec2c509c1 100644 --- a/src/vaults/utils.ts +++ b/src/vaults/utils.ts @@ -4,18 +4,14 @@ import type { VaultRef, VaultAction, CommitId, + VaultName, } from './types'; -import type { FileSystem, POJO } from '../types'; import type { GRPCClientAgent } from '../agent'; import type { NodeId } from '../nodes/types'; -import path from 'path'; import { IdInternal, IdRandom, utils as idUtils } from '@matrixai/id'; -import * as grpc from '@grpc/grpc-js'; import { tagLast, refs, vaultActions } from './types'; import * as nodesUtils from '../nodes/utils'; -import * as vaultsPB from '../proto/js/polykey/v1/vaults/vaults_pb'; -import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb'; /** * Vault history is designed for linear-history @@ -67,27 +63,7 @@ function commitAuthor(nodeId: NodeId): { name: string; email: string } { }; } -// Function isVaultId(arg: any) { -// return isId(arg); -// } -// /** -// * This will return arg as a valid VaultId or throw an error if it can't be converted. -// * This will take a multibase string of the ID or the raw Buffer of the ID. -// * @param arg - The variable we wish to convert -// * @throws vaultsErrors.ErrorInvalidVaultId if the arg can't be converted into a VaultId -// * @returns VaultId -// */ -// function makeVaultId(arg: any): VaultId { -// return makeId(arg); -// } -// function isVaultIdPretty(arg: any): arg is VaultIdPretty { -// return isIdString(arg); -// } -// function makeVaultIdPretty(arg: any): VaultIdPretty { -// return makeIdString(arg); -// } - -// async function fileExists(fs: FileSystem, path: string): Promise { +// Async function fileExists(fs: FileSystem, path: string): Promise { // try { // const fh = await fs.promises.open(path, 'r'); // await fh.close(); @@ -99,112 +75,116 @@ function commitAuthor(nodeId: NodeId): { name: string; email: string } { // return true; // } -// async function* readdirRecursively(fs, dir = '.') { -// const dirents = await fs.promises.readdir(dir); -// for (const dirent of dirents) { -// const res = path.join(dir, dirent.toString()); -// const stat = await fs.promises.stat(res); -// if (stat.isDirectory()) { -// yield* readdirRecursively(fs, res); -// } else if (stat.isFile()) { -// yield res; -// } -// } -// } +// TODO: remove or move? +async function* readdirRecursively(fs, dir = '.') { + throw Error('Not Implemented'); + // Const dirents = await fs.promises.readdir(dir); + // for (const dirent of dirents) { + // const res = path.join(dir, dirent.toString()); + // const stat = await fs.promises.stat(res); + // if (stat.isDirectory()) { + // yield* readdirRecursively(fs, res); + // } else if (stat.isFile()) { + // yield res; + // } + // } +} -// async function request( -// client: GRPCClientAgent, -// nodeId: NodeId, -// vaultNameOrId: VaultId | VaultName, -// ) { -// const requestMessage = new vaultsPB.InfoRequest(); -// const vaultMessage = new vaultsPB.Vault(); -// const nodeMessage = new nodesPB.Node(); -// nodeMessage.setNodeId(nodeId); -// requestMessage.setAction('clone'); -// if (typeof vaultNameOrId === 'string') { -// vaultMessage.setNameOrId(vaultNameOrId); -// } else { -// // To have consistency between GET and POST, send the user -// // readable form of the vault Id -// vaultMessage.setNameOrId(makeVaultIdPretty(vaultNameOrId)); -// } -// requestMessage.setVault(vaultMessage); -// requestMessage.setNode(nodeMessage); -// const response = client.vaultsGitInfoGet(requestMessage); -// let vaultName, remoteVaultId; -// response.stream.on('metadata', async (meta) => { -// // Receive the Id of the remote vault -// vaultName = meta.get('vaultName').pop(); -// if (vaultName) vaultName = vaultName.toString(); -// const vId = meta.get('vaultId').pop(); -// if (vId) remoteVaultId = makeVaultId(vId.toString()); -// }); -// // Collet the response buffers from the GET request -// const infoResponse: Uint8Array[] = []; -// for await (const resp of response) { -// infoResponse.push(resp.getChunk_asU8()); -// } -// const metadata = new grpc.Metadata(); -// if (typeof vaultNameOrId === 'string') { -// metadata.set('vaultNameOrId', vaultNameOrId); -// } else { -// // Metadata only accepts the user readable form of the vault Id -// // as the string form has illegal characters -// metadata.set('vaultNameOrId', makeVaultIdPretty(vaultNameOrId)); -// } -// return [ -// async function ({ -// url, -// method = 'GET', -// headers = {}, -// body = [Buffer.from('')], -// }: { -// url: string; -// method: string; -// headers: POJO; -// body: Buffer[]; -// }) { -// if (method === 'GET') { -// // Send back the GET request info response -// return { -// url: url, -// method: method, -// body: infoResponse, -// headers: headers, -// statusCode: 200, -// statusMessage: 'OK', -// }; -// } else if (method === 'POST') { -// const responseBuffers: Array = []; -// const stream = client.vaultsGitPackGet(metadata); -// const chunk = new vaultsPB.PackChunk(); -// // Body is usually an async generator but in the cases we are using, -// // only the first value is used -// chunk.setChunk(body[0]); -// // Tell the server what commit we need -// await stream.write(chunk); -// let packResponse = (await stream.read()).value; -// while (packResponse != null) { -// responseBuffers.push(packResponse.getChunk_asU8()); -// packResponse = (await stream.read()).value; -// } -// return { -// url: url, -// method: method, -// body: responseBuffers, -// headers: headers, -// statusCode: 200, -// statusMessage: 'OK', -// }; -// } else { -// throw new Error('Method not supported'); -// } -// }, -// vaultName, -// remoteVaultId, -// ]; -// } +// TODO: Is this being removed or moved? +async function request( + client: GRPCClientAgent, + nodeId: NodeId, + vaultNameOrId: VaultId | VaultName, +): Promise { + throw Error('Not Implemented'); + // Const requestMessage = new vaultsPB.InfoRequest(); + // const vaultMessage = new vaultsPB.Vault(); + // const nodeMessage = new nodesPB.Node(); + // nodeMessage.setNodeId(nodeId); + // requestMessage.setAction('clone'); + // if (typeof vaultNameOrId === 'string') { + // vaultMessage.setNameOrId(vaultNameOrId); + // } else { + // // To have consistency between GET and POST, send the user + // // readable form of the vault Id + // vaultMessage.setNameOrId(makeVaultIdPretty(vaultNameOrId)); + // } + // requestMessage.setVault(vaultMessage); + // requestMessage.setNode(nodeMessage); + // const response = client.vaultsGitInfoGet(requestMessage); + // let vaultName, remoteVaultId; + // response.stream.on('metadata', async (meta) => { + // // Receive the Id of the remote vault + // vaultName = meta.get('vaultName').pop(); + // if (vaultName) vaultName = vaultName.toString(); + // const vId = meta.get('vaultId').pop(); + // if (vId) remoteVaultId = makeVaultId(vId.toString()); + // }); + // // Collet the response buffers from the GET request + // const infoResponse: Uint8Array[] = []; + // for await (const resp of response) { + // infoResponse.push(resp.getChunk_asU8()); + // } + // const metadata = new grpc.Metadata(); + // if (typeof vaultNameOrId === 'string') { + // metadata.set('vaultNameOrId', vaultNameOrId); + // } else { + // // Metadata only accepts the user readable form of the vault Id + // // as the string form has illegal characters + // metadata.set('vaultNameOrId', makeVaultIdPretty(vaultNameOrId)); + // } + // return [ + // async function ({ + // url, + // method = 'GET', + // headers = {}, + // body = [Buffer.from('')], + // }: { + // url: string; + // method: string; + // headers: POJO; + // body: Buffer[]; + // }) { + // if (method === 'GET') { + // // Send back the GET request info response + // return { + // url: url, + // method: method, + // body: infoResponse, + // headers: headers, + // statusCode: 200, + // statusMessage: 'OK', + // }; + // } else if (method === 'POST') { + // const responseBuffers: Array = []; + // const stream = client.vaultsGitPackGet(metadata); + // const chunk = new vaultsPB.PackChunk(); + // // Body is usually an async generator but in the cases we are using, + // // only the first value is used + // chunk.setChunk(body[0]); + // // Tell the server what commit we need + // await stream.write(chunk); + // let packResponse = (await stream.read()).value; + // while (packResponse != null) { + // responseBuffers.push(packResponse.getChunk_asU8()); + // packResponse = (await stream.read()).value; + // } + // return { + // url: url, + // method: method, + // body: responseBuffers, + // headers: headers, + // statusCode: 200, + // statusMessage: 'OK', + // }; + // } else { + // throw new Error('Method not supported'); + // } + // }, + // vaultName, + // remoteVaultId, + // ]; +} function isVaultAction(action: any): action is VaultAction { if (typeof action !== 'string') return false; @@ -222,4 +202,6 @@ export { validateCommitId, commitAuthor, isVaultAction, + request, + readdirRecursively, }; diff --git a/tests/acl/ACL.test.ts b/tests/acl/ACL.test.ts index 82c01757c8..a75819f2f8 100644 --- a/tests/acl/ACL.test.ts +++ b/tests/acl/ACL.test.ts @@ -7,7 +7,6 @@ import path from 'path'; import fs from 'fs'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { DB } from '@matrixai/db'; -import { utils as idUtils } from '@matrixai/id'; import { ACL, errors as aclErrors } from '@/acl'; import { utils as keysUtils } from '@/keys'; import { utils as vaultsUtils } from '@/vaults'; diff --git a/tests/agent/GRPCClientAgent.test.ts b/tests/agent/GRPCClientAgent.test.ts index 6ff3bb8f15..3ace075c0c 100644 --- a/tests/agent/GRPCClientAgent.test.ts +++ b/tests/agent/GRPCClientAgent.test.ts @@ -150,7 +150,6 @@ describe(GRPCClientAgent.name, () => { keyManager: keyManager, vaultsPath: vaultsPath, nodeConnectionManager: nodeConnectionManager, - nodeManager: nodeManager, db: db, acl: acl, gestaltGraph: gestaltGraph, diff --git a/tests/nodes/NodeConnection.test.ts b/tests/nodes/NodeConnection.test.ts index 6b9829036c..16dee5e653 100644 --- a/tests/nodes/NodeConnection.test.ts +++ b/tests/nodes/NodeConnection.test.ts @@ -254,7 +254,6 @@ describe(`${NodeConnection.name} test`, () => { keyManager: serverKeyManager, vaultsPath: serverVaultsPath, nodeConnectionManager: dummyNodeConnectionManager, - nodeManager: serverNodeManager, notificationsManager: serverNotificationsManager, db: serverDb, acl: serverACL, diff --git a/tests/notifications/utils.test.ts b/tests/notifications/utils.test.ts index 4f4d18b0b0..8f85d46424 100644 --- a/tests/notifications/utils.test.ts +++ b/tests/notifications/utils.test.ts @@ -2,8 +2,7 @@ import type { Notification, NotificationData } from '@/notifications/types'; import type { VaultActions, VaultName } from '@/vaults/types'; import { createPublicKey } from 'crypto'; import { EmbeddedJWK, jwtVerify, exportJWK } from 'jose'; -import { IdInternal } from '@matrixai/id'; -import { sleep } from '@/utils'; + import * as keysUtils from '@/keys/utils'; import * as notificationsUtils from '@/notifications/utils'; import * as notificationsErrors from '@/notifications/errors'; diff --git a/tests/vaults/VaultInternal.test.ts b/tests/vaults/VaultInternal.test.ts index cb721f3482..aa3ce2bb82 100644 --- a/tests/vaults/VaultInternal.test.ts +++ b/tests/vaults/VaultInternal.test.ts @@ -1,16 +1,19 @@ import type { VaultId } from '@/vaults/types'; import type { Vault } from '@/vaults/Vault'; import type { KeyManager } from '@/keys'; +import type { DBDomain, DBLevel } from '@matrixai/db'; import os from 'os'; import path from 'path'; import fs from 'fs'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { EncryptedFS } from 'encryptedfs'; +import { DB } from '@matrixai/db'; import { VaultInternal } from '@/vaults'; import { generateVaultId } from '@/vaults/utils'; import * as vaultsErrors from '@/vaults/errors'; import { sleep } from '@/utils'; import { utils as keysUtils } from '@/keys'; +import * as vaultsUtils from '@/vaults/utils'; import * as testsUtils from '../utils'; jest.mock('@/keys/utils', () => ({ @@ -20,14 +23,19 @@ jest.mock('@/keys/utils', () => ({ })); describe('VaultInternal', () => { + const logger = new Logger('Vault', LogLevel.WARN, [new StreamHandler()]); + let dataDir: string; - let dbPath: string; + let efsDbPath: string; let vault: VaultInternal; let dbKey: Buffer; let vaultId: VaultId; let efs: EncryptedFS; - const logger = new Logger('Vault', LogLevel.WARN, [new StreamHandler()]); + + let db: DB; + let vaultsDb: DBLevel; + let vaultsDbDomain: DBDomain; const fakeKeyManager = { getNodeId: () => { @@ -37,33 +45,54 @@ describe('VaultInternal', () => { const secret1 = { name: 'secret-1', content: 'secret-content-1' }; const secret2 = { name: 'secret-2', content: 'secret-content-2' }; - beforeAll(async () => { + beforeEach(async () => { dataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), 'polykey-test-'), ); dbKey = await keysUtils.generateKey(); - dbPath = path.join(dataDir, 'db'); - await fs.promises.mkdir(dbPath); + efsDbPath = path.join(dataDir, 'efsDb'); + await fs.promises.mkdir(efsDbPath); efs = await EncryptedFS.createEncryptedFS({ - dbPath, + dbPath: efsDbPath, dbKey, logger, }); await efs.start(); - }); - beforeEach(async () => { + db = await DB.createDB({ + crypto: { + key: await keysUtils.generateKey(), + ops: { + encrypt: keysUtils.encryptWithKey, + decrypt: keysUtils.decryptWithKey, + }, + }, + dbPath: path.join(dataDir, 'db'), + fs: fs, + logger: logger, + }); + vaultsDbDomain = ['vaults']; + vaultsDb = await db.level(vaultsDbDomain[0]); + vaultId = generateVaultId(); - vault = await VaultInternal.create({ + vault = await VaultInternal.createVaultInternal({ vaultId, keyManager: fakeKeyManager, efs, logger, fresh: true, + db, + vaultsDb, + vaultsDbDomain, + vaultName: 'testVault', }); }); - afterAll(async () => { + afterEach(async () => { + await vault.stop(); + await vault.destroy(); + await db.stop(); + await db.destroy(); await efs.stop(); await efs.destroy(); await fs.promises.rm(dataDir, { @@ -73,9 +102,13 @@ describe('VaultInternal', () => { }); test('VaultInternal readiness', async () => { - await vault.destroy(); + await vault.stop(); await expect(async () => { await vault.log(); + }).rejects.toThrow(vaultsErrors.ErrorVaultNotRunning); + await vault.destroy(); + await expect(async () => { + await vault.start(); }).rejects.toThrow(vaultsErrors.ErrorVaultDestroyed); }); test('is type correct', async () => { @@ -99,13 +132,17 @@ describe('VaultInternal', () => { await vault.writeF(async (efs) => { await efs.writeFile('secret-1', 'secret-content'); }); - await vault.destroy(); - vault = await VaultInternal.create({ + await vault.stop(); + vault = await VaultInternal.createVaultInternal({ vaultId, keyManager: fakeKeyManager, efs, logger, fresh: false, + db, + vaultName: 'testVault2', + vaultsDb, + vaultsDbDomain, }); await vault.readF(async (efs) => { expect((await efs.readFile('secret-1')).toString()).toStrictEqual( @@ -155,7 +192,7 @@ describe('VaultInternal', () => { }); test('does not allow changing to an unrecognised commit', async () => { await expect(() => vault.version('unrecognisedcommit')).rejects.toThrow( - vaultsErrors.ErrorVaultReferenceMissing, + vaultsErrors.ErrorVaultReferenceInvalid, ); await vault.writeF(async (efs) => { await efs.writeFile('test1', 'testdata1'); @@ -256,22 +293,6 @@ describe('VaultInternal', () => { const log = await vault.log(); expect(log.length).toEqual(4); }); - test('write locks read', async () => { - await vault.writeF(async (efs) => { - await efs.writeFile('secret-1', 'secret-content'); - }); - - await Promise.all([ - vault.writeF(async (efs) => { - await efs.writeFile('secret-1', 'SUPER-DUPER-SECRET-CONTENT'); - }), - vault.readF(async (efs) => { - expect((await efs.readFile('secret-1')).toString()).toEqual( - 'SUPER-DUPER-SECRET-CONTENT', - ); - }), - ]); - }); test('commit added if mutation in write', async () => { const commit = (await vault.log())[0].commitId; await vault.writeF(async (efs) => { @@ -371,65 +392,6 @@ describe('VaultInternal', () => { // Has a new commit. expect(await vault.log()).toHaveLength(2); }); - test('locking occurs when making a commit.', async () => { - // We want to check if the locking is happening. so we need a way to see if an operation is being blocked. - - let resolveDelay; - const delayPromise = new Promise((resolve, _reject) => { - resolveDelay = resolve; - }); - let firstCommitResolved = false; - let firstCommitResolveTime; - - // @ts-ignore - expect(vault.lock.isLocked()).toBeFalsy(); - - const commit1 = vault.writeF(async (efs) => { - await efs.writeFile(secret1.name, secret1.content); - await delayPromise; // Hold the lock hostage. - firstCommitResolved = true; - firstCommitResolveTime = Date.now(); - }); - - // Now that we are holding the lock hostage, - // @ts-ignore - expect(vault.lock.isLocked()).toBeTruthy(); - // We want to check if any action resolves before the lock is released. - - let secondCommitResolved = false; - let secondCommitResolveTime; - const commit2 = vault.writeF(async (efs) => { - await efs.writeFile(secret2.name, secret2.content); - secondCommitResolved = true; - await sleep(2); - secondCommitResolveTime = Date.now(); - }); - - // Give plenty of time for a commit to resolve. - await sleep(200); - - // Now we want to check for the expected conditions. - // 1. Both commist have not completed. - // commit 1 is holding the lock. - expect(firstCommitResolved).toBeFalsy(); - expect(secondCommitResolved).toBeFalsy(); - - // 2. We release the hostage so both should resolve. - await sleep(200); - resolveDelay(); - await commit1; - await commit2; - expect(firstCommitResolved).toBeTruthy(); - expect(secondCommitResolved).toBeTruthy(); - expect(secondCommitResolveTime).toBeGreaterThan(firstCommitResolveTime); - // @ts-ignore - expect(vault.lock.isLocked()).toBeFalsy(); - - // Commit order should be commit2 -> commit1 -> init - const log = await vault.log(); - expect(log[0].message).toContain(secret2.name); - expect(log[1].message).toContain(secret1.name); - }); test('read operation allowed', async () => { await vault.writeF(async (efs) => { await efs.writeFile(secret1.name, secret1.content); @@ -476,6 +438,86 @@ describe('VaultInternal', () => { }), ]); }); + test('no commit after read', async () => { + await vault.writeF(async (efs) => { + await efs.writeFile(secret1.name, secret1.content); + await efs.writeFile(secret2.name, secret2.content); + }); + const commit = (await vault.log())[0].commitId; + await vault.readF(async (efs) => { + expect((await efs.readFile(secret1.name)).toString()).toEqual( + secret1.content, + ); + }); + const log = await vault.log(); + expect(log).toHaveLength(2); + expect(log[0].commitId).toStrictEqual(commit); + }); + test('only exposes limited commands of VaultInternal', async () => { + // Converting a vault to the interface + const vaultInterface = vault as Vault; + + // Using the avaliable functions. + await vaultInterface.writeF(async (efs) => { + await efs.writeFile('test', 'testContent'); + }); + + await vaultInterface.readF(async (efs) => { + const content = (await efs.readFile('test')).toString(); + expect(content).toStrictEqual('testContent'); + }); + + expect(vaultInterface.vaultDataDir).toBeTruthy(); + expect(vaultInterface.vaultGitDir).toBeTruthy(); + expect(vaultInterface.vaultId).toBeTruthy(); + expect(vaultInterface.writeF).toBeTruthy(); + expect(vaultInterface.writeG).toBeTruthy(); + expect(vaultInterface.readF).toBeTruthy(); + expect(vaultInterface.readG).toBeTruthy(); + expect(vaultInterface.log).toBeTruthy(); + expect(vaultInterface.version).toBeTruthy(); + + // Can we convert back? + const vaultNormal = vaultInterface as VaultInternal; + expect(vaultNormal.destroy).toBeTruthy(); // This exists again. + }); + test('cannot commit when the remote field is set', async () => { + // Write remote metadata + await db.put( + [...vaultsDbDomain, vaultsUtils.encodeVaultId(vaultId)], + VaultInternal.remoteKey, + { remoteNode: '', remoteVault: '' }, + ); + const commit = (await vault.log(undefined, 1))[0]; + await vault.version(commit.commitId); + const files = await vault.readF(async (efs) => { + return await efs.readdir('.'); + }); + expect(files).toEqual([]); + await expect( + vault.writeF(async (efs) => { + await efs.writeFile('test', 'testdata'); + }), + ).rejects.toThrow(vaultsErrors.ErrorVaultRemoteDefined); + }); + // Old locking tests + // TODO: review and remove? + test('write locks read', async () => { + await vault.writeF(async (efs) => { + await efs.writeFile('secret-1', 'secret-content'); + }); + + await Promise.all([ + vault.writeF(async (efs) => { + await efs.writeFile('secret-1', 'SUPER-DUPER-SECRET-CONTENT'); + }), + vault.readF(async (efs) => { + expect((await efs.readFile('secret-1')).toString()).toEqual( + 'SUPER-DUPER-SECRET-CONTENT', + ); + }), + ]); + }); test('read locks write', async () => { await vault.writeF(async (efs) => { await efs.writeFile(secret1.name, secret1.content); @@ -497,20 +539,64 @@ describe('VaultInternal', () => { }), ]); }); - test('no commit after read', async () => { - await vault.writeF(async (efs) => { + test('locking occurs when making a commit.', async () => { + // We want to check if the locking is happening. so we need a way to see if an operation is being blocked. + + let resolveDelay; + const delayPromise = new Promise((resolve, _reject) => { + resolveDelay = resolve; + }); + let firstCommitResolved = false; + let firstCommitResolveTime; + + // @ts-ignore + expect(vault.lock.isLocked()).toBeFalsy(); + + const commit1 = vault.writeF(async (efs) => { await efs.writeFile(secret1.name, secret1.content); - await efs.writeFile(secret2.name, secret2.content); + await delayPromise; // Hold the lock hostage. + firstCommitResolved = true; + firstCommitResolveTime = Date.now(); }); - const commit = (await vault.log())[0].commitId; - await vault.readF(async (efs) => { - expect((await efs.readFile(secret1.name)).toString()).toEqual( - secret1.content, - ); + + // Now that we are holding the lock hostage, + // @ts-ignore + expect(vault.lock.isLocked()).toBeTruthy(); + // We want to check if any action resolves before the lock is released. + + let secondCommitResolved = false; + let secondCommitResolveTime; + const commit2 = vault.writeF(async (efs) => { + await efs.writeFile(secret2.name, secret2.content); + secondCommitResolved = true; + await sleep(2); + secondCommitResolveTime = Date.now(); }); + + // Give plenty of time for a commit to resolve. + await sleep(200); + + // Now we want to check for the expected conditions. + // 1. Both commist have not completed. + // commit 1 is holding the lock. + expect(firstCommitResolved).toBeFalsy(); + expect(secondCommitResolved).toBeFalsy(); + + // 2. We release the hostage so both should resolve. + await sleep(200); + resolveDelay(); + await commit1; + await commit2; + expect(firstCommitResolved).toBeTruthy(); + expect(secondCommitResolved).toBeTruthy(); + expect(secondCommitResolveTime).toBeGreaterThan(firstCommitResolveTime); + // @ts-ignore + expect(vault.lock.isLocked()).toBeFalsy(); + + // Commit order should be commit2 -> commit1 -> init const log = await vault.log(); - expect(log).toHaveLength(2); - expect(log[0].commitId).toStrictEqual(commit); + expect(log[0].message).toContain(secret2.name); + expect(log[1].message).toContain(secret1.name); }); test('locking occurs when making an access.', async () => { await vault.writeF(async (efs) => { @@ -569,54 +655,223 @@ describe('VaultInternal', () => { // @ts-ignore expect(vault.lock.isLocked()).toBeFalsy(); }); - test('only exposes limited commands of VaultInternal', async () => { - // Converting a vault to the interface - const vaultInterface = vault as Vault; - - // Using the avaliable functions. - await vaultInterface.writeF(async (efs) => { - await efs.writeFile('test', 'testContent'); - }); - - await vaultInterface.readF(async (efs) => { - const content = (await efs.readFile('test')).toString(); - expect(content).toStrictEqual('testContent'); - }); - - expect(vaultInterface.vaultDataDir).toBeTruthy(); - expect(vaultInterface.vaultGitDir).toBeTruthy(); - expect(vaultInterface.vaultId).toBeTruthy(); - expect(vaultInterface.writeF).toBeTruthy(); - expect(vaultInterface.writeG).toBeTruthy(); - expect(vaultInterface.readF).toBeTruthy(); - expect(vaultInterface.readG).toBeTruthy(); - expect(vaultInterface.log).toBeTruthy(); - expect(vaultInterface.version).toBeTruthy(); + // Locking tests + const waitDelay = 200; + const runGen = async (gen) => { + for await (const _ of gen) { + // Do nothing + } + }; + test('writeF respects read and write locking', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseWrite = await lock.acquireWrite(); + + let finished = false; + const writeP = vault.writeF(async () => { + finished = true; + }); + await sleep(waitDelay); + expect(finished).toBe(false); + releaseWrite(); + await writeP; + expect(finished).toBe(true); + + const releaseRead = await lock.acquireRead(); + finished = false; + const writeP2 = vault.writeF(async () => { + finished = true; + }); + await sleep(waitDelay); + releaseRead(); + await writeP2; + expect(finished).toBe(true); + }); + test('writeG respects read and write locking', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseWrite = await lock.acquireWrite(); + + let finished = false; + const writeGen = vault.writeG(async function* () { + yield; + finished = true; + yield; + }); + const runP = runGen(writeGen); + await sleep(waitDelay); + expect(finished).toBe(false); + releaseWrite(); + await runP; + expect(finished).toBe(true); + + const releaseRead = await lock.acquireRead(); + finished = false; + const writeGen2 = vault.writeG(async function* () { + yield; + finished = true; + yield; + }); + const runP2 = runGen(writeGen2); + await sleep(waitDelay); + releaseRead(); + await runP2; + expect(finished).toBe(true); + }); + test('readF respects write locking', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseWrite = await lock.acquireWrite(); + + let finished = false; + const writeP = vault.readF(async () => { + finished = true; + }); + await sleep(waitDelay); + expect(finished).toBe(false); + releaseWrite(); + await writeP; + expect(finished).toBe(true); + }); + test('readG respects write locking', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseWrite = await lock.acquireWrite(); + let finished = false; + const writeGen = vault.readG(async function* () { + yield; + finished = true; + yield; + }); + const runP = runGen(writeGen); + await sleep(waitDelay); + expect(finished).toBe(false); + releaseWrite(); + await runP; + expect(finished).toBe(true); + }); + test('readF allows concurrent reads', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseRead = await lock.acquireRead(); + const finished: boolean[] = []; + const doThing = async () => { + finished.push(true); + }; + await Promise.all([ + vault.readF(doThing), + vault.readF(doThing), + vault.readF(doThing), + vault.readF(doThing), + ]); + expect(finished.length).toBe(4); + releaseRead(); + }); + test('readG allows concurrent reads', async () => { + // @ts-ignore: kidnap lock + const lock = vault.lock; + // Hold a write lock + const releaseRead = await lock.acquireRead(); + const finished: boolean[] = []; + const doThing = async function* () { + yield; + finished.push(true); + yield; + }; + await Promise.all([ + runGen(vault.readG(doThing)), + runGen(vault.readG(doThing)), + runGen(vault.readG(doThing)), + runGen(vault.readG(doThing)), + ]); + expect(finished.length).toBe(4); + releaseRead(); + }); + test.todo('pullVault respects write locking'); + // Life- cycle + test('can create with CreateVaultInternal', async () => { + let vault1: VaultInternal | undefined; + try { + const vaultId1 = vaultsUtils.generateVaultId(); + vault1 = await VaultInternal.createVaultInternal({ + db, + efs, + keyManager: fakeKeyManager, + vaultId: vaultId1, + vaultsDb, + vaultsDbDomain, + logger, + }); + // Data exists for vault now. + expect(await efs.readdir('.')).toContain( + vaultsUtils.encodeVaultId(vaultId1), + ); + } finally { + await vault1?.stop(); + await vault1?.destroy(); + } + }); + test('can create an existing vault with CreateVaultInternal', async () => { + let vault1: VaultInternal | undefined; + let vault2: VaultInternal | undefined; + try { + const vaultId1 = vaultsUtils.generateVaultId(); + vault1 = await VaultInternal.createVaultInternal({ + db, + efs, + keyManager: fakeKeyManager, + vaultId: vaultId1, + vaultsDb, + vaultsDbDomain, + logger, + }); + // Data exists for vault now. + expect(await efs.readdir('.')).toContain( + vaultsUtils.encodeVaultId(vaultId1), + ); + await vault1.stop(); + // Data persists + expect(await efs.readdir('.')).toContain( + vaultsUtils.encodeVaultId(vaultId1), + ); - // Can we convert back? - const vaultNormal = vaultInterface as VaultInternal; - expect(vaultNormal.destroy).toBeTruthy(); // This exists again. + // Re-opening the vault + vault2 = await VaultInternal.createVaultInternal({ + db, + efs, + keyManager: fakeKeyManager, + vaultId: vaultId1, + vaultsDb, + vaultsDbDomain, + logger, + }); + + // Data still exists and no new data was created + expect(await efs.readdir('.')).toContain( + vaultsUtils.encodeVaultId(vaultId1), + ); + expect(await efs.readdir('.')).toHaveLength(2); + } finally { + await vault1?.stop(); + await vault1?.destroy(); + await vault2?.stop(); + await vault2?.destroy(); + } }); - test('cannot commit when the remote field is set', async () => { + test.todo('can create with CloneVaultInternal'); + test('stop is idempotent', async () => { + // Should complete with no errors + await vault.stop(); + await vault.stop(); + }); + test('destroy is idempotent', async () => { + await vault.stop(); + await vault.destroy(); await vault.destroy(); - vault = await VaultInternal.create({ - vaultId, - keyManager: fakeKeyManager, - efs, - logger, - remote: true, - fresh: true, - }); - const commit = (await vault.log(undefined, 1))[0]; - await vault.version(commit.commitId); - const files = await vault.readF(async (efs) => { - return await efs.readdir('.'); - }); - expect(files).toEqual([]); - await expect( - vault.writeF(async (efs) => { - await efs.writeFile('test', 'testdata'); - }), - ).rejects.toThrow(vaultsErrors.ErrorVaultImmutable); }); }); diff --git a/tests/vaults/VaultManager.test.ts b/tests/vaults/VaultManager.test.ts index ca0b5f89fb..48445867ab 100644 --- a/tests/vaults/VaultManager.test.ts +++ b/tests/vaults/VaultManager.test.ts @@ -1,29 +1,42 @@ import type { NodeId, NodeIdEncoded } from '@/nodes/types'; -import type { VaultId, VaultName } from '@/vaults/types'; -import type { GestaltGraph } from '@/gestalts'; -import type { ACL } from '@/acl'; -import type { NotificationsManager } from '@/notifications'; -import type { VaultInternal } from '@/vaults'; -import type { KeyManager } from '@/keys'; -import type { NodeConnectionManager, NodeManager } from '@/nodes'; -import type { NodeAddress } from '@/nodes/types'; +import type { + VaultAction, + VaultId, + VaultIdString, + VaultName, +} from '@/vaults/types'; +import type NotificationsManager from '@/notifications/NotificationsManager'; +import type ReverseProxy from '@/network/ReverseProxy'; +import type { Host, Port, TLSConfig } from '@/network/types'; import fs from 'fs'; import os from 'os'; import path from 'path'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -import { IdInternal, utils as idUtils } from '@matrixai/id'; +import { IdInternal } from '@matrixai/id'; import { DB } from '@matrixai/db'; -import { utils as keysUtils } from '@/keys'; -import { PolykeyAgent } from '@'; -import { VaultManager, vaultOps } from '@/vaults'; -import { errors as vaultErrors } from '@/vaults'; -import { utils as nodesUtils } from '@/nodes'; - -jest.mock('@/keys/utils', () => ({ - ...jest.requireActual('@/keys/utils'), - generateDeterministicKeyPair: - jest.requireActual('@/keys/utils').generateKeyPair, -})); +import { destroyed } from '@matrixai/async-init'; +import { running } from '@matrixai/async-init/dist/utils'; +import ACL from '@/acl/ACL'; +import GestaltGraph from '@/gestalts/GestaltGraph'; +import NodeConnectionManager from '@/nodes/NodeConnectionManager'; +import KeyManager from '@/keys/KeyManager'; +import PolykeyAgent from '@/PolykeyAgent'; +import VaultManager from '@/vaults/VaultManager'; +import * as vaultOps from '@/vaults/VaultOps'; +import * as vaultsErrors from '@/vaults/errors'; +import NodeGraph from '@/nodes/NodeGraph'; +import * as nodesUtils from '@/nodes/utils'; +import ForwardProxy from '@/network/ForwardProxy'; +import * as vaultsUtils from '@/vaults/utils'; +import * as keysUtils from '@/keys/utils'; +import { sleep } from '@/utils'; +import * as testsUtils from '../utils'; + +const mockedGenerateDeterministicKeyPair = jest + .spyOn(keysUtils, 'generateDeterministicKeyPair') + .mockImplementation((bits, _) => { + return keysUtils.generateKeyPair(bits); + }); describe('VaultManager', () => { const logger = new Logger('VaultManager Test', LogLevel.WARN, [ @@ -31,13 +44,8 @@ describe('VaultManager', () => { ]); const nonExistentVaultId = IdInternal.fromString('DoesNotExistxxxx'); const password = 'password'; - let gestaltGraph: GestaltGraph; - let vaultManager: VaultManager; - let keyManager: KeyManager; let remoteVaultId: VaultId; - let localKeynodeId: NodeId; - let localKeynodeIdEncoded: NodeIdEncoded; let remoteKeynode1Id: NodeId; let remoteKeynode1IdEncoded: NodeIdEncoded; let remoteKeynode2Id: NodeId; @@ -49,151 +57,43 @@ describe('VaultManager', () => { const secondVaultName = 'SecondTestVault' as VaultName; const thirdVaultName = 'ThirdTestVault' as VaultName; - let localKeynode: PolykeyAgent; - let remoteKeynode1: PolykeyAgent, remoteKeynode2: PolykeyAgent; + let dataDir: string; + let vaultsPath: string; + let db: DB; - let allDataDir: string; + // We only ever use this to get NodeId, No need to create a whole one + const nodeId = testsUtils.generateRandomNodeId(); + const dummyKeyManager = { + getNodeId: () => nodeId, + } as KeyManager; - beforeAll(async () => { - // Creating agents. - allDataDir = await fs.promises.mkdtemp( + beforeEach(async () => { + mockedGenerateDeterministicKeyPair.mockImplementation((bits, _) => { + return keysUtils.generateKeyPair(bits); + }); + dataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), 'polykey-test-'), ); - localKeynode = await PolykeyAgent.createPolykeyAgent({ - password, - logger: logger.getChild('Local Keynode'), - nodePath: path.join(allDataDir, 'localKeynode'), - }); - gestaltGraph = localKeynode.gestaltGraph; - vaultManager = localKeynode.vaultManager; - keyManager = localKeynode.keyManager; - localKeynodeId = localKeynode.keyManager.getNodeId(); - localKeynodeIdEncoded = nodesUtils.encodeNodeId(localKeynodeId); - - remoteKeynode1 = await PolykeyAgent.createPolykeyAgent({ - password, - logger: logger.getChild('Remote Keynode 1'), - nodePath: path.join(allDataDir, 'remoteKeynode1'), - }); - remoteKeynode1Id = remoteKeynode1.keyManager.getNodeId(); - remoteKeynode1IdEncoded = nodesUtils.encodeNodeId(remoteKeynode1Id); - remoteKeynode2 = await PolykeyAgent.createPolykeyAgent({ - password, - logger: logger.getChild('Remote Keynode 2'), - nodePath: path.join(allDataDir, 'remoteKeynode2'), - }); - remoteKeynode2Id = remoteKeynode2.keyManager.getNodeId(); - remoteKeynode2IdEncoded = nodesUtils.encodeNodeId(remoteKeynode2Id); - - // Adding details to each agent. - await localKeynode.nodeManager.setNode(remoteKeynode1Id, { - host: remoteKeynode1.revProxy.getIngressHost(), - port: remoteKeynode1.revProxy.getIngressPort(), - }); - await localKeynode.nodeManager.setNode(remoteKeynode2Id, { - host: remoteKeynode2.revProxy.getIngressHost(), - port: remoteKeynode2.revProxy.getIngressPort(), - }); - await remoteKeynode1.nodeManager.setNode(localKeynodeId, { - host: localKeynode.revProxy.getIngressHost(), - port: localKeynode.revProxy.getIngressPort(), - }); - await remoteKeynode1.nodeManager.setNode(remoteKeynode2Id, { - host: remoteKeynode2.revProxy.getIngressHost(), - port: remoteKeynode2.revProxy.getIngressPort(), - }); - await remoteKeynode2.nodeManager.setNode(localKeynodeId, { - host: localKeynode.revProxy.getIngressHost(), - port: localKeynode.revProxy.getIngressPort(), - }); - await remoteKeynode2.nodeManager.setNode(remoteKeynode1Id, { - host: remoteKeynode1.revProxy.getIngressHost(), - port: remoteKeynode1.revProxy.getIngressPort(), - }); - - await gestaltGraph.setNode({ - id: remoteKeynode1IdEncoded, - chain: {}, - }); - await gestaltGraph.setNode({ - id: remoteKeynode2IdEncoded, - chain: {}, - }); - await remoteKeynode1.gestaltGraph.setNode({ - id: localKeynodeIdEncoded, - chain: {}, - }); - await remoteKeynode1.gestaltGraph.setNode({ - id: remoteKeynode2IdEncoded, - chain: {}, - }); - await remoteKeynode2.gestaltGraph.setNode({ - id: localKeynodeIdEncoded, - chain: {}, - }); - await remoteKeynode2.gestaltGraph.setNode({ - id: remoteKeynode1IdEncoded, - chain: {}, - }); - - remoteVaultId = await remoteKeynode1.vaultManager.createVault(vaultName); - await remoteKeynode1.vaultManager.shareVault(remoteVaultId, localKeynodeId); - await remoteKeynode1.vaultManager.shareVault( - remoteVaultId, - remoteKeynode2Id, - ); - - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - for (const secret of secretNames.slice(0, 2)) { - await vaultOps.addSecret(remoteVault, secret, 'success?'); - } - }, - ); + vaultsPath = path.join(dataDir, 'VAULTS'); + db = await DB.createDB({ + dbPath: path.join(dataDir, 'DB'), + logger: logger.getChild(DB.name), + }); }); afterEach(async () => { - for (const [, vaultId] of await vaultManager.listVaults()) { - await vaultManager.destroyVault(vaultId); - } - for (const [, vaultId] of await remoteKeynode2.vaultManager.listVaults()) { - await remoteKeynode2.vaultManager.destroyVault(vaultId); - } - }); - - afterAll(async () => { - await remoteKeynode2.stop(); - await remoteKeynode2.destroy(); - await remoteKeynode1.stop(); - await remoteKeynode1.destroy(); - await localKeynode.stop(); - await localKeynode.destroy(); - await fs.promises.rm(allDataDir, { - recursive: true, + await db.stop(); + await db.destroy(); + await fs.promises.rm(dataDir, { force: true, + recursive: true, }); }); test('VaultManager readiness', async () => { - const dataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'polykey-test-'), - ); - const db = await DB.createDB({ - dbPath: path.join(dataDir, 'DB'), - crypto: { - key: await keysUtils.generateKey(), - ops: { - encrypt: keysUtils.encryptWithKey, - decrypt: keysUtils.decryptWithKey, - }, - }, - logger: logger.getChild(DB.name), - }); - const vaultManagerReadiness = await VaultManager.createVaultManager({ - vaultsPath: path.join(dataDir, 'VAULTS'), - keyManager: {} as KeyManager, - nodeManager: {} as NodeManager, + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, gestaltGraph: {} as GestaltGraph, nodeConnectionManager: {} as NodeConnectionManager, acl: {} as ACL, @@ -201,588 +101,1470 @@ describe('VaultManager', () => { db, logger: logger.getChild(VaultManager.name), }); - - await expect(vaultManagerReadiness.destroy()).rejects.toThrow( - vaultErrors.ErrorVaultManagerRunning, - ); - // Should be a noop - await vaultManagerReadiness.start(); - await vaultManagerReadiness.stop(); - await vaultManagerReadiness.destroy(); - await expect(vaultManagerReadiness.start()).rejects.toThrow( - vaultErrors.ErrorVaultManagerDestroyed, - ); - await expect(async () => { - await vaultManagerReadiness.listVaults(); - }).rejects.toThrow(vaultErrors.ErrorVaultManagerNotRunning); - await fs.promises.rm(dataDir, { - force: true, - recursive: true, - }); + try { + await expect(vaultManager.destroy()).rejects.toThrow( + vaultsErrors.ErrorVaultManagerRunning, + ); + // Should be a noop + await vaultManager.start(); + await vaultManager.stop(); + await vaultManager.destroy(); + await expect(vaultManager.start()).rejects.toThrow( + vaultsErrors.ErrorVaultManagerDestroyed, + ); + await expect(async () => { + await vaultManager.listVaults(); + }).rejects.toThrow(vaultsErrors.ErrorVaultManagerNotRunning); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); - test('is type correct', () => { - expect(vaultManager).toBeInstanceOf(VaultManager); + test('is type correct', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + expect(vaultManager).toBeInstanceOf(VaultManager); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('can create many vaults and open a vault', async () => { - const vaultNames = [ - 'Vault1', - 'Vault2', - 'Vault3', - 'Vault4', - 'Vault5', - 'Vault6', - 'Vault7', - 'Vault8', - 'Vault9', - 'Vault10', - 'Vault11', - 'Vault12', - 'Vault13', - 'Vault14', - 'Vault15', - ]; - for (const vaultName of vaultNames) { - await vaultManager.createVault(vaultName as VaultName); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultNames = [ + 'Vault1', + 'Vault2', + 'Vault3', + 'Vault4', + 'Vault5', + 'Vault6', + 'Vault7', + 'Vault8', + 'Vault9', + 'Vault10', + 'Vault11', + 'Vault12', + 'Vault13', + 'Vault14', + 'Vault15', + ]; + for (const vaultName of vaultNames) { + await vaultManager.createVault(vaultName as VaultName); + } + expect((await vaultManager.listVaults()).size).toEqual(vaultNames.length); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); } - expect((await vaultManager.listVaults()).size).toEqual(vaultNames.length); }); test('can rename a vault', async () => { - const vaultId = await vaultManager.createVault(vaultName); - await vaultManager.renameVault(vaultId, secondVaultName); - await expect(vaultManager.getVaultId(vaultName)).resolves.toBeUndefined(); - await expect( - vaultManager.getVaultId(secondVaultName), - ).resolves.toStrictEqual(vaultId); - await expect(() => - vaultManager.renameVault(nonExistentVaultId, 'DNE' as VaultName), - ).rejects.toThrow(vaultErrors.ErrorVaultsVaultUndefined); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId = await vaultManager.createVault(vaultName); + // We can rename the vault here + await vaultManager.renameVault(vaultId, secondVaultName); + await expect(vaultManager.getVaultId(vaultName)).resolves.toBeUndefined(); + await expect( + vaultManager.getVaultId(secondVaultName), + ).resolves.toStrictEqual(vaultId); + // Can't rename an non existing vault + await expect(() => + vaultManager.renameVault(nonExistentVaultId, 'DNE' as VaultName), + ).rejects.toThrow(vaultsErrors.ErrorVaultsVaultUndefined); + await vaultManager.createVault(thirdVaultName); + // Can't rename vault to a name that exists + await expect( + vaultManager.renameVault(vaultId, thirdVaultName), + ).rejects.toThrow(vaultsErrors.ErrorVaultsVaultDefined); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('can delete a vault', async () => { - const secondVaultId = await vaultManager.createVault(secondVaultName); - await vaultManager.destroyVault(secondVaultId); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + expect((await vaultManager.listVaults()).size).toBe(0); + const secondVaultId = await vaultManager.createVault(secondVaultName); + // @ts-ignore: protected method + const vault = await vaultManager.getVault(secondVaultId); + await vaultManager.destroyVault(secondVaultId); + // The mapping should be gone. + expect((await vaultManager.listVaults()).size).toBe(0); + // The vault should be destroyed + expect(vault[destroyed]).toBe(true); + // Metadata should be gone + expect(await vaultManager.getVaultMeta(secondVaultId)).toBeUndefined(); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('can list vaults', async () => { - const firstVaultId = await vaultManager.createVault(vaultName); - const secondVaultId = await vaultManager.createVault(secondVaultName); - const vaultNames: Array = []; - const vaultIds: Array = []; - const vaultList = await vaultManager.listVaults(); - vaultList.forEach((vaultId, vaultName) => { - vaultNames.push(vaultName); - vaultIds.push(vaultId.toString()); - }); - expect(vaultNames.sort()).toEqual([vaultName, secondVaultName].sort()); - expect(vaultIds.sort()).toEqual( - [firstVaultId.toString(), secondVaultId.toString()].sort(), - ); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const firstVaultId = await vaultManager.createVault(vaultName); + const secondVaultId = await vaultManager.createVault(secondVaultName); + const vaultNames: Array = []; + const vaultIds: Array = []; + const vaultList = await vaultManager.listVaults(); + vaultList.forEach((vaultId, vaultName) => { + vaultNames.push(vaultName); + vaultIds.push(vaultId.toString()); + }); + expect(vaultNames.sort()).toEqual([vaultName, secondVaultName].sort()); + expect(vaultIds.sort()).toEqual( + [firstVaultId.toString(), secondVaultId.toString()].sort(), + ); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('able to read and load existing metadata', async () => { - const vaultNames = [ - 'Vault1', - 'Vault2', - 'Vault3', - 'Vault4', - 'Vault5', - 'Vault6', - 'Vault7', - 'Vault8', - 'Vault9', - 'Vault10', - ]; - for (const vaultName of vaultNames) { - await vaultManager.createVault(vaultName as VaultName); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultNames = [ + 'Vault1', + 'Vault2', + 'Vault3', + 'Vault4', + 'Vault5', + 'Vault6', + 'Vault7', + 'Vault8', + 'Vault9', + 'Vault10', + ]; + for (const vaultName of vaultNames) { + await vaultManager.createVault(vaultName as VaultName); + } + const vaults = await vaultManager.listVaults(); + const vaultId = vaults.get('Vault1' as VaultName) as VaultId; + expect(vaultId).not.toBeUndefined(); + await vaultManager.stop(); + await vaultManager.start(); + const restartedVaultNames: Array = []; + const vaultList = await vaultManager.listVaults(); + vaultList.forEach((_, vaultName) => { + restartedVaultNames.push(vaultName); + }); + expect(restartedVaultNames.sort()).toEqual(vaultNames.sort()); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); } - const vaults = await vaultManager.listVaults(); - const vaultId = vaults.get('Vault1' as VaultName) as VaultId; - expect(vaultId).not.toBeUndefined(); - await vaultManager.stop(); - await vaultManager.start(); - const restartedVaultNames: Array = []; - const vaultList = await vaultManager.listVaults(); - vaultList.forEach((_, vaultName) => { - restartedVaultNames.push(vaultName); - }); - expect(restartedVaultNames.sort()).toEqual(vaultNames.sort()); }); test.skip('cannot concurrently create vaults with the same name', async () => { - const vaults = Promise.all([ - vaultManager.createVault(vaultName), - vaultManager.createVault(vaultName), - ]); - await expect(() => vaults).rejects.toThrow( - vaultErrors.ErrorVaultsVaultDefined, - ); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaults = Promise.all([ + vaultManager.createVault(vaultName), + vaultManager.createVault(vaultName), + ]); + await expect(() => vaults).rejects.toThrow( + vaultsErrors.ErrorVaultsVaultDefined, + ); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('can concurrently rename the same vault', async () => { - const vaultId = await vaultManager.createVault(vaultName); - await Promise.all([ - vaultManager.renameVault(vaultId, secondVaultName), - vaultManager.renameVault(vaultId, thirdVaultName), - ]); - const vaultNameTest = (await vaultManager.getVaultMeta(vaultId)).name; - expect(vaultNameTest).toBe(thirdVaultName); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId = await vaultManager.createVault(vaultName); + await Promise.all([ + vaultManager.renameVault(vaultId, secondVaultName), + vaultManager.renameVault(vaultId, thirdVaultName), + ]); + const vaultNameTest = (await vaultManager.getVaultMeta(vaultId)) + ?.vaultName; + expect(vaultNameTest).toBe(thirdVaultName); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); test('can concurrently open and rename the same vault', async () => { - const vaultId = await vaultManager.createVault(vaultName); - await Promise.all([ - vaultManager.renameVault(vaultId, secondVaultName), - vaultManager.withVaults([vaultId], async (vault) => vault.vaultId), - ]); - const vaultNameTest = (await vaultManager.getVaultMeta(vaultId)).name; - expect(vaultNameTest).toBe(secondVaultName); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId = await vaultManager.createVault(vaultName); + await Promise.all([ + vaultManager.renameVault(vaultId, secondVaultName), + vaultManager.withVaults([vaultId], async (vault) => vault.vaultId), + ]); + const vaultNameTest = (await vaultManager.getVaultMeta(vaultId)) + ?.vaultName; + expect(vaultNameTest).toBe(secondVaultName); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); + // FIXME: Why is this a test? we don't care if we + // can modify a vault in this testing scope test('can save the commit state of a vault', async () => { - const vaultId = await vaultManager.createVault(vaultName); - await vaultManager.withVaults([vaultId], async (vault) => { - await vault.writeF(async (efs) => { - await efs.writeFile('test', 'test'); - }); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); + try { + const vaultId = await vaultManager.createVault(vaultName); + await vaultManager.withVaults([vaultId], async (vault) => { + await vault.writeF(async (efs) => { + await efs.writeFile('test', 'test'); + }); + }); - await vaultManager.stop(); - await vaultManager.start(); + await vaultManager.stop(); + await vaultManager.start(); - const read = await vaultManager.withVaults( - [vaultId], - async (vaultLoaded) => { - return await vaultLoaded.readF(async (efs) => { - return await efs.readFile('test', { encoding: 'utf8' }); - }); - }, - ); - expect(read).toBe('test'); - }); - test('able to recover metadata after complex operations', async () => { - const vaultNames = ['Vault1', 'Vault2', 'Vault3', 'Vault4', 'Vault5']; - const alteredVaultNames = [ - 'Vault1', - 'Vault2', - 'Vault3', - 'Vault6', - 'Vault10', - ]; - for (const vaultName of vaultNames) { - await vaultManager.createVault(vaultName as VaultName); + const read = await vaultManager.withVaults( + [vaultId], + async (vaultLoaded) => { + return await vaultLoaded.readF(async (efs) => { + return await efs.readFile('test', { encoding: 'utf8' }); + }); + }, + ); + expect(read).toBe('test'); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); } - const v5 = await vaultManager.getVaultId('Vault5' as VaultName); - expect(v5).not.toBeUndefined(); - await vaultManager.destroyVault(v5!); - const v4 = await vaultManager.getVaultId('Vault4' as VaultName); - expect(v4).toBeTruthy(); - await vaultManager.renameVault(v4!, 'Vault10' as VaultName); - const v6 = await vaultManager.createVault('Vault6' as VaultName); - - await vaultManager.withVaults([v6], async (vault6) => { - await vault6.writeF(async (efs) => { - await efs.writeFile('reloaded', 'reload'); - }); + }); + test('Do actions on a vault using `withVault`', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); + try { + const vault1 = await vaultManager.createVault('testVault1' as VaultName); + const vault2 = await vaultManager.createVault('testVault2' as VaultName); + const vaults = [vault1, vault2]; - const vn: Array = []; - (await vaultManager.listVaults()).forEach((_, vaultName) => - vn.push(vaultName), - ); - expect(vn.sort()).toEqual(alteredVaultNames.sort()); - await vaultManager.stop(); - await vaultManager.start(); - await vaultManager.createVault('Vault7' as VaultName); - - const v10 = await vaultManager.getVaultId('Vault10' as VaultName); - expect(v10).not.toBeUndefined(); - alteredVaultNames.push('Vault7'); - expect((await vaultManager.listVaults()).size).toEqual( - alteredVaultNames.length, - ); - const vnAltered: Array = []; - (await vaultManager.listVaults()).forEach((_, vaultName) => - vnAltered.push(vaultName), - ); - expect(vnAltered.sort()).toEqual(alteredVaultNames.sort()); - const file = await vaultManager.withVaults([v6], async (reloadedVault) => { - return await reloadedVault.readF(async (efs) => { - return await efs.readFile('reloaded', { encoding: 'utf8' }); + await vaultManager.withVaults(vaults, async (vault1, vault2) => { + expect(vault1.vaultId).toEqual(vaults[0]); + expect(vault2.vaultId).toEqual(vaults[1]); + await vault1.writeF(async (fs) => { + await fs.writeFile('test', 'test1'); + }); + await vault2.writeF(async (fs) => { + await fs.writeFile('test', 'test2'); + }); }); - }); - expect(file).toBe('reload'); - }); - test('clone vaults from a remote keynode using a vault name', async () => { - await expect(() => - vaultManager.cloneVault( - remoteKeynode1.keyManager.getNodeId(), - 'not-existing' as VaultName, - ), - ).rejects.toThrow(vaultErrors.ErrorVaultsVaultUndefined); - await vaultManager.cloneVault( - remoteKeynode1.keyManager.getNodeId(), - vaultName, - ); - const vaultId = await vaultManager.getVaultId(vaultName); - if (vaultId === undefined) fail('VaultId is not found.'); - const [file, secretsList] = await vaultManager.withVaults( - [vaultId], - async (vaultClone) => { - const file = await vaultClone.readF(async (efs) => { - return await efs.readFile(secretNames[0], { encoding: 'utf8' }); + await vaultManager.withVaults(vaults, async (vault1, vault2) => { + const a = await vault1.readF((fs) => { + return fs.readFile('test'); }); - const secretsList = (await vaultOps.listSecrets(vaultClone)).sort(); - return [file, secretsList]; - }, - ); - expect(file).toBe('success?'); - expect(secretsList).toStrictEqual(secretNames.slice(0, 2).sort()); - }, 100000); - test('clone and pull vaults using a vault id', async () => { - const vaultId = await vaultManager.cloneVault( - remoteKeynode1.keyManager.getNodeId(), - remoteVaultId, - ); - await vaultManager.withVaults([vaultId], async (vaultClone) => { - const file = await vaultClone.readF(async (efs) => { - return await efs.readFile(secretNames[0], { encoding: 'utf8' }); + const b = await vault2.readF((fs) => { + return fs.readFile('test'); + }); + + expect(a.toString()).toEqual('test1'); + expect(b.toString()).toEqual('test2'); }); - expect(file).toBe('success?'); - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.slice(0, 2).sort(), + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + describe('With remote agents', () => { + let allDataDir: string; + let keyManager: KeyManager; + let fwdProxy: ForwardProxy; + let nodeGraph: NodeGraph; + let nodeConnectionManager: NodeConnectionManager; + let remoteKeynode1: PolykeyAgent, remoteKeynode2: PolykeyAgent; + let localNodeId: NodeId; + let localNodeIdEncoded: NodeIdEncoded; + + beforeAll(async () => { + // Creating agents. + allDataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), ); - }); - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - for (const secret of secretNames.slice(2)) { - await vaultOps.addSecret(remoteVault, secret, 'second success?'); - } - }, - ); + remoteKeynode1 = await PolykeyAgent.createPolykeyAgent({ + password, + logger: logger.getChild('Remote Keynode 1'), + nodePath: path.join(allDataDir, 'remoteKeynode1'), + }); + remoteKeynode1Id = remoteKeynode1.keyManager.getNodeId(); + remoteKeynode1IdEncoded = nodesUtils.encodeNodeId(remoteKeynode1Id); + remoteKeynode2 = await PolykeyAgent.createPolykeyAgent({ + password, + logger: logger.getChild('Remote Keynode 2'), + nodePath: path.join(allDataDir, 'remoteKeynode2'), + }); + remoteKeynode2Id = remoteKeynode2.keyManager.getNodeId(); + remoteKeynode2IdEncoded = nodesUtils.encodeNodeId(remoteKeynode2Id); - await vaultManager.pullVault({ vaultId }); + // Adding details to each agent. + await remoteKeynode1.nodeGraph.setNode(remoteKeynode2Id, { + host: remoteKeynode2.revProxy.getIngressHost(), + port: remoteKeynode2.revProxy.getIngressPort(), + }); + await remoteKeynode2.nodeGraph.setNode(remoteKeynode1Id, { + host: remoteKeynode1.revProxy.getIngressHost(), + port: remoteKeynode1.revProxy.getIngressPort(), + }); - await vaultManager.withVaults([vaultId], async (vaultClone) => { - const file = await vaultClone.readF(async (efs) => { - return await efs.readFile(secretNames[2], { encoding: 'utf8' }); + await remoteKeynode1.gestaltGraph.setNode({ + id: remoteKeynode2IdEncoded, + chain: {}, + }); + await remoteKeynode2.gestaltGraph.setNode({ + id: remoteKeynode1IdEncoded, + chain: {}, }); - expect(file).toBe('second success?'); - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.sort(), - ); - }); - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - for (const secret of secretNames.slice(2)) { - await vaultOps.deleteSecret(remoteVault, secret); - } - }, - ); - }); - test('reject cloning and pulling when permissions are not set', async () => { - await remoteKeynode1.vaultManager.unshareVault( - remoteVaultId, - localKeynodeId, - ); - await expect(() => - vaultManager.cloneVault(remoteKeynode1Id, remoteVaultId), - ).rejects.toThrow(vaultErrors.ErrorVaultsPermissionDenied); - expect((await vaultManager.listVaults()).size).toBe(0); - await remoteKeynode1.vaultManager.shareVault(remoteVaultId, localKeynodeId); - const clonedVaultId = await vaultManager.cloneVault( - remoteKeynode1Id, - remoteVaultId, - ); - await vaultManager.withVaults([clonedVaultId], async (clonedVault) => { - const file = await clonedVault.readF(async (efs) => { - return await efs.readFile(secretNames[0], { encoding: 'utf8' }); + remoteVaultId = await remoteKeynode1.vaultManager.createVault(vaultName); + + // FIXME: we need to create state to clone. + // await remoteKeynode1.vaultManager.withVaults( + // [remoteVaultId], + // async (remoteVault) => { + // for (const secret of secretNames.slice(0, 2)) { + // await vaultOps.addSecret(remoteVault, secret, 'success?'); + // } + // }, + // ); + }); + afterAll(async () => { + await remoteKeynode2.stop(); + await remoteKeynode2.destroy(); + await remoteKeynode1.stop(); + await remoteKeynode1.destroy(); + await fs.promises.rm(allDataDir, { + recursive: true, + force: true, }); - expect(file).toBe('success?'); }); - await remoteKeynode1.vaultManager.unshareVault( - remoteVaultId, - localKeynodeId, - ); - await expect(() => - vaultManager.pullVault({ vaultId: clonedVaultId }), - ).rejects.toThrow(vaultErrors.ErrorVaultsPermissionDenied); + beforeEach(async () => { + nodeGraph = await NodeGraph.createNodeGraph({ + db, + keyManager: dummyKeyManager, + logger, + }); + fwdProxy = new ForwardProxy({ + authToken: 'auth', + logger, + }); - await vaultManager.withVaults([clonedVaultId], async (clonedVault) => { - await expect(vaultOps.listSecrets(clonedVault)).resolves.toStrictEqual( - secretNames.slice(0, 2), - ); + keyManager = await KeyManager.createKeyManager({ + keysPath: path.join(allDataDir, 'allKeyManager'), + password: 'password', + logger, + }); + localNodeId = keyManager.getNodeId(); + localNodeIdEncoded = nodesUtils.encodeNodeId(localNodeId); + + const tlsConfig: TLSConfig = { + keyPrivatePem: keyManager.getRootKeyPairPem().privateKey, + certChainPem: await keyManager.getRootCertChainPem(), + }; + + await fwdProxy.start({ tlsConfig }); + const dummyRevProxy = { + getIngressHost: () => 'localhost' as Host, + getIngressPort: () => 0 as Port, + } as ReverseProxy; + + nodeConnectionManager = new NodeConnectionManager({ + keyManager, + nodeGraph, + fwdProxy, + revProxy: dummyRevProxy, + logger, + }); + await nodeConnectionManager.start(); + + await nodeGraph.setNode(remoteKeynode1Id, { + host: remoteKeynode1.revProxy.getIngressHost(), + port: remoteKeynode1.revProxy.getIngressPort(), + }); + await nodeGraph.setNode(remoteKeynode2Id, { + host: remoteKeynode2.revProxy.getIngressHost(), + port: remoteKeynode2.revProxy.getIngressPort(), + }); }); - await remoteKeynode1.vaultManager.shareVault(remoteVaultId, localKeynodeId); - }); - test('throw when trying to commit to a cloned vault', async () => { - const clonedVaultId = await vaultManager.cloneVault( - remoteKeynode1Id, - remoteVaultId, - ); - await vaultManager.withVaults([clonedVaultId], async (clonedVault) => { - await expect( - vaultOps.renameSecret(clonedVault, secretNames[0], secretNames[2]), - ).rejects.toThrow(vaultErrors.ErrorVaultImmutable); + afterEach(async () => { + await nodeConnectionManager.stop(); + await fwdProxy.stop(); + await nodeGraph.stop(); + await nodeGraph.destroy(); + await keyManager.stop(); + await keyManager.destroy(); }); - }); - test( - 'clone and pull from other cloned vaults', - async () => { - const clonedVaultRemote2Id = await remoteKeynode2.vaultManager.cloneVault( - remoteKeynode1Id, - remoteVaultId, - ); - await localKeynode.acl.setNodePerm(remoteKeynode2Id, { - gestalt: { - notify: null, - }, - vaults: {}, + + test('clone vaults from a remote keynode using a vault name', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); - await remoteKeynode2.vaultManager.shareVault( - clonedVaultRemote2Id, - localKeynodeId, - ); - const notification = ( - await localKeynode.notificationsManager.readNotifications() - ).pop(); - expect(notification?.data['type']).toBe('VaultShare'); - expect(notification?.data['vaultId']).toBe( - idUtils.toString(clonedVaultRemote2Id), - ); - expect(notification?.data['vaultName']).toBe(vaultName); - expect(notification?.data['actions']['clone']).toBeNull(); - expect(notification?.data['actions']['pull']).toBeNull(); - await vaultManager.cloneVault(remoteKeynode2Id, clonedVaultRemote2Id); - const vaultIdClone = await vaultManager.getVaultId(vaultName); - expect(vaultIdClone).not.toBeUndefined(); - await vaultManager.withVaults([vaultIdClone!], async (vaultClone) => { - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.slice(0, 2).sort(), + try { + // Setting permissions + await remoteKeynode1.gestaltGraph.setNode({ + id: localNodeIdEncoded, + chain: {}, + }); + await remoteKeynode1.gestaltGraph.setGestaltActionByNode( + localNodeId, + 'scan', + ); + await remoteKeynode1.acl.setVaultAction( + remoteVaultId, + localNodeId, + 'clone', + ); + await remoteKeynode1.acl.setVaultAction( + remoteVaultId, + localNodeId, + 'pull', ); - }); - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - for (const secret of secretNames.slice(2)) { - await vaultOps.addSecret(remoteVault, secret, 'success?'); - } - }, - ); + await expect(() => + vaultManager.cloneVault( + remoteKeynode1Id, + 'not-existing' as VaultName, + ), + ).rejects.toThrow(vaultsErrors.ErrorVaultsVaultUndefined); - await vaultManager.pullVault({ - vaultId: vaultIdClone!, - pullNodeId: remoteKeynode1Id, - pullVaultNameOrId: remoteVaultId, - }); - await vaultManager.withVaults([vaultIdClone!], async (vaultClone) => { - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.sort(), + await vaultManager.cloneVault(remoteKeynode1Id, vaultName); + const vaultId = await vaultManager.getVaultId(vaultName); + if (vaultId === undefined) fail('VaultId is not found.'); + const [file, secretsList] = await vaultManager.withVaults( + [vaultId], + async (vaultClone) => { + const file = await vaultClone.readF(async (efs) => { + return await efs.readFile(secretNames[0], { encoding: 'utf8' }); + }); + const secretsList = (await vaultOps.listSecrets(vaultClone)).sort(); + return [file, secretsList]; + }, ); + expect(file).toBe('success?'); + expect(secretsList).toStrictEqual(secretNames.slice(0, 2).sort()); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('clone and pull vaults using a vault id', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); + try { + const vaultId = await vaultManager.cloneVault( + remoteKeynode1.keyManager.getNodeId(), + remoteVaultId, + ); + await vaultManager.withVaults([vaultId], async (vaultClone) => { + const file = await vaultClone.readF(async (efs) => { + return await efs.readFile(secretNames[0], { encoding: 'utf8' }); + }); + expect(file).toBe('success?'); + expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( + secretNames.slice(0, 2).sort(), + ); + }); - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - for (const secret of secretNames.slice(2)) { - await vaultOps.deleteSecret(remoteVault, secret); - } - }, - ); - }, - global.defaultTimeout * 2, - ); - // Irrelevant for the moment as cloned vaults are immutable but will - // be useful in the future - test.skip('manage pulling from different remotes', async () => { - const clonedVaultRemote2Id = await remoteKeynode2.vaultManager.cloneVault( - remoteKeynode1Id, - remoteVaultId, - ); + await remoteKeynode1.vaultManager.withVaults( + [remoteVaultId], + async (remoteVault) => { + for (const secret of secretNames.slice(2)) { + await vaultOps.addSecret(remoteVault, secret, 'second success?'); + } + }, + ); - await remoteKeynode2.vaultManager.shareVault( - clonedVaultRemote2Id, - localKeynodeId, - ); + await vaultManager.pullVault({ vaultId }); - const vaultCloneId = await vaultManager.cloneVault( - remoteKeynode2Id, - clonedVaultRemote2Id, - ); + await vaultManager.withVaults([vaultId], async (vaultClone) => { + const file = await vaultClone.readF(async (efs) => { + return await efs.readFile(secretNames[2], { encoding: 'utf8' }); + }); + expect(file).toBe('second success?'); + expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( + secretNames.sort(), + ); + }); - await remoteKeynode1.vaultManager.withVaults( - [remoteVaultId], - async (remoteVault) => { - await vaultOps.addSecret(remoteVault, secretNames[2], 'success?'); - }, - ); - await vaultManager.pullVault({ - vaultId: vaultCloneId, - pullNodeId: remoteKeynode1Id, - pullVaultNameOrId: vaultName, + await remoteKeynode1.vaultManager.withVaults( + [remoteVaultId], + async (remoteVault) => { + for (const secret of secretNames.slice(2)) { + await vaultOps.deleteSecret(remoteVault, secret); + } + }, + ); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); - - await vaultManager.withVaults([vaultCloneId], async (vaultClone) => { - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.slice(0, 3).sort(), - ); + test('should reject cloning when permissions are not set', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // Should reject with no permissions set + await expect(() => + vaultManager.cloneVault(remoteKeynode1Id, remoteVaultId), + ).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); + // No new vault created + expect((await vaultManager.listVaults()).size).toBe(0); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); + test('should reject Pulling when permissions are not set', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // Setting permissions + await remoteKeynode1.gestaltGraph.setNode({ + id: localNodeIdEncoded, + chain: {}, + }); + await remoteKeynode1.gestaltGraph.setGestaltActionByNode( + localNodeId, + 'scan', + ); + await remoteKeynode1.acl.setVaultAction( + remoteVaultId, + localNodeId, + 'clone', + ); - await remoteKeynode2.vaultManager.withVaults( - [clonedVaultRemote2Id], - async (clonedVaultRemote2) => { - await vaultOps.addSecret( - clonedVaultRemote2, - secretNames[3], - 'second success?', + const clonedVaultId = await vaultManager.cloneVault( + remoteKeynode1Id, + remoteVaultId, ); - }, - ); - await vaultManager.pullVault({ vaultId: vaultCloneId }); - await vaultManager.withVaults([vaultCloneId], async (vaultClone) => { - expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( - secretNames.sort(), - ); + await expect(() => + vaultManager.pullVault({ vaultId: clonedVaultId }), + ).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); - }); - test('Do actions on a vault using `withVault`', async () => { - const vault1 = await vaultManager.createVault('testVault1' as VaultName); - const vault2 = await vaultManager.createVault('testVault2' as VaultName); - const vaults = [vault1, vault2]; - - await vaultManager.withVaults(vaults, async (vault1, vault2) => { - expect(vault1.vaultId).toEqual(vaults[0]); - expect(vault2.vaultId).toEqual(vaults[1]); - await vault1.writeF(async (fs) => { - await fs.writeFile('test', 'test1'); - }); - await vault2.writeF(async (fs) => { - await fs.writeFile('test', 'test2'); + test.todo("test pulling a vault that isn't remote"); + test.todo("test cloning a vault that doesn't exist"); + // FIXME: + // test( + // 'clone and pull from other cloned vaults', + // async () => { + // const vaultManager = await VaultManager.createVaultManager({ + // vaultsPath, + // keyManager: dummyKeyManager,` + // gestaltGraph: {} as GestaltGraph, + // nodeConnectionManager: {} as NodeConnectionManager, + // acl: {} as ACL, + // notificationsManager: {} as NotificationsManager, + // db, + // logger: logger.getChild(VaultManager.name), + // }); + // try { + // const clonedVaultRemote2Id = + // await remoteKeynode2.vaultManager.cloneVault( + // remoteKeynode1Id, + // remoteVaultId, + // ); + // await localKeynode.acl.setNodePerm(remoteKeynode2Id, { + // gestalt: { + // notify: null, + // }, + // vaults: {}, + // }); + // await remoteKeynode2.vaultManager.shareVault( + // clonedVaultRemote2Id, + // nodeId, + // ); + // const notification = ( + // await localKeynode.notificationsManager.readNotifications() + // ).pop(); + // expect(notification?.data['type']).toBe('VaultShare'); + // expect(notification?.data['vaultId']).toBe( + // idUtils.toString(clonedVaultRemote2Id), + // ); + // expect(notification?.data['vaultName']).toBe(vaultName); + // expect(notification?.data['actions']['clone']).toBeNull(); + // expect(notification?.data['actions']['pull']).toBeNull(); + // await vaultManager.cloneVault(remoteKeynode2Id, clonedVaultRemote2Id); + // const vaultIdClone = await vaultManager.getVaultId(vaultName); + // expect(vaultIdClone).not.toBeUndefined(); + // await vaultManager.withVaults([vaultIdClone!], async (vaultClone) => { + // expect( + // (await vaultOps.listSecrets(vaultClone)).sort(), + // ).toStrictEqual(secretNames.slice(0, 2).sort()); + // }); + // + // await remoteKeynode1.vaultManager.withVaults( + // [remoteVaultId], + // async (remoteVault) => { + // for (const secret of secretNames.slice(2)) { + // await vaultOps.addSecret(remoteVault, secret, 'success?'); + // } + // }, + // ); + // + // await vaultManager.pullVault({ + // vaultId: vaultIdClone!, + // pullNodeId: remoteKeynode1Id, + // pullVaultNameOrId: remoteVaultId, + // }); + // await vaultManager.withVaults([vaultIdClone!], async (vaultClone) => { + // expect( + // (await vaultOps.listSecrets(vaultClone)).sort(), + // ).toStrictEqual(secretNames.sort()); + // }); + // + // await remoteKeynode1.vaultManager.withVaults( + // [remoteVaultId], + // async (remoteVault) => { + // for (const secret of secretNames.slice(2)) { + // await vaultOps.deleteSecret(remoteVault, secret); + // } + // }, + // ); + // } finally { + // await vaultManager?.stop(); + // await vaultManager?.destroy(); + // } + // }, + // global.defaultTimeout * 2, + // ); + // Irrelevant for the moment as cloned vaults are immutable but will + // be useful in the future + test.skip('manage pulling from different remotes', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); - }); + try { + const clonedVaultRemote2Id = + await remoteKeynode2.vaultManager.cloneVault( + remoteKeynode1Id, + remoteVaultId, + ); - await vaultManager.withVaults(vaults, async (vault1, vault2) => { - const a = await vault1.readF((fs) => { - return fs.readFile('test'); - }); - const b = await vault2.readF((fs) => { - return fs.readFile('test'); - }); + await remoteKeynode2.vaultManager.shareVault( + clonedVaultRemote2Id, + nodeId, + ); + + const vaultCloneId = await vaultManager.cloneVault( + remoteKeynode2Id, + clonedVaultRemote2Id, + ); + + await remoteKeynode1.vaultManager.withVaults( + [remoteVaultId], + async (remoteVault) => { + await vaultOps.addSecret(remoteVault, secretNames[2], 'success?'); + }, + ); + await vaultManager.pullVault({ + vaultId: vaultCloneId, + pullNodeId: remoteKeynode1Id, + pullVaultNameOrId: vaultName, + }); + + await vaultManager.withVaults([vaultCloneId], async (vaultClone) => { + expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( + secretNames.slice(0, 3).sort(), + ); + }); - expect(a.toString()).toEqual('test1'); - expect(b.toString()).toEqual('test2'); + await remoteKeynode2.vaultManager.withVaults( + [clonedVaultRemote2Id], + async (clonedVaultRemote2) => { + await vaultOps.addSecret( + clonedVaultRemote2, + secretNames[3], + 'second success?', + ); + }, + ); + await vaultManager.pullVault({ vaultId: vaultCloneId }); + + await vaultManager.withVaults([vaultCloneId], async (vaultClone) => { + expect((await vaultOps.listSecrets(vaultClone)).sort()).toStrictEqual( + secretNames.sort(), + ); + }); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); - }); - // FIXME: remove? not relevant anymore? - test.skip('WorkingDirIndex is maintained across certain actions', async () => { - const vaultId = await vaultManager.createVault('testVault1' as VaultName); - const oid2 = await vaultManager.withVaults([vaultId], async (vault) => { - await vault.writeF(async (fs) => { - await fs.writeFile('test1', 'test1'); + // FIXME: + test.skip('able to recover metadata after complex operations', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); - await vault.writeF(async (fs) => { - await fs.writeFile('test2', 'test2'); + try { + const vaultNames = ['Vault1', 'Vault2', 'Vault3', 'Vault4', 'Vault5']; + const alteredVaultNames = [ + 'Vault1', + 'Vault2', + 'Vault3', + 'Vault6', + 'Vault10', + ]; + for (const vaultName of vaultNames) { + await vaultManager.createVault(vaultName as VaultName); + } + const v5 = await vaultManager.getVaultId('Vault5' as VaultName); + expect(v5).not.toBeUndefined(); + await vaultManager.destroyVault(v5!); + const v4 = await vaultManager.getVaultId('Vault4' as VaultName); + expect(v4).toBeTruthy(); + await vaultManager.renameVault(v4!, 'Vault10' as VaultName); + const v6 = await vaultManager.createVault('Vault6' as VaultName); + + await vaultManager.withVaults([v6], async (vault6) => { + await vault6.writeF(async (efs) => { + await efs.writeFile('reloaded', 'reload'); + }); + }); + + const vn: Array = []; + (await vaultManager.listVaults()).forEach((_, vaultName) => + vn.push(vaultName), + ); + expect(vn.sort()).toEqual(alteredVaultNames.sort()); + await vaultManager.stop(); + await vaultManager.start(); + await vaultManager.createVault('Vault7' as VaultName); + + const v10 = await vaultManager.getVaultId('Vault10' as VaultName); + expect(v10).not.toBeUndefined(); + alteredVaultNames.push('Vault7'); + expect((await vaultManager.listVaults()).size).toEqual( + alteredVaultNames.length, + ); + const vnAltered: Array = []; + (await vaultManager.listVaults()).forEach((_, vaultName) => + vnAltered.push(vaultName), + ); + expect(vnAltered.sort()).toEqual(alteredVaultNames.sort()); + const file = await vaultManager.withVaults( + [v6], + async (reloadedVault) => { + return await reloadedVault.readF(async (efs) => { + return await efs.readFile('reloaded', { encoding: 'utf8' }); + }); + }, + ); + + expect(file).toBe('reload'); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test.skip('throw when trying to commit to a cloned vault', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); - const oid2 = (await vault.log(undefined, 1)).pop()!.commitId; - await vault.writeF(async (fs) => { - await fs.writeFile('test3', 'test3'); + try { + const clonedVaultId = await vaultManager.cloneVault( + remoteKeynode1Id, + remoteVaultId, + ); + await vaultManager.withVaults([clonedVaultId], async (clonedVault) => { + await expect( + vaultOps.renameSecret(clonedVault, secretNames[0], secretNames[2]), + ).rejects.toThrow(vaultsErrors.ErrorVaultRemoteDefined); + }); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + }); + test('handleScanVaults should list all vaults with permissions', async () => { + // 1. we need to set up state. + const acl = await ACL.createACL({ + db, + logger, + }); + const gestaltGraph = await GestaltGraph.createGestaltGraph({ + db, + acl, + logger, + }); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + nodeConnectionManager: {} as NodeConnectionManager, + acl, + gestaltGraph, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // Setting up state. + const nodeId1 = testsUtils.generateRandomNodeId(); + const nodeId2 = testsUtils.generateRandomNodeId(); + await gestaltGraph.setNode({ + id: nodesUtils.encodeNodeId(nodeId1), + chain: {}, }); - await vault.version(oid2); - return oid2; - }); - await vaultManager.closeVault(vaultId); - await vaultManager.withVaults([vaultId], async (vault) => { - const vaultInternal = vault as VaultInternal; - const currentOid = ''; // FIXME: vaultInternal.getworkingDirIndex(); - await vault.readF(async (fs) => { - expect(await fs.readdir('.')).toEqual(['test1', 'test2']); + await gestaltGraph.setNode({ + id: nodesUtils.encodeNodeId(nodeId2), + chain: {}, }); - expect(currentOid).toStrictEqual(oid2); - }); + await gestaltGraph.setGestaltActionByNode(nodeId1, 'scan'); + + const vault1 = await vaultManager.createVault('testVault1' as VaultName); + const vault2 = await vaultManager.createVault('testVault2' as VaultName); + const vault3 = await vaultManager.createVault('testVault3' as VaultName); + + // Setting permissions + await acl.setVaultAction(vault1, nodeId1, 'clone'); + await acl.setVaultAction(vault1, nodeId1, 'pull'); + await acl.setVaultAction(vault2, nodeId1, 'clone'); + // No permissions for vault3 + + // scanning vaults + const gen = vaultManager.handleScanVaults(nodeId1); + const vaults: Record = {}; + for await (const vault of gen) { + vaults[vault.vaultId] = [vault.vaultName, vault.vaultPermissions]; + } + expect(vaults[vault1]).toEqual(['testVault1', ['clone', 'pull']]); + expect(vaults[vault2]).toEqual(['testVault2', ['clone']]); + expect(vaults[vault3]).toBeUndefined(); + + // Should throw due to no permission + await expect(async () => { + for await (const _ of vaultManager.handleScanVaults(nodeId2)) { + // Should throw + } + }).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); + // Should throw due to lack of scan permission + await gestaltGraph.setGestaltActionByNode(nodeId2, 'notify'); + await expect(async () => { + for await (const _ of vaultManager.handleScanVaults(nodeId2)) { + // Should throw + } + }).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); + } finally { + await vaultManager.stop(); + await vaultManager.destroy(); + await gestaltGraph.stop(); + await gestaltGraph.destroy(); + await acl.stop(); + await acl.destroy(); + } }); - describe('Scanning nodes', () => { - let server: PolykeyAgent; - let serverNodeId: NodeId; - let serverNodeAddress: NodeAddress; - let allDataDir: string; + test('ScanVaults should get all vaults with permissions from remote node', async () => { + // 1. we need to set up state. + const remoteAgent = await PolykeyAgent.createPolykeyAgent({ + password: 'password', + nodePath: path.join(dataDir, 'remoteNode'), + logger, + }); + const acl = await ACL.createACL({ + db, + logger, + }); + const gestaltGraph = await GestaltGraph.createGestaltGraph({ + db, + acl, + logger, + }); + const nodeGraph = await NodeGraph.createNodeGraph({ + db, + keyManager: dummyKeyManager, + logger, + }); + const fwdProxy = new ForwardProxy({ + authToken: 'auth', + logger, + }); + const keyManager = await KeyManager.createKeyManager({ + keysPath: path.join(dataDir, 'keys'), + password: 'password', + logger, + }); + await fwdProxy.start({ + tlsConfig: { + keyPrivatePem: keyManager.getRootKeyPairPem().privateKey, + certChainPem: await keyManager.getRootCertChainPem(), + }, + }); + const nodeConnectionManager = new NodeConnectionManager({ + keyManager, + logger, + nodeGraph, + fwdProxy, + revProxy: {} as ReverseProxy, + connConnectTime: 1000, + }); + await nodeConnectionManager.start(); + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager, + nodeConnectionManager, + acl, + gestaltGraph, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // Setting up state. + const targetNodeId = remoteAgent.keyManager.getNodeId(); + const nodeId1 = keyManager.getNodeId(); - beforeAll(async () => { - allDataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'polykey-test-'), - ); - server = await PolykeyAgent.createPolykeyAgent({ - password, - logger, - nodePath: path.join(allDataDir, 'server'), + // Letting nodeGraph know where the remote agent is + await nodeGraph.setNode(targetNodeId, { + host: 'localhost' as Host, + port: remoteAgent.revProxy.getIngressPort(), }); - serverNodeId = server.keyManager.getNodeId(); - serverNodeAddress = { - host: server.revProxy.getIngressHost(), - port: server.revProxy.getIngressPort(), - }; - }, global.polykeyStartupTimeout * 2); - afterAll(async () => { - await server.stop(); - await server.destroy(); - await fs.promises.rm(allDataDir, { force: true, recursive: true }); - }); - test('scans the targets vaults', async () => { - await localKeynode.nodeGraph.setNode(serverNodeId, serverNodeAddress); - await server.gestaltGraph.setNode({ - id: nodesUtils.encodeNodeId(keyManager.getNodeId()), + await remoteAgent.gestaltGraph.setNode({ + id: nodesUtils.encodeNodeId(nodeId1), chain: {}, }); - await server.gestaltGraph.setGestaltActionByNode( - keyManager.getNodeId(), - 'scan', + + const vault1 = await remoteAgent.vaultManager.createVault( + 'testVault1' as VaultName, + ); + const vault2 = await remoteAgent.vaultManager.createVault( + 'testVault2' as VaultName, + ); + const vault3 = await remoteAgent.vaultManager.createVault( + 'testVault3' as VaultName, ); - const vaultName1 = 'vn1' as VaultName; - const vaultName2 = 'vn2' as VaultName; - const vaultName3 = 'vn3' as VaultName; - const v1Id = await server.vaultManager.createVault(vaultName1); - const v2Id = await server.vaultManager.createVault(vaultName2); - const v3Id = await server.vaultManager.createVault(vaultName3); + // Scanning vaults - const vaultList: Array<[VaultName, VaultId]> = []; + // Should throw due to no permission + await expect(async () => { + for await (const _ of vaultManager.scanVaults(targetNodeId)) { + // Should throw + } + }).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); + // Should throw due to lack of scan permission + await remoteAgent.gestaltGraph.setGestaltActionByNode(nodeId1, 'notify'); + await expect(async () => { + for await (const _ of vaultManager.scanVaults(targetNodeId)) { + // Should throw + } + }).rejects.toThrow(vaultsErrors.ErrorVaultsPermissionDenied); - vaultList.push([vaultName1, v1Id]); - vaultList.push([vaultName2, v2Id]); - vaultList.push([vaultName3, v3Id]); + // Setting permissions + await remoteAgent.gestaltGraph.setGestaltActionByNode(nodeId1, 'scan'); + await remoteAgent.acl.setVaultAction(vault1, nodeId1, 'clone'); + await remoteAgent.acl.setVaultAction(vault1, nodeId1, 'pull'); + await remoteAgent.acl.setVaultAction(vault2, nodeId1, 'clone'); + // No permissions for vault3 - const vaults = await vaultManager.scanNodeVaults(serverNodeId); - expect(vaults.sort()).toStrictEqual(vaultList.sort()); + const gen = vaultManager.scanVaults(targetNodeId); + const vaults: Record = {}; + for await (const vault of gen) { + vaults[vault.vaultIdEncoded] = [ + vault.vaultName, + vault.vaultPermissions, + ]; + } - await server.gestaltGraph.unsetGestaltActionByNode( - keyManager.getNodeId(), - 'scan', - ); + expect(vaults[vaultsUtils.encodeVaultId(vault1)]).toEqual([ + 'testVault1', + ['clone', 'pull'], + ]); + expect(vaults[vaultsUtils.encodeVaultId(vault2)]).toEqual([ + 'testVault2', + ['clone'], + ]); + expect(vaults[vaultsUtils.encodeVaultId(vault3)]).toBeUndefined(); + } finally { + await vaultManager.stop(); + await vaultManager.destroy(); + await nodeConnectionManager.stop(); + await fwdProxy.stop(); + await nodeGraph.stop(); + await nodeGraph.destroy(); + await gestaltGraph.stop(); + await gestaltGraph.destroy(); + await acl.stop(); + await acl.destroy(); + await remoteAgent.stop(); + await remoteAgent.destroy(); + } + }); + // Locking tests + test('stopping respects locks', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); - test('fails to scan the targets vaults without permission', async () => { - await localKeynode.nodeGraph.setNode(serverNodeId, serverNodeAddress); - await server.gestaltGraph.setNode({ - id: nodesUtils.encodeNodeId(keyManager.getNodeId()), - chain: {}, + try { + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + // Create the vault + const vaultId = await vaultManager.createVault('vaultName'); + // Getting and holding the lock + const vaultAndLock = vaultMap.get(vaultId.toString() as VaultIdString)!; + const lock = vaultAndLock.lock; + const vault = vaultAndLock.vault!; + const release = await lock.acquireWrite(); + // Try to destroy + const closeP = vaultManager.closeVault(vaultId); + await sleep(1000); + // Shouldn't be closed + expect(vault[running]).toBe(true); + expect( + vaultMap.get(vaultId.toString() as VaultIdString)!.vault, + ).toBeDefined(); + // Release the lock + release(); + await closeP; + expect(vault[running]).toBe(false); + expect(vaultMap.get(vaultId.toString() as VaultIdString)).toBeUndefined(); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('destroying respects locks', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + // Create the vault + const vaultId = await vaultManager.createVault('vaultName'); + // Getting and holding the lock + const vaultAndLock = vaultMap.get(vaultId.toString() as VaultIdString)!; + const lock = vaultAndLock.lock; + const vault = vaultAndLock.vault!; + const release = await lock.acquireWrite(); + // Try to destroy + const destroyP = vaultManager.destroyVault(vaultId); + await sleep(1000); + // Shouldn't be destroyed + expect(vault[destroyed]).toBe(false); + expect( + vaultMap.get(vaultId.toString() as VaultIdString)!.vault, + ).toBeDefined(); + // Release the lock + release(); + await destroyP; + expect(vault[destroyed]).toBe(true); + expect(vaultMap.get(vaultId.toString() as VaultIdString)).toBeUndefined(); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('withVault respects locks', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + // Create the vault + const vaultId = await vaultManager.createVault('vaultName'); + // Getting and holding the lock + const vaultAndLock = vaultMap.get(vaultId.toString() as VaultIdString)!; + const lock = vaultAndLock.lock; + const release = await lock.acquireWrite(); + // Try to use vault + let finished = false; + const withP = vaultManager.withVaults([vaultId], async () => { + finished = true; }); + await sleep(1000); + // Shouldn't be destroyed + expect(finished).toBe(false); + // Release the lock + release(); + await withP; + expect(finished).toBe(true); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + // Object map lifecycle + test('Creation adds a vault', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + await vaultManager.createVault(vaultName); + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + expect(vaultMap.size).toBe(1); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('Concurrently creating vault with same name only creates 1 vault', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + await expect( + Promise.all([ + vaultManager.createVault(vaultName), + vaultManager.createVault(vaultName), + ]), + ).rejects.toThrow(vaultsErrors.ErrorVaultsVaultDefined); + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + expect(vaultMap.size).toBe(1); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('vaults persist', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId = await vaultManager.createVault(vaultName); + await vaultManager.closeVault(vaultId); + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + expect(vaultMap.size).toBe(0); - const vaultName1 = 'vn1' as VaultName; - const vaultName2 = 'vn2' as VaultName; - const vaultName3 = 'vn3' as VaultName; - const v1Id = await server.vaultManager.createVault(vaultName1); - const v2Id = await server.vaultManager.createVault(vaultName2); - const v3Id = await server.vaultManager.createVault(vaultName3); - - const vaultList: Array<[VaultName, VaultId]> = []; + // @ts-ignore: protected method + const vault1 = await vaultManager.getVault(vaultId); + expect(vaultMap.size).toBe(1); - vaultList.push([vaultName1, v1Id]); - vaultList.push([vaultName2, v2Id]); - vaultList.push([vaultName3, v3Id]); + // @ts-ignore: protected method + const vault2 = await vaultManager.getVault(vaultId); + expect(vaultMap.size).toBe(1); + expect(vault1).toEqual(vault2); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('vaults can be removed from map', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId = await vaultManager.createVault(vaultName); + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + expect(vaultMap.size).toBe(1); + // @ts-ignore: protected method + const vault1 = await vaultManager.getVault(vaultId); + await vaultManager.closeVault(vaultId); + expect(vaultMap.size).toBe(0); + // @ts-ignore: protected method + const vault2 = await vaultManager.getVault(vaultId); + expect(vault1).not.toEqual(vault2); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('stopping vaultManager empties map and stops all vaults', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + try { + const vaultId1 = await vaultManager.createVault('vault1'); + const vaultId2 = await vaultManager.createVault('vault2'); + // @ts-ignore: kidnapping the map + const vaultMap = vaultManager.vaultMap; + expect(vaultMap.size).toBe(2); + // @ts-ignore: protected method + const vault1 = await vaultManager.getVault(vaultId1); + // @ts-ignore: protected method + const vault2 = await vaultManager.getVault(vaultId2); + await vaultManager.stop(); + expect(vaultMap.size).toBe(0); + expect(vault1[running]).toBe(false); + expect(vault2[running]).toBe(false); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } + }); + test('destroying vaultManager destroys all data', async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); + let vaultManager2: VaultManager | undefined; + try { + const vaultId = await vaultManager.createVault('vault1'); + await vaultManager.stop(); + await vaultManager.destroy(); + // Vaults DB should be empty + const vaultsDb = await db.level(VaultManager.constructor.name); + expect(await db.count(vaultsDb)).toBe(0); + vaultManager2 = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), + }); - await expect(() => - vaultManager.scanNodeVaults(serverNodeId), - ).rejects.toThrow(vaultErrors.ErrorVaultsPermissionDenied); + // @ts-ignore: protected method + await expect(vaultManager2.getVault(vaultId)).rejects.toThrow( + vaultsErrors.ErrorVaultsVaultUndefined, + ); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + await vaultManager2?.stop(); + await vaultManager2?.destroy(); + } + }); + test("withVaults should throw if vaultId doesn't exist", async () => { + const vaultManager = await VaultManager.createVaultManager({ + vaultsPath, + keyManager: dummyKeyManager, + gestaltGraph: {} as GestaltGraph, + nodeConnectionManager: {} as NodeConnectionManager, + acl: {} as ACL, + notificationsManager: {} as NotificationsManager, + db, + logger: logger.getChild(VaultManager.name), }); + try { + const vaultId = vaultsUtils.generateVaultId(); + await expect( + vaultManager.withVaults([vaultId], async () => { + // Do nothing + }), + ).rejects.toThrow(vaultsErrors.ErrorVaultsVaultUndefined); + } finally { + await vaultManager?.stop(); + await vaultManager?.destroy(); + } }); + test.todo('generateVaultId handles vault conflicts'); + test.todo('pullVault respects locking'); }); diff --git a/tests/vaults/VaultOps.test.ts b/tests/vaults/VaultOps.test.ts index 8c6907bf98..962b3de919 100644 --- a/tests/vaults/VaultOps.test.ts +++ b/tests/vaults/VaultOps.test.ts @@ -1,12 +1,14 @@ import type { VaultId } from '@/vaults/types'; import type { Vault } from '@/vaults/Vault'; import type { KeyManager } from '@/keys'; +import type { DBDomain, DBLevel } from '@matrixai/db'; import fs from 'fs'; import path from 'path'; import os from 'os'; import { EncryptedFS } from 'encryptedfs'; import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { utils as idUtils } from '@matrixai/id'; +import { DB } from '@matrixai/db'; import * as errors from '@/vaults/errors'; import { VaultInternal, vaultOps } from '@/vaults'; import * as vaultsUtils from '@/vaults/utils'; @@ -21,6 +23,9 @@ describe('VaultOps', () => { let vaultId: VaultId; let vaultInternal: VaultInternal; let vault: Vault; + let db: DB; + let vaultsDb: DBLevel; + let vaultsDbDomain: DBDomain; const dummyKeyManager = { getNodeId: () => { return testUtils.generateRandomNodeId(); @@ -30,7 +35,7 @@ describe('VaultOps', () => { let mockedGenerateKeyPair: jest.SpyInstance; let mockedGenerateDeterministicKeyPair: jest.SpyInstance; - beforeAll(async () => { + beforeEach(async () => { const globalKeyPair = await testUtils.setupGlobalKeypair(); mockedGenerateKeyPair = jest .spyOn(keysUtils, 'generateKeyPair') @@ -42,7 +47,7 @@ describe('VaultOps', () => { dataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), 'polykey-test-'), ); - const dbPath = path.join(dataDir, 'db'); + const dbPath = path.join(dataDir, 'efsDb'); const dbKey = await keysUtils.generateKey(); baseEfs = await EncryptedFS.createEncryptedFS({ dbKey, @@ -50,34 +55,43 @@ describe('VaultOps', () => { logger, }); await baseEfs.start(); - }); - - afterAll(async () => { - mockedGenerateKeyPair.mockRestore(); - mockedGenerateDeterministicKeyPair.mockRestore(); - await baseEfs.stop(); - await baseEfs.destroy(); - await fs.promises.rm(dataDir, { - force: true, - recursive: true, - }); - }); - beforeEach(async () => { vaultId = vaultsUtils.generateVaultId(); await baseEfs.mkdir(path.join(idUtils.toString(vaultId), 'contents'), { recursive: true, }); - vaultInternal = await VaultInternal.create({ + db = await DB.createDB({ dbPath: path.join(dataDir, 'db') }); + vaultsDbDomain = ['vaults']; + vaultsDb = await db.level(vaultsDbDomain[0]); + vaultInternal = await VaultInternal.createVaultInternal({ keyManager: dummyKeyManager, vaultId, efs: baseEfs, logger: logger.getChild(VaultInternal.name), fresh: true, + db, + vaultsDbDomain, + vaultsDb, + vaultName: 'VaultName', }); vault = vaultInternal as Vault; }); + afterEach(async () => { + await vaultInternal.stop(); + await vaultInternal.destroy(); + await db.stop(); + await db.destroy(); + mockedGenerateKeyPair.mockRestore(); + mockedGenerateDeterministicKeyPair.mockRestore(); + await baseEfs.stop(); + await baseEfs.destroy(); + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + test('adding a secret', async () => { await vaultOps.addSecret(vault, 'secret-1', 'secret-content'); const dir = await vault.readF(async (efs) => { diff --git a/tests/vaults/utils.test.ts b/tests/vaults/utils.test.ts index d41cec6c8e..85a866f88a 100644 --- a/tests/vaults/utils.test.ts +++ b/tests/vaults/utils.test.ts @@ -27,10 +27,6 @@ describe('Vaults utils', () => { }); }); - test('VaultId type guard works', async () => { - const vaultId = vaultsUtils.generateVaultId(); - expect(vaultsUtils.decodeVaultId(vaultId)).toBeTruthy(); - }); test('EFS can be read recursively', async () => { const key = await keysUtils.generateKey(256); const efs = await EncryptedFS.createEncryptedFS({