From 982035a866b43aec93fcd5a60a293d92c1545ae1 Mon Sep 17 00:00:00 2001 From: Benjamin Goering <171782+gobengo@users.noreply.github.com> Date: Wed, 22 Mar 2023 15:30:25 -0700 Subject: [PATCH] fix: DbDelegationsStorageWithR2 saving to desired r2 key (#600) Motivation: * part of https://github.com/web3-storage/w3protocol/pull/578 --- packages/access-api/src/models/delegations.js | 90 ++++++++----------- .../access-api/src/types/access-api-cf-db.ts | 19 ++-- .../test/delegations-storage.test.js | 4 +- 3 files changed, 54 insertions(+), 59 deletions(-) diff --git a/packages/access-api/src/models/delegations.js b/packages/access-api/src/models/delegations.js index ccac96973..381b96bb0 100644 --- a/packages/access-api/src/models/delegations.js +++ b/packages/access-api/src/models/delegations.js @@ -9,35 +9,34 @@ import { */ /** - * @template {import('../types/access-api-cf-db').DelegationsV2Table | import('../types/access-api-cf-db').DelegationsV3Table} DelegationRow + * @template {import('../types/access-api-cf-db').DelegationsV2Row | import('../types/access-api-cf-db').DelegationsV3Row} DelegationRow * @typedef {Omit} DelegationRowUpdate */ /** - * @typedef V2Tables - * @property {import('../types/access-api-cf-db').DelegationsV2Table} delegations_v2 + * @template Tables + * @typedef {import("../types/database").Database} Database */ -/** - * @typedef {import("../types/database").Database} DelegationsDatabase - */ - -// @todo remove reference to v2 -export const delegationsTable = /** @type {const} */ ('delegations_v2') +export const delegationsV2TableName = /** @type {const} */ ('delegations_v2') /** * DelegationsStorage that persists using SQL. - * * should work with cloudflare D1 + * * should work with cloudflare D1 v2 schema + * + * @deprecated - use AccessApiD1TablesV3 and DbDelegationsStorageWithR2 + * + * @template {Database} DB */ export class DbDelegationsStorage { - /** @type {DelegationsDatabase} */ + /** @type {DB} */ #db #tables = { - delegations: delegationsTable, + delegations: delegationsV2TableName, } /** - * @param {DelegationsDatabase} db + * @param {DB} db */ constructor(db) { this.#db = db @@ -117,7 +116,7 @@ class UnexpectedDelegation extends Error { } /** - * @param {Pick} row + * @param {Pick} row * @returns {Ucanto.Delegation} */ function rowToDelegation(row) { @@ -153,7 +152,7 @@ function rowToDelegation(row) { /** * @param {Ucanto.Delegation} d - * @returns {DelegationRowUpdate} + * @returns {DelegationRowUpdate} */ export function createDelegationRowUpdate(d) { return { @@ -166,6 +165,7 @@ export function createDelegationRowUpdate(d) { /** * @param {Ucanto.Delegation} d + * @returns {DelegationRowUpdate} */ export function createDelegationRowUpdateV3(d) { return { @@ -192,36 +192,27 @@ export function delegationsTableBytesToArrayBuffer(sqlValue) { } } -/** - * @typedef {`delegations_v3`} DelegationsTableWithoutBytesName - */ - -/** @type {DelegationsTableWithoutBytesName} */ -export const delegationsV3Table = `delegations_v3` +export const delegationsV3Table = /** @type {const} */ (`delegations_v3`) /** - * @typedef {import('../types/access-api-cf-db').AccessApiD1TablesV2} AccessApiD1TablesV2 - * @typedef {import('../types/access-api-cf-db').AccessApiD1TablesV3} AccessApiD1TablesV3 + * @template {Database} DB */ - export class DbDelegationsStorageWithR2 { // @todo abstract away R2 specifics into DagStore: ~AsyncMap /** @type {R2Bucket} */ #dags - /** @type {import('../types/database').Database} */ + /** @type {DB} */ #db - /** @type {keyof AccessApiD1TablesV3} */ #delegationsTableName = delegationsV3Table + /* @type {(d: { cid: string }) => string} */ + #getDagsKey = carFileKeyer /** - * @param {import('../types/database').Database} db + * @param {DB} db * @param {R2Bucket} dags - * @param {keyof AccessApiD1TablesV3} delegationsTableName */ - // eslint-disable-next-line no-useless-constructor - constructor(db, dags, delegationsTableName = delegationsV3Table) { + constructor(db, dags) { this.#db = db - this.#delegationsTableName = delegationsTableName this.#dags = dags // eslint-disable-next-line no-void void ( @@ -239,8 +230,9 @@ export class DbDelegationsStorageWithR2 { if (delegations.length === 0) { return } - await writeDelegations(this.#dags, delegations) + await writeDelegations(this.#dags, delegations, this.#getDagsKey) const values = delegations.map((d) => createDelegationRowUpdateV3(d)) + // @todo - if this fails, undo writeDelegations that dont need to be stored await this.#db .insertInto(this.#delegationsTableName) .values(values) @@ -287,17 +279,17 @@ export class DbDelegationsStorageWithR2 { } /** - * @param {Pick} row + * @param {Pick} row * @param {R2Bucket} dags + * @param {(d: { cid: string }) => string} keyer - builds k/v key strings for each delegation * @returns {Promise} */ - async #rowToDelegation(row, dags = this.#dags) { - const { cid } = row - const carBytesR2 = await dags.get(cid.toString()) + async #rowToDelegation(row, dags = this.#dags, keyer = this.#getDagsKey) { + const cidString = row.cid.toString() + const carBytesR2 = await dags.get(keyer({ cid: cidString })) if (!carBytesR2) { - throw new Error(`failed to read car bytes for cid ${cid.toString()}`) + throw new Error(`failed to read car bytes for cid ${cidString}`) } - // @todo stream car reading const carBytes = new Uint8Array(await carBytesR2.arrayBuffer()) const delegations = bytesToDelegations(carBytes) if (delegations.length !== 1) { @@ -311,13 +303,13 @@ export class DbDelegationsStorageWithR2 { } /** - * @typedef {import('../types/access-api-cf-db').DelegationsV3Table} DelegationsV3Table - * @typedef {import('../types/access-api-cf-db').DelegationsV2Table} DelegationsV2Table + * @typedef {import('../types/access-api-cf-db').DelegationsV3Row} DelegationsV3Row + * @typedef {import('../types/access-api-cf-db').DelegationsV2Row} DelegationsV2Row */ /** * @template {string} TableName - * @template {Record} Tables + * @template {Record} Tables * @param {import('../types/database').Database} db * @param {TableName} delegationsTable * @returns {Promise} - count of table @@ -331,26 +323,22 @@ async function count(db, delegationsTable) { } /** - * @param {Ucanto.Delegation} ucan + * @param {{ cid: string }} ucan */ -function delegationCarFileKeyer(ucan) { - return `${ucan.cid.toString()}.car` +function carFileKeyer(ucan) { + return /** @type {const} */ (`${ucan.cid.toString()}.car`) } /** * @param {R2Bucket} bucket * @param {Iterable} delegations - * @param {(d: Ucanto.Delegation) => string} keyer - builds k/v key strings for each delegation + * @param {(d: { cid: string }) => string} keyer - builds k/v key strings for each delegation */ -async function writeDelegations( - bucket, - delegations, - keyer = delegationCarFileKeyer -) { +async function writeDelegations(bucket, delegations, keyer) { return writeEntries( bucket, [...delegations].map((delegation) => { - const key = delegation.cid.toString() + const key = keyer({ cid: delegation.cid.toString() }) const carBytes = delegationsToBytes([delegation]) const value = carBytes return /** @type {[key: string, value: Uint8Array]} */ ([key, value]) diff --git a/packages/access-api/src/types/access-api-cf-db.ts b/packages/access-api/src/types/access-api-cf-db.ts index 9f61d1b48..7df30445e 100644 --- a/packages/access-api/src/types/access-api-cf-db.ts +++ b/packages/access-api/src/types/access-api-cf-db.ts @@ -5,7 +5,10 @@ export { R2Bucket } from '@miniflare/r2' // v2 -export interface DelegationsV2Table { +/** + * @deprecated - use DelegationsV3Row + */ +export interface DelegationsV2Row { cid: string bytes: Uint8Array | number[] audience: URI<'did:'> @@ -15,13 +18,17 @@ export interface DelegationsV2Table { updated_at: ColumnType } -export interface AccessApiD1TablesV2 { - delegations_v2: DelegationsV2Table +/** + * @deprecated - use DelegationsV3Tables + */ +export interface DelegationsV2Tables { + delegations_v2: DelegationsV2Row } // v3 +// * drop 'bytes' column in favor of storing them in R2 (see DbDelegationsStorageWithR2) -export interface DelegationsV3Table { +export interface DelegationsV3Row { cid: string audience: `did:${string}` issuer: `did:${string}` @@ -30,6 +37,6 @@ export interface DelegationsV3Table { updated_at: ColumnType } -export interface AccessApiD1TablesV3 { - delegations_v3: DelegationsV3Table +export interface DelegationsV3Tables { + delegations_v3: DelegationsV3Row } diff --git a/packages/access-api/test/delegations-storage.test.js b/packages/access-api/test/delegations-storage.test.js index c6483cd07..8e9abb3ed 100644 --- a/packages/access-api/test/delegations-storage.test.js +++ b/packages/access-api/test/delegations-storage.test.js @@ -3,7 +3,7 @@ import { createDelegationRowUpdate, DbDelegationsStorage, DbDelegationsStorageWithR2, - delegationsTable, + delegationsV2TableName, } from '../src/models/delegations.js' import { createD1Database } from '../src/utils/d1.js' import * as assert from 'node:assert' @@ -31,7 +31,7 @@ describe('DbDelegationsStorage', () => { ) // insert row with empty bytes await db - .insertInto(delegationsTable) + .insertInto(delegationsV2TableName) .values([ { ...row,