From 405abcbc1cfc6a3886446a5bdde2e8a7dc47f10a Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 2 Feb 2024 10:08:30 -0500 Subject: [PATCH 01/14] WIP(arweave): initial skeleton --- package.json | 1 + src/caching/Arweave/ArweaveClient.ts | 53 ++++++++++++++++++++++++++++ src/caching/Arweave/index.ts | 1 + src/caching/index.ts | 1 + yarn.lock | 26 ++++++++++++-- 5 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 src/caching/Arweave/ArweaveClient.ts create mode 100644 src/caching/Arweave/index.ts diff --git a/package.json b/package.json index 13a5fff0..eac1de1f 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "@pinata/sdk": "^2.1.0", "@types/mocha": "^10.0.1", "@uma/sdk": "^0.34.1", + "arweave": "^1.14.4", "axios": "^0.27.2", "big-number": "^2.0.0", "decimal.js": "^10.3.1", diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts new file mode 100644 index 00000000..1df1a93d --- /dev/null +++ b/src/caching/Arweave/ArweaveClient.ts @@ -0,0 +1,53 @@ +// import { Struct } from "superstruct"; +// import { CachingMechanismInterface } from "../../interfaces"; +import Arweave from "arweave"; +import winston from "winston"; +import { JWKInterface } from "arweave/node/lib/wallet"; +import { assert } from "../../utils"; + +export class ArweaveClient { + private client: Arweave; + + public constructor( + private arweaveJWT: JWKInterface, + private logger: winston.Logger, + publicGatewayURL = "arweave.net" + ) { + this.client = new Arweave({ + host: publicGatewayURL, + port: 443, + protocol: "https", + timeout: 20000, + logging: false, + }); + this.logger.info("Arweave client initialized"); + } + + // get( + // key?: string | undefined, + // structValidator?: Struct | undefined, + // overrides?: OverrideType | undefined + // ): Promise { + // throw new Error("Method not implemented."); + // } + async set(_key: string, value: ObjectType): Promise { + const transaction = await this.client.createTransaction({ data: JSON.stringify(value) }, this.arweaveJWT); + // Add tags to the transaction + transaction.addTag("Content-Type", "application/json"); + // Sign the transaction + await this.client.transactions.sign(transaction, this.arweaveJWT); + // Send the transaction + const result = await this.client.transactions.post(transaction); + // Ensure that the result is successful + assert(result.status === 200, "Server failed to receive arweave transaction"); + return transaction.id; + } + + getAddress(): Promise { + return this.client.wallets.jwkToAddress(this.arweaveJWT); + } + async getBalance(): Promise { + const address = await this.getAddress(); + return this.client.wallets.getBalance(address); + } +} diff --git a/src/caching/Arweave/index.ts b/src/caching/Arweave/index.ts new file mode 100644 index 00000000..0a9fba95 --- /dev/null +++ b/src/caching/Arweave/index.ts @@ -0,0 +1 @@ +export * from "./ArweaveClient"; \ No newline at end of file diff --git a/src/caching/index.ts b/src/caching/index.ts index 9387e433..5118e901 100644 --- a/src/caching/index.ts +++ b/src/caching/index.ts @@ -1 +1,2 @@ export * from "./IPFS"; +export * from "./Arweave"; diff --git a/yarn.lock b/yarn.lock index 03633aa4..b48b11f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3255,6 +3255,13 @@ anymatch@~3.1.1, anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" +arconnect@^0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/arconnect/-/arconnect-0.4.2.tgz#83de7638fb46183e82d7ec7efb5594c5f7cdc806" + integrity sha512-Jkpd4QL3TVqnd3U683gzXmZUVqBUy17DdJDuL/3D9rkysLgX6ymJ2e+sR+xyZF5Rh42CBqDXWNMmCjBXeP7Gbw== + dependencies: + arweave "^1.10.13" + arg@^4.1.0: version "4.1.3" resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" @@ -3376,12 +3383,22 @@ arrify@^2.0.0, arrify@^2.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== +arweave@^1.10.13, arweave@^1.14.4: + version "1.14.4" + resolved "https://registry.yarnpkg.com/arweave/-/arweave-1.14.4.tgz#5ba22136aa0e7fd9495258a3931fb770c9d6bf21" + integrity sha512-tmqU9fug8XAmFETYwgUhLaD3WKav5DaM4p1vgJpEj/Px2ORPPMikwnSySlFymmL2qgRh2ZBcZsg11+RXPPGLsA== + dependencies: + arconnect "^0.4.2" + asn1.js "^5.4.1" + base64-js "^1.5.1" + bignumber.js "^9.0.2" + asap@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1.js@^5.0.1, asn1.js@^5.2.0: +asn1.js@^5.0.1, asn1.js@^5.2.0, asn1.js@^5.4.1: version "5.4.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== @@ -3553,7 +3570,7 @@ base-x@^3.0.2, base-x@^3.0.8: dependencies: safe-buffer "^5.0.1" -base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1: +base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -3622,6 +3639,11 @@ bignumber.js@^8.0.1: resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-8.1.1.tgz#4b072ae5aea9c20f6730e4e5d529df1271c4d885" integrity sha512-QD46ppGintwPGuL1KqmwhR0O+N2cZUg8JG/VzwI2e28sM9TqHjQB10lI4QAaMHVbLzwVLLAwEglpKPViWX+5NQ== +bignumber.js@^9.0.2: + version "9.1.2" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c" + integrity sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug== + binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" From 57a407d437e36b4eef3d81ee895c461bbe9ce4a8 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:05:07 -0500 Subject: [PATCH 02/14] chore: enforce v18 --- .nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nvmrc b/.nvmrc index b6a7d89c..25bf17fc 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -16 +18 \ No newline at end of file From d21e00824923ea8ab9401fc92e73a53d4bb5e76b Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:05:41 -0500 Subject: [PATCH 03/14] improve: modify arweave client --- src/caching/Arweave/ArweaveClient.ts | 93 ++++++++++++++++++++++------ src/caching/Arweave/index.ts | 2 +- src/utils/FormattingUtils.ts | 18 ++++++ src/utils/JSONUtils.ts | 8 +++ 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts index 1df1a93d..df7123ad 100644 --- a/src/caching/Arweave/ArweaveClient.ts +++ b/src/caching/Arweave/ArweaveClient.ts @@ -1,9 +1,11 @@ // import { Struct } from "superstruct"; // import { CachingMechanismInterface } from "../../interfaces"; import Arweave from "arweave"; -import winston from "winston"; import { JWKInterface } from "arweave/node/lib/wallet"; -import { assert } from "../../utils"; +import { ethers } from "ethers"; +import winston from "winston"; +import { jsonReplacerWithBigNumbers, parseWinston } from "../../utils"; +import { Struct, is } from "superstruct"; export class ArweaveClient { private client: Arweave; @@ -11,43 +13,98 @@ export class ArweaveClient { public constructor( private arweaveJWT: JWKInterface, private logger: winston.Logger, - publicGatewayURL = "arweave.net" + gatewayURL = "arweave.net", + protocol = "https", + port = 443 ) { this.client = new Arweave({ - host: publicGatewayURL, - port: 443, - protocol: "https", + host: gatewayURL, + port, + protocol, timeout: 20000, logging: false, }); this.logger.info("Arweave client initialized"); } - // get( - // key?: string | undefined, - // structValidator?: Struct | undefined, - // overrides?: OverrideType | undefined - // ): Promise { - // throw new Error("Method not implemented."); - // } - async set(_key: string, value: ObjectType): Promise { - const transaction = await this.client.createTransaction({ data: JSON.stringify(value) }, this.arweaveJWT); + /** + * Stores an arbitrary record in the Arweave network. The record is stored as a JSON string and uses + * JSON.stringify to convert the record to a string. The record has all of its big numbers converted + * to strings for convenience. + * @param value The value to store + * @returns The transaction ID of the stored value + * @ + */ + async set(value: Record): Promise { + const transaction = await this.client.createTransaction( + { data: JSON.stringify(value, jsonReplacerWithBigNumbers) }, + this.arweaveJWT + ); // Add tags to the transaction transaction.addTag("Content-Type", "application/json"); // Sign the transaction await this.client.transactions.sign(transaction, this.arweaveJWT); // Send the transaction const result = await this.client.transactions.post(transaction); + this.logger.debug({ + at: "ArweaveClient:set", + message: `Arweave transaction posted with ${transaction.id}`, + }); // Ensure that the result is successful - assert(result.status === 200, "Server failed to receive arweave transaction"); + if (result.status !== 200) { + this.logger.error({ + at: "ArweaveClient:set", + message: `Arweave transaction failed with ${transaction.id}`, + result, + address: await this.getAddress(), + balance: (await this.getBalance()).toString(), + }); + throw new Error("Server failed to receive arweave transaction"); + } return transaction.id; } + /** + * Retrieves a record from the Arweave network. The record is expected to be a JSON string and is + * parsed using JSON.parse. All numeric strings are converted to big numbers for convenience. + * @param transactionID The transaction ID of the record to retrieve + * @param structValidator An optional struct validator to validate the retrieved value. If the value does not match the struct, null is returned. + * @returns The record if it exists, otherwise null + */ + async get(transactionID: string, validator?: Struct): Promise { + const rawData = await this.client.transactions.getData(transactionID, { decode: true, string: true }); + if (!rawData) { + return null; + } + // Parse the retrieved data - if it is an Uint8Array, it is a buffer and needs to be converted to a string + const data = JSON.parse(typeof rawData === "string" ? rawData : Buffer.from(rawData).toString("utf-8")); + // Ensure that the result is successful. If it is not, the retrieved value is not our expected type + // but rather a {status: string, statusText: string} object. We can detect that and return null. + if (data.status === 400) { + return null; + } + if (validator && !is(data, validator)) { + this.logger.warn("Retrieved value from Arweave does not match the expected type"); + return null; + } + return data as T; + } + + /** + * Returns the address of the signer of the JWT + * @returns The address of the signer in this client + */ getAddress(): Promise { return this.client.wallets.jwkToAddress(this.arweaveJWT); } - async getBalance(): Promise { + + /** + * The balance of the signer + * @returns The balance of the signer in winston units + */ + async getBalance(): Promise { const address = await this.getAddress(); - return this.client.wallets.getBalance(address); + const balanceInFloat = await this.client.wallets.getBalance(address); + return parseWinston(balanceInFloat); } } diff --git a/src/caching/Arweave/index.ts b/src/caching/Arweave/index.ts index 0a9fba95..6798fe4b 100644 --- a/src/caching/Arweave/index.ts +++ b/src/caching/Arweave/index.ts @@ -1 +1 @@ -export * from "./ArweaveClient"; \ No newline at end of file +export * from "./ArweaveClient"; diff --git a/src/utils/FormattingUtils.ts b/src/utils/FormattingUtils.ts index 7ac39f09..5b6ee7fa 100644 --- a/src/utils/FormattingUtils.ts +++ b/src/utils/FormattingUtils.ts @@ -166,3 +166,21 @@ export const ConvertDecimals = (fromDecimals: number, toDecimals: number): ((amo return amount.mul(toBN("10").pow(toBN((-1 * diff).toString()))); }; }; + +/** + * Converts a numeric decimal-inclusive string to winston, the base unit of Arweave + * @param numericString The numeric string to convert + * @returns The winston representation of the numeric string as a BigNumber + */ +export function parseWinston(numericString: string): ethers.BigNumber { + return ethers.utils.parseUnits(numericString, 12); +} + +/** + * Converts a winston value to a numeric string + * @param winstonValue The winston value to convert + * @returns The numeric string representation of the winston value + */ +export function formatWinston(winstonValue: ethers.BigNumber): string { + return ethers.utils.formatUnits(winstonValue, 12); +} diff --git a/src/utils/JSONUtils.ts b/src/utils/JSONUtils.ts index e78a37d8..775d2b34 100644 --- a/src/utils/JSONUtils.ts +++ b/src/utils/JSONUtils.ts @@ -1,4 +1,5 @@ import { BigNumber } from "ethers"; +import { isDefined } from "./TypeGuards"; /** * This function converts a JSON string into a JSON object. The caveat is that if @@ -50,6 +51,13 @@ export function jsonReplacerWithBigNumbers(_key: string, value: unknown): unknow if (BigNumber.isBigNumber(value)) { return value.toString(); } + // There's a legacy issues that returns BigNumbers as { type: "BigNumber", hex: "0x..." } + // so we need to check for that as well. + const recordValue = value as { type: string; hex: string }; + if (recordValue.type === "BigNumber" && isDefined(recordValue.hex)) { + return BigNumber.from(recordValue.hex).toString(); + } + // Return the value as is return value; } From 11ec8d0a4f318b0dcfee0b70836fdeb35f4b099b Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:05:53 -0500 Subject: [PATCH 04/14] ci(arweave): setup integration tests --- package.json | 1 + test/arweaveClient.ts | 121 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 test/arweaveClient.ts diff --git a/package.json b/package.json index eac1de1f..8b780286 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "copy-abi": "abi=utils/abi/contracts; dstdir=\"./dist/${DIR}/${abi}\"; mkdir -p \"${dstdir}\"; cp ./src/${abi}/*.json \"${dstdir}\"", "test": "hardhat test", "test:watch": "hardhat watch test", + "test:run:arweave": "npx -y arlocal", "lint": "eslint --fix src test e2e && yarn prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"", "lint-check": "eslint src test e2e && yarn prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"", "prepare": "yarn build && husky install", diff --git a/test/arweaveClient.ts b/test/arweaveClient.ts new file mode 100644 index 00000000..3138a59a --- /dev/null +++ b/test/arweaveClient.ts @@ -0,0 +1,121 @@ +import Arweave from "arweave"; +import { JWKInterface } from "arweave/node/lib/wallet"; +import axios from "axios"; +import { expect } from "chai"; +import winston from "winston"; +import { ArweaveClient } from "../src/caching"; +import { parseWinston, toBN } from "../src/utils"; +import { object, string } from "superstruct"; + +const INITIAL_FUNDING_AMNT = "5000000000"; +const LOCAL_ARWEAVE_NODE = { + protocol: "http", + host: "localhost", + port: 1984, +}; +const LOCAL_ARWEAVE_URL = `${LOCAL_ARWEAVE_NODE.protocol}://${LOCAL_ARWEAVE_NODE.host}:${LOCAL_ARWEAVE_NODE.port}`; + +const mineBlock = () => axios.get(`${LOCAL_ARWEAVE_URL}/mine`); + +describe("", () => { + let jwk: JWKInterface; + let client: ArweaveClient; + // Before running any of the tests, we need to fund the address with some AR + // so that we can post to our testnet node + before(async () => { + // Generate a new JWK for our tests + jwk = await Arweave.init({}).wallets.generate(); + // Resolve the address of the JWK + const address = await Arweave.init({}).wallets.jwkToAddress(jwk); + // Call into the local arweave node to fund the address + await axios.get(`${LOCAL_ARWEAVE_URL}/mint/${address}/${INITIAL_FUNDING_AMNT}`); + // Wait for the transaction to be mined + await mineBlock(); + }); + + beforeEach(() => { + // Create a new Arweave client + client = new ArweaveClient( + jwk, + // Define default winston logger + winston.createLogger({ + level: "info", + format: winston.format.json(), + defaultMeta: { service: "arweave-client" }, + transports: [new winston.transports.Console()], + }), + LOCAL_ARWEAVE_NODE.host, + LOCAL_ARWEAVE_NODE.protocol, + LOCAL_ARWEAVE_NODE.port + ); + }); + + it(`should have ${INITIAL_FUNDING_AMNT} initial AR in the address`, async () => { + const balance = (await client.getBalance()).toString(); + expect(balance.toString()).to.equal(parseWinston(INITIAL_FUNDING_AMNT).toString()); + }); + + it("should be able to set a basic record and view it on the network", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + console.log(txID); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const retrievedValue = await client.get(txID!); + expect(retrievedValue).to.deep.equal(value); + }); + + it("should successfully set a record with a BigNumber", async () => { + const value = { test: "value", bigNumber: toBN("1000000000000000000") }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const retrievedValue = await client.get(txID!); + + const expectedValue = { test: "value", bigNumber: "1000000000000000000" }; + expect(retrievedValue).to.deep.equal(expectedValue); + }); + + it("should fail to get a non-existent record", async () => { + const retrievedValue = await client.get("non-existent"); + expect(retrievedValue).to.be.null; + }); + + it("should validate the record with a struct validator", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const validatorStruct = object({ test: string() }); + + const retrievedValue = await client.get(txID!, validatorStruct); + expect(retrievedValue).to.deep.equal(value); + }); + + it("should fail validation of the record with a struct validator that doesn't match the returned type", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const validatorStruct = object({ invalid: string() }); + + const retrievedValue = await client.get(txID!, validatorStruct); + expect(retrievedValue).to.eq(null); + }); +}); From eff7e6e7865de8bca775cb87a64bb51b381f5ace Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:08:01 -0500 Subject: [PATCH 05/14] ci: update git action --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b683562..3c4a7761 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,8 @@ jobs: with: node-version: 16 - run: yarn install + - name: Run local arweave node + run: yarn run:local:arweave - run: yarn test env: NODE_URL_1: ${{ secrets.NODE_URL_1 }} From ef46fe546b6753833baabdb7207eecb366985cad Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:08:40 -0500 Subject: [PATCH 06/14] chore: bump nvm --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c4a7761..33b1ad6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - run: yarn install - name: Run local arweave node run: yarn run:local:arweave From 1288d4059651ebb00b03950ad322347e132349ef Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:12:08 -0500 Subject: [PATCH 07/14] nit: fix arweave --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 33b1ad6e..936fcecf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: node-version: 18 - run: yarn install - name: Run local arweave node - run: yarn run:local:arweave + run: yarn test:run:arweave - run: yarn test env: NODE_URL_1: ${{ secrets.NODE_URL_1 }} From 19a1b38fa39cb1d9f2bc9df22dcaa567745dba15 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:13:32 -0500 Subject: [PATCH 08/14] nit: add EOL --- .nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nvmrc b/.nvmrc index 25bf17fc..3c032078 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18 \ No newline at end of file +18 From 026529f8ebcd6d7e52839387dbba043be71d1eb5 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:15:34 -0500 Subject: [PATCH 09/14] nit: run in background --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 936fcecf..74a65e1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: node-version: 18 - run: yarn install - name: Run local arweave node - run: yarn test:run:arweave + run: yarn test:run:arweave & - run: yarn test env: NODE_URL_1: ${{ secrets.NODE_URL_1 }} From 0fe8ec08a656039347c4f6d25ada7c8bfccae4c6 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Fri, 9 Feb 2024 16:18:23 -0500 Subject: [PATCH 10/14] nit: name the arweave client tests --- test/arweaveClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/arweaveClient.ts b/test/arweaveClient.ts index 3138a59a..72b64b4a 100644 --- a/test/arweaveClient.ts +++ b/test/arweaveClient.ts @@ -17,7 +17,7 @@ const LOCAL_ARWEAVE_URL = `${LOCAL_ARWEAVE_NODE.protocol}://${LOCAL_ARWEAVE_NODE const mineBlock = () => axios.get(`${LOCAL_ARWEAVE_URL}/mine`); -describe("", () => { +describe("ArweaveClient", () => { let jwk: JWKInterface; let client: ArweaveClient; // Before running any of the tests, we need to fund the address with some AR From d8d5603c510996e0a8815200db97cc76ee33604b Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Mon, 12 Feb 2024 13:16:09 -0500 Subject: [PATCH 11/14] nit: remove comments --- src/caching/Arweave/ArweaveClient.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts index df7123ad..2674c180 100644 --- a/src/caching/Arweave/ArweaveClient.ts +++ b/src/caching/Arweave/ArweaveClient.ts @@ -1,5 +1,3 @@ -// import { Struct } from "superstruct"; -// import { CachingMechanismInterface } from "../../interfaces"; import Arweave from "arweave"; import { JWKInterface } from "arweave/node/lib/wallet"; import { ethers } from "ethers"; From c2264ce88ff1c2fb4732eb41857ca45acf0b8852 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Mon, 12 Feb 2024 13:16:27 -0500 Subject: [PATCH 12/14] improve: enforce validation --- src/caching/Arweave/ArweaveClient.ts | 7 ++++--- test/arweaveClient.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts index 2674c180..66a410e0 100644 --- a/src/caching/Arweave/ArweaveClient.ts +++ b/src/caching/Arweave/ArweaveClient.ts @@ -69,7 +69,7 @@ export class ArweaveClient { * @param structValidator An optional struct validator to validate the retrieved value. If the value does not match the struct, null is returned. * @returns The record if it exists, otherwise null */ - async get(transactionID: string, validator?: Struct): Promise { + async get(transactionID: string, validator: Struct): Promise { const rawData = await this.client.transactions.getData(transactionID, { decode: true, string: true }); if (!rawData) { return null; @@ -81,11 +81,12 @@ export class ArweaveClient { if (data.status === 400) { return null; } - if (validator && !is(data, validator)) { + // If the validator does not match the retrieved value, return null and log a warning + if (!is(data, validator)) { this.logger.warn("Retrieved value from Arweave does not match the expected type"); return null; } - return data as T; + return data; } /** diff --git a/test/arweaveClient.ts b/test/arweaveClient.ts index 72b64b4a..3f863491 100644 --- a/test/arweaveClient.ts +++ b/test/arweaveClient.ts @@ -65,7 +65,7 @@ describe("ArweaveClient", () => { await mineBlock(); await mineBlock(); - const retrievedValue = await client.get(txID!); + const retrievedValue = await client.get(txID!, object()); expect(retrievedValue).to.deep.equal(value); }); @@ -78,14 +78,14 @@ describe("ArweaveClient", () => { await mineBlock(); await mineBlock(); - const retrievedValue = await client.get(txID!); + const retrievedValue = await client.get(txID!, object()); const expectedValue = { test: "value", bigNumber: "1000000000000000000" }; expect(retrievedValue).to.deep.equal(expectedValue); }); it("should fail to get a non-existent record", async () => { - const retrievedValue = await client.get("non-existent"); + const retrievedValue = await client.get("non-existent", object()); expect(retrievedValue).to.be.null; }); From a9e4f16f8cf97f43aff8abd682852a59db5a7fac Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Tue, 13 Feb 2024 13:22:23 -0500 Subject: [PATCH 13/14] improve: add topic tags --- .gitignore | 1 + src/caching/Arweave/ArweaveClient.ts | 12 ++++++++++-- src/constants.ts | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c723fd70..0d9ff47f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ cache yarn-error.log yarn-debug.log* gasReporterOutput.json +logs diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts index 66a410e0..29d0ca84 100644 --- a/src/caching/Arweave/ArweaveClient.ts +++ b/src/caching/Arweave/ArweaveClient.ts @@ -2,8 +2,9 @@ import Arweave from "arweave"; import { JWKInterface } from "arweave/node/lib/wallet"; import { ethers } from "ethers"; import winston from "winston"; -import { jsonReplacerWithBigNumbers, parseWinston } from "../../utils"; +import { isDefined, jsonReplacerWithBigNumbers, parseWinston } from "../../utils"; import { Struct, is } from "superstruct"; +import { ARWEAVE_TAG_APP_NAME } from "../../constants"; export class ArweaveClient { private client: Arweave; @@ -30,16 +31,23 @@ export class ArweaveClient { * JSON.stringify to convert the record to a string. The record has all of its big numbers converted * to strings for convenience. * @param value The value to store + * @param topicTag An optional topic tag to add to the transaction * @returns The transaction ID of the stored value * @ */ - async set(value: Record): Promise { + async set(value: Record, topicTag?: string | undefined): Promise { const transaction = await this.client.createTransaction( { data: JSON.stringify(value, jsonReplacerWithBigNumbers) }, this.arweaveJWT ); + // Add tags to the transaction transaction.addTag("Content-Type", "application/json"); + transaction.addTag("App-Name", ARWEAVE_TAG_APP_NAME); + if (isDefined(topicTag)) { + transaction.addTag("Topic", topicTag); + } + // Sign the transaction await this.client.transactions.sign(transaction, this.arweaveJWT); // Send the transaction diff --git a/src/constants.ts b/src/constants.ts index 9ec84a5d..b2a32a28 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -16,6 +16,9 @@ export const HUBPOOL_CHAIN_ID = 1; // List of versions where certain UMIP features were deprecated export const TRANSFER_THRESHOLD_MAX_CONFIG_STORE_VERSION = 1; +// A hardcoded identifier used, by default, to tag all Arweave records. +export const ARWEAVE_TAG_APP_NAME = "across-protocol"; + /** * A default list of chain Ids that the protocol supports. This is outlined * in the UMIP (https://github.com/UMAprotocol/UMIPs/pull/590) and is used From 578f74cc8b0b205c99694c0c1b8b477743109b28 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Tue, 13 Feb 2024 13:28:24 -0500 Subject: [PATCH 14/14] chore: add metadata to arweave --- src/caching/Arweave/ArweaveClient.ts | 23 ++++++++++++++++++ test/arweaveClient.ts | 36 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts index 29d0ca84..62cb91c0 100644 --- a/src/caching/Arweave/ArweaveClient.ts +++ b/src/caching/Arweave/ArweaveClient.ts @@ -97,6 +97,29 @@ export class ArweaveClient { return data; } + /** + * Retrieves the metadata of a transaction + * @param transactionID The transaction ID of the record to retrieve + * @returns The metadata of the transaction if it exists, otherwise null + */ + async getMetadata(transactionID: string): Promise | null> { + const transaction = await this.client.transactions.get(transactionID); + if (!isDefined(transaction)) { + return null; + } + const tags = Object.fromEntries( + transaction.tags.map((tag) => [ + tag.get("name", { decode: true, string: true }), + tag.get("value", { decode: true, string: true }), + ]) + ); + return { + contentType: tags["Content-Type"], + appName: tags["App-Name"], + topic: tags.Topic, + }; + } + /** * Returns the address of the signer of the JWT * @returns The address of the signer in this client diff --git a/test/arweaveClient.ts b/test/arweaveClient.ts index 3f863491..01221908 100644 --- a/test/arweaveClient.ts +++ b/test/arweaveClient.ts @@ -6,6 +6,7 @@ import winston from "winston"; import { ArweaveClient } from "../src/caching"; import { parseWinston, toBN } from "../src/utils"; import { object, string } from "superstruct"; +import { ARWEAVE_TAG_APP_NAME } from "../src/constants"; const INITIAL_FUNDING_AMNT = "5000000000"; const LOCAL_ARWEAVE_NODE = { @@ -118,4 +119,39 @@ describe("ArweaveClient", () => { const retrievedValue = await client.get(txID!, validatorStruct); expect(retrievedValue).to.eq(null); }); + + it("should retrieve the metadata of a transaction", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const metadata = await client.getMetadata(txID!); + expect(metadata).to.deep.equal({ + contentType: "application/json", + appName: ARWEAVE_TAG_APP_NAME, + topic: undefined, + }); + }); + + it("should retrieve the metadata of a transaction with a topic tag", async () => { + const value = { test: "value" }; + const topicTag = "test-topic"; + const txID = await client.set(value, topicTag); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const metadata = await client.getMetadata(txID!); + expect(metadata).to.deep.equal({ + contentType: "application/json", + appName: ARWEAVE_TAG_APP_NAME, + topic: topicTag, + }); + }); });