diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b6835629..74a65e1c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,8 +10,10 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - run: yarn install + - name: Run local arweave node + run: yarn test:run:arweave & - run: yarn test env: NODE_URL_1: ${{ secrets.NODE_URL_1 }} diff --git a/.gitignore b/.gitignore index c723fd706..0d9ff47f4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ cache yarn-error.log yarn-debug.log* gasReporterOutput.json +logs diff --git a/.nvmrc b/.nvmrc index b6a7d89c6..3c032078a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -16 +18 diff --git a/package.json b/package.json index 586c54996..9ee878ef0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "copy-abi": "abi=utils/abi/contracts; dstdir=\"./dist/${DIR}/${abi}\"; mkdir -p \"${dstdir}\"; cp ./src/${abi}/*.json \"${dstdir}\"", "test": "hardhat test", "test:watch": "hardhat watch test", + "test:run:arweave": "npx -y arlocal", "lint": "eslint --fix src test e2e && yarn prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"", "lint-check": "eslint src test e2e && yarn prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"", "prepare": "yarn build && husky install", @@ -103,6 +104,7 @@ "@pinata/sdk": "^2.1.0", "@types/mocha": "^10.0.1", "@uma/sdk": "^0.34.1", + "arweave": "^1.14.4", "axios": "^0.27.2", "big-number": "^2.0.0", "decimal.js": "^10.3.1", diff --git a/src/caching/Arweave/ArweaveClient.ts b/src/caching/Arweave/ArweaveClient.ts new file mode 100644 index 000000000..62cb91c04 --- /dev/null +++ b/src/caching/Arweave/ArweaveClient.ts @@ -0,0 +1,140 @@ +import Arweave from "arweave"; +import { JWKInterface } from "arweave/node/lib/wallet"; +import { ethers } from "ethers"; +import winston from "winston"; +import { isDefined, jsonReplacerWithBigNumbers, parseWinston } from "../../utils"; +import { Struct, is } from "superstruct"; +import { ARWEAVE_TAG_APP_NAME } from "../../constants"; + +export class ArweaveClient { + private client: Arweave; + + public constructor( + private arweaveJWT: JWKInterface, + private logger: winston.Logger, + gatewayURL = "arweave.net", + protocol = "https", + port = 443 + ) { + this.client = new Arweave({ + host: gatewayURL, + port, + protocol, + timeout: 20000, + logging: false, + }); + this.logger.info("Arweave client initialized"); + } + + /** + * Stores an arbitrary record in the Arweave network. The record is stored as a JSON string and uses + * JSON.stringify to convert the record to a string. The record has all of its big numbers converted + * to strings for convenience. + * @param value The value to store + * @param topicTag An optional topic tag to add to the transaction + * @returns The transaction ID of the stored value + * @ + */ + async set(value: Record, topicTag?: string | undefined): Promise { + const transaction = await this.client.createTransaction( + { data: JSON.stringify(value, jsonReplacerWithBigNumbers) }, + this.arweaveJWT + ); + + // Add tags to the transaction + transaction.addTag("Content-Type", "application/json"); + transaction.addTag("App-Name", ARWEAVE_TAG_APP_NAME); + if (isDefined(topicTag)) { + transaction.addTag("Topic", topicTag); + } + + // Sign the transaction + await this.client.transactions.sign(transaction, this.arweaveJWT); + // Send the transaction + const result = await this.client.transactions.post(transaction); + this.logger.debug({ + at: "ArweaveClient:set", + message: `Arweave transaction posted with ${transaction.id}`, + }); + // Ensure that the result is successful + if (result.status !== 200) { + this.logger.error({ + at: "ArweaveClient:set", + message: `Arweave transaction failed with ${transaction.id}`, + result, + address: await this.getAddress(), + balance: (await this.getBalance()).toString(), + }); + throw new Error("Server failed to receive arweave transaction"); + } + return transaction.id; + } + + /** + * Retrieves a record from the Arweave network. The record is expected to be a JSON string and is + * parsed using JSON.parse. All numeric strings are converted to big numbers for convenience. + * @param transactionID The transaction ID of the record to retrieve + * @param structValidator An optional struct validator to validate the retrieved value. If the value does not match the struct, null is returned. + * @returns The record if it exists, otherwise null + */ + async get(transactionID: string, validator: Struct): Promise { + const rawData = await this.client.transactions.getData(transactionID, { decode: true, string: true }); + if (!rawData) { + return null; + } + // Parse the retrieved data - if it is an Uint8Array, it is a buffer and needs to be converted to a string + const data = JSON.parse(typeof rawData === "string" ? rawData : Buffer.from(rawData).toString("utf-8")); + // Ensure that the result is successful. If it is not, the retrieved value is not our expected type + // but rather a {status: string, statusText: string} object. We can detect that and return null. + if (data.status === 400) { + return null; + } + // If the validator does not match the retrieved value, return null and log a warning + if (!is(data, validator)) { + this.logger.warn("Retrieved value from Arweave does not match the expected type"); + return null; + } + return data; + } + + /** + * Retrieves the metadata of a transaction + * @param transactionID The transaction ID of the record to retrieve + * @returns The metadata of the transaction if it exists, otherwise null + */ + async getMetadata(transactionID: string): Promise | null> { + const transaction = await this.client.transactions.get(transactionID); + if (!isDefined(transaction)) { + return null; + } + const tags = Object.fromEntries( + transaction.tags.map((tag) => [ + tag.get("name", { decode: true, string: true }), + tag.get("value", { decode: true, string: true }), + ]) + ); + return { + contentType: tags["Content-Type"], + appName: tags["App-Name"], + topic: tags.Topic, + }; + } + + /** + * Returns the address of the signer of the JWT + * @returns The address of the signer in this client + */ + getAddress(): Promise { + return this.client.wallets.jwkToAddress(this.arweaveJWT); + } + + /** + * The balance of the signer + * @returns The balance of the signer in winston units + */ + async getBalance(): Promise { + const address = await this.getAddress(); + const balanceInFloat = await this.client.wallets.getBalance(address); + return parseWinston(balanceInFloat); + } +} diff --git a/src/caching/Arweave/index.ts b/src/caching/Arweave/index.ts new file mode 100644 index 000000000..6798fe4b7 --- /dev/null +++ b/src/caching/Arweave/index.ts @@ -0,0 +1 @@ +export * from "./ArweaveClient"; diff --git a/src/caching/index.ts b/src/caching/index.ts index 9387e433f..5118e9015 100644 --- a/src/caching/index.ts +++ b/src/caching/index.ts @@ -1 +1,2 @@ export * from "./IPFS"; +export * from "./Arweave"; diff --git a/src/constants.ts b/src/constants.ts index e46b51b58..8f6219f81 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -17,6 +17,9 @@ export const HUBPOOL_CHAIN_ID = 1; // List of versions where certain UMIP features were deprecated export const TRANSFER_THRESHOLD_MAX_CONFIG_STORE_VERSION = 1; +// A hardcoded identifier used, by default, to tag all Arweave records. +export const ARWEAVE_TAG_APP_NAME = "across-protocol"; + /** * A default list of chain Ids that the protocol supports. This is outlined * in the UMIP (https://github.com/UMAprotocol/UMIPs/pull/590) and is used diff --git a/src/utils/FormattingUtils.ts b/src/utils/FormattingUtils.ts index 7ac39f090..5b6ee7fa8 100644 --- a/src/utils/FormattingUtils.ts +++ b/src/utils/FormattingUtils.ts @@ -166,3 +166,21 @@ export const ConvertDecimals = (fromDecimals: number, toDecimals: number): ((amo return amount.mul(toBN("10").pow(toBN((-1 * diff).toString()))); }; }; + +/** + * Converts a numeric decimal-inclusive string to winston, the base unit of Arweave + * @param numericString The numeric string to convert + * @returns The winston representation of the numeric string as a BigNumber + */ +export function parseWinston(numericString: string): ethers.BigNumber { + return ethers.utils.parseUnits(numericString, 12); +} + +/** + * Converts a winston value to a numeric string + * @param winstonValue The winston value to convert + * @returns The numeric string representation of the winston value + */ +export function formatWinston(winstonValue: ethers.BigNumber): string { + return ethers.utils.formatUnits(winstonValue, 12); +} diff --git a/src/utils/JSONUtils.ts b/src/utils/JSONUtils.ts index e78a37d8a..775d2b340 100644 --- a/src/utils/JSONUtils.ts +++ b/src/utils/JSONUtils.ts @@ -1,4 +1,5 @@ import { BigNumber } from "ethers"; +import { isDefined } from "./TypeGuards"; /** * This function converts a JSON string into a JSON object. The caveat is that if @@ -50,6 +51,13 @@ export function jsonReplacerWithBigNumbers(_key: string, value: unknown): unknow if (BigNumber.isBigNumber(value)) { return value.toString(); } + // There's a legacy issues that returns BigNumbers as { type: "BigNumber", hex: "0x..." } + // so we need to check for that as well. + const recordValue = value as { type: string; hex: string }; + if (recordValue.type === "BigNumber" && isDefined(recordValue.hex)) { + return BigNumber.from(recordValue.hex).toString(); + } + // Return the value as is return value; } diff --git a/test/arweaveClient.ts b/test/arweaveClient.ts new file mode 100644 index 000000000..012219088 --- /dev/null +++ b/test/arweaveClient.ts @@ -0,0 +1,157 @@ +import Arweave from "arweave"; +import { JWKInterface } from "arweave/node/lib/wallet"; +import axios from "axios"; +import { expect } from "chai"; +import winston from "winston"; +import { ArweaveClient } from "../src/caching"; +import { parseWinston, toBN } from "../src/utils"; +import { object, string } from "superstruct"; +import { ARWEAVE_TAG_APP_NAME } from "../src/constants"; + +const INITIAL_FUNDING_AMNT = "5000000000"; +const LOCAL_ARWEAVE_NODE = { + protocol: "http", + host: "localhost", + port: 1984, +}; +const LOCAL_ARWEAVE_URL = `${LOCAL_ARWEAVE_NODE.protocol}://${LOCAL_ARWEAVE_NODE.host}:${LOCAL_ARWEAVE_NODE.port}`; + +const mineBlock = () => axios.get(`${LOCAL_ARWEAVE_URL}/mine`); + +describe("ArweaveClient", () => { + let jwk: JWKInterface; + let client: ArweaveClient; + // Before running any of the tests, we need to fund the address with some AR + // so that we can post to our testnet node + before(async () => { + // Generate a new JWK for our tests + jwk = await Arweave.init({}).wallets.generate(); + // Resolve the address of the JWK + const address = await Arweave.init({}).wallets.jwkToAddress(jwk); + // Call into the local arweave node to fund the address + await axios.get(`${LOCAL_ARWEAVE_URL}/mint/${address}/${INITIAL_FUNDING_AMNT}`); + // Wait for the transaction to be mined + await mineBlock(); + }); + + beforeEach(() => { + // Create a new Arweave client + client = new ArweaveClient( + jwk, + // Define default winston logger + winston.createLogger({ + level: "info", + format: winston.format.json(), + defaultMeta: { service: "arweave-client" }, + transports: [new winston.transports.Console()], + }), + LOCAL_ARWEAVE_NODE.host, + LOCAL_ARWEAVE_NODE.protocol, + LOCAL_ARWEAVE_NODE.port + ); + }); + + it(`should have ${INITIAL_FUNDING_AMNT} initial AR in the address`, async () => { + const balance = (await client.getBalance()).toString(); + expect(balance.toString()).to.equal(parseWinston(INITIAL_FUNDING_AMNT).toString()); + }); + + it("should be able to set a basic record and view it on the network", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + console.log(txID); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const retrievedValue = await client.get(txID!, object()); + expect(retrievedValue).to.deep.equal(value); + }); + + it("should successfully set a record with a BigNumber", async () => { + const value = { test: "value", bigNumber: toBN("1000000000000000000") }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const retrievedValue = await client.get(txID!, object()); + + const expectedValue = { test: "value", bigNumber: "1000000000000000000" }; + expect(retrievedValue).to.deep.equal(expectedValue); + }); + + it("should fail to get a non-existent record", async () => { + const retrievedValue = await client.get("non-existent", object()); + expect(retrievedValue).to.be.null; + }); + + it("should validate the record with a struct validator", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const validatorStruct = object({ test: string() }); + + const retrievedValue = await client.get(txID!, validatorStruct); + expect(retrievedValue).to.deep.equal(value); + }); + + it("should fail validation of the record with a struct validator that doesn't match the returned type", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const validatorStruct = object({ invalid: string() }); + + const retrievedValue = await client.get(txID!, validatorStruct); + expect(retrievedValue).to.eq(null); + }); + + it("should retrieve the metadata of a transaction", async () => { + const value = { test: "value" }; + const txID = await client.set(value); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const metadata = await client.getMetadata(txID!); + expect(metadata).to.deep.equal({ + contentType: "application/json", + appName: ARWEAVE_TAG_APP_NAME, + topic: undefined, + }); + }); + + it("should retrieve the metadata of a transaction with a topic tag", async () => { + const value = { test: "value" }; + const topicTag = "test-topic"; + const txID = await client.set(value, topicTag); + expect(txID).to.not.be.undefined; + + // Wait for the transaction to be mined + await mineBlock(); + await mineBlock(); + + const metadata = await client.getMetadata(txID!); + expect(metadata).to.deep.equal({ + contentType: "application/json", + appName: ARWEAVE_TAG_APP_NAME, + topic: topicTag, + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 84f13666f..a50c947e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3262,6 +3262,13 @@ anymatch@~3.1.1, anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" +arconnect@^0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/arconnect/-/arconnect-0.4.2.tgz#83de7638fb46183e82d7ec7efb5594c5f7cdc806" + integrity sha512-Jkpd4QL3TVqnd3U683gzXmZUVqBUy17DdJDuL/3D9rkysLgX6ymJ2e+sR+xyZF5Rh42CBqDXWNMmCjBXeP7Gbw== + dependencies: + arweave "^1.10.13" + arg@^4.1.0: version "4.1.3" resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" @@ -3383,12 +3390,22 @@ arrify@^2.0.0, arrify@^2.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== +arweave@^1.10.13, arweave@^1.14.4: + version "1.14.4" + resolved "https://registry.yarnpkg.com/arweave/-/arweave-1.14.4.tgz#5ba22136aa0e7fd9495258a3931fb770c9d6bf21" + integrity sha512-tmqU9fug8XAmFETYwgUhLaD3WKav5DaM4p1vgJpEj/Px2ORPPMikwnSySlFymmL2qgRh2ZBcZsg11+RXPPGLsA== + dependencies: + arconnect "^0.4.2" + asn1.js "^5.4.1" + base64-js "^1.5.1" + bignumber.js "^9.0.2" + asap@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1.js@^5.0.1, asn1.js@^5.2.0: +asn1.js@^5.0.1, asn1.js@^5.2.0, asn1.js@^5.4.1: version "5.4.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== @@ -3569,7 +3586,7 @@ base-x@^3.0.2, base-x@^3.0.8: dependencies: safe-buffer "^5.0.1" -base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1: +base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -3638,6 +3655,11 @@ bignumber.js@^8.0.1: resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-8.1.1.tgz#4b072ae5aea9c20f6730e4e5d529df1271c4d885" integrity sha512-QD46ppGintwPGuL1KqmwhR0O+N2cZUg8JG/VzwI2e28sM9TqHjQB10lI4QAaMHVbLzwVLLAwEglpKPViWX+5NQ== +bignumber.js@^9.0.2: + version "9.1.2" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c" + integrity sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug== + binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"