feat(arweave): add arweave client for persisting/retrieving data (#547)

.github/workflows/test.yml

@@ -10,8 +10,10 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
       - run: yarn install
+      - name: Run local arweave node
+        run: yarn test:run:arweave &
       - run: yarn test
         env:
           NODE_URL_1: ${{ secrets.NODE_URL_1 }}

.gitignore

@@ -31,3 +31,4 @@ cache
 yarn-error.log
 yarn-debug.log*
 gasReporterOutput.json
+logs

.nvmrc

@@ -1 +1 @@
-16
+18

package.json

@@ -22,6 +22,7 @@
     "copy-abi": "abi=utils/abi/contracts; dstdir=\"./dist/${DIR}/${abi}\"; mkdir -p \"${dstdir}\"; cp ./src/${abi}/*.json \"${dstdir}\"",
     "test": "hardhat test",
     "test:watch": "hardhat watch test",
+    "test:run:arweave": "npx -y arlocal",
     "lint": "eslint --fix src test e2e && yarn prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"",
     "lint-check": "eslint src test e2e && yarn prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"e2e/**/*.ts\"",
     "prepare": "yarn build && husky install",
@@ -103,6 +104,7 @@
     "@pinata/sdk": "^2.1.0",
     "@types/mocha": "^10.0.1",
     "@uma/sdk": "^0.34.1",
+    "arweave": "^1.14.4",
     "axios": "^0.27.2",
     "big-number": "^2.0.0",
     "decimal.js": "^10.3.1",

src/caching/Arweave/ArweaveClient.ts

@@ -0,0 +1,140 @@
+import Arweave from "arweave";
+import { JWKInterface } from "arweave/node/lib/wallet";
+import { ethers } from "ethers";
+import winston from "winston";
+import { isDefined, jsonReplacerWithBigNumbers, parseWinston } from "../../utils";
+import { Struct, is } from "superstruct";
+import { ARWEAVE_TAG_APP_NAME } from "../../constants";
+
+export class ArweaveClient {
+  private client: Arweave;
+
+  public constructor(
+    private arweaveJWT: JWKInterface,
+    private logger: winston.Logger,
+    gatewayURL = "arweave.net",
+    protocol = "https",
+    port = 443
+  ) {
+    this.client = new Arweave({
+      host: gatewayURL,
+      port,
+      protocol,
+      timeout: 20000,
+      logging: false,
+    });
+    this.logger.info("Arweave client initialized");
+  }
+
+  /**
+   * Stores an arbitrary record in the Arweave network. The record is stored as a JSON string and uses
+   * JSON.stringify to convert the record to a string. The record has all of its big numbers converted
+   * to strings for convenience.
+   * @param value The value to store
+   * @param topicTag An optional topic tag to add to the transaction
+   * @returns The transaction ID of the stored value
+   * @
+   */
+  async set(value: Record<string, unknown>, topicTag?: string | undefined): Promise<string | undefined> {
+    const transaction = await this.client.createTransaction(
+      { data: JSON.stringify(value, jsonReplacerWithBigNumbers) },
+      this.arweaveJWT
+    );
+
+    // Add tags to the transaction
+    transaction.addTag("Content-Type", "application/json");
+    transaction.addTag("App-Name", ARWEAVE_TAG_APP_NAME);
+    if (isDefined(topicTag)) {
+      transaction.addTag("Topic", topicTag);
+    }
+
+    // Sign the transaction
+    await this.client.transactions.sign(transaction, this.arweaveJWT);
+    // Send the transaction
+    const result = await this.client.transactions.post(transaction);
+    this.logger.debug({
+      at: "ArweaveClient:set",
+      message: `Arweave transaction posted with ${transaction.id}`,
+    });
+    // Ensure that the result is successful
+    if (result.status !== 200) {
+      this.logger.error({
+        at: "ArweaveClient:set",
+        message: `Arweave transaction failed with ${transaction.id}`,
+        result,
+        address: await this.getAddress(),
+        balance: (await this.getBalance()).toString(),
+      });
+      throw new Error("Server failed to receive arweave transaction");
+    }
+    return transaction.id;
+  }
+
+  /**
+   * Retrieves a record from the Arweave network. The record is expected to be a JSON string and is
+   * parsed using JSON.parse. All numeric strings are converted to big numbers for convenience.
+   * @param transactionID The transaction ID of the record to retrieve
+   * @param structValidator An optional struct validator to validate the retrieved value. If the value does not match the struct, null is returned.
+   * @returns The record if it exists, otherwise null
+   */
+  async get<T>(transactionID: string, validator: Struct<T>): Promise<T | null> {
+    const rawData = await this.client.transactions.getData(transactionID, { decode: true, string: true });
+    if (!rawData) {
+      return null;
+    }
+    // Parse the retrieved data - if it is an Uint8Array, it is a buffer and needs to be converted to a string
+    const data = JSON.parse(typeof rawData === "string" ? rawData : Buffer.from(rawData).toString("utf-8"));
+    // Ensure that the result is successful. If it is not, the retrieved value is not our expected type
+    // but rather a {status: string, statusText: string} object. We can detect that and return null.
+    if (data.status === 400) {
+      return null;
+    }
+    // If the validator does not match the retrieved value, return null and log a warning
+    if (!is(data, validator)) {
+      this.logger.warn("Retrieved value from Arweave does not match the expected type");
+      return null;
+    }
+    return data;
+  }
+
+  /**
+   * Retrieves the metadata of a transaction
+   * @param transactionID The transaction ID of the record to retrieve
+   * @returns The metadata of the transaction if it exists, otherwise null
+   */
+  async getMetadata(transactionID: string): Promise<Record<string, string> | null> {
+    const transaction = await this.client.transactions.get(transactionID);
+    if (!isDefined(transaction)) {
+      return null;
+    }
+    const tags = Object.fromEntries(
+      transaction.tags.map((tag) => [
+        tag.get("name", { decode: true, string: true }),
+        tag.get("value", { decode: true, string: true }),
+      ])
+    );
+    return {
+      contentType: tags["Content-Type"],
+      appName: tags["App-Name"],
+      topic: tags.Topic,
+    };
+  }
+
+  /**
+   * Returns the address of the signer of the JWT
+   * @returns The address of the signer in this client
+   */
+  getAddress(): Promise<string> {
+    return this.client.wallets.jwkToAddress(this.arweaveJWT);
+  }
+
+  /**
+   * The balance of the signer
+   * @returns The balance of the signer in winston units
+   */
+  async getBalance(): Promise<ethers.BigNumber> {
+    const address = await this.getAddress();
+    const balanceInFloat = await this.client.wallets.getBalance(address);
+    return parseWinston(balanceInFloat);
+  }
+}

src/caching/Arweave/index.ts

@@ -0,0 +1 @@
+export * from "./ArweaveClient";

src/caching/index.ts

@@ -1 +1,2 @@
 export * from "./IPFS";
+export * from "./Arweave";

src/constants.ts

@@ -17,6 +17,9 @@ export const HUBPOOL_CHAIN_ID = 1;
 // List of versions where certain UMIP features were deprecated
 export const TRANSFER_THRESHOLD_MAX_CONFIG_STORE_VERSION = 1;
 
+// A hardcoded identifier used, by default, to tag all Arweave records.
+export const ARWEAVE_TAG_APP_NAME = "across-protocol";
+
 /**
  * A default list of chain Ids that the protocol supports. This is outlined
  * in the UMIP (https://github.com/UMAprotocol/UMIPs/pull/590) and is used

src/utils/FormattingUtils.ts

@@ -166,3 +166,21 @@ export const ConvertDecimals = (fromDecimals: number, toDecimals: number): ((amo
     return amount.mul(toBN("10").pow(toBN((-1 * diff).toString())));
   };
 };
+
+/**
+ * Converts a numeric decimal-inclusive string to winston, the base unit of Arweave
+ * @param numericString The numeric string to convert
+ * @returns The winston representation of the numeric string as a BigNumber
+ */
+export function parseWinston(numericString: string): ethers.BigNumber {
+  return ethers.utils.parseUnits(numericString, 12);
+}
+
+/**
+ * Converts a winston value to a numeric string
+ * @param winstonValue The winston value to convert
+ * @returns The numeric string representation of the winston value
+ */
+export function formatWinston(winstonValue: ethers.BigNumber): string {
+  return ethers.utils.formatUnits(winstonValue, 12);
+}

src/utils/JSONUtils.ts

@@ -1,4 +1,5 @@
 import { BigNumber } from "ethers";
+import { isDefined } from "./TypeGuards";
 
 /**
  * This function converts a JSON string into a JSON object. The caveat is that if
@@ -50,6 +51,13 @@ export function jsonReplacerWithBigNumbers(_key: string, value: unknown): unknow
   if (BigNumber.isBigNumber(value)) {
     return value.toString();
   }
+  // There's a legacy issues that returns BigNumbers as { type: "BigNumber", hex: "0x..." }
+  // so we need to check for that as well.
+  const recordValue = value as { type: string; hex: string };
+  if (recordValue.type === "BigNumber" && isDefined(recordValue.hex)) {
+    return BigNumber.from(recordValue.hex).toString();
+  }
+  // Return the value as is
   return value;
 }
 

test/arweaveClient.ts

@@ -0,0 +1,157 @@
+import Arweave from "arweave";
+import { JWKInterface } from "arweave/node/lib/wallet";
+import axios from "axios";
+import { expect } from "chai";
+import winston from "winston";
+import { ArweaveClient } from "../src/caching";
+import { parseWinston, toBN } from "../src/utils";
+import { object, string } from "superstruct";
+import { ARWEAVE_TAG_APP_NAME } from "../src/constants";
+
+const INITIAL_FUNDING_AMNT = "5000000000";
+const LOCAL_ARWEAVE_NODE = {
+  protocol: "http",
+  host: "localhost",
+  port: 1984,
+};
+const LOCAL_ARWEAVE_URL = `${LOCAL_ARWEAVE_NODE.protocol}://${LOCAL_ARWEAVE_NODE.host}:${LOCAL_ARWEAVE_NODE.port}`;
+
+const mineBlock = () => axios.get(`${LOCAL_ARWEAVE_URL}/mine`);
+
+describe("ArweaveClient", () => {
+  let jwk: JWKInterface;
+  let client: ArweaveClient;
+  // Before running any of the tests, we need to fund the address with some AR
+  // so that we can post to our testnet node
+  before(async () => {
+    // Generate a new JWK for our tests
+    jwk = await Arweave.init({}).wallets.generate();
+    // Resolve the address of the JWK
+    const address = await Arweave.init({}).wallets.jwkToAddress(jwk);
+    // Call into the local arweave node to fund the address
+    await axios.get(`${LOCAL_ARWEAVE_URL}/mint/${address}/${INITIAL_FUNDING_AMNT}`);
+    // Wait for the transaction to be mined
+    await mineBlock();
+  });
+
+  beforeEach(() => {
+    // Create a new Arweave client
+    client = new ArweaveClient(
+      jwk,
+      // Define default winston logger
+      winston.createLogger({
+        level: "info",
+        format: winston.format.json(),
+        defaultMeta: { service: "arweave-client" },
+        transports: [new winston.transports.Console()],
+      }),
+      LOCAL_ARWEAVE_NODE.host,
+      LOCAL_ARWEAVE_NODE.protocol,
+      LOCAL_ARWEAVE_NODE.port
+    );
+  });
+
+  it(`should have ${INITIAL_FUNDING_AMNT} initial AR in the address`, async () => {
+    const balance = (await client.getBalance()).toString();
+    expect(balance.toString()).to.equal(parseWinston(INITIAL_FUNDING_AMNT).toString());
+  });
+
+  it("should be able to set a basic record and view it on the network", async () => {
+    const value = { test: "value" };
+    const txID = await client.set(value);
+    console.log(txID);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const retrievedValue = await client.get(txID!, object());
+    expect(retrievedValue).to.deep.equal(value);
+  });
+
+  it("should successfully set a record with a BigNumber", async () => {
+    const value = { test: "value", bigNumber: toBN("1000000000000000000") };
+    const txID = await client.set(value);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const retrievedValue = await client.get(txID!, object());
+
+    const expectedValue = { test: "value", bigNumber: "1000000000000000000" };
+    expect(retrievedValue).to.deep.equal(expectedValue);
+  });
+
+  it("should fail to get a non-existent record", async () => {
+    const retrievedValue = await client.get("non-existent", object());
+    expect(retrievedValue).to.be.null;
+  });
+
+  it("should validate the record with a struct validator", async () => {
+    const value = { test: "value" };
+    const txID = await client.set(value);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const validatorStruct = object({ test: string() });
+
+    const retrievedValue = await client.get(txID!, validatorStruct);
+    expect(retrievedValue).to.deep.equal(value);
+  });
+
+  it("should fail validation of the record with a struct validator that doesn't match the returned type", async () => {
+    const value = { test: "value" };
+    const txID = await client.set(value);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const validatorStruct = object({ invalid: string() });
+
+    const retrievedValue = await client.get(txID!, validatorStruct);
+    expect(retrievedValue).to.eq(null);
+  });
+
+  it("should retrieve the metadata of a transaction", async () => {
+    const value = { test: "value" };
+    const txID = await client.set(value);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const metadata = await client.getMetadata(txID!);
+    expect(metadata).to.deep.equal({
+      contentType: "application/json",
+      appName: ARWEAVE_TAG_APP_NAME,
+      topic: undefined,
+    });
+  });
+
+  it("should retrieve the metadata of a transaction with a topic tag", async () => {
+    const value = { test: "value" };
+    const topicTag = "test-topic";
+    const txID = await client.set(value, topicTag);
+    expect(txID).to.not.be.undefined;
+
+    // Wait for the transaction to be mined
+    await mineBlock();
+    await mineBlock();
+
+    const metadata = await client.getMetadata(txID!);
+    expect(metadata).to.deep.equal({
+      contentType: "application/json",
+      appName: ARWEAVE_TAG_APP_NAME,
+      topic: topicTag,
+    });
+  });
+});