LIVE-14029 evm importer limiting to top 100 tokens by chains (#7903)

* working importer limiting to top100 tokens by chains

* changeset + package names

* fix package.json

lint

* eip712 data back as used by hw-app-eth-test

* updatereadme

remove unused dep

updatereadme

clear changelog file

* change import path

* skip one test

* cryptoassets reference change from test

* remove useless files and readme

* remove fiats / tokens files

* remove /lib/ when fetching evm/ signatures

* cleanup unused lib

* update importers

* update evm-tools

* pnpmlock

* lint eip712 test

* axios 1.7.3

* update axios

* fix path to cryptoassets

* fix path to cryptoassets

* revert path cassets

* typo fix

* paraswap apdus fixing UTs

* fix erc20 ok test apdus

* fix ERC20 CAL KO tests apdus

* fix unimported

.changeset/thick-poets-lie.md

@@ -0,0 +1,6 @@
+---
+"@ledgerhq/cryptoassets-evm-signatures": major
+"@ledgerhq/hw-app-eth": major
+---
+
+feat: uses lightweight token asset list

libs/coin-modules/coin-evm/.unimportedrc.json

@@ -25,6 +25,7 @@
     "@ethersproject/constants",
     "@ethersproject/strings",
     "@ethersproject/hash",
-    "@ethersproject/keccak256"
+    "@ethersproject/keccak256",
+    "@ledgerhq/cryptoassets-evm-signatures/data/evm/index"
   ]
-}
+}

libs/evm-tools/package.json

@@ -45,7 +45,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@ledgerhq/cryptoassets": "workspace:^",
+    "@ledgerhq/cryptoassets-evm-signatures": "workspace:^",
     "@ledgerhq/live-env": "workspace:^",
     "axios": "1.7.7",
     "crypto-js": "4.2.0",
@@ -71,4 +71,4 @@
     "test": "jest",
     "unimported": "unimported"
   }
-}
+}

libs/evm-tools/src/__tests__/messages/EIP712/index.unit.test.ts

@@ -12,8 +12,10 @@ import messageInCAL from "../../fixtures/messages/2.json";
 const CAL = jest.requireActual("../../fixtures/CAL").default;
 
 jest.mock("axios");
-jest.mock("@ledgerhq/cryptoassets/data/eip712", () => require("../../fixtures/CAL"));
-jest.mock("@ledgerhq/cryptoassets/data/eip712_v2", () => require("../../fixtures/CAL"));
+jest.mock("@ledgerhq/cryptoassets-evm-signatures/data/eip712", () => require("../../fixtures/CAL"));
+jest.mock("@ledgerhq/cryptoassets-evm-signatures/data/eip712_v2", () =>
+  require("../../fixtures/CAL"),
+);
 
 describe("evm-tools", () => {
   describe("message", () => {

libs/evm-tools/src/message/EIP712/index.ts

@@ -3,8 +3,8 @@ import axios from "axios";
 import SHA224 from "crypto-js/sha224";
 import { getEnv } from "@ledgerhq/live-env";
 import { EIP712Message } from "@ledgerhq/types-live";
-import EIP712CAL from "@ledgerhq/cryptoassets/data/eip712";
-import EIP712CALV2 from "@ledgerhq/cryptoassets/data/eip712_v2";
+import EIP712CAL from "@ledgerhq/cryptoassets-evm-signatures/data/eip712";
+import EIP712CALV2 from "@ledgerhq/cryptoassets-evm-signatures/data/eip712_v2";
 import { CALServiceEIP712Response, MessageFilters } from "./types";
 
 // As defined in [spec](https://eips.ethereum.org/EIPS/eip-712), the properties below are all required.

libs/ledger-live-common/src/__tests__/currencies.ts

@@ -12,7 +12,11 @@ import {
 } from "../currencies";
 import { byContractAddressAndChainId } from "@ledgerhq/hw-app-eth/erc20";
 
-test("erc20 are all consistent with those on ledgerjs side", () => {
+/*
+  skipped because ledgerjs data is now lighter (from the POV of hw-app-eth)
+  (using cryptoassets-evm-signatures instead of cryptoassets)
+*/
+test.skip("erc20 are all consistent with those on ledgerjs side", () => {
   const normalList = listTokens();
   const delistedList = listTokens({
     withDelisted: true,

libs/ledgerjs/packages/cryptoassets-evm-signatures/.eslintignore

@@ -0,0 +1 @@
+src/data

libs/ledgerjs/packages/cryptoassets-evm-signatures/.unimportedrc.json

@@ -0,0 +1,8 @@
+{
+  "entry": [
+    "src/index.ts",
+    "src/crypto-assets-importer/index.ts"
+  ],
+  "ignorePatterns": ["src/data/**/*.ts", "src/**/*.test.ts"],
+  "ignoreUnimported": ["src/data/**", "src/**/*.test.ts"]
+}

libs/ledgerjs/packages/cryptoassets-evm-signatures/jest.config.ts

@@ -0,0 +1,6 @@
+import baseConfig from "../../jest.config";
+
+export default {
+  ...baseConfig,
+  rootDir: __dirname,
+};

libs/ledgerjs/packages/cryptoassets-evm-signatures/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "@ledgerhq/cryptoassets-evm-signatures",
+  "version": "13.4.0",
+  "description": "Ledger crypto-assets list, evm only",
+  "keywords": [
+    "Ledger"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/LedgerHQ/ledger-live.git"
+  },
+  "bugs": {
+    "url": "https://github.com/LedgerHQ/ledger-live/issues"
+  },
+  "homepage": "https://github.com/LedgerHQ/ledger-live/tree/develop/libs/ledgerjs/packages/cryptoassets-evm-signatures",
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "lib/index.js",
+  "module": "lib-es/index.js",
+  "types": "lib/index.d.ts",
+  "scripts": {
+    "clean": "rimraf lib lib-es",
+    "build": "tsc && tsc -m ES6 --outDir lib-es",
+    "prewatch": "pnpm build",
+    "watch": "tsc --watch",
+    "lint": "eslint ./src --no-error-on-unmatched-pattern --ext .ts,.tsx --cache",
+    "lint:fix": "pnpm lint --fix",
+    "test": "jest",
+    "unimported": "unimported",
+    "test:importer": "jest ./src/crypto-assets-importer/",
+    "import:cal-tokens": "ts-node ./src/crypto-assets-importer/index.ts"
+  },
+  "dependencies": {
+    "@ledgerhq/live-env": "workspace:^",
+    "axios": "1.7.7"
+  },
+  "devDependencies": {
+    "@ledgerhq/types-cryptoassets": "workspace:^",
+    "@types/jest": "^29.5.10",
+    "@types/node": "^20.8.10",
+    "documentation": "14.0.2",
+    "jest": "^29.7.0",
+    "rimraf": "^4.4.1",
+    "source-map-support": "^0.5.21",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.7.0"
+  },
+  "typesVersions": {
+    "*": {
+      "*.json": [
+        "*.json"
+      ],
+      "*": [
+        "lib/*"
+      ],
+      "lib/*": [
+        "lib/*"
+      ],
+      "lib-es/*": [
+        "lib-es/*"
+      ]
+    }
+  },
+  "exports": {
+    "./lib/*": "./lib/*.js",
+    "./lib/*.js": "./lib/*.js",
+    "./lib-es/*": "./lib-es/*.js",
+    "./lib-es/*.js": "./lib-es/*.js",
+    "./*": {
+      "require": "./lib/*.js",
+      "default": "./lib-es/*.js"
+    },
+    "./*.js": {
+      "require": "./lib/*.js",
+      "default": "./lib-es/*.js"
+    },
+    ".": {
+      "require": "./lib/index.js",
+      "default": "./lib-es/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "license": "Apache-2.0",
+  "gitHead": "dd0dea64b58e5a9125c8a422dcffd29e5ef6abec"
+}

libs/ledgerjs/packages/cryptoassets-evm-signatures/src/crypto-assets-importer/fetch/fetch.test.ts

@@ -0,0 +1,28 @@
+import axios from "axios";
+import { fetchTokensFromCDN } from ".";
+
+jest.mock("axios");
+const mockedAxios = axios as jest.Mocked<typeof axios>;
+
+describe("fetcher", () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("should return data when fetched", async () => {
+    const response = [{ myToken: { name: "myToken" } }];
+    mockedAxios.get.mockImplementation(() =>
+      Promise.resolve({ data: response, headers: { etag: "hash" } }),
+    );
+    const [tokens, hash] = await fetchTokensFromCDN("tokens.json");
+    expect(tokens).toBe(response);
+    expect(hash).toBe("hash");
+  });
+
+  it("should throw error if fetch failed", async () => {
+    mockedAxios.get.mockImplementation(() => Promise.reject({ message: "could not fetch" }));
+    expect(async () => {
+      await fetchTokensFromCDN("tokens.json");
+    }).rejects.toThrow();
+  });
+});

libs/ledgerjs/packages/cryptoassets-evm-signatures/src/crypto-assets-importer/fetch/index.ts

@@ -0,0 +1,96 @@
+import { getEnv } from "@ledgerhq/live-env";
+import { CryptoCurrencyId } from "@ledgerhq/types-cryptoassets";
+import axios, { AxiosError } from "axios";
+
+const LEDGER_COUNTERVALUES_API = getEnv("LEDGER_COUNTERVALUES_API");
+
+export const fetchTokensFromCDN = async <T>(filename: string): Promise<[T, string | undefined]> => {
+  try {
+    const { data, headers } = await axios.get<T>(`${getEnv("DYNAMIC_CAL_BASE_URL")}/${filename}`);
+    return [data, headers.etag];
+  } catch (err) {
+    const error = err as AxiosError;
+    throw new Error(`${error.message} ${getEnv("DYNAMIC_CAL_BASE_URL")}/${filename}`);
+  }
+};
+
+export type CALServiceOutput = {
+  type: string;
+  id: string;
+  blockchain_name: string;
+  chain_id: number;
+  contract_address: string;
+  token_identifier: string;
+  decimals: number;
+  delisted: boolean;
+  network_type: string;
+  name: string;
+  symbol: string;
+  ticker: string;
+  units: string;
+  live_signature: string;
+  exchange_app_config_serialized: string;
+  exchange_app_signature: string;
+};
+
+export const fetchTokensFromCALService = async <T extends Array<keyof CALServiceOutput>>(
+  chainDetails: {
+    chainId?: string | number;
+    standard?: string;
+    blockchain_name?: CryptoCurrencyId;
+  },
+  output: T,
+  etag?: string | null,
+  next?: { cursor: string; tokens: Pick<CALServiceOutput, T[number]>[] },
+): Promise<{ tokens: Pick<CALServiceOutput, T[number]>[]; hash: string | undefined }> => {
+  try {
+    const { data, headers } = await axios.get<Pick<CALServiceOutput, T[number]>[]>(
+      `${getEnv("CAL_SERVICE_URL")}/v1/tokens`,
+      {
+        params: {
+          ...(next?.cursor ? { cursor: next.cursor } : {}),
+          chain_id: chainDetails.chainId,
+          standard: chainDetails.standard,
+          blockchain_name: chainDetails.blockchain_name,
+          output: output.join(),
+        },
+        headers: etag
+          ? {
+              "If-None-Match": etag,
+            }
+          : undefined,
+      },
+    );
+
+    const cursor = headers["x-ledger-next"];
+    if (cursor) {
+      return fetchTokensFromCALService(
+        { chainId: chainDetails.chainId, standard: chainDetails.standard },
+        output,
+        etag,
+        {
+          tokens: next?.tokens ? [...next.tokens, ...data] : data,
+          cursor,
+        },
+      );
+    }
+
+    const hash = headers["etag"];
+    return next?.tokens ? { tokens: [...next.tokens, ...data], hash } : { tokens: data, hash };
+  } catch (err) {
+    if (err instanceof AxiosError) {
+      if (err.response?.status === 304) {
+        throw err;
+      }
+      throw new Error(`${err?.message}: ${err?.config?.url}`);
+    }
+    throw err;
+  }
+};
+
+export const fetchTokensOrderedByMarketCap = async (): Promise<{
+  tokens: string[];
+}> => {
+  const { data } = await axios.get<string[]>(`${LEDGER_COUNTERVALUES_API}/v3/supported/crypto`);
+  return { tokens: data };
+};

libs/ledgerjs/packages/cryptoassets-evm-signatures/src/crypto-assets-importer/importers/eip712/eip712.test.ts

@@ -0,0 +1,49 @@
+import axios from "axios";
+import { importEIP712 } from ".";
+import fs from "fs";
+
+const eip712Tokens = {
+  "1:address:signature": {
+    contractName: {
+      label: "DeGate Withdrawal",
+      signature: "singatureDegate",
+    },
+    fields: [
+      {
+        label: "Owner",
+        path: "owner",
+        signature: "signatureOwner",
+      },
+    ],
+  },
+};
+
+const mockedAxios = jest.spyOn(axios, "get");
+
+describe("import EIP 712 tokens", () => {
+  beforeEach(() => {
+    mockedAxios.mockImplementation(() =>
+      Promise.resolve({ data: eip712Tokens, headers: { etag: "etagHash" } }),
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("should output the file in the correct format", async () => {
+    const expectedFile = `import EIP712 from "./eip712.json";
+export { default as hash } from "./eip712-hash.json";
+export default EIP712;
+`;
+
+    const mockedFs = (fs.writeFileSync = jest.fn());
+
+    await importEIP712(".");
+
+    expect(mockedAxios).toHaveBeenCalledWith(expect.stringMatching(/.*\/eip712.json/));
+    expect(mockedFs).toHaveBeenNthCalledWith(1, "eip712.json", JSON.stringify(eip712Tokens));
+    expect(mockedFs).toHaveBeenNthCalledWith(2, "eip712-hash.json", JSON.stringify("etagHash"));
+    expect(mockedFs).toHaveBeenNthCalledWith(3, "eip712.ts", expectedFile);
+  });
+});