From 749db48934a00bb965e2c8d56bcf7f5370f457c1 Mon Sep 17 00:00:00 2001 From: Matt <90358481+xbtmatt@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:27:01 -0800 Subject: [PATCH] Retry mildly flaky sleep tests Fix lint/tsconfig issues with config files Force `unstable_cache` to store the candlestick cached data as raw JSON database data instead of bigint/structured data Add comment Remove non-existent import Don't output pre-commit logs if the hook passes Lint/format --- .github/workflows/pre-commit.yaml | 2 +- src/typescript/frontend/.eslintrc.js | 3 +- ...ywright.config.ts => playwright.config.js} | 2 +- .../frontend/src/app/candlesticks/route.ts | 13 ++-- .../src/components/charts/PrivateChart.tsx | 12 +++- src/typescript/frontend/tests/tsconfig.json | 3 +- .../indexer-v2/queries/app/candlesticks.ts | 65 ------------------- .../sdk/src/indexer-v2/queries/app/index.ts | 1 - .../sdk/src/indexer-v2/queries/app/market.ts | 13 ++-- src/typescript/sdk/tests/unit/sleep.test.ts | 2 + 10 files changed, 28 insertions(+), 88 deletions(-) rename src/typescript/frontend/{playwright.config.ts => playwright.config.js} (98%) delete mode 100644 src/typescript/sdk/src/indexer-v2/queries/app/candlesticks.ts diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 6a9f76ed77..d1e19c5874 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -61,7 +61,7 @@ jobs: path: './src/python/hooks/.venv' - uses: 'pre-commit/action@v3.0.0' with: - extra_args: '--all-files --config cfg/pre-commit-config.yaml --verbose' + extra_args: '--all-files --config cfg/pre-commit-config.yaml' name: 'pre-commit' 'on': pull_request: null diff --git a/src/typescript/frontend/.eslintrc.js b/src/typescript/frontend/.eslintrc.js index 6a41e681a6..f133c9fa8f 100644 --- a/src/typescript/frontend/.eslintrc.js +++ b/src/typescript/frontend/.eslintrc.js @@ -22,7 +22,8 @@ module.exports = { "node_modules/**", ".eslintrc.js", "config-overrides.js", - "playwright.config.ts", + "next.config.mjs", + "playwright.config.js", "postcss.config.js", "tailwind.config.js", "tests/**", diff --git a/src/typescript/frontend/playwright.config.ts b/src/typescript/frontend/playwright.config.js similarity index 98% rename from src/typescript/frontend/playwright.config.ts rename to src/typescript/frontend/playwright.config.js index 061bba49bb..1233feb126 100644 --- a/src/typescript/frontend/playwright.config.ts +++ b/src/typescript/frontend/playwright.config.js @@ -24,7 +24,7 @@ export default defineConfig({ baseURL: "http://127.0.0.1:3001", /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "retain-on-first-failure", + trace: "on-first-retry", launchOptions: { env: { diff --git a/src/typescript/frontend/src/app/candlesticks/route.ts b/src/typescript/frontend/src/app/candlesticks/route.ts index a0509baa1d..dd051cd46e 100644 --- a/src/typescript/frontend/src/app/candlesticks/route.ts +++ b/src/typescript/frontend/src/app/candlesticks/route.ts @@ -1,11 +1,13 @@ import { fetchPeriodicEventsSince } from "@/queries/market"; import { type Period, toPeriod } from "@sdk/index"; -import { type PeriodicStateEventModel } from "@sdk/indexer-v2/types"; -import { type PeriodTypeFromDatabase } from "@sdk/indexer-v2/types/json-types"; +import { stringifyJSONWithBigInts } from "@sdk/indexer-v2/json-bigint"; +import { + type DatabaseJsonType, + type PeriodTypeFromDatabase, +} from "@sdk/indexer-v2/types/json-types"; import { parseInt } from "lodash"; import { unstable_cache } from "next/cache"; import { type NextRequest } from "next/server"; -import { stringifyJSON } from "utils"; const CANDLESTICKS_LIMIT = 500; @@ -18,7 +20,7 @@ type QueryParams = { const getCandlesticks = async (params: QueryParams) => { const { marketID, start, period, limit } = params; - const aggregate: PeriodicStateEventModel[] = []; + const aggregate: DatabaseJsonType["periodic_state_events"][] = []; while (aggregate.length < limit) { const data = await fetchPeriodicEventsSince({ @@ -75,6 +77,5 @@ export async function GET(request: NextRequest) { const marketID = parseInt(marketIDStr); const data = await getCachedCandlesticks({ marketID, start, period, limit }); - - return new Response(stringifyJSON(data)); + return new Response(stringifyJSONWithBigInts(data)); } diff --git a/src/typescript/frontend/src/components/charts/PrivateChart.tsx b/src/typescript/frontend/src/components/charts/PrivateChart.tsx index 61fc592793..54cd62a901 100644 --- a/src/typescript/frontend/src/components/charts/PrivateChart.tsx +++ b/src/typescript/frontend/src/components/charts/PrivateChart.tsx @@ -30,7 +30,11 @@ import { useEventStore } from "context/event-store-context"; import { getPeriodStartTimeFromTime } from "@sdk/utils"; import { getAptosConfig } from "lib/utils/aptos-client"; import { getSymbolEmojisInString, symbolToEmojis, toMarketEmojiData } from "@sdk/emoji_data"; -import { type PeriodicStateEventModel, type MarketMetadataModel } from "@sdk/indexer-v2/types"; +import { + type PeriodicStateEventModel, + type MarketMetadataModel, + toPeriodicStateEventModel, +} from "@sdk/indexer-v2/types"; import { getMarketResource } from "@sdk/markets"; import { Aptos } from "@aptos-labs/ts-sdk"; import { PeriodDuration, periodEnumToRawDuration, Trigger } from "@sdk/const"; @@ -40,7 +44,8 @@ import { periodicStateTrackerToLatestBar, toBar, } from "@/store/event/candlestick-bars"; -import { parseJSON } from "utils"; +import { parseJSONWithBigInts } from "@sdk/indexer-v2/json-bigint"; +import { type DatabaseJsonType } from "@sdk/indexer-v2/types/json-types"; const configurationData: DatafeedConfiguration = { supported_resolutions: TV_CHARTING_LIBRARY_RESOLUTIONS, @@ -169,7 +174,8 @@ export const Chart = (props: ChartContainerProps) => { }); const data: PeriodicStateEventModel[] = await fetch(`/candlesticks?${params.toString()}`) .then((res) => res.text()) - .then((res) => parseJSON(res)); + .then((res) => parseJSONWithBigInts(res)) + .then((res) => res.map(toPeriodicStateEventModel)); const isFetchForMostRecentBars = end.getTime() - new Date().getTime() > 1000; diff --git a/src/typescript/frontend/tests/tsconfig.json b/src/typescript/frontend/tests/tsconfig.json index c30c5f46b2..105efec3f0 100644 --- a/src/typescript/frontend/tests/tsconfig.json +++ b/src/typescript/frontend/tests/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../tsconfig.json", "include": [ - "**/*.ts", - "../playwright.config.ts" + "**/*.ts" ] } diff --git a/src/typescript/sdk/src/indexer-v2/queries/app/candlesticks.ts b/src/typescript/sdk/src/indexer-v2/queries/app/candlesticks.ts deleted file mode 100644 index 24bc6d499f..0000000000 --- a/src/typescript/sdk/src/indexer-v2/queries/app/candlesticks.ts +++ /dev/null @@ -1,65 +0,0 @@ -"use server"; - -import { type Period } from "../../../const"; -import { LIMIT } from "../../../queries/const"; -import { type PeriodicStateEventModel } from "../../types"; -import { fetchPeriodicEventsSince } from "./market"; - -/** - * Query to exhaustively fetch all candlesticks (periodic state events) in a given time range. - * Since this function's data will be used for frontend charting, it's - * written with non-blocking pseudo-recursive setTimeout calls to avoid - * blocking the main javascript execution thread. - * - * @param marketID The market ID to fetch data for. - * @param start The start time of the range to fetch data from. - * @param end The end time of the range to fetch data from. - * @param period The period of the candlesticks to fetch. - * @param limit Optional limit to the number of elements to fetch per request. Defaults to `LIMIT`. - * @param fetchDelay Optional delay in milliseconds between each fetch request. - * @returns an Array. - */ -/* eslint-disable-next-line import/no-unused-modules */ -export async function fetchAllCandlesticksInTimeRange(args: { - marketID: string; - start: Date; - end: Date; - period: Period; - limit?: number; - fetchDelay?: number; -}): Promise { - const { marketID, start, end, period, limit = LIMIT, fetchDelay = 0 } = args; - const aggregate: PeriodicStateEventModel[] = []; - let keepFetching = true; - - /* eslint-disable consistent-return */ - const fetchData = async ( - resolve: (value: PeriodicStateEventModel[] | PromiseLike) => void, - reject: (reason?: string | Error) => void - ) => { - /* eslint-enable consistent-return */ - if (!keepFetching) { - return resolve(aggregate); - } - try { - const data = await fetchPeriodicEventsSince({ - marketID, - period, - start, - offset: aggregate.length, - limit, - }); - const filtered = data.filter((d) => d.transaction.timestamp.getTime() <= end.getTime()); - keepFetching = filtered.length === limit; - aggregate.push(...filtered); - - setTimeout(() => fetchData(resolve, reject), fetchDelay); - } catch (err) { - return reject(err as string | Error); - } - }; - - return new Promise((resolve, reject) => { - fetchData(resolve, reject); - }); -} diff --git a/src/typescript/sdk/src/indexer-v2/queries/app/index.ts b/src/typescript/sdk/src/indexer-v2/queries/app/index.ts index 0009eab6e4..e3dc8266e4 100644 --- a/src/typescript/sdk/src/indexer-v2/queries/app/index.ts +++ b/src/typescript/sdk/src/indexer-v2/queries/app/index.ts @@ -1,4 +1,3 @@ export * from "./home"; export * from "./market"; export * from "./pools"; -export * from "./candlesticks"; diff --git a/src/typescript/sdk/src/indexer-v2/queries/app/market.ts b/src/typescript/sdk/src/indexer-v2/queries/app/market.ts index bf079dfe26..2897f1d69d 100644 --- a/src/typescript/sdk/src/indexer-v2/queries/app/market.ts +++ b/src/typescript/sdk/src/indexer-v2/queries/app/market.ts @@ -4,15 +4,10 @@ if (process.env.NODE_ENV !== "test") { import { LIMIT, ORDER_BY } from "../../../queries"; import { type AnyNumberString } from "../../../types"; -import { TableName } from "../../types/json-types"; +import { type DatabaseJsonType, TableName } from "../../types/json-types"; import { postgrest, toQueryArray } from "../client"; import { queryHelper, queryHelperSingle } from "../utils"; -import { - toChatEventModel, - toMarketStateModel, - toPeriodicStateEventModel, - toSwapEventModel, -} from "../../types"; +import { toChatEventModel, toMarketStateModel, toSwapEventModel } from "../../types"; import { type PeriodicStateEventQueryArgs, type MarketStateQueryArgs } from "../../types/common"; import { type SymbolEmoji } from "../../../emoji_data/types"; @@ -68,8 +63,10 @@ const selectMarketState = ({ searchEmojis }: { searchEmojis: SymbolEmoji[] }) => export const fetchSwapEvents = queryHelper(selectSwapsByMarketID, toSwapEventModel); export const fetchChatEvents = queryHelper(selectChatsByMarketID, toChatEventModel); +// Note the lack of a conversion function here- this is because we must cache the data +// with no bigints, so we store it as the raw database JSON data. export const fetchPeriodicEventsSince = queryHelper( selectPeriodicEventsSince, - toPeriodicStateEventModel + (r: DatabaseJsonType["periodic_state_events"]) => r ); export const fetchMarketState = queryHelperSingle(selectMarketState, toMarketStateModel); diff --git a/src/typescript/sdk/tests/unit/sleep.test.ts b/src/typescript/sdk/tests/unit/sleep.test.ts index 1ac48e344f..8066adee47 100644 --- a/src/typescript/sdk/tests/unit/sleep.test.ts +++ b/src/typescript/sdk/tests/unit/sleep.test.ts @@ -1,5 +1,7 @@ import { getTime, sleep, UNIT_OF_TIME_MULTIPLIERS, UnitOfTime } from "../../src"; +jest.retryTimes(3); + describe("sleep utility function with units of time", () => { it("converts units of time to milliseconds", () => { expect(UNIT_OF_TIME_MULTIPLIERS[UnitOfTime.Microseconds]).toEqual(0.001);