Skip to content

Commit

Permalink
Add abstracted caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@CRBl69
CRBl69 committed Dec 2, 2024
1 parent b9b3ac8 commit 3e35c17
Show file tree
Hide file tree
Showing 8 changed files with 353 additions and 232 deletions.
184 changes: 32 additions & 152 deletions src/typescript/frontend/src/app/candlesticks/route.ts
View file
Original file line number Diff line number Diff line change
@@ -1,110 +1,34 @@
// cspell:word timespan

import { type AnyNumberString, getPeriodStartTimeFromTime, toPeriod } from "@sdk/index";
import { type Period, toPeriod } from "@sdk/index";
import { parseInt } from "lodash";
import { type NextRequest } from "next/server";
import {
type CandlesticksSearchParams,
type GetCandlesticksParams,
getPeriodDurationSeconds,
HISTORICAL_CACHE_DURATION,
indexToParcelEndDate,
indexToParcelStartDate,
isValidCandlesticksSearchParams,
jsonStrAppend,
NORMAL_CACHE_DURATION,
PARCEL_SIZE,
toIndex,
} from "./utils";
import { unstable_cache } from "next/cache";
import { getLatestProcessedEmojicoinTimestamp } from "@sdk/indexer-v2/queries/utils";
import { parseJSON, stringifyJSON } from "utils";
import { fetchMarketRegistration, fetchPeriodicEventsSince } from "@/queries/market";

/**
* @property `data` the stringified version of {@link CandlesticksDataType}.
* @property `count` the number of rows returned.
*/
type GetCandlesticksResponse = {
data: string;
count: number;
};
import { fetchPeriodicEventsTo, tryFetchMarketRegistration } from "@/queries/market";
import { Parcel } from "lib/parcel";

type CandlesticksDataType = Awaited<ReturnType<typeof fetchPeriodicEventsSince>>;
type CandlesticksDataType = Awaited<ReturnType<typeof fetchPeriodicEventsTo>>;

const getCandlesticks = async (params: GetCandlesticksParams) => {
const { marketID, index, period } = params;
const getCandlesticksParcel = async (
{ to, count }: { to: number; count: number },
query: { marketID: number; period: Period }
) => {
const endDate = new Date(to * 1000);

const start = indexToParcelStartDate(index, period);

const periodDurationMilliseconds = getPeriodDurationSeconds(period) * 1000;
const timespan = periodDurationMilliseconds * PARCEL_SIZE;
const end = new Date(start.getTime() + timespan);

// PARCEL_SIZE determines the max number of rows, so we don't need to pass a `LIMIT` value.
// `start` and `end` determine the level of pagination, so no need to specify `offset` either.
const data = await fetchPeriodicEventsSince({
marketID,
period,
start,
end,
const data = await fetchPeriodicEventsTo({
...query,
end: endDate,
amount: count,
});

return {
data: stringifyJSON(data),
count: data.length,
};
return data;
};

/**
* Returns the market registration event for a market if it exists.
*
* If it doesn't exist, it throws an error so that the value isn't cached in the
* `unstable_cache` call.
*
* @see {@link getCachedMarketRegistrationMs}
*/
const getMarketRegistrationMs = async (marketID: AnyNumberString) =>
fetchMarketRegistration({ marketID }).then((res) => {
if (res) {
return Number(res.market.time / 1000n);
}
throw new Error("Market is not yet registered.");
});

const getCachedMarketRegistrationMs = unstable_cache(
getMarketRegistrationMs,
["market-registrations"],
{
revalidate: HISTORICAL_CACHE_DURATION,
}
);

/**
* Fetch all of the parcels of candlesticks that have completely ended.
* The only difference between this and {@link getNormalCachedCandlesticks} is the cache tag and
* thus how long the data is cached for.
*/
const getHistoricCachedCandlesticks = unstable_cache(getCandlesticks, ["candlesticks-historic"], {
revalidate: HISTORICAL_CACHE_DURATION,
});

/**
* Fetch all candlestick parcels that haven't completed yet.
* The only difference between this and {@link getHistoricCachedCandlesticks} is the cache tag and
* thus how long the data is cached for.
*/
const getNormalCachedCandlesticks = unstable_cache(getCandlesticks, ["candlesticks"], {
revalidate: NORMAL_CACHE_DURATION,
});

const getCachedLatestProcessedEmojicoinTimestamp = unstable_cache(
getLatestProcessedEmojicoinTimestamp,
["processor-timestamp"],
{ revalidate: 5 }
);

/* eslint-disable-next-line import/no-unused-modules */
export async function GET(request: NextRequest) {
const searchParams = request.nextUrl.searchParams;
const params: CandlesticksSearchParams = {
Expand All @@ -122,70 +46,26 @@ export async function GET(request: NextRequest) {
const to = parseInt(params.to);
const period = toPeriod(params.period);
const countBack = parseInt(params.countBack);
const numParcels = parseInt(params.amount);

const index = toIndex(to, period);

// Ensure that the last start date as calculated per the search params is valid.
// This is specifically the last parcel's start date- aka the last parcel's first candlestick's
// start time.
const lastParcelStartDate = indexToParcelStartDate(index + numParcels - 1, period);
if (lastParcelStartDate > new Date()) {
return new Response("The last parcel's start date cannot be later than the current time.", {
status: 400,
});
}

let data: string = "[]";

const processorTimestamp = new Date(await getCachedLatestProcessedEmojicoinTimestamp());

let totalCount = 0;
let i = 0;
const queryHelper = new Parcel<
CandlesticksDataType[number],
{ marketID: number; period: Period }
>({
parcelSize: 500,
normalRevalidate: NORMAL_CACHE_DURATION,
historicRevalidate: HISTORICAL_CACHE_DURATION,
fetchHistoricThreshold: () => getLatestProcessedEmojicoinTimestamp().then((r) => r.getTime()),
fetchFirst: (query) => tryFetchMarketRegistration(query.marketID),
cacheKey: "candlesticks",
getKey: (s) => Number(s.periodicMetadata.startTime / 1000n / 1000n),
fetchFn: getCandlesticksParcel,
step: getPeriodDurationSeconds(period),
});

let registrationPeriodBoundaryStart: Date;
try {
registrationPeriodBoundaryStart = await getCachedMarketRegistrationMs(marketID).then(
(time) => new Date(Number(getPeriodStartTimeFromTime(time, period)))
);
} catch {
return new Response("Market has not been registered yet.", { status: 400 });
const data = await queryHelper.getUnparsedData(to, countBack, { marketID, period });
return new Response(data);
} catch (e) {
return new Response(e as string, { status: 400 });
}

while (totalCount <= countBack) {
const localIndex = index - i;
const endDate = indexToParcelEndDate(localIndex, period);
let res: GetCandlesticksResponse;
if (endDate < processorTimestamp) {
res = await getHistoricCachedCandlesticks({
marketID,
index: localIndex,
period,
});
} else {
res = await getNormalCachedCandlesticks({
marketID,
index: localIndex,
period,
});
}

if (i == 0) {
const parsed = parseJSON<CandlesticksDataType>(res.data);
const filtered = parsed.filter(
(val) => val.periodicMetadata.startTime < BigInt(to) * 1_000_000n
);
totalCount += filtered.length;
data = jsonStrAppend(data, stringifyJSON(filtered));
} else {
totalCount += res.count;
data = jsonStrAppend(data, res.data);
}
if (endDate < registrationPeriodBoundaryStart) {
break;
}
i++;
}

return new Response(data);
}
38 changes: 0 additions & 38 deletions src/typescript/frontend/src/app/candlesticks/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,8 @@
import { isPeriod, type Period, PeriodDuration, periodEnumToRawDuration } from "@sdk/index";

/**
* Parcel size is the amount of candlestick periods that will be in a single parcel.
* That is, a parcel for 1m candlesticks will be `PARCEL_SIZE` minutes of time.
*
* Note that this is *NOT* the number of candlesticks in the database- as there may be gaps in the
* on-chain data (and thus the database).
*
* More specifically, each parcel will have anywhere from 0 to PARCEL_SIZE number of candlesticks
* and will always span `PARCEL_SIZE` candlesticks/periods worth of time.
*/
export const PARCEL_SIZE = 500;

export const indexToParcelStartDate = (index: number, period: Period): Date =>
new Date((PARCEL_SIZE * (index * periodEnumToRawDuration(period))) / 1000);
export const indexToParcelEndDate = (index: number, period: Period): Date =>
new Date((PARCEL_SIZE * ((index + 1) * periodEnumToRawDuration(period))) / 1000);

export const getPeriodDurationSeconds = (period: Period) =>
(periodEnumToRawDuration(period) / PeriodDuration.PERIOD_1M) * 60;

export const toIndex = (end: number, period: Period): number => {
const periodDuration = getPeriodDurationSeconds(period);
const parcelDuration = periodDuration * PARCEL_SIZE;

const index = Math.floor(end / parcelDuration);

return index;
};

export const jsonStrAppend = (a: string, b: string): string => {
if (a === "[]") return b;
if (b === "[]") return a;
return `${a.substring(0, a.length - 1)},${b.substring(1)}`;
};

export type GetCandlesticksParams = {
marketID: number;
index: number;
period: Period;
};

/**
* The search params used in the `GET` request at `candlesticks/api`.
*
Expand Down
44 changes: 30 additions & 14 deletions src/typescript/frontend/src/app/chats/route.ts
View file
Original file line number Diff line number Diff line change
@@ -1,43 +1,59 @@
import { fetchChatEvents } from "@/queries/market";
import { fetchChatEvents, tryFetchFirstChatEvent } from "@/queries/market";
import { Parcel } from "lib/parcel";
import type { NextRequest } from "next/server";
import { stringifyJSON } from "utils";
import { isNumber } from "utils";

type ChatSearchParams = {
marketID: string | null;
fromMarketNonce: string | null;
toMarketNonce: string | null;
};

export type ValidChatSearchParams = {
marketID: string;
fromMarketNonce: string;
toMarketNonce: string;
};

const isNumber = (s: string) => !isNaN(parseInt(s));

const isValidChatSearchParams = (params: ChatSearchParams): params is ValidChatSearchParams => {
const { marketID, fromMarketNonce } = params;
const { marketID, toMarketNonce } = params;
// prettier-ignore
return (
marketID !== null && isNumber(marketID) &&
fromMarketNonce !== null && isNumber(fromMarketNonce)
toMarketNonce !== null && isNumber(toMarketNonce)
);
};

type Chat = Awaited<ReturnType<typeof fetchChatEvents>>[number];

export async function GET(request: NextRequest) {
const searchParams = request.nextUrl.searchParams;
const params: ChatSearchParams = {
marketID: searchParams.get("marketID"),
fromMarketNonce: searchParams.get("fromMarketNonce"),
toMarketNonce: searchParams.get("toMarketNonce"),
};

if (!isValidChatSearchParams(params)) {
return new Response("Invalid chat search params.", { status: 400 });
}

const marketID = Number(params.marketID);
const fromMarketNonce = Number(params.fromMarketNonce);

const res = await fetchChatEvents({ marketID, fromMarketNonce });

return new Response(stringifyJSON(res));
const toMarketNonce = Number(params.toMarketNonce);

const queryHelper = new Parcel<Chat, { marketID: number }>({
parcelSize: 20,
normalRevalidate: 5,
historicRevalidate: 365 * 24 * 60 * 60,
fetchHistoricThreshold: (query) =>
fetchChatEvents({ marketID: query.marketID, amount: 1 }).then((r) =>
Number(r[0].market.marketNonce)
),
fetchFirst: (query) => tryFetchFirstChatEvent(query.marketID),
cacheKey: "chats",
getKey: (s) => Number(s.market.marketNonce),
fetchFn: ({ to, count }, { marketID }) =>
fetchChatEvents({ marketID, toMarketNonce: to, amount: count }),
});

const res = await queryHelper.getUnparsedData(toMarketNonce, 50, { marketID });

return new Response(res);
}
Loading

0 comments on commit 3e35c17

Please sign in to comment.