Make candlesticks fetching a cachable route

src/typescript/frontend/src/app/candlesticks/route.ts

@@ -0,0 +1,80 @@
+import { fetchPeriodicEventsSince } from "@/queries/market";
+import { type Period, toPeriod } from "@sdk/index";
+import { type PeriodicStateEventModel } from "@sdk/indexer-v2/types";
+import { type PeriodTypeFromDatabase } from "@sdk/indexer-v2/types/json-types";
+import { parseInt } from "lodash";
+import { unstable_cache } from "next/cache";
+import { type NextRequest } from "next/server";
+import { stringifyJSON } from "utils";
+
+const CANDLESTICKS_LIMIT = 500;
+
+type QueryParams = {
+  marketID: number;
+  start: Date;
+  period: Period;
+  limit: number;
+};
+
+const getCandlesticks = async (params: QueryParams) => {
+  const { marketID, start, period, limit } = params;
+  const aggregate: PeriodicStateEventModel[] = [];
+
+  while (aggregate.length < limit) {
+    const data = await fetchPeriodicEventsSince({
+      marketID,
+      period,
+      start,
+      offset: aggregate.length,
+      limit: limit - aggregate.length,
+    });
+    aggregate.push(...data);
+    if (data.length < limit) {
+      break;
+    }
+  }
+
+  return aggregate;
+};
+
+const getCachedCandlesticks = unstable_cache(getCandlesticks, ["candlesticks"], { revalidate: 1 });
+
+/* eslint-disable-next-line import/no-unused-modules */
+export async function GET(request: NextRequest) {
+  const searchParams = request.nextUrl.searchParams;
+  const marketIDStr = searchParams.get("marketID");
+  const startStr = searchParams.get("start");
+  const periodStr = searchParams.get("period");
+  const limitStr = searchParams.get("limit");
+
+  if (!marketIDStr || !startStr || !periodStr || !limitStr) {
+    return new Response("", { status: 400 });
+  }
+
+  if (isNaN(parseInt(marketIDStr))) {
+    return new Response("", { status: 400 });
+  }
+
+  if (isNaN(parseInt(startStr))) {
+    return new Response("", { status: 400 });
+  }
+
+  let period: Period;
+  try {
+    period = toPeriod(periodStr as PeriodTypeFromDatabase);
+  } catch {
+    return new Response("", { status: 400 });
+  }
+
+  if (isNaN(parseInt(limitStr)) || parseInt(limitStr) > CANDLESTICKS_LIMIT) {
+    return new Response("", { status: 400 });
+  }
+
+  const start = new Date(parseInt(startStr) * 1000);
+  const limit = parseInt(limitStr);
+  const marketID = parseInt(marketIDStr);
+
+  const data = getCachedCandlesticks({ marketID, start, period, limit });
+
+  return new Response(stringifyJSON(data));
+}

src/typescript/frontend/src/app/pools/api/route.ts

@@ -2,10 +2,10 @@ import { symbolBytesToEmojis } from "@sdk/emoji_data/utils";
 import { getValidSortByForPoolsPage } from "@sdk/indexer-v2/queries/query-params";
 import { handleEmptySearchBytes, safeParsePageWithDefault } from "lib/routes/home-page-params";
 import { stringifyJSON } from "utils";
-import { REVALIDATION_TIME } from "lib/server-env";
 import { getPoolData } from "./getPoolDataQuery";
 
 export const revalidate = 1;
+export const fetchCache = "default-cache";
 
 export async function GET(request: Request) {
   const { searchParams } = new URL(request.url);

src/typescript/frontend/src/components/charts/PrivateChart.tsx

@@ -30,17 +30,17 @@ import { useEventStore } from "context/event-store-context";
 import { getPeriodStartTimeFromTime } from "@sdk/utils";
 import { getAptosConfig } from "lib/utils/aptos-client";
 import { getSymbolEmojisInString, symbolToEmojis, toMarketEmojiData } from "@sdk/emoji_data";
-import { type MarketMetadataModel } from "@sdk/indexer-v2/types";
+import { type PeriodicStateEventModel, type MarketMetadataModel } from "@sdk/indexer-v2/types";
 import { getMarketResource } from "@sdk/markets";
 import { Aptos } from "@aptos-labs/ts-sdk";
-import { periodEnumToRawDuration, Trigger } from "@sdk/const";
-import { fetchAllCandlesticksInTimeRange } from "@/queries/candlesticks";
+import { PeriodDuration, periodEnumToRawDuration, Trigger } from "@sdk/const";
 import {
   type LatestBar,
   marketToLatestBars,
   periodicStateTrackerToLatestBar,
   toBar,
 } from "@/store/event/candlestick-bars";
+import { parseJSON } from "utils";
 
 const configurationData: DatafeedConfiguration = {
   supported_resolutions: TV_CHARTING_LIBRARY_RESOLUTIONS,
@@ -145,119 +145,117 @@ export const Chart = async (props: ChartContainerProps) => {
 
         setTimeout(() => onSymbolResolvedCallback(symbolInfo), 0);
       },
-      getBars: async (symbolInfo, resolution, periodParams, onHistoryCallback, onErrorCallback) => {
+      getBars: async (
+        _symbolInfo,
+        resolution,
+        periodParams,
+        onHistoryCallback,
+        _onErrorCallback
+      ) => {
         const { from, to } = periodParams;
-        try {
-          const period = ResolutionStringToPeriod[resolution.toString()];
-          const periodDuration = periodEnumToRawDuration(period);
-          const start = new Date(from * 1000);
-          const end = new Date(to * 1000);
-          // TODO: Consider that if our data is internally consistent and we run into performance/scalability issues
-          // with this implementation below (fetching without regard for anything in state), we can store the values in
-          // state and coalesce that with the data we fetch from the server.
-          const data = await fetchAllCandlesticksInTimeRange({
-            marketID: props.marketID,
-            start,
-            end,
-            period,
-          });
+        const period = ResolutionStringToPeriod[resolution.toString()];
+        const periodDuration = periodEnumToRawDuration(period);
+        const periodDurationSeconds = (periodDuration / PeriodDuration.PERIOD_1M) * 60;
+        const end = new Date(to * 1000);
+        // The start timestamp is rounded so that all the people who load the webpage at a similar time get served
+        // the same cached response.
+        const start = from - (from % periodDurationSeconds);
+        const data: PeriodicStateEventModel[] = await fetch(
+          `/candlesticks?marketID=${props.marketID}&start=${start}&period=${period}&limit=500`
+        )
+          .then((res) => res.text())
+          .then((res) => parseJSON(res));
 
-          const isFetchForMostRecentBars = end.getTime() - new Date().getTime() > 1000;
+        const isFetchForMostRecentBars = end.getTime() - new Date().getTime() > 1000;
 
-          // If the end time is in the future, it means that `getBars` is being called for the most recent candlesticks,
-          // and thus we should append the latest candlestick to this dataset to ensure the chart is up to date.
-          let latestBar: LatestBar | undefined;
-          if (isFetchForMostRecentBars) {
-            // Fetch the current candlestick data from the Aptos fullnode. This fetch call should *never* be cached.
-            // Also, we specifically call this client-side because the server will get rate-limited if we call the
-            // fullnode from the server for each client.
-            const marketResource = await getMarketResource({
-              aptos: new Aptos(getAptosConfig()),
-              marketAddress: props.marketAddress,
-            });
+        // If the end time is in the future, it means that `getBars` is being called for the most recent candlesticks,
+        // and thus we should append the latest candlestick to this dataset to ensure the chart is up to date.
+        let latestBar: LatestBar | undefined;
+        if (isFetchForMostRecentBars) {
+          // Fetch the current candlestick data from the Aptos fullnode. This fetch call should *never* be cached.
+          // Also, we specifically call this client-side because the server will get rate-limited if we call the
+          // fullnode from the server for each client.
+          const marketResource = await getMarketResource({
+            aptos: new Aptos(getAptosConfig()),
+            marketAddress: props.marketAddress,
+          });
 
-            // Convert the market view data to `latestBar[]` and set the latest bars in our EventStore to those values.
-            const latestBars = marketToLatestBars(marketResource);
-            const marketEmojiData = toMarketEmojiData(marketResource.metadata.emojiBytes);
-            const marketMetadata: MarketMetadataModel = {
-              marketID: marketResource.metadata.marketID,
-              time: 0n,
-              marketNonce: marketResource.sequenceInfo.nonce,
-              trigger: Trigger.PackagePublication, // Make up some bunk trigger, since it should be clear it's made up.
-              symbolEmojis: marketEmojiData.emojis.map((e) => e.emoji),
-              marketAddress: marketResource.metadata.marketAddress,
-              ...marketEmojiData,
-            };
-            setLatestBars({ marketMetadata, latestBars });
+          // Convert the market view data to `latestBar[]` and set the latest bars in our EventStore to those values.
+          const latestBars = marketToLatestBars(marketResource);
+          const marketEmojiData = toMarketEmojiData(marketResource.metadata.emojiBytes);
+          const marketMetadata: MarketMetadataModel = {
+            marketID: marketResource.metadata.marketID,
+            time: 0n,
+            marketNonce: marketResource.sequenceInfo.nonce,
+            trigger: Trigger.PackagePublication, // Make up some bunk trigger, since it should be clear it's made up.
+            symbolEmojis: marketEmojiData.emojis.map((e) => e.emoji),
+            marketAddress: marketResource.metadata.marketAddress,
+            ...marketEmojiData,
+          };
+          setLatestBars({ marketMetadata, latestBars });
 
-            // Get the period-specific state tracker for the current resolution/period type and set the latest bar on
-            // the chart- *not* just in state- to the latest bar from that tracker.
-            const tracker = marketResource.periodicStateTrackers.find(
-              // These are most likely indexed in order, but in case they aren't, we use `find` here.
-              (p) => Number(p.period) === periodDuration
-            );
-            if (!tracker) {
-              throw new Error("This should never happen.");
-            }
-            const nonce = marketResource.sequenceInfo.nonce;
-            latestBar = periodicStateTrackerToLatestBar(tracker, nonce);
+          // Get the period-specific state tracker for the current resolution/period type and set the latest bar on
+          // the chart- *not* just in state- to the latest bar from that tracker.
+          const tracker = marketResource.periodicStateTrackers.find(
+            // These are most likely indexed in order, but in case they aren't, we use `find` here.
+            (p) => Number(p.period) === periodDuration
+          );
+          if (!tracker) {
+            throw new Error("This should never happen.");
           }
-          // Filter the data so that all resulting bars are within the specified time range.
-          // Also, update the `open` price to the previous bar's `close` price if it exists.
-          // NOTE: Since `getBars` is called multiple times, this will result in several
-          // bars having incorrect `open` values. This isn't a big deal but may result in
-          // some visual inconsistencies in the chart.
-          const bars: Bar[] = data.reduce((acc: Bar[], event) => {
-            const bar = toBar(event);
-            if (bar.time >= from * 1000 && bar.time <= to * 1000) {
-              if (acc.at(-1)) {
-                bar.open = acc.at(-1)!.close;
-              }
-              acc.push(bar);
-            }
-            return acc;
-          }, []);
-
-          // Push the latest bar to the bars array if it exists and update its `open` value to be the previous bar's
-          // `close` if it's not the first/only bar.
-          if (latestBar) {
-            if (bars.at(-1)) {
-              latestBar.open = bars.at(-1)!.close;
+          const nonce = marketResource.sequenceInfo.nonce;
+          latestBar = periodicStateTrackerToLatestBar(tracker, nonce);
+        }
+        // Filter the data so that all resulting bars are within the specified time range.
+        // Also, update the `open` price to the previous bar's `close` price if it exists.
+        // NOTE: Since `getBars` is called multiple times, this will result in several
+        // bars having incorrect `open` values. This isn't a big deal but may result in
+        // some visual inconsistencies in the chart.
+        const bars: Bar[] = data.reduce((acc: Bar[], event) => {
+          const bar = toBar(event);
+          if (bar.time >= from * 1000 && bar.time <= to * 1000) {
+            if (acc.at(-1)) {
+              bar.open = acc.at(-1)!.close;
             }
-            bars.push(latestBar);
+            acc.push(bar);
           }
+          return acc;
+        }, []);
 
-          if (bars.length === 0) {
-            if (isFetchForMostRecentBars) {
-              // If this is the most recent bar fetch and there is literally zero trading activity thus far,
-              // we should create a single empty bar to get rid of the `No chart data` ghost error from showing.
-              const time = BigInt(new Date().getTime()) * 1000n;
-              const timeAsPeriod = getPeriodStartTimeFromTime(time, periodDuration) / 1000n;
-              bars.push({
-                time: Number(timeAsPeriod),
-                open: 0,
-                high: 0,
-                low: 0,
-                close: 0,
-                volume: 0,
-              });
-            } else {
-              onHistoryCallback([], {
-                noData: true,
-              });
-              return;
-            }
+        // Push the latest bar to the bars array if it exists and update its `open` value to be the previous bar's
+        // `close` if it's not the first/only bar.
+        if (latestBar) {
+          if (bars.at(-1)) {
+            latestBar.open = bars.at(-1)!.close;
           }
+          bars.push(latestBar);
+        }
 
-          onHistoryCallback(bars, {
-            noData: bars.length === 0,
-          });
-        } catch (e) {
-          if (e instanceof Error) {
-            console.warn("[getBars]: Get error", e);
-            onErrorCallback(e.message);
+        if (bars.length === 0) {
+          if (isFetchForMostRecentBars) {
+            // If this is the most recent bar fetch and there is literally zero trading activity thus far,
+            // we should create a single empty bar to get rid of the `No chart data` ghost error from showing.
+            const time = BigInt(new Date().getTime()) * 1000n;
+            const timeAsPeriod = getPeriodStartTimeFromTime(time, periodDuration) / 1000n;
+            bars.push({
+              time: Number(timeAsPeriod.toString()),
+              open: 0,
+              high: 0,
+              low: 0,
+              close: 0,
+              volume: 0,
+            });
+          } else {
+            onHistoryCallback([], {
+              noData: true,
+            });
+            return;
           }
         }
+
+        onHistoryCallback(bars, {
+          noData: bars.length === 0,
+        });
       },
       subscribeBars: async (
         symbolInfo,

src/typescript/frontend/src/middleware.ts

@@ -3,6 +3,7 @@ import {
   COOKIE_FOR_HASHED_ADDRESS,
 } from "components/pages/verify/session-info";
 import { authenticate } from "components/pages/verify/verify";
+import { IS_ALLOWLIST_ENABLED } from "lib/env";
 import { NextResponse, type NextRequest } from "next/server";
 import { ROUTES } from "router/routes";
 import { normalizePossibleMarketPath } from "utils/pathname-helpers";
@@ -19,6 +20,10 @@ export default async function middleware(request: NextRequest) {
     return NextResponse.next();
   }
 
+  if (!IS_ALLOWLIST_ENABLED) {
+    return NextResponse.next();
+  }
+
   const possibleMarketPath = normalizePossibleMarketPath(pathname, request.url);
   if (possibleMarketPath) {
     return NextResponse.redirect(possibleMarketPath);

src/typescript/sdk/src/indexer-v2/types/json-types.ts

@@ -10,7 +10,7 @@ import type {
 } from "../../emojicoin_dot_fun/types";
 import { type Flatten } from "../../types";
 
-type PeriodTypeFromDatabase =
+export type PeriodTypeFromDatabase =
   | "period_1m"
   | "period_5m"
   | "period_15m"