[ECO-2404] Add localStorage caching of events

src/typescript/frontend/src/context/wallet-context/AptosContextProvider.tsx

@@ -242,7 +242,7 @@ export function AptosContextProvider({ children }: PropsWithChildren) {
           ...models.liquidityEvents,
           ...models.marketLatestStateEvents,
         ];
-        flattenedEvents.forEach(pushEventFromClient);
+        flattenedEvents.forEach((e) => pushEventFromClient(e, true));
         parseChangesAndSetBalances(response);
       }
 

src/typescript/frontend/src/lib/store/event/event-store.ts

@@ -18,16 +18,22 @@ import {
   handleLatestBarForSwapEvent,
   pushPeriodicStateEvents,
   toMappedMarketEvents,
+  initialState,
 } from "./utils";
-import { initialStatePatch } from "./local-storage";
 import { periodEnumToRawDuration } from "@sdk/const";
 import { createWebSocketClientStore, type WebSocketClientStore } from "../websocket/store";
 import { DEBUG_ASSERT, extractFilter } from "@sdk/utils";
+import {
+  updateLocalStorage,
+  cleanReadLocalStorage,
+  clearLocalStorage,
+  LOCAL_STORAGE_EVENT_TYPES,
+} from "./local-storage";
 
 export const createEventStore = () => {
-  return createStore<EventStore & WebSocketClientStore>()(
+  const store = createStore<EventStore & WebSocketClientStore>()(
     immer((set, get) => ({
-      ...initialStatePatch(),
+      ...initialState(),
       getMarket: (emojis) => get().markets.get(emojis.join("")),
       getRegisteredMarkets: () => {
         return get().markets;
@@ -77,7 +83,7 @@ export const createEventStore = () => {
           });
         });
       },
-      pushEventFromClient: (event: AnyEventModel) => {
+      pushEventFromClient: (event: AnyEventModel, pushToLocalStorage = false) => {
         if (get().guids.has(event.guid)) return;
         set((state) => {
           state.guids.add(event.guid);
@@ -91,17 +97,32 @@ export const createEventStore = () => {
             if (isSwapEventModel(event)) {
               market.swapEvents.unshift(event);
               handleLatestBarForSwapEvent(market, event);
+              if (pushToLocalStorage) {
+                updateLocalStorage("swap", event);
+              }
             } else if (isChatEventModel(event)) {
               market.chatEvents.unshift(event);
+              if (pushToLocalStorage) {
+                updateLocalStorage("chat", event);
+              }
             } else if (isLiquidityEventModel(event)) {
               market.liquidityEvents.unshift(event);
+              if (pushToLocalStorage) {
+                updateLocalStorage("liquidity", event);
+              }
             } else if (isMarketLatestStateEventModel(event)) {
               market.stateEvents.unshift(event);
               state.stateFirehose.unshift(event);
+              if (pushToLocalStorage) {
+                updateLocalStorage("market", event);
+              }
             } else if (isPeriodicStateEventModel(event)) {
               const period = periodEnumToRawDuration(event.periodicMetadata.period);
               market[period].candlesticks.unshift(event);
               handleLatestBarForPeriodicStateEvent(market, event);
+              if (pushToLocalStorage) {
+                updateLocalStorage("periodic", event);
+              }
             }
           }
         });
@@ -146,4 +167,18 @@ export const createEventStore = () => {
       ...createWebSocketClientStore(set, get),
     }))
   );
+
+  const state = store.getState();
+  for (const eventType of LOCAL_STORAGE_EVENT_TYPES) {
+    try {
+      const events = cleanReadLocalStorage(eventType);
+      for (const event of events) {
+        state.pushEventFromClient(event);
+      }
+    } catch (e) {
+      console.error(e);
+      clearLocalStorage(eventType);
+    }
+  }
+  return store;
 };

src/typescript/frontend/src/lib/store/event/local-storage.ts

@@ -1,177 +1,39 @@
-import {
-  type DatabaseModels,
-  type AnyEventModel,
-  type MarketLatestStateEventModel,
-  type MarketRegistrationEventModel,
-  type GlobalStateEventModel,
-} from "@sdk/indexer-v2/types";
-import { type TableName } from "@sdk/indexer-v2/types/json-types";
-import { LOCALSTORAGE_EXPIRY_TIME_MS } from "const";
+import { type AnyEventModel } from "@sdk/indexer-v2/types";
 import { parseJSON, stringifyJSON } from "utils";
-import { type MarketEventStore, type EventState, type SymbolString } from "./types";
-import { createInitialMarketState } from "./utils";
-import { type DeepWritable } from "@sdk/utils/utility-types";
 
-const shouldKeepItem = <T extends AnyEventModel>(event: T) => {
-  const eventTime = Number(event.transaction.timestamp);
-  const now = new Date().getTime();
-  return eventTime > now - LOCALSTORAGE_EXPIRY_TIME_MS;
-};
+export const LOCAL_STORAGE_EXPIRATION_TIME = 1000 * 60; // 10 minutes.
 
-export const addToLocalStorage = <T extends AnyEventModel>(event: T) => {
-  const { eventName } = event;
-  const events = localStorage.getItem(eventName) ?? "[]";
-  const filtered: T[] = parseJSON<T[]>(events).filter(shouldKeepItem);
-  const guids = new Set(filtered.map((e) => e.guid));
-  if (!guids.has(event.guid)) {
-    filtered.push(event);
-  }
-  localStorage.setItem(eventName, stringifyJSON(filtered));
-};
+export const LOCAL_STORAGE_EVENT_TYPES = [
+  "swap",
+  "chat",
+  "liquidity",
+  "market",
+  "periodic",
+] as const;
+type EventLocalStorageKey = (typeof LOCAL_STORAGE_EVENT_TYPES)[number];
 
-type EventArraysByModelType = {
-  Swap: Array<DatabaseModels[TableName.SwapEvents]>;
-  Chat: Array<DatabaseModels[TableName.ChatEvents]>;
-  MarketRegistration: Array<DatabaseModels[TableName.MarketRegistrationEvents]>;
-  PeriodicState: Array<DatabaseModels[TableName.PeriodicStateEvents]>;
-  State: Array<DatabaseModels[TableName.MarketLatestStateEvent]>;
-  GlobalState: Array<DatabaseModels[TableName.GlobalStateEvents]>;
-  Liquidity: Array<DatabaseModels[TableName.LiquidityEvents]>;
+const shouldKeep = (e: AnyEventModel) => {
+  const now = new Date().getTime();
+  // The time at which all events that occurred prior to are considered stale.
+  const staleTimeBoundary = new Date(now - LOCAL_STORAGE_EXPIRATION_TIME);
+  return e.transaction.timestamp > staleTimeBoundary;
 };
 
-const emptyEventArraysByModelType: () => EventArraysByModelType = () => ({
-  Swap: [] as EventArraysByModelType["Swap"],
-  Chat: [] as EventArraysByModelType["Chat"],
-  MarketRegistration: [] as EventArraysByModelType["MarketRegistration"],
-  PeriodicState: [] as EventArraysByModelType["PeriodicState"],
-  State: [] as EventArraysByModelType["State"],
-  GlobalState: [] as EventArraysByModelType["GlobalState"],
-  Liquidity: [] as EventArraysByModelType["Liquidity"],
-});
-
-type MarketEventTypes =
-  | DatabaseModels["swap_events"]
-  | DatabaseModels["chat_events"]
-  | MarketLatestStateEventModel
-  | DatabaseModels["liquidity_events"]
-  | DatabaseModels["periodic_state_events"];
-
-export const initialStatePatch = (): EventState => {
-  return {
-    guids: new Set<string>(),
-    stateFirehose: [],
-    marketRegistrations: [],
-    markets: new Map(),
-    globalStateEvents: [],
-  };
+export const updateLocalStorage = (key: EventLocalStorageKey, event: AnyEventModel) => {
+  const str = localStorage.getItem(key) ?? "[]";
+  const data: AnyEventModel[] = parseJSON(str);
+  data.unshift(event);
+  localStorage.setItem(key, stringifyJSON(data));
 };
 
-export const initialStateFromLocalStorage = (): EventState => {
-  // Purge stale events then load up the remaining ones.
-  const events = getEventsFromLocalStorage();
-
-  // Sort each event that has a market by its market.
-  const markets: Map<SymbolString, DeepWritable<MarketEventStore>> = new Map();
-  const guids: Set<AnyEventModel["guid"]> = new Set();
-
-  const addGuidAndGetMarket = (event: MarketEventTypes) => {
-    // Before ensuring the market is initialized, add the incoming event to the set of guids.
-    guids.add(event.guid);
-
-    const { market } = event;
-    const symbol = market.symbolData.symbol;
-    if (!markets.has(symbol)) {
-      markets.set(symbol, createInitialMarketState(market));
-    }
-    return markets.get(symbol)!;
-  };
-
-  const marketRegistrations: MarketRegistrationEventModel[] = [];
-  const globalStateEvents: GlobalStateEventModel[] = [];
-
-  events.Chat.forEach((e) => {
-    addGuidAndGetMarket(e).chatEvents.push(e);
-  });
-  events.Liquidity.forEach((e) => {
-    addGuidAndGetMarket(e).liquidityEvents.push(e);
-  });
-  events.State.forEach((e) => {
-    addGuidAndGetMarket(e).stateEvents.push(e);
-  });
-  events.Swap.forEach((e) => {
-    addGuidAndGetMarket(e).swapEvents.push(e);
-  });
-  events.PeriodicState.forEach((e) => {
-    addGuidAndGetMarket(e)[e.periodicMetadata.period].candlesticks.push(e);
-  });
-  events.MarketRegistration.forEach((e) => {
-    marketRegistrations.push(e);
-    guids.add(e.guid);
-  });
-  events.GlobalState.forEach((e) => {
-    globalStateEvents.push(e);
-    guids.add(e.guid);
-  });
-
-  const stateFirehose: MarketLatestStateEventModel[] = [];
-
-  for (const { stateEvents } of markets.values()) {
-    stateFirehose.push(...(stateEvents as Array<MarketLatestStateEventModel>));
-  }
-
-  // Sort the state firehose by bump time, then market ID, then market nonce.
-  stateFirehose.sort(({ market: a }, { market: b }) => {
-    if (a.time === b.time) {
-      if (a.marketID === b.marketID) {
-        if (a.marketNonce === b.marketNonce) return 0;
-        if (a.marketNonce < b.marketNonce) return 1;
-        return -1;
-      } else if (a.marketID < b.marketID) {
-        return 1;
-      }
-      return -1;
-    } else if (a.time < b.time) {
-      return -1;
-    }
-    return 1;
-  });
-
-  return {
-    guids,
-    stateFirehose,
-    marketRegistrations,
-    markets: markets as unknown as Map<SymbolString, MarketEventStore>,
-    globalStateEvents,
-  };
+export const cleanReadLocalStorage = (key: EventLocalStorageKey) => {
+  const str = localStorage.getItem(key) ?? "[]";
+  const data: AnyEventModel[] = parseJSON(str);
+  const relevantItems = data.filter(shouldKeep);
+  localStorage.setItem(key, stringifyJSON(relevantItems));
+  return relevantItems;
 };
 
-/**
- * Purges old local storage events and returns any that remain.
- */
-export const getEventsFromLocalStorage = () => {
-  const res = emptyEventArraysByModelType();
-  const guids = new Set<string>();
-
-  // Filter the events in local storage, then return them.
-  Object.entries(res).forEach((entry) => {
-    const eventName = entry[0] as keyof EventArraysByModelType;
-    const existing = localStorage.getItem(eventName) ?? "[]";
-    const filtered =
-      parseJSON<EventArraysByModelType[typeof eventName]>(existing).filter(shouldKeepItem);
-    const reduced = filtered.reduce(
-      (acc, curr) => {
-        if (!guids.has(curr.guid)) {
-          acc.push(curr);
-          guids.add(curr.guid);
-        }
-        return acc;
-      },
-      [] as typeof filtered
-    );
-    const events = entry[1] as typeof filtered;
-    events.push(...reduced);
-    localStorage.setItem(eventName, stringifyJSON(filtered));
-  });
-
-  return res;
+export const clearLocalStorage = (key: EventLocalStorageKey) => {
+  localStorage.setItem(key, "[]");
 };

src/typescript/frontend/src/lib/store/event/types.ts

@@ -68,7 +68,7 @@ export type EventActions = {
   getRegisteredMarkets: () => Readonly<EventState["markets"]>;
   loadMarketStateFromServer: (states: Array<DatabaseModels["market_state"]>) => void;
   loadEventsFromServer: (events: Array<AnyEventModel>) => void;
-  pushEventFromClient: (event: AnyEventModel) => void;
+  pushEventFromClient: (event: AnyEventModel, localize?: boolean) => void;
   setLatestBars: ({ marketMetadata, latestBars }: SetLatestBarsArgs) => void;
   subscribeToPeriod: ({ marketEmojis, period, cb }: PeriodSubscription) => void;
   unsubscribeFromPeriod: ({ marketEmojis, period }: Omit<PeriodSubscription, "cb">) => void;

src/typescript/frontend/src/lib/store/event/utils.ts

@@ -162,3 +162,13 @@ export const toMappedMarketEvents = <T extends EventModelWithMarket>(events: Arr
   events.forEach((event) => map.get(event.market.symbolData.symbol)!.push(event));
   return map;
 };
+
+export const initialState = (): EventState => {
+  return {
+    guids: new Set<string>(),
+    stateFirehose: [],
+    marketRegistrations: [],
+    markets: new Map(),
+    globalStateEvents: [],
+  };
+};

src/typescript/frontend/src/utils/index.ts

@@ -13,13 +13,21 @@ export { isDisallowedEventKey } from "./check-is-disallowed-event-key";
 export { getEmptyListTr } from "./get-empty-list-tr";
 
 export const stringifyJSON = (data: object) =>
-  JSON.stringify(data, (_, value) => (typeof value === "bigint" ? value.toString() + "n" : value));
+  JSON.stringify(data, (_, value) => {
+    if (typeof value === "bigint") return value.toString() + "n";
+    return value;
+  });
 
 export const parseJSON = <T>(json: string): T =>
   JSON.parse(json, (_, value) => {
     if (typeof value === "string" && /^\d+n$/.test(value)) {
       return BigInt(value.substring(0, value.length - 1));
     }
+    // This matches the below pattern: 1234-12-31T23:59:59.666Z
+    const dateRegex = /^\d{4}-\d{2}-\d2T\d{2}:\d{2}:\d{2}.\d*Z$/;
+    if (typeof value === "string" && dateRegex.test(value)) {
+      return new Date(value);
+    }
     return value as T;
   });