store data in indexeddb instead, migrate data

packages/app/src/App.tsx

@@ -1,14 +1,14 @@
 import 'core-js/actual';
 import '@atlaskit/css-reset';
-import { RivetApp } from './components/RivetApp.js';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { RivetAppLoader } from './components/RivetAppLoader';
 
 const queryClient = new QueryClient();
 
 function App() {
   return (
     <QueryClientProvider client={queryClient}>
-      <RivetApp />
+      <RivetAppLoader />
     </QueryClientProvider>
   );
 }

packages/app/src/components/RivetApp.tsx

@@ -34,6 +34,7 @@ import { HelpModal } from './HelpModal';
 import { openedProjectsSortedIdsState } from '../state/savedGraphs';
 import { NoProject } from './NoProject';
 import { swallowPromise, syncWrapper } from '../utils/syncWrapper';
+import { allInitializeStoreFns } from '../state/storage';
 
 const styles = css`
   overflow: hidden;

packages/app/src/components/RivetAppLoader.tsx

@@ -0,0 +1,22 @@
+import { useState } from 'react';
+import { allInitializeStoreFns } from '../state/storage';
+import useAsyncEffect from 'use-async-effect';
+import { RivetApp } from './RivetApp';
+
+export const RivetAppLoader = () => {
+  const [isLoading, setIsLoading] = useState(true);
+
+  useAsyncEffect(async () => {
+    for (const initializeFn of allInitializeStoreFns) {
+      await initializeFn();
+    }
+
+    setIsLoading(false);
+  }, []);
+
+  if (isLoading) {
+    return <div>Loading...</div>;
+  }
+
+  return <RivetApp />;
+};

packages/app/src/state/execution.ts

@@ -2,9 +2,9 @@ import { atom } from 'jotai';
 import { atomWithStorage } from 'jotai/utils';
 import { type ExecutionRecorder } from '@ironclad/rivet-core';
 import { defaultExecutorState } from './settings';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
-const storage = createStorage('execution');
+const { storage } = createHybridStorage('execution');
 
 export const remoteUploadAllowedState = atom<boolean>(false);
 

packages/app/src/state/graph.ts

@@ -15,9 +15,9 @@ import { mapValues } from 'lodash-es';
 import { projectState } from './savedGraphs';
 import { pluginRefreshCounterState } from './plugins';
 import { type CalculatedRevision } from '../utils/ProjectRevisionCalculator';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
-const storage = createStorage('graph');
+const { storage } = createHybridStorage('graph');
 
 // Basic atoms
 export const historicalGraphState = atom<CalculatedRevision | null>(null);

packages/app/src/state/graphBuilder.ts

@@ -10,10 +10,10 @@ import {
   type NodeGraph,
 } from '@ironclad/rivet-core';
 import { type WireDef } from '../components/WireLayer.js';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 import { type SearchedItem, type SearchableItem } from '../hooks/useSearchProject';
 
-const storage = createStorage('graphBuilder');
+const { storage } = createHybridStorage('graphBuilder');
 
 export const viewingNodeChangesState = atom<NodeId | undefined>(undefined);
 

packages/app/src/state/savedGraphs.ts

@@ -14,7 +14,7 @@ import {
 } from '@ironclad/rivet-core';
 import { blankProject } from '../utils/blankProject.js';
 import { entries, values } from '../../../core/src/utils/typeSafety';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 /** Project context values stored in the IDE and not in the project file. Available in Context nodes. */
 export type ProjectContext = Record<
   string,
@@ -24,7 +24,7 @@ export type ProjectContext = Record<
   }
 >;
 
-const storage = createStorage('project');
+const { storage } = createHybridStorage('project');
 
 // What's the data of the last loaded project?
 export const projectState = atomWithStorage<Omit<Project, 'data'>>(

packages/app/src/state/settings.ts

@@ -3,10 +3,10 @@ import { atomWithStorage } from 'jotai/utils';
 import { type Settings } from '@ironclad/rivet-core';
 import { isInTauri } from '../utils/tauri';
 import { DEFAULT_CHAT_NODE_TIMEOUT } from '../../../core/src/utils/defaults';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
 // Legacy storage key for recoil-persist to avoid breaking existing users' settings
-const storage = createStorage('recoil-persist');
+const { storage } = createHybridStorage('recoil-persist');
 
 export const settingsState = atomWithStorage<Settings>(
   'settings',

packages/app/src/state/storage.ts

@@ -1,27 +1,193 @@
+import { getError } from '@ironclad/rivet-core';
 import { createJSONStorage } from 'jotai/utils';
 import type { SyncStorage } from 'jotai/vanilla/utils/atomWithStorage';
+import prettyBytes from 'pretty-bytes';
+import { toast } from 'react-toastify';
+import { debounce } from 'lodash-es';
 
-export const createStorage = (mainKey?: string): SyncStorage<any> => {
-  const storage = createJSONStorage<any>(() => localStorage);
+// In-memory storage that acts as a buffer
+const memoryStorage = new Map<string, any>();
 
-  if (!mainKey) {
-    return storage;
+// Interface for the actual storage backend
+interface AsyncStorageBackend {
+  getItem: (key: string) => Promise<string | null>;
+  setItem: (key: string, value: string) => Promise<void>;
+  removeItem: (key: string) => Promise<void>;
+}
+
+// IndexedDB implementation example
+class IndexedDBStorage implements AsyncStorageBackend {
+  private dbName = 'jotai-store';
+  private storeName = 'state';
+  private dbPromise: Promise<IDBDatabase>;
+
+  constructor() {
+    this.dbPromise = this.initDB();
+  }
+
+  private initDB(): Promise<IDBDatabase> {
+    return new Promise((resolve, reject) => {
+      const request = indexedDB.open(this.dbName, 1);
+
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => resolve(request.result);
+
+      request.onupgradeneeded = (event) => {
+        const db = (event.target as IDBOpenDBRequest).result;
+        db.createObjectStore(this.storeName);
+      };
+    });
+  }
+
+  async getItem(key: string): Promise<string | null> {
+    const db = await this.dbPromise;
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.storeName, 'readonly');
+      const store = transaction.objectStore(this.storeName);
+      const request = store.get(key);
+
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => resolve(request.result);
+    });
+  }
+
+  async setItem(key: string, value: string): Promise<void> {
+    const db = await this.dbPromise;
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.storeName, 'readwrite');
+      const store = transaction.objectStore(this.storeName);
+      const request = store.put(value, key);
+
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => resolve();
+    });
+  }
+
+  async removeItem(key: string): Promise<void> {
+    const db = await this.dbPromise;
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.storeName, 'readwrite');
+      const store = transaction.objectStore(this.storeName);
+      const request = store.delete(key);
+
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => resolve();
+    });
   }
+}
+
+// Create the hybrid storage
+export const createHybridStorage = (
+  mainKey?: string,
+  asyncStorage: AsyncStorageBackend = new IndexedDBStorage(),
+): {
+  storage: SyncStorage<any>;
+} => {
+  const jsonStorage = createJSONStorage<any>(() => localStorage);
 
-  return {
+  // Debounced save function to avoid too frequent writes
+  const debouncedSave = debounce(async (key: string, value: any) => {
+    try {
+      await asyncStorage.setItem(key, JSON.stringify(value));
+    } catch (e) {
+      console.error('Error saving to async storage:', e);
+      toast.error(`Error saving to persistent storage: ${e}`);
+    }
+  }, 1000);
+
+  const storage: SyncStorage<any> = {
     getItem: (key: string, initialValue) => {
-      const mainObject = storage.getItem(mainKey, {});
+      if (!mainKey) {
+        return memoryStorage.get(key) ?? initialValue;
+      }
+      const mainObject = memoryStorage.get(mainKey) ?? {};
       return mainObject[key] ?? initialValue;
     },
     setItem: (key: string, value): void => {
-      const mainObject = storage.getItem(mainKey, {});
-      mainObject[key] = value;
-      storage.setItem(mainKey, mainObject);
+      try {
+        if (!mainKey) {
+          memoryStorage.set(key, value);
+          debouncedSave(key, value);
+          return;
+        }
+
+        const mainObject = memoryStorage.get(mainKey) ?? {};
+        mainObject[key] = value;
+        memoryStorage.set(mainKey, mainObject);
+        debouncedSave(mainKey, mainObject);
+      } catch (e) {
+        const err = getError(e);
+        toast.error(`Error setting storage item: ${err}`);
+      }
     },
     removeItem: (key: string): void => {
-      const mainObject = storage.getItem(mainKey, {});
+      if (!mainKey) {
+        memoryStorage.delete(key);
+        asyncStorage.removeItem(key).catch(console.error);
+        return;
+      }
+
+      const mainObject = memoryStorage.get(mainKey) ?? {};
       delete mainObject[key];
-      storage.setItem(mainKey, mainObject);
+      memoryStorage.set(mainKey, mainObject);
+      debouncedSave(mainKey, mainObject);
     },
   };
+
+  // Initialize function to load data from async storage with localStorage fallback
+  const initializeStore = async () => {
+    try {
+      if (!mainKey) {
+        // Load all individual keys
+        // You might need a way to know all possible keys here
+        return;
+      }
+
+      // Try loading from IndexedDB first
+      const storedData = await asyncStorage.getItem(mainKey);
+
+      if (storedData) {
+        const parsedData = JSON.parse(storedData);
+        memoryStorage.set(mainKey, parsedData);
+      } else {
+        // Fallback to localStorage if data doesn't exist in IndexedDB
+        try {
+          const localData = jsonStorage.getItem(mainKey, null);
+          if (localData) {
+            console.log('Migrating data from localStorage to IndexedDB...');
+            memoryStorage.set(mainKey, localData);
+            // Save to async storage
+            await asyncStorage.setItem(mainKey, JSON.stringify(localData));
+            console.log('Migration complete');
+
+            // Optionally, clear localStorage to free up space
+            // Uncomment if you want to clear after successful migration
+            // localStorage.removeItem(mainKey);
+          }
+        } catch (localError) {
+          console.error('Error reading from localStorage:', localError);
+        }
+      }
+    } catch (e) {
+      console.error('Error initializing store:', e);
+      toast.error(`Error loading persistent storage: ${e}`);
+
+      // If IndexedDB fails completely, fall back to localStorage
+      try {
+        const localData = jsonStorage.getItem(mainKey!, null);
+        if (localData) {
+          memoryStorage.set(mainKey!, localData);
+          console.warn('Failed to load from IndexedDB, using localStorage data');
+        }
+      } catch (localError) {
+        console.error('Error reading from localStorage:', localError);
+      }
+    }
+  };
+
+  allInitializeStoreFns.add(initializeStore);
+
+  return { storage };
 };
+
+export const allInitializeStoreFns = new Set<() => Promise<void>>();

packages/app/src/state/trivet.ts

@@ -1,7 +1,7 @@
 import { type TrivetResults, type TrivetTestSuite } from '@ironclad/trivet';
 import { atom } from 'jotai';
 import { atomWithStorage } from 'jotai/utils';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
 export type TrivetState = {
   testSuites: TrivetTestSuite[];
@@ -11,7 +11,7 @@ export type TrivetState = {
   runningTests: boolean;
 };
 
-const storage = createStorage('trivet');
+const { storage } = createHybridStorage('trivet');
 
 // Convert to persisted atom using atomWithStorage
 export const trivetState = atomWithStorage<TrivetState>(

packages/app/src/state/ui.ts

@@ -1,8 +1,8 @@
 import { atom } from 'jotai';
 import { atomWithStorage } from 'jotai/utils';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
-const storage = createStorage('ui');
+const { storage } = createHybridStorage('ui');
 
 export const debuggerPanelOpenState = atom<boolean>(false);
 

packages/app/src/state/userInput.ts

@@ -1,9 +1,9 @@
 import { atom } from 'jotai';
 import { atomWithStorage } from 'jotai/utils';
 import { type ArrayDataValue, type NodeId, type ProcessId, type StringDataValue } from '@ironclad/rivet-core';
-import { createStorage } from './storage.js';
+import { createHybridStorage } from './storage.js';
 
-const storage = createStorage('userInput');
+const { storage } = createHybridStorage('userInput');
 
 export const userInputModalOpenState = atom<boolean>(false);