From 8e224d75b308d000f28e344d38254e82f9c23db0 Mon Sep 17 00:00:00 2001 From: vwh Date: Wed, 14 Aug 2024 06:57:28 +0300 Subject: [PATCH] optimize sqlite functions --- src/hooks/useQueryData.ts | 40 ++++++++++----- src/lib/sqlite.ts | 104 +++++++++++++++++++++++--------------- 2 files changed, 92 insertions(+), 52 deletions(-) diff --git a/src/hooks/useQueryData.ts b/src/hooks/useQueryData.ts index 405985d..6253694 100644 --- a/src/hooks/useQueryData.ts +++ b/src/hooks/useQueryData.ts @@ -1,6 +1,5 @@ import { useState, useEffect, useCallback, useMemo } from "react"; import useSQLiteStore from "@/store/useSQLiteStore"; - import { mapQueryResults } from "@/lib/sqlite"; import type { QueryExecResult } from "sql.js"; import type { TableRow } from "@/types"; @@ -58,7 +57,7 @@ export function useQueryData( type: row[2] as string })); - // optimize blob columns + // Optimize blob columns const columnSelects = columnInfo .map((col) => col.type.toUpperCase() === "BLOB" @@ -72,20 +71,22 @@ export function useQueryData( filterQuery ? ` WHERE ${filterQuery}` : "" }`; const countResult: QueryExecResult[] = query(countQueryString); - const totalRows = countResult[0].values[0][0] as number; - setTotalRows(totalRows); + const newTotalRows = countResult[0].values[0][0] as number; + setTotalRows(newTotalRows); + // Main query let queryString = `SELECT ${columnSelects} FROM "${tableName}"`; if (filterQuery) queryString += ` WHERE ${filterQuery}`; if (orderBy) queryString += ` ORDER BY "${orderBy}" ${orderByDirection}`; - queryString += ` LIMIT ${rowsPerPage} OFFSET ${page};`; const tableResult: QueryExecResult[] = query(queryString); - const { data, columns } = mapQueryResults(tableResult); - setColumns(columns); - setData(data); + const { data: newData, columns: newColumns } = + mapQueryResults(tableResult); + + setColumns(newColumns); + setData(newData); setQueryError(null); setCustomQuery(queryString); unShiftToQueryHistory(queryString); @@ -97,22 +98,37 @@ export function useQueryData( }; fetchData(); - }, [db, tableName, page, rowsPerPage, filterQuery, orderByDirection]); + }, [ + db, + tableName, + page, + rowsPerPage, + filterQuery, + orderBy, + orderByDirection, + query, + setQueryError, + setCustomQuery, + unShiftToQueryHistory, + setTotalRows + ]); const handleCustomQuery = useCallback(() => { if (customQuery.trim() === "") { setQueryError(null); return; } + setIsQueryLoading(true); try { const tableName = tables[parseInt(selectedTable)].name; const customResult: QueryExecResult[] = query( customQuery.replace("@", `"${tableName}"`) ); - const { data, columns } = mapQueryResults(customResult); - setColumns(columns); - setData(data); + const { data: newData, columns: newColumns } = + mapQueryResults(customResult); + setColumns(newColumns); + setData(newData); setIsCustomQuery(true); setQueryError(null); } catch (error) { diff --git a/src/lib/sqlite.ts b/src/lib/sqlite.ts index 1836131..d123edb 100644 --- a/src/lib/sqlite.ts +++ b/src/lib/sqlite.ts @@ -4,12 +4,19 @@ import type { TableRow } from "@/types"; const SQL_WASM_PATH = "/sql.wasm"; +// Initialize SQL.js once +let SQL: Awaited>; +const initSQL = async () => { + if (!SQL) { + SQL = await initSqlJs({ locateFile: () => SQL_WASM_PATH }); + } + return SQL; +}; + export const loadDatabase = async (file: File): Promise => { try { - const [arrayBuffer, SQL] = await Promise.all([ - file.arrayBuffer(), - initSqlJs({ locateFile: () => SQL_WASM_PATH }) - ]); + const SQL = await initSQL(); + const arrayBuffer = await file.arrayBuffer(); return new SQL.Database(new Uint8Array(arrayBuffer)); } catch (error) { console.error("Failed to load database:", error); @@ -19,50 +26,59 @@ export const loadDatabase = async (file: File): Promise => { export const getTableNames = (database: Database): string[] => { try { - const result = database.exec( + const stmt = database.prepare( "SELECT name FROM sqlite_master WHERE type='table';" ); - return (result[0]?.values.flat() as string[]) || []; + const names: string[] = []; + while (stmt.step()) { + names.push(stmt.get()[0] as string); + } + stmt.free(); + return names; } catch (error) { console.error("Failed to get table names:", error); return []; } }; -export const getTableSchema = async (database: Database, tableName: string) => { +export const getTableSchema = (database: Database, tableName: string) => { try { - const [tableInfoResult, foreignKeyInfoResult] = database.exec(` - PRAGMA table_info("${tableName}"); - PRAGMA foreign_key_list("${tableName}"); - `); - - const tableSchema = tableInfoResult.values.reduce( - (acc, row) => { - acc[row[1] as string] = { - type: row[2] ? (row[2] as string).toUpperCase() : (row[2] as string), - isPrimaryKey: (row[5] as number) === 1, // 1 means the column is a primary key - isForeignKey: false, - nullable: (row[3] as number) === 0 // 0 means the column is nullable - }; - return acc; - }, - {} as Record< - string, - { - type: string; - isPrimaryKey: boolean; - isForeignKey: boolean; - nullable: boolean; - } - > + const tableSchema: Record< + string, + { + type: string; + isPrimaryKey: boolean; + isForeignKey: boolean; + nullable: boolean; + } + > = {}; + + const tableInfoStmt = database.prepare( + `PRAGMA table_info("${tableName}");` ); + while (tableInfoStmt.step()) { + const row = tableInfoStmt.getAsObject(); + tableSchema[row.name as string] = { + type: row.type + ? (row.type as string).toUpperCase() + : (row.type as string), + isPrimaryKey: row.pk === 1, + isForeignKey: false, + nullable: row.notnull === 0 + }; + } + tableInfoStmt.free(); - foreignKeyInfoResult?.values.forEach((row) => { - const columnName = row[3] as string; - if (tableSchema[columnName]) { - tableSchema[columnName].isForeignKey = true; + const foreignKeyStmt = database.prepare( + `PRAGMA foreign_key_list("${tableName}");` + ); + while (foreignKeyStmt.step()) { + const row = foreignKeyStmt.getAsObject(); + if (tableSchema[row.from as string]) { + tableSchema[row.from as string].isForeignKey = true; } - }); + } + foreignKeyStmt.free(); return tableSchema; } catch (error) { @@ -71,7 +87,6 @@ export const getTableSchema = async (database: Database, tableName: string) => { } }; -// Map query results to data and columns export const mapQueryResults = ( result: QueryExecResult[] ): { @@ -112,11 +127,20 @@ const exportFromQuery = ( tableName: string ): void => { try { - const result = database.exec(query); - if (result.length === 0) { + const stmt = database.prepare(query); + const columns: string[] = stmt.getColumnNames(); + const data: TableRow[] = []; + + while (stmt.step()) { + const row = stmt.getAsObject(); + data.push(row as TableRow); + } + stmt.free(); + + if (data.length === 0) { throw new Error(`Query "${query}" returned no results.`); } - const { data, columns } = mapQueryResults(result); + const csvContent = arrayToCSV(columns, data); const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); saveAs(blob, `${tableName}.csv`);