diff --git a/frontend/app/api/history-index/route.ts b/frontend/app/api/history-index/route.ts index dc199a9..a027f55 100644 --- a/frontend/app/api/history-index/route.ts +++ b/frontend/app/api/history-index/route.ts @@ -26,7 +26,7 @@ export async function POST(req: Request) { } const result = await response.json(); - return Response.json(result); + return Response.json('Success'); } catch (error) { console.error('Full index error:', error); return Response.json({ error: 'Failed to process full index' }, { status: 500 }); diff --git a/frontend/components/modal/search-model.tsx b/frontend/components/modal/search-model.tsx index bf2f2bd..ff267ec 100644 --- a/frontend/components/modal/search-model.tsx +++ b/frontend/components/modal/search-model.tsx @@ -7,10 +7,11 @@ import { Input } from '@/components/ui/input'; import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; import { Loader2, MessageCircle } from 'lucide-react'; -import { ScrollArea } from '@/components/ui/scroll-area'; import { isUserFullIndexed } from '@/lib/store/search'; import { User } from '@/lib/types'; import { Alert, AlertDescription } from '@/components/ui/alert'; +import { resolveTime } from '@/lib/utils'; +import { toast } from 'sonner'; interface SearchResult { id: string; @@ -29,6 +30,7 @@ interface SearchResult { title: string; url: string; text: string; + create_time: Date; } export function SearchDialog({ openSearch: open, onOpenModelChange: onOpenChange, user: user }: SearchDialogProps) { @@ -50,6 +52,7 @@ export function SearchDialog({ openSearch: open, onOpenModelChange: onOpenChange const [isIndexing, setIsIndexing] = useState(false); const handleFullIndex = async () => { + if (isIndexing) return; setIsIndexing(true); try { const response = await fetch('/api/history-index', { @@ -62,7 +65,12 @@ export function SearchDialog({ openSearch: open, onOpenModelChange: onOpenChange const result = await response.json(); if (result === 'Success') { - setIsIndexed(true); + toast.success('Historical messages have started to index', { + description: + 'MemFre is building your search index in the background. It will take several minutes to complete. You can use the search function after it is completed.', + duration: 5000, + }); + onOpenChange(false); } } catch (error) { console.error('Failed to trigger full index:', error); @@ -137,31 +145,36 @@ export function SearchDialog({ openSearch: open, onOpenModelChange: onOpenChange - +
{isLoading ? (
Searching ...
) : ( -
+
{results.map((result) => (
handleResultClick(result.url)} > -
+

{result.title}

-

{result.text}

+

+ {result.text} +

+
+
+ {resolveTime(result.create_time)}
))}
)} - +
)} diff --git a/frontend/components/sidebar/sidebar-header.tsx b/frontend/components/sidebar/sidebar-header.tsx index e01adb1..9127450 100644 --- a/frontend/components/sidebar/sidebar-header.tsx +++ b/frontend/components/sidebar/sidebar-header.tsx @@ -25,16 +25,16 @@ export function SidebarHeader({ user }: UserProps) { {siteConfig.name}
- {/* */} +
- {/* */} +
); } diff --git a/frontend/components/sidebar/sidebar-item.tsx b/frontend/components/sidebar/sidebar-item.tsx index 6c2dd38..4f90ed7 100644 --- a/frontend/components/sidebar/sidebar-item.tsx +++ b/frontend/components/sidebar/sidebar-item.tsx @@ -55,7 +55,7 @@ export function SidebarItem({ search: search, children }: SidebarItemProps) { {search.messages.length} {search.messages.length > 1 ? 'messages' : 'message'} - {!isActive && {resolveTime(search)}} + {!isActive && {resolveTime(search.createdAt)}} {isActive &&
{children}
} diff --git a/frontend/lib/tools/auto.ts b/frontend/lib/tools/auto.ts index 056a54a..4d10de7 100644 --- a/frontend/lib/tools/auto.ts +++ b/frontend/lib/tools/auto.ts @@ -275,9 +275,9 @@ export async function autoAnswer( // }); await saveMessages(userId, messages, fullAnswer, texts, images, videos, fullRelated, SearchCategory.ALL); - // indexMessage(userId, messages[0].title, messages[0].id, query + '\n\n' + fullAnswer).catch((error) => { - // console.error(`Failed to index message for user ${userId}:`, error); - // }); + indexMessage(userId, messages[0].title, messages[0].id, query + '\n\n' + fullAnswer).catch((error) => { + console.error(`Failed to index message for user ${userId}:`, error); + }); onStream?.(null, true); } catch (error) { console.error('Error:', error); diff --git a/frontend/lib/tools/chat.ts b/frontend/lib/tools/chat.ts index 4255c68..a7de54b 100644 --- a/frontend/lib/tools/chat.ts +++ b/frontend/lib/tools/chat.ts @@ -75,9 +75,9 @@ export async function chat( // console.error(`Failed to increment search count for user ${userId}:`, error); // }); await saveMessages(userId, messages, fullAnswer, [], [], [], '', SearchCategory.ALL); - // indexMessage(userId, messages[0].title, messages[0].id, query + '\n\n' + fullAnswer).catch((error) => { - // console.error(`Failed to index message for user ${userId}:`, error); - // }); + indexMessage(userId, messages[0].title, messages[0].id, query + '\n\n' + fullAnswer).catch((error) => { + console.error(`Failed to index message for user ${userId}:`, error); + }); onStream?.(null, true); } catch (error) { diff --git a/frontend/lib/tools/index-message.ts b/frontend/lib/tools/index-message.ts index 5fba311..84b7e73 100644 --- a/frontend/lib/tools/index-message.ts +++ b/frontend/lib/tools/index-message.ts @@ -20,6 +20,7 @@ export async function indexMessage(userId: string, title: string, url: string, t title: title, text: text, url: url, + timestamp: new Date(), }), }); diff --git a/frontend/lib/utils.ts b/frontend/lib/utils.ts index f0f0e6d..41f1760 100644 --- a/frontend/lib/utils.ts +++ b/frontend/lib/utils.ts @@ -19,8 +19,8 @@ export function formatDate(input: string | number): string { }); } -export function resolveTime(search) { - const createdAt = new Date(search.createdAt); +export function resolveTime(timestamp: Date | number): string { + const createdAt = new Date(timestamp); const now = new Date(); const timeDiff = now.getTime() - createdAt.getTime(); @@ -33,7 +33,7 @@ export function resolveTime(search) { if (diffInHours <= 24) return `${diffInHours} hours ago`; if (diffInDays <= 7) return `${diffInDays} days ago`; - return format(new Date(search.createdAt), 'MMM d, yyyy h:mm a'); + return format(new Date(timestamp), 'MMM d, yyyy'); } export function formatDateTime(input: string | number): string { diff --git a/vector/history-api.ts b/vector/history-api.ts index 3fbb029..730a78c 100644 --- a/vector/history-api.ts +++ b/vector/history-api.ts @@ -22,21 +22,6 @@ async function handleRequest(req: Request): Promise { const path = new URL(req.url).pathname; const { method } = req; - // if (method === "OPTIONS") { - // const origin = req.headers.get("Origin"); - // if (origin && allowedOrigins.includes(origin)) { - // return new Response("OK", { - // headers: { - // "Access-Control-Allow-Origin": origin, - // "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", - // "Access-Control-Allow-Headers": "Authorization, Content-Type, Token", - // }, - // }); - // } else { - // return new Response("Forbidden", { status: 403 }); - // } - // } - let authResponse = checkAuth(req, path); if (authResponse) { return authResponse; @@ -46,7 +31,7 @@ async function handleRequest(req: Request): Promise { const { query, userId } = await req.json(); try { const result = await db.search(userId, query, { - selectFields: ["title", "text", "url", "image"], + selectFields: ["title", "text", "url", "image", "create_time"], }); return Response.json(result); } catch (unknownError) { @@ -129,12 +114,18 @@ async function handleRequest(req: Request): Promise { await markUserIndexing(userId); - const success = await processAllUserSearchMessages(userId); - if (!success) { - return Response.json("Failed to index all users", { status: 500 }); - } - await markUserFullIndexed(userId); - await clearUserIndexing(userId); + Promise.resolve().then(async () => { + try { + const success = await processAllUserSearchMessages(userId); + if (success) { + await markUserFullIndexed(userId); + } + } catch (error) { + console.error("Background indexing error:", error); + } finally { + await clearUserIndexing(userId); + } + }); return Response.json("Success"); } catch (error) { log({ diff --git a/vector/ingest.ts b/vector/ingest.ts index 3f8dc42..dacfd8c 100644 --- a/vector/ingest.ts +++ b/vector/ingest.ts @@ -112,7 +112,8 @@ export async function addVectors( image: string, title: string, url: string, - documents: Document[] + documents: Document[], + timestamp?: number ): Promise>> { const texts = documents.map(({ pageContent }) => pageContent); if (texts.length === 0) { @@ -121,17 +122,19 @@ export async function addVectors( const embeddings = await getEmbedding().embedDocuments(texts); const data: Array> = []; + for (let i = 0; i < documents.length; i += 1) { if (documents[i].pageContent.length < 10) { continue; } + const newImage = image ? extractImage(documents[i].pageContent) : null; const record = { - create_time: Date.now(), + create_time: timestamp || Date.now(), title: title, url: url, - image: newImage ? newImage : image, + image: newImage || image, text: documents[i].pageContent, vector: embeddings[i] as number[], }; diff --git a/vector/memfree_index.ts b/vector/memfree_index.ts index 3d0e96d..118c777 100644 --- a/vector/memfree_index.ts +++ b/vector/memfree_index.ts @@ -67,6 +67,7 @@ export async function processAllUserSearchMessages( try { let lastIndexedTime = Number(await redis.get(LAST_INDEXED_TIME_KEY + userId)) || 0; + let hasError = false; console.time("processSearchMessages"); while (true) { @@ -111,7 +112,8 @@ export async function processAllUserSearchMessages( "", search.title, search.id, - documents + documents, + new Date(search.createdAt).getTime() ); console.log("data length", data.length); await appendData(userId, data); @@ -119,14 +121,17 @@ export async function processAllUserSearchMessages( console.log("search title ", search.title, "search id ", search.id); } catch (error) { console.error(`Failed to process search ${search.id}:`, error); + hasError = true; } }) ); - lastIndexedTime = lastTimestamp; - await redis.set(LAST_INDEXED_TIME_KEY + userId, lastIndexedTime); + if (!hasError) { + lastIndexedTime = lastTimestamp; + await redis.set(LAST_INDEXED_TIME_KEY + userId, lastIndexedTime); + } - if (!hasMore) { + if (!hasMore || hasError) { break; } } @@ -135,7 +140,7 @@ export async function processAllUserSearchMessages( await compact(userId); - return true; + return !hasError; } catch (error) { console.error("Failed to process search messages:", error); return false; diff --git a/vector/redis.ts b/vector/redis.ts index 08e0034..4cd23d6 100644 --- a/vector/redis.ts +++ b/vector/redis.ts @@ -19,8 +19,8 @@ export const TOTAL_SEARCH_COUNT_KEY = "t_s_count:"; export const SEARCH_KEY = "search:"; export const USER_SEARCH_KEY = "user:search:"; -export const LAST_INDEXED_TIME_KEY = "user:last_indexed_time:"; +export const LAST_INDEXED_TIME_KEY = "user:last_indexed_time:"; export const USER_FULL_INDEXED = "user:f-indexed:"; export const USER_INDEXING = "user:indexing:"; @@ -44,6 +44,10 @@ export async function clearUserIndexing(userId: string): Promise { await redis.del(USER_INDEXING + userId); } +export async function clearUserIndexTime(userId: string): Promise { + await redis.del(LAST_INDEXED_TIME_KEY + userId); +} + export async function markUserFullIndexed(userId: string): Promise { await redis.set(USER_FULL_INDEXED + userId, "1"); }