From ec6097407bc51f4d440cb9258c66c9ef90ecc76e Mon Sep 17 00:00:00 2001 From: Arvin Xu Date: Fri, 13 Dec 2024 00:33:54 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20perf:=20downset=20tokenize?= =?UTF-8?q?r=20threshold=20to=20reduce=20browser=20memory=20usage=20(#5000?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⚡️ perf: down token account * Update index.ts --- src/utils/tokenizer/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/tokenizer/index.ts b/src/utils/tokenizer/index.ts index 1c6071a6c116..6c0c49672736 100644 --- a/src/utils/tokenizer/index.ts +++ b/src/utils/tokenizer/index.ts @@ -1,9 +1,9 @@ export const encodeAsync = async (str: string): Promise => { if (str.length === 0) return 0; - // 50_000 is the limit of the client - // if the string is longer than 100_000, we will use the server - if (str.length <= 50_000) { + // 30_000 is the limit of the client + // if the string is longer than 30_000, we will use the server + if (str.length <= 30_000) { const { clientEncodeAsync } = await import('./client'); return await clientEncodeAsync(str);