Replace null tokens and new lines in imported text

BibleNLP · Jan 23, 2024 · 0bd47b6 · 0bd47b6
1 parent 61a4233
commit 0bd47b6
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/app/core/vectordb/postgres4langchain.py b/app/core/vectordb/postgres4langchain.py
@@ -136,6 +136,12 @@ def add_to_collection(self, docs: List[schema.Document], **kwargs) -> None:
         """Loads the document object as per chroma DB formats into the collection"""
         data_list = []
         for doc in docs:
+            doc.text = (doc.text
+                        .replace("\n", " ")
+                        .replace("\r", " ")
+                        .replace("\t", " ")
+                        .replace('\x00', '')
+            )
             cur = self.db_conn.cursor()
             cur.execute(
                 "SELECT 1 FROM embeddings WHERE source_id = %s", (doc.docId,))