Filter out whitespace-only strings before embedding (#384)

NVIDIA · Jan 29, 2025 · aaa8501 · aaa8501
1 parent f8c2e8c
commit aaa8501
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/nv_ingest/modules/transforms/embed_extractions.py b/src/nv_ingest/modules/transforms/embed_extractions.py
@@ -474,7 +474,9 @@ def _generate_embeddings(
                 continue
 
             cudf_content_getter = cudf_content_extractor[content_type]
-            content_mask = (content_mask & (cudf_content_getter(mdf["metadata"]) != "")).fillna(False)
+            # Embedding NIMs will complain if text has only whitespaces.
+            content_text_mask = cudf_content_getter(mdf["metadata"]).str.strip() != ""
+            content_mask = (content_mask & content_text_mask).fillna(False)
             if not content_mask.any():
                 continue