add eps_thresholds (#390)

Signed-off-by: Sarah Yurick <[email protected]>
NVIDIA · Nov 26, 2024 · c460374 · c460374
1 parent 9173db3
commit c460374
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/nemo_curator/scripts/semdedup/extract_dedup_data.py b/nemo_curator/scripts/semdedup/extract_dedup_data.py
@@ -45,7 +45,7 @@ def main(args):
         logger=logger,
     )
 
-    semantic_dedup.compute_semantic_match_dfs()
+    semantic_dedup.compute_semantic_match_dfs(semdedup_config.eps_thresholds)
     for eps in semdedup_config.eps_thresholds:
         dedup_id_dataset = semantic_dedup.extract_dedup_data(eps_to_extract=eps)
         print(dedup_id_dataset.df.head(10))