From 143b114ae1f68885250e1375b78f8614f3463a23 Mon Sep 17 00:00:00 2001 From: Praateek Date: Wed, 30 Oct 2024 09:15:41 -0700 Subject: [PATCH] remove debug statements Signed-off-by: Praateek --- nemo_curator/modules/fuzzy_dedup.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nemo_curator/modules/fuzzy_dedup.py b/nemo_curator/modules/fuzzy_dedup.py index 63106129..9441a6ba 100644 --- a/nemo_curator/modules/fuzzy_dedup.py +++ b/nemo_curator/modules/fuzzy_dedup.py @@ -1141,11 +1141,6 @@ def _batched_merge_and_write( text_part_offset:end_text_offset ] - print("==== HERE ====") - print(subset_text_df.compute().to_pandas().to_dict("records")) - print(subset_bucket_df.compute().to_pandas().to_dict("records")) - print("==== HERE OVER ====") - try: # NOTE: If we have more text-df partitions than bucket-map # partitions, we are more likely to see an OverflowError