fix bugs

Signed-off-by: rjzamora <[email protected]>
NVIDIA · Sep 17, 2024 · 51e54ff · 51e54ff
1 parent 8ba6e7a
commit 51e54ff
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/nemo_curator/scripts/fuzzy_deduplication/minhash_lsh.py b/nemo_curator/scripts/fuzzy_deduplication/minhash_lsh.py
@@ -54,7 +54,7 @@ def main(args):
             dask_cudf.read_parquet(data_path, blocksize="2GB", aggregate_files=True)
         )
     df = dask_cudf.concat(dfs, ignore_unknown_divisions=True)
-    df = df[~df.id_field.isna()]
+    df = df[~df[id_field].isna()]
     df = df.map_partitions(
         convert_str_id_to_int,
         id_column=id_field,

diff --git a/nemo_curator/utils/fuzzy_dedup_utils/shuffle_utils.py b/nemo_curator/utils/fuzzy_dedup_utils/shuffle_utils.py
@@ -74,7 +74,7 @@ def rearange_by_column_direct(
         return new_collection(
             RearrangeByColumn(
                 frame=df.expr,
-                partitioning_index=df[col].expr,
+                partitioning_index=col,
                 npartitions_out=npartitions,
                 ignore_index=ignore_index,
                 method="tasks",