Fix id field (NVIDIA#250)

Signed-off-by: Ryan Wolf <[email protected]> Signed-off-by: Vinay Raman <[email protected]>
vinay-raman · Nov 12, 2024 · 092da34 · 092da34
1 parent b0adda5
commit 092da34
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/nemo_curator/scripts/fuzzy_deduplication/minhash_lsh.py b/nemo_curator/scripts/fuzzy_deduplication/minhash_lsh.py
@@ -54,7 +54,7 @@ def main(args):
             dask_cudf.read_parquet(data_path, blocksize="2GB", aggregate_files=True)
         )
     df = dask_cudf.concat(dfs, ignore_unknown_divisions=True)
-    df = df[~df.id_field.isna()]
+    df = df[~df[id_field].isna()]
     df = df.map_partitions(
         convert_str_id_to_int,
         id_column=id_field,