From cdfb47ae10612615e09e06ca1c908bff97b4bb50 Mon Sep 17 00:00:00 2001 From: Walter Teng Date: Wed, 13 Nov 2024 15:57:59 +0800 Subject: [PATCH] update obsolete flag (#351) Signed-off-by: Walter Teng <16046667+davzoku@users.noreply.github.com> --- nemo_curator/scripts/find_exact_duplicates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_curator/scripts/find_exact_duplicates.py b/nemo_curator/scripts/find_exact_duplicates.py index 4a3bde88..e71ef5e1 100644 --- a/nemo_curator/scripts/find_exact_duplicates.py +++ b/nemo_curator/scripts/find_exact_duplicates.py @@ -60,7 +60,7 @@ def main(args): df = read_data( files[:num_files] if num_files else files, file_type="jsonl", - backend="pandas" if args.no_gpu else "cudf", + backend="pandas" if args.device != "gpu" else "cudf", files_per_partition=args.files_per_partition, add_filename=False, )[[id_field, text_field]]