add classifiers and semdedup

Signed-off-by: Sarah Yurick <[email protected]>

nemo_curator/scripts/classifiers/aegis_classifier_inference.py

@@ -17,6 +17,8 @@
 import warnings
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
+
+from nemo_curator import __version__
 from nemo_curator.classifiers import AegisClassifier
 from nemo_curator.datasets import DocumentDataset
 
@@ -103,7 +105,10 @@ def main():
 
 
 def attach_args():
-    parser = ArgumentHelper.parse_distributed_classifier_args(max_chars_default=6000)
+    parser = ArgumentHelper.parse_distributed_classifier_args(
+        description=f"\nNVIDIA NeMo Curator -- v{__version__}\n\nRun AEGIS classifier inference",
+        max_chars_default=6000,
+    )
 
     parser.add_argument(
         "--aegis-variant",

nemo_curator/scripts/classifiers/domain_classifier_inference.py

@@ -17,6 +17,8 @@
 import warnings
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
+
+from nemo_curator import __version__
 from nemo_curator.classifiers import DomainClassifier
 from nemo_curator.datasets import DocumentDataset
 
@@ -29,7 +31,9 @@
 
 
 def main():
-    args = ArgumentHelper.parse_distributed_classifier_args().parse_args()
+    args = ArgumentHelper.parse_distributed_classifier_args(
+        description=f"\nNVIDIA NeMo Curator -- v{__version__}\n\nRun domain classifier inference"
+    ).parse_args()
     print(f"Arguments parsed = {args}", flush=True)
     client_args = ArgumentHelper.parse_client_args(args)
     client_args["cluster_type"] = "gpu"

nemo_curator/scripts/classifiers/fineweb_edu_classifier_inference.py

@@ -19,7 +19,6 @@
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
 
 from nemo_curator import __version__
-
 from nemo_curator.classifiers import FineWebEduClassifier
 from nemo_curator.datasets import DocumentDataset
 

nemo_curator/scripts/classifiers/quality_classifier_inference.py

@@ -17,6 +17,8 @@
 import warnings
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
+
+from nemo_curator import __version__
 from nemo_curator.classifiers import QualityClassifier
 from nemo_curator.datasets import DocumentDataset
 
@@ -29,7 +31,9 @@
 
 
 def main():
-    args = ArgumentHelper.parse_distributed_classifier_args().parse_args()
+    args = ArgumentHelper.parse_distributed_classifier_args(
+        description=f"\nNVIDIA NeMo Curator -- v{__version__}\n\nRun quality classifier inference"
+    ).parse_args()
     print(f"Arguments parsed = {args}", flush=True)
 
     client_args = ArgumentHelper.parse_client_args(args)

nemo_curator/scripts/semdedup/clustering.py

@@ -18,6 +18,7 @@
 
 import dask_cudf
 
+from nemo_curator import __version__
 from nemo_curator.datasets import DocumentDataset
 from nemo_curator.log import create_logger
 from nemo_curator.modules.config import SemDedupConfig
@@ -81,6 +82,7 @@ def main(args):
 def attach_args():
     parser = ArgumentHelper.parse_semdedup_args(
         description=(
+            f"NVIDIA NeMo Curator -- v{__version__} "
             "Performs clustering on the computed embeddings of a collection of documents. "
             "This script requires that the embeddings have been created beforehand using: "
             "semdedup_extract_embeddings"

nemo_curator/scripts/semdedup/compute_embeddings.py

@@ -16,6 +16,7 @@
 import os
 import time
 
+from nemo_curator import __version__
 from nemo_curator.datasets import DocumentDataset
 from nemo_curator.log import create_logger
 from nemo_curator.modules.config import SemDedupConfig
@@ -87,6 +88,7 @@ def main(args):
 def attach_args():
     parser = ArgumentHelper.parse_semdedup_args(
         description=(
+            f"NVIDIA NeMo Curator -- v{__version__} "
             "Computes the embeddings of a collection of documents using the specified model. "
             "The model is specified in the config file using embedding_model_name_or_path (e.g. 'sentence-transformers/paraphrase-MiniLM-L6-v2'). "
             "The embeddings are saved in the specified cache directory under the embeddings_save_loc directory. "

nemo_curator/scripts/semdedup/extract_dedup_data.py

@@ -2,6 +2,7 @@
 import os
 from datetime import datetime
 
+from nemo_curator import __version__
 from nemo_curator.log import create_logger
 from nemo_curator.modules.config import SemDedupConfig
 from nemo_curator.modules.semantic_dedup import SemanticClusterLevelDedup
@@ -62,6 +63,7 @@ def main(args):
 def attach_args():
     parser = ArgumentHelper.parse_semdedup_args(
         description=(
+            f"NVIDIA NeMo Curator -- v{__version__} "
             "Extracts deduplicated data from the clustered embeddings of a collection of documents. "
             "This script requires that embeddings and clustering have been performed beforehand using the specified configurations. "
             "earlier using semdedup_extract_embeddings and semdedup_cluster_embeddings."

nemo_curator/utils/script_utils.py

@@ -30,11 +30,12 @@ def __init__(self, parser: argparse.ArgumentParser):
 
     def attach_version_arg(self):
         self.parser.add_argument(
-            "--version", "-v",
+            "--version",
+            "-v",
             action="version",
             version=f"NVIDIA NeMo Curator -- v{__version__}",
-            help="Show the version and exit."
-    )
+            help="Show the version and exit.",
+        )
 
     @staticmethod
     def attach_bool_arg(