Fix broken NeMo dependencies

.github/workflows/test.yml

@@ -37,9 +37,8 @@ jobs:
 
         # Installing wheel beforehand due to fasttext issue:
         # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666
-        # Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94
         run: |
-          pip install wheel cython
+          pip install wheel
           pip install --no-cache-dir .
           pip install pytest
       - name: Run tests

Dockerfile

@@ -38,7 +38,7 @@ RUN conda create -y --name curator -c conda-forge -c nvidia \
   libcusparse \
   libcusolver && \
   source activate curator && \
-  pip install --upgrade cython pytest pip
+  pip install --upgrade pytest pip
 
 RUN \
 --mount=type=bind,source=/opt/NeMo-Curator/nemo_curator/__init__.py,target=/opt/NeMo-Curator/nemo_curator/__init__.py,from=curator-update \

README.md

@@ -83,14 +83,12 @@ You can get NeMo-Curator in 3 ways.
 #### PyPi
 
 ```bash
-pip install cython
 pip install --extra-index-url https://pypi.nvidia.com nemo-curator[all]
 ```
 
 #### Source
 ```bash
 git clone https://github.com/NVIDIA/NeMo-Curator.git
-pip install cython
 pip install --extra-index-url https://pypi.nvidia.com "./NeMo-Curator[all]"
 ```
 

docs/user-guide/image/gettingstarted.rst

@@ -33,7 +33,6 @@ NeMo Curator's PyPi page can be found `here <https://pypi.org/project/nemo-curat
 
 .. code-block:: bash
 
-    pip install cython
     pip install nemo-curator[image]
 
 #####################
@@ -44,7 +43,6 @@ NeMo Curator's GitHub can be found `here <https://github.com/NVIDIA/NeMo-Curator
 .. code-block:: bash
 
     git clone https://github.com/NVIDIA/NeMo-Curator.git
-    pip install cython
     pip install ./NeMo-Curator[image]
 
 ############################

nemo_curator/filters/code.py

@@ -15,6 +15,7 @@
 import csv
 import warnings
 
+import sentencepiece
 from bs4 import BeautifulSoup
 from comment_parser import comment_parser
 
@@ -102,19 +103,18 @@ def keep_document(self, score):
 class TokenizerFertilityFilter(DocumentFilter):
 
     def __init__(self, path_to_tokenizer=None, min_char_to_token_ratio=2.5):
-        from nemo.collections.common.tokenizers import SentencePieceTokenizer
-
         if path_to_tokenizer is None:
             raise ValueError(
                 "Must provide a valid path to a SentencePiece " "tokenizer"
             )
-        self._tokenizer = SentencePieceTokenizer(path_to_tokenizer)
+        self._tokenizer = sentencepiece.SentencePieceProcessor()
+        self._tokenizer.Load(path_to_tokenizer)
         self._threshold = min_char_to_token_ratio
 
         self._name = "tokenizer_fertility"
 
     def score_document(self, source):
-        tokens = self._tokenizer.text_to_tokens(source)
+        tokens = self._tokenizer.encode_as_pieces(source)
         num_chars = len(source)
         num_tokens = len(tokens)
         if num_tokens == 0:

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "crossfit>=0.0.6",
     "dask-mpi>=2021.11.0",
     "dask[complete]>=2021.7.1",
+    "datasets",
     "distributed>=2021.7.1",
     "fasttext==0.9.2",
     "ftfy==6.1.1",
@@ -54,14 +55,14 @@ dependencies = [
     "lxml_html_clean",
     "mecab-python3",
     "mwparserfromhell==0.6.5",
-    "nemo_toolkit[nlp]>=1.23.0",
     "numpy<2",
     "openai",
     "peft",
     "presidio-analyzer==2.2.351",
     "presidio-anonymizer==2.2.351",
     "pycld2",
     "resiliparse",
+    "sentencepiece",
     "spacy>=3.6.0, <3.8.0",
     "unidic-lite==1.0.8",
     "usaddress==0.5.10",

tutorials/image-curation/image-curation.ipynb

@@ -49,7 +49,7 @@
    },
    "outputs": [],
    "source": [
-    "!pip install cython ipywidgets aiofiles\n",
+    "!pip install ipywidgets aiofiles\n",
     "# Install from source by default\n",
     "!pip install --extra-index-url https://pypi.nvidia.com ../../[image]\n",
     "%env DASK_DATAFRAME__QUERY_PLANNING False"

tutorials/synthetic-data-hello-world/Synthetic Data Generation - Hello World Examples.ipynb

@@ -58,11 +58,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install -qU wheel cython"
+    "!pip install -qU wheel"
    ]
   },
   {