Nit improvements to package installation (#219)

CONTRIBUTING.md

@@ -55,7 +55,7 @@ pip install -r requirements-devel.txt
 > the fairseq2n installation command above to get the most up-to-date binary. If
 > you observe runtime or test failures after the installation, it might be
 > because the latest nightlies are not published yet. If the problem persists
-> after about 12 hours, please create a
+> for more than 12 hours, please create a
 > [GitHub issue](https://github.com/facebookresearch/fairseq2/issues/new/choose).
 
 ## Testing Your Work

INSTALL_FROM_SOURCE.md

@@ -47,7 +47,14 @@ to learn more about other environment options.
 
 > [!IMPORTANT]
 > We strongly recommend creating a new environment from scratch instead of
-reusing an existing one to avoid dependency conflicts.
+> reusing an existing one to avoid dependency conflicts.
+
+
+> [!IMPORTANT]
+> Manually building fairseq2 or any other C++ project in a Conda environment can
+> become tricky and fail due to environment-specific conflicts with the host
+> system libraries. Unless necessary, we recommend using a Python virtual
+> environment to build fairseq2.
 
 
 ## 3. Install Dependencies

native/CMakeLists.txt

@@ -14,7 +14,7 @@ endif()
 
 if(DEFINED ENV{CONDA_PREFIX} AND NOT DEFINED ENV{CONDA_BUILD_SYSROOT})
     message(FATAL_ERROR
-        "It looks like you are in a Conda environment, but the compiler packages are not installed. Please run `conda install -c conda-forge compilers` first."
+        "It looks like you are in a Conda environment, but the `compilers` package is not installed. Please run `conda install -c conda-forge compilers=1.2.0` first."
     )
 endif()
 

native/cmake/modules/FindTorch.cmake

@@ -129,6 +129,10 @@ endif()
 
 if(TORCH_CUDA_LIBRARY)
     __torch_determine_cuda_version()
+
+    set(TORCH_VARIANT "CUDA ${TORCH_CUDA_VERSION_MAJOR}.${TORCH_CUDA_VERSION_MINOR}")
+else()
+    set(TORCH_VARIANT "CPU-only")
 endif()
 
 if(NOT TARGET torch_cxx11_abi)
@@ -147,8 +151,9 @@ if(NOT TARGET torch_cxx11_abi)
         )
 
         if(result EQUAL 0)
-            target_compile_definitions(torch_cxx11_abi INTERFACE
-                _GLIBCXX_USE_CXX11_ABI=$<BOOL:${TORCH_CXX11_ABI}>
+            target_compile_definitions(torch_cxx11_abi
+                INTERFACE
+                    _GLIBCXX_USE_CXX11_ABI=$<BOOL:${TORCH_CXX11_ABI}>
             )
         endif()
 

native/python/requirements-build.txt

@@ -1,6 +1,5 @@
 cmake~=3.26
 ninja~=1.11
-packaging~=23.1
 pip~=23.2
 setuptools~=67.8
 tbb-devel==2021.8;platform_machine=='x86_64'

native/python/setup.py

@@ -8,7 +8,6 @@
 from typing import Final, List, Optional
 
 import torch
-from packaging import version
 from setuptools import Command, find_packages, setup
 from setuptools.command.install import install as install_base
 from setuptools.dist import Distribution as DistributionBase
@@ -168,6 +167,6 @@ def get_inputs(self) -> List[str]:
         # PyTorch has no ABI compatibility between releases; this means we have
         # to ensure that we depend on the exact same version that we used to
         # build fairseq2n.
-        "torch==" + version.parse(torch.__version__).public,
+        "torch==" + torch.__version__.split("+", 1)[0],  # Trim the label.
     ],
 )

native/python/src/fairseq2n/__init__.py

@@ -6,17 +6,16 @@
 
 import platform
 import site
-import sys
 from ctypes import CDLL, RTLD_GLOBAL
-from ctypes.util import find_library
 from os import environ
 from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import List, Optional, Tuple
 
 from fairseq2n.config import (
     _CUDA_VERSION,
     _SUPPORTS_CUDA,
     _SUPPORTS_IMAGE,
+    _TORCH_VARIANT,
     _TORCH_VERSION,
 )
 
@@ -40,7 +39,12 @@ def get_cmake_prefix_path() -> Path:
 
 def torch_version() -> str:
     """Return the version of PyTorch that was used to build fairseq2n."""
-    return _TORCH_VERSION.split("+", 1)[0]
+    return _TORCH_VERSION
+
+
+def torch_variant() -> str:
+    """Return the variant of PyTorch that was used to build fairseq2n."""
+    return _TORCH_VARIANT
 
 
 def supports_image() -> bool:
@@ -170,22 +174,15 @@ def _check_torch_version() -> None:
     # Trim the local version label.
     source_version = torch.__version__.split("+", 1)[0]
 
-    target_version = torch_version()
-
     if source_var := torch.version.cuda:
         # Use only the major and minor version segments.
         source_variant = "CUDA " + ".".join(source_var.split(".", 2)[:2])
     else:
         source_variant = "CPU-only"
 
-    if target_var := cuda_version():
-        target_variant = "CUDA " + f"{target_var[0]}.{target_var[1]}"
-    else:
-        target_variant = "CPU-only"
-
-    if source_version != target_version or source_variant != target_variant:
+    if source_version != _TORCH_VERSION or source_variant != _TORCH_VARIANT:
         raise RuntimeError(
-            f"fairseq2 requires a {target_variant} build of PyTorch {target_version}, but the installed version is a {source_variant} build of PyTorch {source_version}. Either follow the instructions at https://pytorch.org/get-started/locally to update PyTorch, or the instructions at https://github.com/facebookresearch/fairseq2#variants to update fairseq2."
+            f"fairseq2 requires a {_TORCH_VARIANT} build of PyTorch {_TORCH_VERSION}, but the installed version is a {source_variant} build of PyTorch {source_version}. Either follow the instructions at https://pytorch.org/get-started/locally to update PyTorch, or the instructions at https://github.com/facebookresearch/fairseq2#variants to update fairseq2."
         )
 
 

native/python/src/fairseq2n/config.py.in

@@ -6,10 +6,10 @@
 
 from typing import Final
 
-_TORCH_VERSION: Final = "@TORCH_PEP440_VERSION@"
+_TORCH_VERSION: Final = "@TORCH_VERSION@"
+_TORCH_VARIANT: Final = "@TORCH_VARIANT@"
 
 _SUPPORTS_IMAGE: Final = @SUPPORTS_IMAGE@
 
 _SUPPORTS_CUDA: Final = @USES_CUDA@
-
 _CUDA_VERSION: Final = @CUDA_VERSION@

native/src/fairseq2n/config.h.in

@@ -11,17 +11,17 @@
 
 namespace fairseq2n {
 
-constexpr std::int32_t version_major = @PROJECT_VERSION_MAJOR@;
-constexpr std::int32_t version_minor = @PROJECT_VERSION_MINOR@;
-constexpr std::int32_t version_patch = @PROJECT_VERSION_PATCH@;
+inline constexpr std::int32_t version_major = @PROJECT_VERSION_MAJOR@;
+inline constexpr std::int32_t version_minor = @PROJECT_VERSION_MINOR@;
+inline constexpr std::int32_t version_patch = @PROJECT_VERSION_PATCH@;
 
-constexpr char torch_version[] = "@TORCH_PEP440_VERSION@";
+inline constexpr char torch_version[] = "@TORCH_VERSION@";
+inline constexpr char torch_variant[] = "@TORCH_VARIANT@"
 
-constexpr bool supports_image = @SUPPORTS_IMAGE@;
+inline constexpr bool supports_image = @SUPPORTS_IMAGE@;
 
-constexpr bool supports_cuda = @USES_CUDA@;
-
-constexpr std::optional<std::int32_t> cuda_version_major = @CUDA_VERSION_MAJOR@;
-constexpr std::optional<std::int32_t> cuda_version_minor = @CUDA_VERSION_MINOR@;
+inline constexpr bool supports_cuda = @USES_CUDA@;
+inline constexpr std::optional<std::int32_t> cuda_version_major = @CUDA_VERSION_MAJOR@;
+inline constexpr std::optional<std::int32_t> cuda_version_minor = @CUDA_VERSION_MINOR@;
 
 }  // namespace fairseq2n

native/third-party/libjpeg-turbo.cmake

@@ -24,7 +24,7 @@ macro(fairseq2n_add_libjpeg_turbo)
 
         ExternalProject_Add(
             #NAME
-                jpeg_turbo_proj
+                jpeg_turbo
             PREFIX
                 ${prefix}
             GIT_REPOSITORY
@@ -50,7 +50,7 @@ macro(fairseq2n_add_libjpeg_turbo)
 
         add_library(jpeg_turbo_static STATIC IMPORTED)
 
-        add_dependencies(jpeg_turbo_static jpeg_turbo_proj)
+        add_dependencies(jpeg_turbo_static jpeg_turbo)
 
         set_property(TARGET jpeg_turbo_static PROPERTY IMPORTED_LOCATION ${JPEG_TURBO_LIBRARY})
 

setup.py

@@ -44,13 +44,10 @@
     python_requires=">=3.8",
     install_requires=[
         "fairseq2n" + fairseq2n_version_spec,
-        "jiwer~=3.0",
         "numpy~=1.23",
         "overrides~=7.3",
         "packaging~=23.1",
         "pyyaml~=6.0",
-        "sacrebleu~=2.3",
-        "torch>=1.12.1",
         "torcheval~=0.0.6",
         "tqdm~=4.62",
         "typing_extensions~=4.3;python_version<'3.10'",

tests/integration/parquet/test_parquet_dataloader.py

@@ -19,9 +19,6 @@
     import pandas as pd
     import pyarrow as pa
     import pyarrow.parquet as pq
-
-    arrow_found = True
-
     from numpy.typing import NDArray
 
     from recipes.parquet.parquet_dataloader import (
@@ -30,7 +27,7 @@
         parquet_iterator,
     )
 except ImportError:
-    arrow_found = False
+    pytest.skip("arrow not found", allow_module_level=True)
 
 
 def gen_random_string(length: int) -> str:
@@ -99,7 +96,6 @@ def multi_partition_file() -> Generator[str, None, None]:
     shutil.rmtree(tmpdir)
 
 
-@pytest.mark.skipif(not arrow_found, reason="arrow not found")
 class TestParquetDataloader:
     def test_simple_dataload(self, multi_partition_file: str) -> None:
         config = ParquetBasicDataloaderConfig(