From 19aa9a30d743869af460ef9125c7027365b8e350 Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Thu, 14 Dec 2023 20:26:38 -0500 Subject: [PATCH] Nit improvements to package installation (#219) --- CONTRIBUTING.md | 2 +- INSTALL_FROM_SOURCE.md | 9 +++++++- native/CMakeLists.txt | 2 +- native/cmake/modules/FindTorch.cmake | 9 ++++++-- native/python/requirements-build.txt | 1 - native/python/setup.py | 3 +-- native/python/src/fairseq2n/__init__.py | 23 ++++++++----------- native/python/src/fairseq2n/config.py.in | 4 ++-- native/src/fairseq2n/config.h.in | 18 +++++++-------- native/third-party/libjpeg-turbo.cmake | 4 ++-- setup.py | 3 --- .../parquet/test_parquet_dataloader.py | 6 +---- 12 files changed, 42 insertions(+), 42 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 64a591b9d..514ea97d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ pip install -r requirements-devel.txt > the fairseq2n installation command above to get the most up-to-date binary. If > you observe runtime or test failures after the installation, it might be > because the latest nightlies are not published yet. If the problem persists -> after about 12 hours, please create a +> for more than 12 hours, please create a > [GitHub issue](https://github.com/facebookresearch/fairseq2/issues/new/choose). ## Testing Your Work diff --git a/INSTALL_FROM_SOURCE.md b/INSTALL_FROM_SOURCE.md index b27a52fe0..18b0daf20 100644 --- a/INSTALL_FROM_SOURCE.md +++ b/INSTALL_FROM_SOURCE.md @@ -47,7 +47,14 @@ to learn more about other environment options. > [!IMPORTANT] > We strongly recommend creating a new environment from scratch instead of -reusing an existing one to avoid dependency conflicts. +> reusing an existing one to avoid dependency conflicts. + + +> [!IMPORTANT] +> Manually building fairseq2 or any other C++ project in a Conda environment can +> become tricky and fail due to environment-specific conflicts with the host +> system libraries. Unless necessary, we recommend using a Python virtual +> environment to build fairseq2. ## 3. Install Dependencies diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index aae3795a3..e00b15327 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -14,7 +14,7 @@ endif() if(DEFINED ENV{CONDA_PREFIX} AND NOT DEFINED ENV{CONDA_BUILD_SYSROOT}) message(FATAL_ERROR - "It looks like you are in a Conda environment, but the compiler packages are not installed. Please run `conda install -c conda-forge compilers` first." + "It looks like you are in a Conda environment, but the `compilers` package is not installed. Please run `conda install -c conda-forge compilers=1.2.0` first." ) endif() diff --git a/native/cmake/modules/FindTorch.cmake b/native/cmake/modules/FindTorch.cmake index 573f4f5ff..859984662 100644 --- a/native/cmake/modules/FindTorch.cmake +++ b/native/cmake/modules/FindTorch.cmake @@ -129,6 +129,10 @@ endif() if(TORCH_CUDA_LIBRARY) __torch_determine_cuda_version() + + set(TORCH_VARIANT "CUDA ${TORCH_CUDA_VERSION_MAJOR}.${TORCH_CUDA_VERSION_MINOR}") +else() + set(TORCH_VARIANT "CPU-only") endif() if(NOT TARGET torch_cxx11_abi) @@ -147,8 +151,9 @@ if(NOT TARGET torch_cxx11_abi) ) if(result EQUAL 0) - target_compile_definitions(torch_cxx11_abi INTERFACE - _GLIBCXX_USE_CXX11_ABI=$ + target_compile_definitions(torch_cxx11_abi + INTERFACE + _GLIBCXX_USE_CXX11_ABI=$ ) endif() diff --git a/native/python/requirements-build.txt b/native/python/requirements-build.txt index 11c7b4a0d..b30ce934a 100644 --- a/native/python/requirements-build.txt +++ b/native/python/requirements-build.txt @@ -1,6 +1,5 @@ cmake~=3.26 ninja~=1.11 -packaging~=23.1 pip~=23.2 setuptools~=67.8 tbb-devel==2021.8;platform_machine=='x86_64' diff --git a/native/python/setup.py b/native/python/setup.py index b50d2a287..20535fc31 100644 --- a/native/python/setup.py +++ b/native/python/setup.py @@ -8,7 +8,6 @@ from typing import Final, List, Optional import torch -from packaging import version from setuptools import Command, find_packages, setup from setuptools.command.install import install as install_base from setuptools.dist import Distribution as DistributionBase @@ -168,6 +167,6 @@ def get_inputs(self) -> List[str]: # PyTorch has no ABI compatibility between releases; this means we have # to ensure that we depend on the exact same version that we used to # build fairseq2n. - "torch==" + version.parse(torch.__version__).public, + "torch==" + torch.__version__.split("+", 1)[0], # Trim the label. ], ) diff --git a/native/python/src/fairseq2n/__init__.py b/native/python/src/fairseq2n/__init__.py index df74fce55..e4c03784b 100644 --- a/native/python/src/fairseq2n/__init__.py +++ b/native/python/src/fairseq2n/__init__.py @@ -6,17 +6,16 @@ import platform import site -import sys from ctypes import CDLL, RTLD_GLOBAL -from ctypes.util import find_library from os import environ from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import List, Optional, Tuple from fairseq2n.config import ( _CUDA_VERSION, _SUPPORTS_CUDA, _SUPPORTS_IMAGE, + _TORCH_VARIANT, _TORCH_VERSION, ) @@ -40,7 +39,12 @@ def get_cmake_prefix_path() -> Path: def torch_version() -> str: """Return the version of PyTorch that was used to build fairseq2n.""" - return _TORCH_VERSION.split("+", 1)[0] + return _TORCH_VERSION + + +def torch_variant() -> str: + """Return the variant of PyTorch that was used to build fairseq2n.""" + return _TORCH_VARIANT def supports_image() -> bool: @@ -170,22 +174,15 @@ def _check_torch_version() -> None: # Trim the local version label. source_version = torch.__version__.split("+", 1)[0] - target_version = torch_version() - if source_var := torch.version.cuda: # Use only the major and minor version segments. source_variant = "CUDA " + ".".join(source_var.split(".", 2)[:2]) else: source_variant = "CPU-only" - if target_var := cuda_version(): - target_variant = "CUDA " + f"{target_var[0]}.{target_var[1]}" - else: - target_variant = "CPU-only" - - if source_version != target_version or source_variant != target_variant: + if source_version != _TORCH_VERSION or source_variant != _TORCH_VARIANT: raise RuntimeError( - f"fairseq2 requires a {target_variant} build of PyTorch {target_version}, but the installed version is a {source_variant} build of PyTorch {source_version}. Either follow the instructions at https://pytorch.org/get-started/locally to update PyTorch, or the instructions at https://github.com/facebookresearch/fairseq2#variants to update fairseq2." + f"fairseq2 requires a {_TORCH_VARIANT} build of PyTorch {_TORCH_VERSION}, but the installed version is a {source_variant} build of PyTorch {source_version}. Either follow the instructions at https://pytorch.org/get-started/locally to update PyTorch, or the instructions at https://github.com/facebookresearch/fairseq2#variants to update fairseq2." ) diff --git a/native/python/src/fairseq2n/config.py.in b/native/python/src/fairseq2n/config.py.in index a7ea08ced..e346d4252 100644 --- a/native/python/src/fairseq2n/config.py.in +++ b/native/python/src/fairseq2n/config.py.in @@ -6,10 +6,10 @@ from typing import Final -_TORCH_VERSION: Final = "@TORCH_PEP440_VERSION@" +_TORCH_VERSION: Final = "@TORCH_VERSION@" +_TORCH_VARIANT: Final = "@TORCH_VARIANT@" _SUPPORTS_IMAGE: Final = @SUPPORTS_IMAGE@ _SUPPORTS_CUDA: Final = @USES_CUDA@ - _CUDA_VERSION: Final = @CUDA_VERSION@ diff --git a/native/src/fairseq2n/config.h.in b/native/src/fairseq2n/config.h.in index a97642cab..1c14b2b6d 100644 --- a/native/src/fairseq2n/config.h.in +++ b/native/src/fairseq2n/config.h.in @@ -11,17 +11,17 @@ namespace fairseq2n { -constexpr std::int32_t version_major = @PROJECT_VERSION_MAJOR@; -constexpr std::int32_t version_minor = @PROJECT_VERSION_MINOR@; -constexpr std::int32_t version_patch = @PROJECT_VERSION_PATCH@; +inline constexpr std::int32_t version_major = @PROJECT_VERSION_MAJOR@; +inline constexpr std::int32_t version_minor = @PROJECT_VERSION_MINOR@; +inline constexpr std::int32_t version_patch = @PROJECT_VERSION_PATCH@; -constexpr char torch_version[] = "@TORCH_PEP440_VERSION@"; +inline constexpr char torch_version[] = "@TORCH_VERSION@"; +inline constexpr char torch_variant[] = "@TORCH_VARIANT@" -constexpr bool supports_image = @SUPPORTS_IMAGE@; +inline constexpr bool supports_image = @SUPPORTS_IMAGE@; -constexpr bool supports_cuda = @USES_CUDA@; - -constexpr std::optional cuda_version_major = @CUDA_VERSION_MAJOR@; -constexpr std::optional cuda_version_minor = @CUDA_VERSION_MINOR@; +inline constexpr bool supports_cuda = @USES_CUDA@; +inline constexpr std::optional cuda_version_major = @CUDA_VERSION_MAJOR@; +inline constexpr std::optional cuda_version_minor = @CUDA_VERSION_MINOR@; } // namespace fairseq2n diff --git a/native/third-party/libjpeg-turbo.cmake b/native/third-party/libjpeg-turbo.cmake index a81806722..20f8ae992 100644 --- a/native/third-party/libjpeg-turbo.cmake +++ b/native/third-party/libjpeg-turbo.cmake @@ -24,7 +24,7 @@ macro(fairseq2n_add_libjpeg_turbo) ExternalProject_Add( #NAME - jpeg_turbo_proj + jpeg_turbo PREFIX ${prefix} GIT_REPOSITORY @@ -50,7 +50,7 @@ macro(fairseq2n_add_libjpeg_turbo) add_library(jpeg_turbo_static STATIC IMPORTED) - add_dependencies(jpeg_turbo_static jpeg_turbo_proj) + add_dependencies(jpeg_turbo_static jpeg_turbo) set_property(TARGET jpeg_turbo_static PROPERTY IMPORTED_LOCATION ${JPEG_TURBO_LIBRARY}) diff --git a/setup.py b/setup.py index b18128dd4..d192d8012 100644 --- a/setup.py +++ b/setup.py @@ -44,13 +44,10 @@ python_requires=">=3.8", install_requires=[ "fairseq2n" + fairseq2n_version_spec, - "jiwer~=3.0", "numpy~=1.23", "overrides~=7.3", "packaging~=23.1", "pyyaml~=6.0", - "sacrebleu~=2.3", - "torch>=1.12.1", "torcheval~=0.0.6", "tqdm~=4.62", "typing_extensions~=4.3;python_version<'3.10'", diff --git a/tests/integration/parquet/test_parquet_dataloader.py b/tests/integration/parquet/test_parquet_dataloader.py index fe00187c3..bb1478574 100644 --- a/tests/integration/parquet/test_parquet_dataloader.py +++ b/tests/integration/parquet/test_parquet_dataloader.py @@ -19,9 +19,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq - - arrow_found = True - from numpy.typing import NDArray from recipes.parquet.parquet_dataloader import ( @@ -30,7 +27,7 @@ parquet_iterator, ) except ImportError: - arrow_found = False + pytest.skip("arrow not found", allow_module_level=True) def gen_random_string(length: int) -> str: @@ -99,7 +96,6 @@ def multi_partition_file() -> Generator[str, None, None]: shutil.rmtree(tmpdir) -@pytest.mark.skipif(not arrow_found, reason="arrow not found") class TestParquetDataloader: def test_simple_dataload(self, multi_partition_file: str) -> None: config = ParquetBasicDataloaderConfig(