From 8f62fd174083479ab2f1eb9f1be2847d3934950e Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 19 Jan 2024 09:50:36 -0600 Subject: [PATCH] Meson build, replacing setuptools Limitation: The sdist (source distribution for PyPI) now contains the whole of mmCoreAndDevices and more (33 MiB). While technically functional, we should fix this before merging this change. Update mmCoreAndDevices to latest (which has meson.build for MMCore and MMDevice). Add Meson build file. Use meson-python so that `python -m build` just works via pyproject.toml. Move the single source of truth for the version number from _version.py (now generated) to meson.build. The Meson build has several advantages: - Build details of MMDevice and MMCore come from their own build files, rather than being duplicated here in setup.py - Editable installs truly work (even if C++ files are edited) (Caveat: beware of importing pymmcore from the source root) - Since we are using a true C++ build system, controlling build options is much easier and cleaner than it was with setuptools The main disadvantage is that MANIFEST.in can no longer be used to control what gets included in the sdist (meson-python uses `meson dist` to produce the sdist, which includes all version-controlled files). However, once we are ready to use MMDevice and MMCore from independent repositories, this will no longer be an issue (and ends up being simpler than the error-prone MANIFEST.in). --- .github/workflows/ci.yml | 23 ++----- .gitignore | 8 --- MANIFEST.in | 12 ---- maintainer-notes.md | 42 +++++-------- meson.build | 82 ++++++++++++++++++++++++ pyproject.toml | 22 +++---- setup.py | 121 ------------------------------------ src/pymmcore/_version.py | 1 - src/pymmcore/_version.py.in | 1 + src/pymmcore/meson.build | 89 ++++++++++++++++++++++++++ 10 files changed, 205 insertions(+), 196 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 meson.build delete mode 100644 setup.py delete mode 100644 src/pymmcore/_version.py create mode 100644 src/pymmcore/_version.py.in create mode 100644 src/pymmcore/meson.build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 713dcac..34bee69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,18 +14,6 @@ concurrency: cancel-in-progress: true jobs: - # check that sdist contains all files and that extra files - # are explicitly ignored in manifest or pyproject - check-manifest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: "recursive" - - name: Check manifest - run: pipx run check-manifest - test: name: Test ${{ matrix.os }} py${{ matrix.python-version }} np${{ matrix.numpy }} runs-on: ${{ matrix.os }} @@ -50,6 +38,10 @@ jobs: with: submodules: "recursive" + - uses: ilammy/msvc-dev-cmd@v1 + with: + toolset: "14.2" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -92,10 +84,6 @@ jobs: uses: pypa/cibuildwheel@v2.19 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" - # Python on Linux is usually configured to add debug information, - # which increases binary size by ~11-fold. Remove for the builds we - # distribute. - CIBW_ENVIRONMENT_LINUX: "LDFLAGS=-Wl,--strip-debug" - uses: actions/upload-artifact@v4 with: @@ -113,8 +101,7 @@ jobs: - name: Build sdist run: | - pip install -U pip build check-manifest - check-manifest + pip install -U pip build python -m build --sdist - uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 216189a..a063963 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,7 @@ venv/ build/ dist/ -*.pdb -*.py[cod] - wheelhouse/ -src/pymmcore/pymmcore_swig_wrap.h -src/pymmcore/pymmcore_swig_wrap.cpp -src/pymmcore/_pymmcore_swig.* -src/pymmcore/pymmcore_swig.py -pymmcore.egg-info .mypy_cache/ diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 6a329c0..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,12 +0,0 @@ -recursive-include * *.pyi -recursive-include mmCoreAndDevices/MMDevice *.h *.cpp -recursive-include mmCoreAndDevices/MMCore *.h *.cpp - -prune mmCoreAndDevices/MMDevice/unittest -prune mmCoreAndDevices/MMCore/unittest -prune mmCoreAndDevices/MMCoreJ_wrap -prune mmCoreAndDevices/DeviceAdapters -prune mmCoreAndDevices/m4 -prune mmCoreAndDevices/.github -recursive-exclude mmCoreAndDevices *.txt *.md *.am .project \ - *.vcxproj* *.sln *.props *.cdt* secret-device-* meson.build *.wrap diff --git a/maintainer-notes.md b/maintainer-notes.md index f5107e8..235fc63 100644 --- a/maintainer-notes.md +++ b/maintainer-notes.md @@ -28,9 +28,10 @@ maintaining separate branches; this can ease transition when the device interface version changes. Such branches should be named `mmcore-x.y.z.w`. When upgrading the MMCore version (by bumping the mmCoreAndDevices submodule -commit), the pymmcore version in `_version.py` should be updated in synchrony. -The versioning for the python package is taken dynamically from that file -in the `[tool.setuptools.dynamic]` table in `pyproject.toml`. +commit), the pymmcore version in `meson.build` should be updated in synchrony. +The versioning for the python package is taken dynamically from that file, via +the generated `_version.py` and the `project.dynamic` field in +`pyproject.toml`. ## Building Binary Wheels and Source Distributions @@ -63,12 +64,14 @@ The package can be built in a few ways: This will build an sdist and wheel for the current platform and Python version, and place them in the `dist` directory. -3. Use `pip install -e .` +3. Use `pip install --no-build-isolation -e .` This will build the extension module in-place and allow you to run tests, - but will not build a wheel or sdist. Note that if you do this, you will - need to rerun it each time you change the extension module. - - + but will not build a wheel or sdist. `meson-python` (the build backend) will + arrange to automatically rebuild the extension module each time it is + imported. This method requires that you first manually install all of the + build requirements listed in `pyproject.toml`. See the + [meson-python docs](https://meson-python.readthedocs.io/en/latest/how-to-guides/editable-installs.html) + for more information. ## Release procedure @@ -79,11 +82,11 @@ prefixed to the version: ```bash git checkout main -vim src/pymmcore/_version.py # Remove .dev0 +vim meson.build # Remove .dev0 git commit -a -m 'Version 1.2.3.42.4' git tag -a v1.2.3.42.4 -m Release -vim src/pymmcore/_version.py # Set version to 1.2.3.42.5.dev0 +vim meson.build # Set version to 1.2.3.42.5.dev0 git commit -a -m 'Version back to dev' git push upstream --follow-tags @@ -101,8 +104,9 @@ and the binary wheels attached. - The minimum version of python supported is declared in `pypyproject.toml`, in the `[project.requires-python]` section. -- SWIG 4.x is required and automatically fetched via `pyproject.toml` under - `[build-system.requires]`. +- Meson (via `meson-python`), Ninja, and SWIG 4.x are required and + automatically fetched via `pyproject.toml` under `[build-system.requires]`. +- A C++ toolchain is required and must be available on your system. - The build-time versions of numpy are in `pyproject.toml`, in the `[build-system.requires]` section. - The run-time numpy dependency is declared in `pyproject.toml`, in the @@ -111,7 +115,7 @@ and the binary wheels attached. determined by the settings in the `[tool.cibuildwheel]` section of `pyproject.toml`. - _We_ should provide wheels for all Python versions we claim to support, - built agains the oldest NumPy version that we claim to support. Thus, any + built against the oldest NumPy version that we claim to support. Thus, any issue with the build or our CI will limit the lowest supported versions. ## ABI Compatibility @@ -125,18 +129,6 @@ and the binary wheels attached. [`oldest-supported-numpy`](https://github.com/scipy/oldest-supported-numpy) in our build requires. -## Building with debug symbols on Windows - -Since there is no easy way to pass compile and linker options to `build_clib`, -the easiest hack is to edit the local `setuptools` installation's -`_distutils/_msvccompiler.py` to add the compiler flag `/Zi` and linker flag -`/DEBUG:FULL` (see the method `initialize`). This produces `vc140.pdb`. - -(The "normal" method would be to run `setup.py build_clib` and `setup.py -build_ext` with the `--debug` option, and run with `python_d.exe`. But then we -would need a debug build of NumPy, which is hard to build on Windows.) - - ### Legacy Build Notes Many of these notes are probably obviated by the use of cibuildwheel... but diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..ea19e3f --- /dev/null +++ b/meson.build @@ -0,0 +1,82 @@ +# Copyright 2020-2024 Board of Regents of the University of Wisconsin System +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License, version 2.1, as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Mark A. Tsuchida + +project( + 'pymmcore', + 'cpp', + version: '11.1.1.71.1.dev0', + meson_version: '>=1.3.0', + default_options: [ + 'cpp_std=c++14', + 'warning_level=3', + ], + # Until MMDevice and MMCore are available individually, we need to use them + # from the same git submodule, so this is a bit of a hack: + subproject_dir: 'mmCoreAndDevices', +) + +cxx = meson.get_compiler('cpp') +if cxx.get_id() in ['gcc', 'clang'] + add_project_arguments('-Wno-deprecated', language: 'cpp') # throw() + # Disable warnings triggered by SWIG-generated code: + add_project_arguments('-Wno-unused-parameter', language: 'cpp') + add_project_arguments('-Wno-unused-variable', language: 'cpp') +endif +if cxx.get_id() in ['msvc', 'clang-cl'] + add_project_arguments('-DNOMINMAX', language: 'cpp') + add_project_arguments('-D_CRT_SECURE_NO_WARNINGS', language: 'cpp') + # Disable warnings triggered by SWIG-generated code: + add_project_arguments('/wd4100', language: 'cpp') + add_project_arguments('/wd4101', language: 'cpp') + add_project_arguments('/wd4127', language: 'cpp') + add_project_arguments('/wd4456', language: 'cpp') + add_project_arguments('/wd4706', language: 'cpp') +endif + +fs = import('fs') + +python = import('python').find_installation(pure: false) + +threads_dep = dependency('threads') + +numpy_abs_incdir = run_command( + python, '-c', 'import numpy; print(numpy.get_include())', + check: true, +).stdout().strip() +# The "correct" way would be to "detect" NumPy as a dependency. Since we are +# cutting corners, we need to use a relative path as if the NumPy headers are +# part of this project. +numpy_incdirs = include_directories(fs.relative_to(numpy_abs_incdir, '.')) + +swig = find_program('swig', native: true) + +# For now, use MMCore as a subproject. This may be changed to using as a +# proper dependency via a wrap, but that will likely require better SWIG +# support by Meson in order to get the SWIG include directories from the +# dependency object. +mmcore_proj = subproject( + 'MMCore', + default_options: { + 'default_library': 'static', + 'tests': 'disabled', # Avoid Catch2 subproject in sdist + }, +) +mmcore_dep = mmcore_proj.get_variable('mmcore') + +swig_include_dirs = mmcore_proj.get_variable('swig_include_dirs') + +subdir('src/pymmcore') diff --git a/pyproject.toml b/pyproject.toml index 52a9590..1a628c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,12 @@ # https://peps.python.org/pep-0517/ [build-system] -requires = ["setuptools ==72.1.0", "swig >=4.1", "numpy>=2.0.0"] -build-backend = "setuptools.build_meta" +requires = [ + "meson-python", + "ninja", + "swig >=4.1", + "numpy >=2.0.0", +] +build-backend = "mesonpy" # https://peps.python.org/pep-0621/ [project] @@ -34,15 +39,10 @@ test = ["pytest"] homepage = "https://micro-manager.org" repository = "https://github.com/micro-manager/pymmcore" -[tool.setuptools.dynamic] -version = { attr = "pymmcore._version.__version__" } - -[tool.setuptools.package-dir] -"" = "src" - -[tool.setuptools.package-data] -"*" = ["py.typed", ".pyi"] - +[tool.meson-python.args] +setup = ['-Dstrip=true'] +install = ['--tags=python-runtime,runtime'] +dist = ['--include-subprojects'] [tool.cibuildwheel] # Skip 32-bit builds, musllinux, and PyPy wheels on all platforms diff --git a/setup.py b/setup.py deleted file mode 100644 index ee6e8cc..0000000 --- a/setup.py +++ /dev/null @@ -1,121 +0,0 @@ -# Setup for pymmcore -# -# Copyright (C) 2020-2021 Board of Regents of the University of Wisconsin -# System -# -# This library is free software; you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License, version 2.1, as published -# by the Free Software Foundation. -# -# This library is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Author: Mark A. Tsuchida - -import os -import os.path -import platform -from pathlib import Path - -import numpy -import setuptools.command.build_ext -import setuptools.command.build_py -from setuptools import Extension, setup - -PKG_NAME = "pymmcore" -SWIG_MOD_NAME = "pymmcore_swig" -IS_WINDOWS = platform.system() == "Windows" -ROOT = Path(__file__).parent -MMCorePath = ROOT / "mmCoreAndDevices" / "MMCore" -MMDevicePath = ROOT / "mmCoreAndDevices" / "MMDevice" - -# We build MMCore from sources, into the Python extension. MMCore depends on -# MMDevice. However, we need to build MMDevice separately from MMCore, because -# it requires different preprocessor macros to be defined. For this purpose, we -# make use of a rather obscure feature of distutils/setuptools called -# build_clib. There may be other ways to do this.... - - -# Customize 'build_py' to run 'build_ext' first; otherwise the SWIG-generated -# .py file gets missed. -class build_py(setuptools.command.build_py.build_py): - def run(self): - self.run_command("build_ext") - super().run() - - -# Customize 'build_ext' to trigger 'build_clib' first. -class build_ext(setuptools.command.build_ext.build_ext): - def run(self): - self.run_command("build_clib") - super().run() - - -define_macros = [ - ("MMDEVICE_CLIENT_BUILD", None), -] + ([ - ("NOMINMAX", None), - ("_CRT_SECURE_NO_WARNINGS", None), -] if IS_WINDOWS else []) - -mmdevice_build_info = { - "sources": [str(x.relative_to(ROOT)) for x in MMDevicePath.glob("*.cpp")], - "include_dirs": ["mmCoreAndDevices/MMDevice"], - "macros": define_macros, -} - -omit = ["unittest"] + (["Unix"] if IS_WINDOWS else ["Windows"]) -mmcore_sources = [ - str(x.relative_to(ROOT)) - for x in MMCorePath.rglob("*.cpp") - if all(o not in str(x) for o in omit) -] - -mmcore_libraries = ["MMDevice"] -if not IS_WINDOWS: - mmcore_libraries.extend(["dl"]) - -if not IS_WINDOWS: - cflags = [ - "-std=c++14", - "-fvisibility=hidden", - "-Wno-deprecated", # Hide warnings for throw() specififiers - "-Wno-unused-variable", # Hide warnings for SWIG-generated code - ] - if "CFLAGS" in os.environ: - cflags.append(os.environ["CFLAGS"]) - os.environ["CFLAGS"] = " ".join(cflags) - - -mmcore_extension = Extension( - f"{PKG_NAME}._{SWIG_MOD_NAME}", - sources=mmcore_sources + [os.path.join( - "src", PKG_NAME, f"{SWIG_MOD_NAME}.i", - )], - swig_opts=[ - "-c++", - "-python", - "-builtin", - "-I./mmCoreAndDevices/MMDevice", - "-I./mmCoreAndDevices/MMCore", - ], - include_dirs=[ - numpy.get_include(), - "./mmCoreAndDevices/MMDevice", - "./mmCoreAndDevices/MMCore", - ], - libraries=mmcore_libraries, - define_macros=define_macros, -) - -setup( - ext_modules=[mmcore_extension], - libraries=[("MMDevice", mmdevice_build_info)], - cmdclass={"build_ext": build_ext, "build_py": build_py}, -) diff --git a/src/pymmcore/_version.py b/src/pymmcore/_version.py deleted file mode 100644 index 164cbaf..0000000 --- a/src/pymmcore/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "11.1.1.71.3.dev0" diff --git a/src/pymmcore/_version.py.in b/src/pymmcore/_version.py.in new file mode 100644 index 0000000..b238b3b --- /dev/null +++ b/src/pymmcore/_version.py.in @@ -0,0 +1 @@ +__version__ = "@VERSION@" diff --git a/src/pymmcore/meson.build b/src/pymmcore/meson.build new file mode 100644 index 0000000..3cf7979 --- /dev/null +++ b/src/pymmcore/meson.build @@ -0,0 +1,89 @@ +# Copyright 2020-2024 Board of Regents of the University of Wisconsin System +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License, version 2.1, as published +# by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Mark A. Tsuchida + +swig_incdir_args = [] +foreach abspath : swig_include_dirs + # Use relative path for readability only. + swig_incdir_args += '-I' + fs.relative_to( + abspath, + meson.project_build_root(), + ) +endforeach + +swig_gen = custom_target( + 'swig-pymmcore', + input: 'pymmcore_swig.i', + output: [ + # Note: Order is significant and used below! + 'pymmcore_swig_wrap.cpp', # @OUTPUT0@, swig_gen[0] + 'pymmcore_swig_wrap.h', # @OUTPUT1@, swig_gen[1] + 'pymmcore_swig.py', + ], + depfile: 'pymmcore_swig_wrap.d', + command: [ + swig, + '-c++', + '-python', # -py3 removed in SWIG 4.1 + '-builtin', + swig_incdir_args, + '-MD', '-MF', '@DEPFILE@', + '-o', '@OUTPUT0@', + '-oh', '@OUTPUT1@', + '-outdir', '@OUTDIR@', + '@INPUT@', + ], + # Would be nice to install just the .py, but there seems to be no way to + # split out a file from the target. So leave in the .h/.cpp. + install: true, + install_dir: python.get_install_dir() / 'pymmcore', + install_tag: 'python-runtime', +) +swig_gen_cpp_sources = [swig_gen[0], swig_gen[1]] + +python.extension_module( + '_pymmcore_swig', + swig_gen_cpp_sources, + dependencies: [ + mmcore_dep, + threads_dep, + ], + include_directories: [ + numpy_incdirs, + ], + install: true, + subdir: 'pymmcore', +) + +version_py = configure_file( + configuration: {'VERSION': meson.project_version()}, + input: '_version.py.in', + output: '_version.py', + install: true, + install_dir: python.get_install_dir() / 'pymmcore', + install_tag: 'python-runtime', +) + +pymmcore_sources = files( + '__init__.py', + '__init__.pyi', + 'py.typed', +) + +python.install_sources( + pymmcore_sources, + subdir: 'pymmcore', +)