diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9b0f82d..5cb8306 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,33 +2,65 @@ name: tests on: [push, pull_request] -env: - REGISTRY: ghcr.io/commaai - BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $REGISTRY/rednose:latest -t rednose -f Dockerfile . - RUN: docker run rednose bash -c - jobs: - test: - runs-on: ubuntu-latest + static_analysis: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - name: Build docker image - run: eval ${{ env.BUILD }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Installing requirements + run: + python -m pip install -r requirements.txt - name: Static analysis - run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all" - - name: Unit Tests - run: ${{ env.RUN }} "pytest" + run: pre-commit run --all + + scons_test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"] + python: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Installing python requirements + run: + python -m pip install -r requirements.txt + - name: Installing ubuntu requirements + if: ${{startsWith(matrix.os, 'ubuntu')}} + run: + scripts/ubuntu_dependencies.sh + - name: Installing macos requirements + if: ${{startsWith(matrix.os, 'macos')}} + run: + scripts/macos_dependencies.sh + - name: Building + run: scons -j$(nproc || sysctl -n hw.logicalcpu) - docker_push: - name: docker push - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/rednose' + full_test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"] + python: ["3.11", "3.12"] steps: - uses: actions/checkout@v4 - - name: Build Docker image - run: eval ${{ env.BUILD }} - - name: Push to dockerhub - run: | - docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - docker tag rednose ${{ env.REGISTRY }}/rednose:latest - docker push ${{ env.REGISTRY }}/rednose:latest + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Installing ubuntu requirements + if: ${{startsWith(matrix.os, 'ubuntu')}} + run: + scripts/ubuntu_dependencies.sh + - name: Installing macos requirements + if: ${{startsWith(matrix.os, 'macos')}} + run: + scripts/macos_dependencies.sh + - name: Installing rednose + run: pip install -e .[dev] + - name: Running all tests + run: pytest diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..b4db7b7 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,51 @@ +name: wheels + +on: [push, pull_request] + +jobs: + wheels: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64", arch: "x86_64"}, + {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"}, + {os: "macos-14", target: "macosx_arm64", arch: "arm64"} ] + python: [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ] + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Building wheel + uses: pypa/cibuildwheel@v2.19.2 + env: + CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}" + CIBW_ARCHS: "${{ matrix.platform.arch }}" + CIBW_BEFORE_ALL_LINUX: "dnf install -y clang libffi-devel eigen3-devel python${{ matrix.python.py }}-devel" + CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" + CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package} + + + - uses: actions/setup-python@v5 + if: ${{ matrix.platform.arch != 'aarch64' }} + with: + python-version: ${{ matrix.python.py }} + + - name: Installing the wheel + if: ${{ matrix.platform.arch != 'aarch64' }} + run: | + pip install --break-system-packages ./wheelhouse/*.whl + + - name: Saving wheel + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }} + path: ./wheelhouse/*.whl diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 57bd4c1..0000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y capnproto libcapnp-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl libeigen3-dev python3-pip python3-dev - -WORKDIR /project - -ENV PYTHONPATH=/project - -COPY . . -RUN rm -rf .git -RUN pip3 install --break-system-packages --no-cache-dir -r requirements.txt -RUN python3 setup.py install -RUN scons -c && scons -j$(nproc) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..360135c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +exclude rednose/**/*.o +include SConstruct +include rednose/SConscript +include rednose/examples/SConscript +recursive-include site_scons * diff --git a/SConstruct b/SConstruct index c668d1b..3fe4232 100644 --- a/SConstruct +++ b/SConstruct @@ -1,22 +1,35 @@ import os +import platform import subprocess import sysconfig import numpy as np arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() +if platform.system() == "Darwin": + arch = "Darwin" common = '' +py_include = sysconfig.get_paths()['include'] -python_path = sysconfig.get_paths()['include'] +libpath = [] cpppath = [ '#', '#rednose', '#rednose/examples/generated', '/usr/lib/include', - python_path, + py_include, np.get_include(), ] +if arch == "Darwin": + brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip() + libpath += [ + f"{brew_prefix}/lib", + ] + cpppath += [ + f"{brew_prefix}/include", + ] + env = Environment( ENV=os.environ, CC='clang', @@ -32,7 +45,7 @@ env = Environment( "-Werror=format-extra-args", "-Wshadow", ], - LIBPATH=["#rednose/examples/generated"], + LIBPATH=libpath + ["#rednose/examples/generated"], CFLAGS="-std=gnu11", CXXFLAGS="-std=c++1z", CPPPATH=cpppath, @@ -43,17 +56,18 @@ env = Environment( # Cython build enviroment envCython = env.Clone() envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"] +envCython["CPPPATH"] += [py_include, np.get_include()] envCython["LIBS"] = [] if arch == "Darwin": envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] elif arch == "aarch64": envCython["LINKFLAGS"] = ["-shared"] - envCython["LIBS"] = [os.path.basename(python_path)] + envCython["LIBS"] = [os.path.basename(py_include)] else: envCython["LINKFLAGS"] = ["-pthread", "-shared"] Export('env', 'envCython', 'common') SConscript(['#rednose/SConscript']) -SConscript(['#examples/SConscript']) +SConscript(['#rednose/examples/SConscript']) diff --git a/pyproject.toml b/pyproject.toml index fa6ae11..1337e81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,31 @@ -# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml +[project] +name = "rednose" +requires-python = ">= 3.11" +version = "0.1.0" + +dependencies = [ + "sympy", + "numpy<2.0", + "scipy", + "tqdm", + "cffi", + "scons", + "Cython", + "setuptools", +] + +[project.optional-dependencies] +dev = [ + "ruff", + "pre-commit", + "pytest", + "pytest-xdist", +] + +[build-system] +requires = ["setuptools", "scons", "Cython", "numpy<2.0", "sympy", "pytest", "cffi"] +build-backend = "setuptools.build_meta" + [tool.ruff] line-length = 160 target-version="py311" diff --git a/examples/SConscript b/rednose/examples/SConscript similarity index 87% rename from examples/SConscript rename to rednose/examples/SConscript index 3d178ad..57ba76e 100644 --- a/examples/SConscript +++ b/rednose/examples/SConscript @@ -1,5 +1,7 @@ Import('env') +env.AppendENVPath('PYTHONPATH', env["REDNOSE_ROOT"]) + gen_dir = Dir('generated/').abspath env.RednoseCompileFilter( diff --git a/examples/__init__.py b/rednose/examples/__init__.py similarity index 100% rename from examples/__init__.py rename to rednose/examples/__init__.py diff --git a/examples/kinematic_kf.png b/rednose/examples/kinematic_kf.png similarity index 100% rename from examples/kinematic_kf.png rename to rednose/examples/kinematic_kf.png diff --git a/examples/kinematic_kf.py b/rednose/examples/kinematic_kf.py similarity index 100% rename from examples/kinematic_kf.py rename to rednose/examples/kinematic_kf.py diff --git a/examples/live_kf.py b/rednose/examples/live_kf.py similarity index 100% rename from examples/live_kf.py rename to rednose/examples/live_kf.py diff --git a/examples/test_compare.py b/rednose/examples/test_compare.py similarity index 100% rename from examples/test_compare.py rename to rednose/examples/test_compare.py diff --git a/examples/test_kinematic_kf.py b/rednose/examples/test_kinematic_kf.py similarity index 100% rename from examples/test_kinematic_kf.py rename to rednose/examples/test_kinematic_kf.py diff --git a/requirements.txt b/requirements.txt index 81875a8..d5a0215 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,58 @@ -ruff -sympy -numpy -scipy -tqdm -cffi -scons -pre-commit -Cython -pytest -pytest-xdist +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras -o requirements.txt pyproject.toml +cffi==1.16.0 + # via rednose (pyproject.toml) +cfgv==3.4.0 + # via pre-commit +cython==3.0.10 + # via rednose (pyproject.toml) +distlib==0.3.8 + # via virtualenv +execnet==2.1.1 + # via pytest-xdist +filelock==3.15.4 + # via virtualenv +identify==2.6.0 + # via pre-commit +iniconfig==2.0.0 + # via pytest +mpmath==1.3.0 + # via sympy +nodeenv==1.9.1 + # via pre-commit +numpy==1.26.4 + # via + # rednose (pyproject.toml) + # scipy +packaging==24.1 + # via pytest +platformdirs==4.2.2 + # via virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==3.7.1 + # via rednose (pyproject.toml) +pycparser==2.22 + # via cffi +pytest==8.2.2 + # via + # rednose (pyproject.toml) + # pytest-xdist +pytest-xdist==3.6.1 + # via rednose (pyproject.toml) +pyyaml==6.0.1 + # via pre-commit +ruff==0.5.3 + # via rednose (pyproject.toml) +scipy==1.14.0 + # via rednose (pyproject.toml) +scons==4.8.0 + # via rednose (pyproject.toml) +setuptools==71.0.4 + # via rednose (pyproject.toml) +sympy==1.13.1 + # via rednose (pyproject.toml) +tqdm==4.66.4 + # via rednose (pyproject.toml) +virtualenv==20.26.3 + # via pre-commit diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh new file mode 100755 index 0000000..10534fc --- /dev/null +++ b/scripts/macos_dependencies.sh @@ -0,0 +1,6 @@ +#!/bin/bash +brew bundle --file=- <<-EOS +brew "eigen" +cask "gcc-arm-embedded" +brew "gcc@13" +EOS diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh new file mode 100755 index 0000000..652f236 --- /dev/null +++ b/scripts/ubuntu_dependencies.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ ! $(id -u) -eq 0 ]]; then + if [[ -z $(which sudo) ]]; then + echo "Please install sudo or run as root" + exit 1 + fi + SUDO="sudo" +fi + +$SUDO apt-get update +$SUDO apt-get install -y --no-install-recommends clang libeigen3-dev diff --git a/setup.py b/setup.py index 492b58f..823c18e 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,31 @@ -import os -from setuptools import setup, find_packages +from setuptools import Command, setup, Distribution +from setuptools.command.build import build +import subprocess + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +class SconsBuild(Command): + def initialize_options(self) -> None: + pass + + def finalize_options(self) -> None: + pass + + def run(self) -> None: + subprocess.run(["scons -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode() + + +class CustomBuild(build): + sub_commands = [('scons_build', None)] + build.sub_commands -here = os.path.abspath(os.path.dirname(__file__)) setup( - name='rednose', - version='0.0.1', - url='https://github.com/commaai/rednose', - author='comma.ai', - author_email='harald@comma.ai', - packages=find_packages(), - platforms='any', - license='MIT', - package_data={'': ['helpers/chi2_lookup_table.npy', 'templates/*']}, - install_requires=[ - 'sympy', - 'numpy', - 'scipy', - 'tqdm', - 'cffi', - ], - ext_modules=[], - description="Kalman filter library", - long_description='See https://github.com/commaai/rednose', -) + packages = ["rednose", "rednose.examples", "rednose.helpers"], + package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']}, + include_package_data=True, + cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}, + distclass=BinaryDistribution, + )