Real python package

.github/workflows/tests.yml

@@ -2,33 +2,65 @@ name: tests
 
 on: [push, pull_request]
 
-env:
-  REGISTRY: ghcr.io/commaai
-  BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $REGISTRY/rednose:latest -t rednose -f Dockerfile .
-  RUN: docker run rednose bash -c
-
 jobs:
-  test:
-    runs-on: ubuntu-latest
+  static_analysis:
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
-    - name: Build docker image
-      run: eval ${{ env.BUILD }}
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Installing requirements
+      run:
+        python -m pip install -r requirements.txt
     - name: Static analysis
-      run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"
-    - name: Unit Tests
-      run: ${{ env.RUN }} "pytest"
+      run: pre-commit run --all
+
+  scons_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        python: ["3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Installing python requirements
+      run:
+        python -m pip install -r requirements.txt
+    - name: Installing ubuntu requirements
+      if: ${{startsWith(matrix.os, 'ubuntu')}}
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Installing macos requirements
+      if: ${{startsWith(matrix.os, 'macos')}}
+      run:
+        scripts/macos_dependencies.sh
+    - name: Building
+      run: scons -j$(nproc || sysctl -n hw.logicalcpu)
 
-  docker_push:
-    name: docker push
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/rednose'
+  full_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        python: ["3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
-    - name: Build Docker image
-      run: eval ${{ env.BUILD }}
-    - name: Push to dockerhub
-      run: |
-        docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-        docker tag rednose ${{ env.REGISTRY }}/rednose:latest
-        docker push ${{ env.REGISTRY }}/rednose:latest
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Installing ubuntu requirements
+      if: ${{startsWith(matrix.os, 'ubuntu')}}
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Installing macos requirements
+      if: ${{startsWith(matrix.os, 'macos')}}
+      run:
+        scripts/macos_dependencies.sh
+    - name: Installing rednose
+      run: pip install -e .[dev]
+    - name: Running all tests
+      run: pytest

.github/workflows/wheels.yml

@@ -0,0 +1,51 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64",  arch: "x86_64"},
+                    {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+                    {os: "macos-14",      target: "macosx_arm64",      arch: "arm64"} ]
+        python:   [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+
+    - name: Building wheel
+      uses: pypa/[email protected]
+      env:
+        CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+        CIBW_ARCHS: "${{ matrix.platform.arch }}"
+        CIBW_BEFORE_ALL_LINUX: "dnf install -y clang libffi-devel eigen3-devel python${{ matrix.python.py }}-devel"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+        CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+        CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package}
+
+
+    - uses: actions/setup-python@v5
+      if: ${{ matrix.platform.arch != 'aarch64' }}
+      with:
+        python-version: ${{ matrix.python.py }}
+
+    - name: Installing the wheel
+      if: ${{ matrix.platform.arch != 'aarch64' }}
+      run: |
+        pip install --break-system-packages ./wheelhouse/*.whl
+
+    - name: Saving wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+        path: ./wheelhouse/*.whl

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude rednose/**/*.o
+include SConstruct
+include rednose/SConscript
+include rednose/examples/SConscript
+recursive-include site_scons *

SConstruct

@@ -1,22 +1,35 @@
 import os
+import platform
 import subprocess
 import sysconfig
 import numpy as np
 
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
 
 common = ''
+py_include = sysconfig.get_paths()['include']
 
-python_path = sysconfig.get_paths()['include']
+libpath = []
 cpppath = [
   '#',
   '#rednose',
   '#rednose/examples/generated',
   '/usr/lib/include',
-  python_path,
+  py_include,
   np.get_include(),
 ]
 
+if arch == "Darwin":
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+  libpath += [
+    f"{brew_prefix}/lib",
+  ]
+  cpppath += [
+    f"{brew_prefix}/include",
+  ]
+
 env = Environment(
   ENV=os.environ,
   CC='clang',
@@ -32,7 +45,7 @@ env = Environment(
     "-Werror=format-extra-args",
     "-Wshadow",
   ],
-  LIBPATH=["#rednose/examples/generated"],
+  LIBPATH=libpath + ["#rednose/examples/generated"],
   CFLAGS="-std=gnu11",
   CXXFLAGS="-std=c++1z",
   CPPPATH=cpppath,
@@ -43,17 +56,18 @@ env = Environment(
 # Cython build enviroment
 envCython = env.Clone()
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [py_include, np.get_include()]
 
 envCython["LIBS"] = []
 if arch == "Darwin":
   envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
 elif arch == "aarch64":
   envCython["LINKFLAGS"] = ["-shared"]
-  envCython["LIBS"] = [os.path.basename(python_path)]
+  envCython["LIBS"] = [os.path.basename(py_include)]
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 Export('env', 'envCython', 'common')
 
 SConscript(['#rednose/SConscript'])
-SConscript(['#examples/SConscript'])
+SConscript(['#rednose/examples/SConscript'])

pyproject.toml

@@ -1,4 +1,31 @@
-# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
+[project]
+name = "rednose"
+requires-python = ">= 3.11"
+version = "0.1.0"
+
+dependencies = [
+  "sympy",
+  "numpy<2.0",
+  "scipy",
+  "tqdm",
+  "cffi",
+  "scons",
+  "Cython",
+  "setuptools",
+]
+
+[project.optional-dependencies]
+dev = [
+  "ruff",
+  "pre-commit",
+  "pytest",
+  "pytest-xdist",
+]
+
+[build-system]
+requires = ["setuptools", "scons", "Cython", "numpy<2.0", "sympy", "pytest", "cffi"]
+build-backend = "setuptools.build_meta"
+
 [tool.ruff]
 line-length = 160
 target-version="py311"

examples/SConscript → rednose/examples/SConscript

@@ -1,5 +1,7 @@
 Import('env')
 
+env.AppendENVPath('PYTHONPATH', env["REDNOSE_ROOT"])
+
 gen_dir = Dir('generated/').abspath
 
 env.RednoseCompileFilter(

requirements.txt

@@ -1,11 +1,58 @@
-ruff
-sympy
-numpy
-scipy
-tqdm
-cffi
-scons
-pre-commit
-Cython
-pytest
-pytest-xdist
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --all-extras -o requirements.txt pyproject.toml
+cffi==1.16.0
+    # via rednose (pyproject.toml)
+cfgv==3.4.0
+    # via pre-commit
+cython==3.0.10
+    # via rednose (pyproject.toml)
+distlib==0.3.8
+    # via virtualenv
+execnet==2.1.1
+    # via pytest-xdist
+filelock==3.15.4
+    # via virtualenv
+identify==2.6.0
+    # via pre-commit
+iniconfig==2.0.0
+    # via pytest
+mpmath==1.3.0
+    # via sympy
+nodeenv==1.9.1
+    # via pre-commit
+numpy==1.26.4
+    # via
+    #   rednose (pyproject.toml)
+    #   scipy
+packaging==24.1
+    # via pytest
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==3.7.1
+    # via rednose (pyproject.toml)
+pycparser==2.22
+    # via cffi
+pytest==8.2.2
+    # via
+    #   rednose (pyproject.toml)
+    #   pytest-xdist
+pytest-xdist==3.6.1
+    # via rednose (pyproject.toml)
+pyyaml==6.0.1
+    # via pre-commit
+ruff==0.5.3
+    # via rednose (pyproject.toml)
+scipy==1.14.0
+    # via rednose (pyproject.toml)
+scons==4.8.0
+    # via rednose (pyproject.toml)
+setuptools==71.0.4
+    # via rednose (pyproject.toml)
+sympy==1.13.1
+    # via rednose (pyproject.toml)
+tqdm==4.66.4
+    # via rednose (pyproject.toml)
+virtualenv==20.26.3
+    # via pre-commit

scripts/macos_dependencies.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+brew bundle --file=- <<-EOS
+brew "eigen"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS

scripts/ubuntu_dependencies.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ ! $(id -u) -eq 0 ]]; then
+  if [[ -z $(which sudo) ]]; then
+    echo "Please install sudo or run as root"
+    exit 1
+  fi
+  SUDO="sudo"
+fi
+
+$SUDO apt-get update
+$SUDO apt-get install -y --no-install-recommends clang libeigen3-dev