diff --git a/.gitignore b/.gitignore index 0908f2e2..c27fd589 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ benchmarks/results target/ *.so **/target/ -**/dist/ \ No newline at end of file +**/dist/ + +bindings/python/build +bindings/python/src/outlines_core \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0a20ecac..52fc1582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -304,6 +304,7 @@ dependencies = [ name = "python-bindings" version = "0.1.0" dependencies = [ + "outlines-core", "pyo3", ] diff --git a/README.md b/README.md index 18f13af6..df4d1a7e 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,25 @@ ## developing -- build only the outlines-core package `cd outlines-core && cargo build` -- dev build of python bindings `cd bindings/python && maturin develop`. If you have the conda `outlines-dev` environment activated, the outlines-core module is installed within the env automatically +There's a [justfile](https://github.com/casey/just) for most dev & build tasks -There's also a [justfile](https://github.com/casey/just) for running these easier: -- `just dev-core` -- `just dev-python` +- build only the outlines-core rust crate `cd outlines-core && cargo build` +- install an editable pip package with the recepie `just dev-python` which is: +```bash +cd bindings/python && pip install -e . +``` +- to build the python package, run `just build-python`, which is equivalent to: +```bash +cd bindings/python && \ +ln -sf ../../outlines-core outlines-core-lib && \ +sed -i '' 's|path = "../../outlines-core"|path = "outlines-core-lib"|' Cargo.toml && \ +python -m build && \ +rm outlines-core-lib && \ +sed -i '' 's|path = "outlines-core-lib"|path = "../../outlines-core"|' Cargo.toml +``` + +### Developer Notes + +- Setup a virtual environment before running the build or dev commands + +- If you get the `LookupError: setuptools-scm was unable to detect version for...` error, set the env var `SETUPTOOLS_SCM_PRETEND_VERSION=0.1.0-dev` before running the build or dev command. diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index da2038be..a97f9dda 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -5,8 +5,6 @@ edition = "2021" [dependencies] pyo3 = "0.22.0" -# outlines-core = { path = "../../outlines-core" } - [profile.release-lto] inherits = "release" lto = true @@ -15,3 +13,6 @@ lto = true name = "_lib" crate-type = ["cdylib"] path = "rust/lib.rs" + +[dependencies.outlines-core] +path = "../../outlines-core" diff --git a/bindings/python/Manifest.in b/bindings/python/Manifest.in index 855d922d..8b5bea03 100644 --- a/bindings/python/Manifest.in +++ b/bindings/python/Manifest.in @@ -1,6 +1,9 @@ -graft python +graft src graft rust -graft tests include Cargo.toml + global-exclude */__pycache__/* global-exclude *.pyc + +recursive-include outlines-core-lib * +recursive-exclude outlines-core/target * \ No newline at end of file diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 65a21838..356cb06f 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -4,11 +4,14 @@ build-backend = "setuptools.build_meta" [project] name = "outlines_core" -authors= [{name = "Outlines Developers"}] +authors = [ + { name = "Outlines Developers" }, + { name = "Huggingface Developers" }, +] description = "Structured Text Generation in Rust" requires-python = ">=3.8" -license = {text = "Apache-2.0"} -keywords=[ +license = { text = "Apache-2.0" } +keywords = [ "machine learning", "deep learning", "language models", @@ -24,17 +27,17 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "interegular", - "numpy<2.0.0", - "cloudpickle", - "diskcache", - "pydantic>=2.0", - "numba", - "referencing", - "jsonschema", - "tqdm", - "datasets", - "typing_extensions", + "interegular", + "numpy<2.0.0", + "cloudpickle", + "diskcache", + "pydantic>=2.0", + "numba", + "referencing", + "jsonschema", + "tqdm", + "datasets", + "typing_extensions", ] dynamic = ["version"] @@ -61,7 +64,7 @@ documentation = "https://outlines-dev.github.io/outlines-core/" repository = "https://github.com/outlines-dev/outlines-core/" [project.readme] -file="README.md" +file = "README.md" content-type = "text/markdown" [tool.setuptools] @@ -109,21 +112,12 @@ module = [ ignore_missing_imports = true [tool.coverage.run] -omit = [ - "src/outlines_core/_version.py", - "tests/*", -] +omit = ["src/outlines_core/_version.py", "tests/*"] branch = true [tool.coverage.report] -omit = [ - "tests/*", -] -exclude_lines = [ - "pragma: no cover", - "if TYPE_CHECKING:", - "\\.\\.\\.", -] +omit = ["tests/*"] +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "\\.\\.\\."] show_missing = true [tool.diff_cover] @@ -131,4 +125,4 @@ compare_branch = "origin/main" diff_range_notation = ".." [[tool.setuptools-rust.ext-modules]] -target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package. +target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package. diff --git a/bindings/python/rust/lib.rs b/bindings/python/rust/lib.rs index 69ad5356..b1bfb828 100644 --- a/bindings/python/rust/lib.rs +++ b/bindings/python/rust/lib.rs @@ -4,9 +4,9 @@ // use pyo3::types::{PyAnyMethods, PyDict, PyModule, PyModuleMethods, PySet}; use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyResult}; - #[pymodule] mod _lib { + use outlines_core::FLAG; use pyo3::prelude::*; /// Formats the sum of two numbers as string. @@ -14,6 +14,16 @@ mod _lib { fn sum_as_string(a: usize, b: usize) -> PyResult { Ok((a + b).to_string()) } + + #[pyfunction] + fn show_me_the_flag() -> PyResult { + Ok(FLAG.to_string()) + } + + #[pyfunction] + fn anotherone() -> PyResult { + Ok("This is another one".to_string()) + } } // #[pymodule] diff --git a/justfile b/justfile index 491a1066..21f1be7a 100644 --- a/justfile +++ b/justfile @@ -8,4 +8,10 @@ dev-python: cd bindings/python && pip install -e . build-python: - cd bindings/python && python -m build + cd bindings/python && \ + ln -sf ../../outlines-core outlines-core-lib && \ + sed -i '' 's|path = "../../outlines-core"|path = "outlines-core-lib"|' Cargo.toml && \ + python -m build && \ + rm outlines-core-lib && \ + sed -i '' 's|path = "outlines-core-lib"|path = "../../outlines-core"|' Cargo.toml + diff --git a/outlines-core/src/lib.rs b/outlines-core/src/lib.rs index 14e3dbb2..a81aa77f 100644 --- a/outlines-core/src/lib.rs +++ b/outlines-core/src/lib.rs @@ -3,6 +3,8 @@ use std::collections::BTreeSet; use std::sync::Arc; use std::thread; +pub const FLAG: bool = true; + pub fn create_fsm_index_end_to_end_rust( fsm_transitions: &BTreeMap<(i32, i32), i32>, alphabet_symbol_mapping: &BTreeMap,