diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ef626e93a..3af589e0f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -20,15 +20,36 @@ jobs: - run: cargo build --locked - run: cargo test - uses: dtolnay/install-buck2@latest - - name: Build example project (Ubuntu, macOS) + - name: Build example projects (Ubuntu, macOS) run: |- - cd example - reindeer --third-party-dir third-party buckify + pushd examples/01-intro + reindeer buckify buck2 run "//project:test" + popd + + pushd examples/02-hybrid + reindeer buckify + buck2 run "//project:test" + cargo run -p project + popd + + pushd examples/03-complex-fixups + reindeer buckify + buck2 run "//project:test" + popd if: matrix.os == 'ubuntu' || matrix.os == 'macos' - - name: Build example project (Windows) + - name: Build example projects (Windows) run: |- - cd example - & reindeer --third-party-dir third-party buckify + cd examples/01-intro + & reindeer buckify + & buck2 run "//project:test" + + cd examples/02-hybrid + & reindeer buckify + & buck2 run "//project:test" + & cargo run -p project + + cd examples/03-complex-fixups + & reindeer buckify & buck2 run "//project:test" if: matrix.os == 'windows' diff --git a/docs/MANUAL.md b/docs/MANUAL.md index d9a416aca..88270a2e6 100644 --- a/docs/MANUAL.md +++ b/docs/MANUAL.md @@ -30,16 +30,19 @@ environment matches this.) ## An Example -See the [example directory](../example/README.md) as a starting point. This has: +See the [first example directory](../examples/01-intro/) as a starting point. +This has: - Your first-party code in "project" (though of course it can be anywhere and everywhere), and -- A [third-party](../example/third-party) directory which is managed by Reindeer -- A [`setup.sh`](../example/setup.sh) script to get you bootstrapped +- A [third-party](../examples/01-intro/third-party) directory which is managed + by Reindeer +- A [`setup.sh`](../examples/01-intro/setup.sh) script to get you bootstrapped Running `setup.sh` will build Reindeer (using Cargo) and then use it to vendor the small number of third-party dependencies defined in -[`third-party/Cargo.toml`](../third-party/Cargo.toml) and generate build rules +[`third-party/Cargo.toml`](../examples/01-intro/third-party/Cargo.toml) and +generate build rules for them in `third-party/BUCK`. I recommend using this as a starting template for your own project, at least diff --git a/example/third-party/macros/rust_third_party.bzl b/example/third-party/macros/rust_third_party.bzl deleted file mode 100644 index 4c43d4ee5..000000000 --- a/example/third-party/macros/rust_third_party.bzl +++ /dev/null @@ -1,7 +0,0 @@ -def third_party_rust_cxx_library(name, **kwargs): - # @lint-ignore BUCKLINT - native.cxx_library(name = name, **kwargs) - -def third_party_rust_prebuilt_cxx_library(name, **kwargs): - # @lint-ignore BUCKLINT - native.prebuilt_cxx_library(name = name, **kwargs) diff --git a/examples/01-intro/.buckconfig b/examples/01-intro/.buckconfig new file mode 100644 index 000000000..829d0c9ce --- /dev/null +++ b/examples/01-intro/.buckconfig @@ -0,0 +1,24 @@ +[cells] + root = . + prelude = prelude + toolchains = toolchains + none = none + +[cell_aliases] + config = prelude + ovr_config = prelude + fbcode = none + fbsource = none + fbcode_macros = none + buck = none + +# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this +# section and vendor a copy of the prelude to the `prelude` directory of your project. +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default + +[build] + execution_platforms = prelude//platforms:default diff --git a/example/.buckroot b/examples/01-intro/.buckroot similarity index 100% rename from example/.buckroot rename to examples/01-intro/.buckroot diff --git a/examples/01-intro/.gitignore b/examples/01-intro/.gitignore new file mode 100644 index 000000000..c5ffced9a --- /dev/null +++ b/examples/01-intro/.gitignore @@ -0,0 +1,2 @@ +/buck-out +/target diff --git a/examples/01-intro/BUCK b/examples/01-intro/BUCK new file mode 100644 index 000000000..a6f2a23ca --- /dev/null +++ b/examples/01-intro/BUCK @@ -0,0 +1,7 @@ +# A list of available rules and their signatures can be found here: https://buck2.build/docs/prelude/globals/ + +genrule( + name = "hello_world", + out = "out.txt", + cmd = "echo BUILT BY BUCK2> $OUT", +) diff --git a/example/README.md b/examples/01-intro/README.md similarity index 61% rename from example/README.md rename to examples/01-intro/README.md index 0551595f0..814e9228f 100644 --- a/example/README.md +++ b/examples/01-intro/README.md @@ -8,42 +8,42 @@ generate a BUCK file of build rules for them. This will require a Rust installation (stable, but probably fairly recent), and Buck to actually make use of the generated files. -You can learn more about Buck at [buck.build](https://buck.build). The -[getting started](https://buck.build/setup/getting_started.html) page should +You can learn more about Buck at [buck.build](https://buck2.build). The +[getting started](https://buck2.build/docs/about/getting_started/) page should help with getting it installed. ## Buck and its Configuration The `.buckconfig` file both configures Buck and tells it where the top of the -"cell" is (current working tree). This only contains some very minimal Rust -configuration; most notable is overriding the default to Rust 2018. - -(`.buck-java11` won't generally be needed.) +"cell" is (current working tree). ## Reindeer configuration The files and directories Reindeer cares about are under `third-party/`: -- reindeer.toml - Reindeer's configuration. The directory containing this file +- `reindeer.toml` - Reindeer's configuration. The directory containing this + file is also the base for any relative paths mentioned in the file. -- Cargo.toml - You edit this to specify which packages you want to import, along +- `Cargo.toml` - You edit this to specify which packages you want to import, + along with other settings like features, patches and so on, using the full syntax Cargo allows -- Cargo.lock - The resolved dependencies -- BUCK - The generated Buck build rules (once generated) -- .gitignore - This is used to ignore various inconvenient files in vendored +- `Cargo.lock` - The resolved dependencies +- `BUCK` - The generated Buck build rules (once generated) +- `.gitignore` - This is used to ignore various inconvenient files in vendored code. Reindeer itself will look at this to edit them out of the vendored checksum.json files so that Cargo doesn't get upset. In addition to these files, there are a few significant directories: -- vendor/ - where all the vendored code goes -- fixups/ - fixups tell Reindeer how to handle packages with build scripts and +- `vendor/` - where all the vendored code goes +- `fixups/` - fixups tell Reindeer how to handle packages with build scripts + and other special cases; most packages won't need anything here -- macros/ - Buck macros which map from the rules Reindeer generates to the +- `macros/` - Buck macros which map from the rules Reindeer generates to the actual build environment. This directory is not hard-coded and could be anywhere. The macros included here are quite minimal. -- top/ - Cargo needs a dummy package for the Cargo.toml (it doesn't allow a +- `top/` - Cargo needs a dummy package for the Cargo.toml (it doesn't allow a package which is _all_ dependencies) ## Project diff --git a/examples/01-intro/project/BUCK b/examples/01-intro/project/BUCK new file mode 100644 index 000000000..de22a6f15 --- /dev/null +++ b/examples/01-intro/project/BUCK @@ -0,0 +1,7 @@ +rust_binary( + name = "test", + srcs = ["src/main.rs"], + deps = [ + "//third-party:once_cell", + ], +) diff --git a/examples/01-intro/project/src/main.rs b/examples/01-intro/project/src/main.rs new file mode 100644 index 000000000..810e10952 --- /dev/null +++ b/examples/01-intro/project/src/main.rs @@ -0,0 +1,16 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use once_cell::sync::Lazy; + +const MAGIC: &str = "this is a magic string"; + +static SPECIAL: Lazy = Lazy::new(|| MAGIC.to_string()); + +fn main() { + println!("static {}", &*SPECIAL); +} diff --git a/examples/01-intro/reindeer.toml b/examples/01-intro/reindeer.toml new file mode 100644 index 000000000..8158847c1 --- /dev/null +++ b/examples/01-intro/reindeer.toml @@ -0,0 +1,48 @@ +## +## Reindeer Config +## +## This file sets all of Reindeer's basic configuration. This file also marks +## the top of the reindeer-managed directory, as all other paths are relative to +## this one (both paths mentioned in this file, and implicit). +## +## Reindeer is under active development, and the layout and options in this file +## may change. + +# Write output to third-party/BUCK +# This also sets the default input (manifest_path) to third-party/Cargo.toml +third_party_dir = "third-party" + +# If a fixups.toml file is needed (eg, the package has a build.rs), then +# generate a template fixups.toml to be edited. +fixup_templates = true + +# Configuration for generated BUCK file +[buck] +# Name of the generated file +file_name = "BUCK" # default + +# Rules used for various kinds of targets. These rules don't directly +# correspond with BUCK rules - they have extra attributes such as +# per-platform settings. The intent is that you provide a set of macro +# definitions which resolve them to appropriate underlying rules +# suitable for your environment. (This may also work for Buck-like +# build systems such as Bazel.) +rust_library = "cargo.rust_library" # A plain Rust library +rust_binary = "cargo.rust_binary" # A Rust executable +buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources + +# Load the macros to which the rules above will resolve. +buckfile_imports = """ +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") +""" + +# Banner comment for the generated BUCK File. +generated_file_header = """ +## +## \u0040generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## +""" diff --git a/example/setup.sh b/examples/01-intro/setup.sh similarity index 80% rename from example/setup.sh rename to examples/01-intro/setup.sh index a83bd0caa..917803a76 100755 --- a/example/setup.sh +++ b/examples/01-intro/setup.sh @@ -4,11 +4,11 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -# Generate file 'example/third-party/BUCK'. +# Generate file 'third-party/BUCK'. set -e -(cd ..; cargo build) +(cd ../..; cargo build) # Build a BUCK file to build third-party crates. # @@ -19,4 +19,4 @@ set -e # typically commit these fixups and the generated third-party/BUCK in the same # commit as above. -../target/debug/reindeer --third-party-dir third-party buckify +../../target/debug/reindeer --third-party-dir third-party buckify diff --git a/example/third-party/.gitignore b/examples/01-intro/third-party/.gitignore similarity index 91% rename from example/third-party/.gitignore rename to examples/01-intro/third-party/.gitignore index c18f3dc43..cd2eec533 100644 --- a/example/third-party/.gitignore +++ b/examples/01-intro/third-party/.gitignore @@ -1,9 +1,6 @@ # Ignore Cargo-related stuff -.cargo/registry -.cargo/git -/registry -/git -.package-cache +.cargo +target # Various cruft in vendored packages vendor/*/target diff --git a/examples/01-intro/third-party/BUCK b/examples/01-intro/third-party/BUCK new file mode 100644 index 000000000..bea2e1f09 --- /dev/null +++ b/examples/01-intro/third-party/BUCK @@ -0,0 +1,38 @@ +## +## @generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## + +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") + +alias( + name = "once_cell", + actual = ":once_cell-1.20.2", + visibility = ["PUBLIC"], +) + +http_archive( + name = "once_cell-1.20.2.crate", + sha256 = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", + strip_prefix = "once_cell-1.20.2", + urls = ["https://static.crates.io/crates/once_cell/1.20.2/download"], + visibility = [], +) + +cargo.rust_library( + name = "once_cell-1.20.2", + srcs = [":once_cell-1.20.2.crate"], + crate = "once_cell", + crate_root = "once_cell-1.20.2.crate/src/lib.rs", + edition = "2021", + features = [ + "alloc", + "default", + "race", + "std", + ], + visibility = [], +) diff --git a/examples/01-intro/third-party/Cargo.lock b/examples/01-intro/third-party/Cargo.lock new file mode 100644 index 000000000..dbc1b80c6 --- /dev/null +++ b/examples/01-intro/third-party/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "rust-third-party" +version = "0.0.0" +dependencies = [ + "once_cell", +] diff --git a/example/third-party/Cargo.toml b/examples/01-intro/third-party/Cargo.toml similarity index 89% rename from example/third-party/Cargo.toml rename to examples/01-intro/third-party/Cargo.toml index 12ee25d62..8dea07814 100644 --- a/example/third-party/Cargo.toml +++ b/examples/01-intro/third-party/Cargo.toml @@ -15,7 +15,6 @@ path = "top/main.rs" # List of packages to be imported, with version constraints, features # and all options Cargo supports. [dependencies] -blake3 = { version = "0.1", features = ["c_avx512"] } once_cell = "1.4" # Local patches - typically git references diff --git a/examples/01-intro/third-party/top/main.rs b/examples/01-intro/third-party/top/main.rs new file mode 100644 index 000000000..f328e4d9d --- /dev/null +++ b/examples/01-intro/third-party/top/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/examples/01-intro/toolchains/BUCK b/examples/01-intro/toolchains/BUCK new file mode 100644 index 000000000..b3ac5bc3c --- /dev/null +++ b/examples/01-intro/toolchains/BUCK @@ -0,0 +1,5 @@ +load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") + +# All the default toolchains, suitable for a quick demo or early prototyping. +# Most real projects should copy/paste the implementation to configure them. +system_demo_toolchains() \ No newline at end of file diff --git a/example/.buckconfig b/examples/02-hybrid/.buckconfig similarity index 100% rename from example/.buckconfig rename to examples/02-hybrid/.buckconfig diff --git a/example/toolchains/.buckconfig b/examples/02-hybrid/.buckroot similarity index 100% rename from example/toolchains/.buckconfig rename to examples/02-hybrid/.buckroot diff --git a/example/.buckversion b/examples/02-hybrid/.buckversion similarity index 100% rename from example/.buckversion rename to examples/02-hybrid/.buckversion diff --git a/examples/02-hybrid/.gitignore b/examples/02-hybrid/.gitignore new file mode 100644 index 000000000..c5ffced9a --- /dev/null +++ b/examples/02-hybrid/.gitignore @@ -0,0 +1,2 @@ +/buck-out +/target diff --git a/examples/02-hybrid/Cargo.lock b/examples/02-hybrid/Cargo.lock new file mode 100644 index 000000000..bf344e218 --- /dev/null +++ b/examples/02-hybrid/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "project" +version = "0.0.0" +dependencies = [ + "once_cell", +] diff --git a/examples/02-hybrid/Cargo.toml b/examples/02-hybrid/Cargo.toml new file mode 100644 index 000000000..808e5f0ed --- /dev/null +++ b/examples/02-hybrid/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] + +members = ["project"] +resolver = "2" + +# Local patches - typically git references +[patch.crates-io] diff --git a/examples/02-hybrid/README.md b/examples/02-hybrid/README.md new file mode 100644 index 000000000..23844f0ed --- /dev/null +++ b/examples/02-hybrid/README.md @@ -0,0 +1,40 @@ +# Hybrid Cargo + Buck project + +This example is for using both Cargo and Buck on the same project. This is +useful if you want to: + +- use Cargo to do things like `cargo fmt`; +- have it work out of the box with `rust-analyzer`, so it's easy to contribute; +- ultimately build your code with Buck; +- or support building with either. + +The key is the `reindeer.toml` in the root, which stitches together a Cargo +workspace and a Buck package (`third-party`) to buckify Cargo dependencies +into. + +## Running the example + +```sh +cargo install --path . +cd examples/02-hybrid + +# Note that both buck and cargo can build this project +buck2 run //project:test +cargo run -p project + +# Add a dependency if you like +cargo add -p project rand + +# Buckify deps. Do this every time you modify the Cargo.tomls, in general. +reindeer buckify + +# Modify project/BUCK to add `//third-party:rand` to the deps list. +# Buck should still build the project, and the rand crate should be available +# in main.rs. +``` + +## Missing from this example + +This example does no vendoring or anything fancy with fixups. You will have to +follow the other examples to get a non-trivial Cargo project built with Buck. + diff --git a/examples/02-hybrid/project/BUCK b/examples/02-hybrid/project/BUCK new file mode 100644 index 000000000..691f18fb0 --- /dev/null +++ b/examples/02-hybrid/project/BUCK @@ -0,0 +1,9 @@ +rust_binary( + name = "test", + srcs = ["src/main.rs"], + deps = [ + "//third-party:once_cell", + # see README.md + # "//third-party:rand", + ], +) diff --git a/examples/02-hybrid/project/Cargo.toml b/examples/02-hybrid/project/Cargo.toml new file mode 100644 index 000000000..a53a65bd3 --- /dev/null +++ b/examples/02-hybrid/project/Cargo.toml @@ -0,0 +1,11 @@ +[package] +# Pseudo-package whose dependencies are imported and buckified +name = "project" +version = "0.0.0" +edition = "2021" +publish = false + +# List of packages to be imported, with version constraints, features +# and all options Cargo supports. +[dependencies] +once_cell = "1.4" diff --git a/examples/02-hybrid/project/src/main.rs b/examples/02-hybrid/project/src/main.rs new file mode 100644 index 000000000..d94b35a98 --- /dev/null +++ b/examples/02-hybrid/project/src/main.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use once_cell::sync::Lazy; + +const MAGIC: &str = "this is a magic string"; + +static SPECIAL: Lazy = Lazy::new(|| MAGIC.to_string()); + +fn main() { + println!("static {}", &*SPECIAL); + + // 1. Uncomment this line + // 2. cargo add -p project rand + // 3. reindeer buckify + // 4. modify the BUCK file to include "//third-party:rand" + // 5. buck2 run //project:test + // + // println!("random {}", rand::random::()); +} diff --git a/examples/02-hybrid/reindeer.toml b/examples/02-hybrid/reindeer.toml new file mode 100644 index 000000000..57ee7f008 --- /dev/null +++ b/examples/02-hybrid/reindeer.toml @@ -0,0 +1,31 @@ +## +## Reindeer Config for a hybrid Cargo + Buck project +## + +# Paths +# +# This example has a workspace Cargo.toml. For all default workspace members, +# it will recursively buckify their dependencies to third-party/BUCK. +manifest_path = "Cargo.toml" +third_party_dir = "third-party" +fixup_templates = true + +# Configuration for generated BUCK file +# See other examples +[buck] +file_name = "BUCK" +rust_library = "cargo.rust_library" +rust_binary = "cargo.rust_binary" +buildscript_genrule = "buildscript_run" +buckfile_imports = """ +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") +""" +generated_file_header = """ +## +## \u0040generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## +""" diff --git a/examples/02-hybrid/setup.sh b/examples/02-hybrid/setup.sh new file mode 100755 index 000000000..400437c57 --- /dev/null +++ b/examples/02-hybrid/setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Generate file 'third-party/BUCK'. + +set -e + +(cd ../..; cargo build) + +# Build a BUCK file to build third-party crates. +# +# This will resolve all the dependencies, and create or update +# third-party/Cargo.lock as required. +# +# It will create a template fixup.toml which you can edit as needed. You would +# typically commit these fixups and the generated third-party/BUCK in the same +# commit as above. + +../../target/debug/reindeer buckify diff --git a/examples/02-hybrid/third-party/.gitignore b/examples/02-hybrid/third-party/.gitignore new file mode 100644 index 000000000..0bee95aae --- /dev/null +++ b/examples/02-hybrid/third-party/.gitignore @@ -0,0 +1,5 @@ +# Ignore Cargo +# If you need to adjust settings in .cargo/config.toml, put one in a parent directory. +.cargo +target + diff --git a/examples/02-hybrid/third-party/BUCK b/examples/02-hybrid/third-party/BUCK new file mode 100644 index 000000000..bea2e1f09 --- /dev/null +++ b/examples/02-hybrid/third-party/BUCK @@ -0,0 +1,38 @@ +## +## @generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## + +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") + +alias( + name = "once_cell", + actual = ":once_cell-1.20.2", + visibility = ["PUBLIC"], +) + +http_archive( + name = "once_cell-1.20.2.crate", + sha256 = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", + strip_prefix = "once_cell-1.20.2", + urls = ["https://static.crates.io/crates/once_cell/1.20.2/download"], + visibility = [], +) + +cargo.rust_library( + name = "once_cell-1.20.2", + srcs = [":once_cell-1.20.2.crate"], + crate = "once_cell", + crate_root = "once_cell-1.20.2.crate/src/lib.rs", + edition = "2021", + features = [ + "alloc", + "default", + "race", + "std", + ], + visibility = [], +) diff --git a/examples/02-hybrid/toolchains/.buckconfig b/examples/02-hybrid/toolchains/.buckconfig new file mode 100644 index 000000000..e69de29bb diff --git a/example/toolchains/BUCK b/examples/02-hybrid/toolchains/BUCK similarity index 100% rename from example/toolchains/BUCK rename to examples/02-hybrid/toolchains/BUCK diff --git a/examples/03-complex-fixups/.buckconfig b/examples/03-complex-fixups/.buckconfig new file mode 100644 index 000000000..126e8b381 --- /dev/null +++ b/examples/03-complex-fixups/.buckconfig @@ -0,0 +1,21 @@ +[repositories] +root = . +prelude = prelude +toolchains = toolchains +none = none + +[repository_aliases] +config = prelude +fbcode = none +fbsource = none +buck = none + +[external_cells] +prelude = bundled + +[parser] +target_platform_detector_spec = target:root//...->prelude//platforms:default + +[rust] +default_edition = 2021 +remap_src_paths = yes diff --git a/examples/03-complex-fixups/.buckroot b/examples/03-complex-fixups/.buckroot new file mode 100644 index 000000000..e69de29bb diff --git a/examples/03-complex-fixups/.buckversion b/examples/03-complex-fixups/.buckversion new file mode 100644 index 000000000..a0f9a4b4b --- /dev/null +++ b/examples/03-complex-fixups/.buckversion @@ -0,0 +1 @@ +latest diff --git a/example/.gitignore b/examples/03-complex-fixups/.gitignore similarity index 100% rename from example/.gitignore rename to examples/03-complex-fixups/.gitignore diff --git a/examples/03-complex-fixups/PACKAGE b/examples/03-complex-fixups/PACKAGE new file mode 100644 index 000000000..4c7c54e57 --- /dev/null +++ b/examples/03-complex-fixups/PACKAGE @@ -0,0 +1,22 @@ +load("@prelude//cfg/modifier:set_cfg_modifiers.bzl", "set_cfg_modifiers") +load("@prelude//rust:with_workspace.bzl", "with_rust_workspace") +load("@root//config:set_cfg_constructor.bzl", "set_cfg_constructor") + +# This enables the Config Modifiers RFC. +# Described at +# https://github.com/facebook/buck2/blob/main/docs/rfcs/cfg-modifiers/api.md +# +# This function can only be called from the repository root `PACKAGE` file. +ALIASES = { + "debug": "root//config/mode:debug", + "release": "root//config/mode:release", +} +set_cfg_constructor(aliases = ALIASES) + +set_cfg_modifiers( + cfg_modifiers = [ + # default is debug mode. + # override with -m release + "root//config/mode:debug", + ], +) diff --git a/examples/03-complex-fixups/README.md b/examples/03-complex-fixups/README.md new file mode 100644 index 000000000..44000399f --- /dev/null +++ b/examples/03-complex-fixups/README.md @@ -0,0 +1,133 @@ +# Example Reindeer+Buck project + +## Getting Started + +The `setup.sh` script will build Reindeer, vendor some Cargo packages and +generate a BUCK file of build rules for them. + +This will require a Rust installation (stable, but probably fairly recent), and +Buck to actually make use of the generated files. + +## Vendoring + +This example enables vendoring of the source files for the third party +crates (see `reindeer.toml` for how). If you run `reindeer vendor` and then +`reindeer buckify`, you get all the crate sources under `third-party/vendor`. +The idea is to check all these files into source control. + +Note the file `third-party/.gitignore`. This is used to ignore various +inconvenient files in vendored code. Reindeer itself will look at this to +edit them out of the vendored checksum.json files so that Cargo doesn't +get upset. + +There is another option if vendoring would include so many files that it wrecks +your Git performance and you don't have the capacity to tune it: +`vendor = "local-registry"`. This saves tarballs of the crates, instead of +expanding them, and relies on having +[`cargo-local-registry`](https://github.com/dhovart/cargo-local-registry) +installed and Buck2 version `2025-01-15` or later. + +## Fixups + +This example has a few dependencies to illustrate the fixups system. + +### `libc` [toml](third-party/fixups/libc/) + +This one is easy. The `libc` crate has a build script, aka `build.rs`, which is +used to enable `--cfg` flags on certain targets. + +If you delete the `third-party/fixups/libc` directory, and then `reindeer buckify` again, you'll see a template `fixups.toml` generated, suggesting that +you add some directives to handle the build script. You can then replace the +whole TOML file with: + +```toml +[[buildscript]] +[buildscript.rustc_flags] +``` + +### `crossbeam-utils` [toml](third-party/fixups/crossbeam-utils/) + +This is an indirect dependency of `crossbeam-queue`. +It has a build script, which you can see +[here](https://github.com/crossbeam-rs/crossbeam/blob/17fb8417a83a2694b6f8a37198cd20b34b621baf/crossbeam-utils/build.rs). + +So we can add a directive to run that build script and have Buck add the flags +it emits to rustc when we compile the crate: + +```toml +[[buildscript]] +[buildscript.rustc_flags] +``` + +However, that build script has the following line: + +```rust + env!("CARGO_PKG_NAME"), +``` + +Which will fail without some modification. Buck does not provide such +environment variables by default. You can tell reindeer to add them: + +```toml +cargo_env = true + +[[buildscript]] +[buildscript.rustc_flags] +``` + +### `blake3` [toml](third-party/fixups/blake3/) + +This one is huge, basically replacing a long `build.rs` using the `cc` +crate to invoke a C compiler. The `cc` invocations are manually translated to +`[buildscript.cxx_library]` fixups, which reindeer turns into `cxx_library` targets +for Buck to build. It's more complex still, because the blake3 crate works on +many platforms, so there are many variations of the cxx_library to generate +with different source files and C flags. + +If you come across a crate like this, for example `zstd` or `winapi`, you should check +whether someone has already added fixups for it in the [Buck2 repository's fixups +folder](https://github.com/facebook/buck2/tree/main/shim/third-party/rust/fixups). + +## Extra BUCK file imports + +Reindeer is very customizable, but it can't do everything. This example also +demonstrates adding extra macro imports to the BUCK file, and wrapping the +prelude's rules with your own Starlark code. + +A sample use case for this is customizing the opt-levels of third party code by +modifying the `rustc_flags` attribute as it passes through. See the code in + +- `config/`, specifying the debug/release constraints +- `PACKAGE`, enabling config modifiers and setting up aliases like `buck2 build -m release` +- `third-party/macros/rust_third_party.bzl`, which defines macros that will be + imported by `third-party/BUCK`, and selects opt-level based on the + `//config/mode:debug / release` constraints +- `reindeer.toml`, which tells reindeer to generate a macro call for each crate to use our + custom macros. + +## Reindeer configuration + +`reindeer.toml` - Reindeer's configuration. The directory containing this +file is also the base for any relative paths mentioned in the file. + +The files and directories Reindeer cares about are under `third-party/`: + +- `Cargo.toml` - You edit this to specify which packages you want to import, + along + with other settings like features, patches and so on, using the full syntax + Cargo allows +- `Cargo.lock` - The resolved dependencies +- `BUCK` - The generated Buck build rules (once generated) +- `.gitignore` - This is used to ignore various inconvenient files in vendored + code. Reindeer itself will look at this to edit them out of the vendored + checksum.json files so that Cargo doesn't get upset. + +## Project + +The `project/` directory represents some end-user code. There's nothing notable +about it aside from its references to `//third-party:...` for its third-party +dependencies. + +Once everything is set up, you should be able to build it with +`buck build //project:test` to build the executable or `buck run //project:test` +to just build and run in situ. diff --git a/examples/03-complex-fixups/config/mode/BUCK b/examples/03-complex-fixups/config/mode/BUCK new file mode 100644 index 000000000..70f24d80b --- /dev/null +++ b/examples/03-complex-fixups/config/mode/BUCK @@ -0,0 +1,15 @@ +config_setting( + name = "debug", + constraint_values = [ + "//config/mode/constraints:debug", + ], + visibility = ["PUBLIC"], +) + +config_setting( + name = "release", + constraint_values = [ + "//config/mode/constraints:release", + ], + visibility = ["PUBLIC"], +) diff --git a/examples/03-complex-fixups/config/mode/constraints/BUCK b/examples/03-complex-fixups/config/mode/constraints/BUCK new file mode 100644 index 000000000..f09877e18 --- /dev/null +++ b/examples/03-complex-fixups/config/mode/constraints/BUCK @@ -0,0 +1,17 @@ +constraint_setting( + name = "mode", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "debug", + constraint_setting = ":mode", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "release", + constraint_setting = ":mode", + visibility = ["PUBLIC"], +) + diff --git a/examples/03-complex-fixups/config/set_cfg_constructor.bzl b/examples/03-complex-fixups/config/set_cfg_constructor.bzl new file mode 100644 index 000000000..7f4900d24 --- /dev/null +++ b/examples/03-complex-fixups/config/set_cfg_constructor.bzl @@ -0,0 +1,21 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@prelude//cfg/modifier:cfg_constructor.bzl", "cfg_constructor_post_constraint_analysis", "cfg_constructor_pre_constraint_analysis") +load("@prelude//cfg/modifier:common.bzl", "MODIFIER_METADATA_KEY") + +def set_cfg_constructor(aliases = dict()): + project_root_cell = read_root_config("cell_aliases", "root") + current_root_cell = read_config("cell_aliases", "root") + if project_root_cell == current_root_cell: + native.set_cfg_constructor( + stage0 = cfg_constructor_pre_constraint_analysis, + stage1 = cfg_constructor_post_constraint_analysis, + key = MODIFIER_METADATA_KEY, + aliases = struct(**aliases), + extra_data = struct(), + ) diff --git a/example/project/BUCK b/examples/03-complex-fixups/project/BUCK similarity index 67% rename from example/project/BUCK rename to examples/03-complex-fixups/project/BUCK index 050f01d29..0e9f9d3af 100644 --- a/example/project/BUCK +++ b/examples/03-complex-fixups/project/BUCK @@ -4,5 +4,7 @@ rust_binary( deps = [ "//third-party:blake3", "//third-party:once_cell", + "//third-party:libc", + "//third-party:crossbeam-queue", ], ) diff --git a/example/project/test.rs b/examples/03-complex-fixups/project/test.rs similarity index 82% rename from example/project/test.rs rename to examples/03-complex-fixups/project/test.rs index 9e87acf7e..db801c066 100644 --- a/example/project/test.rs +++ b/examples/03-complex-fixups/project/test.rs @@ -7,6 +7,10 @@ use once_cell::sync::Lazy; +// Use these crates without really using them +extern crate libc; +extern crate crossbeam_queue; + const MAGIC: &str = "this is a magic string"; static SPECIAL: Lazy = Lazy::new(|| { diff --git a/example/third-party/reindeer.toml b/examples/03-complex-fixups/reindeer.toml similarity index 86% rename from example/third-party/reindeer.toml rename to examples/03-complex-fixups/reindeer.toml index 5e3a2f7cf..3dcba05a5 100644 --- a/example/third-party/reindeer.toml +++ b/examples/03-complex-fixups/reindeer.toml @@ -8,6 +8,9 @@ ## Reindeer is under active development, and the layout and options in this file ## may change. +# Write output to third-party/BUCK +third_party_dir = "third-party" + # Parse Rust code to work out the precise set of source files for each crate. # This uses `srcfiles` which only works on Rust 2018 and without some macro # constructions. It works in almost all cases, but you may need to have a @@ -17,17 +20,12 @@ precise_srcs = true # Possible patterns for license files - lots of packages have them without # registering them in the Cargo metadata, or have more than the single file it # allows. Just look in the top-level dir for now. -license_patterns = [ - "LICENSE*", "COPYING*", "UNLICENSE*", -] +license_patterns = ["LICENSE*", "COPYING*", "UNLICENSE*"] # If a fixups.toml file is needed (eg, the package has a build.rs), then # generate a template fixups.toml to be edited. fixup_templates = true -# Emit Cargo pkg metadata into rules (experimental, not used) -#emit_metadata = false - # Include an explicit public top-level target which depends on all other library # targets, so that you can to a test/check build. include_top_level = true @@ -42,8 +40,7 @@ include_top_level = true gitignore_checksum_exclude = [".gitignore"] # Additional globs to ignore (prefer .gitignore for consistentcy with source # control) -checksum_exclude = [ -] +checksum_exclude = [] # Platforms we want to support. # @@ -67,7 +64,7 @@ checksum_exclude = [ # x86_64-unknown-linux-gnu [platform.linux-x86_64] -x86_64-unknown-linux-gnu = [] # true for a boolean test +x86_64-unknown-linux-gnu = [] # true for a boolean test target_arch = ["x86_64"] target_endian = ["little"] target_env = ["gnu"] @@ -155,17 +152,17 @@ file_name = "BUCK" # default # definitions which resolve them to appropriate underlying rules # suitable for your environment. (This may also work for Buck-like # build systems such as Bazel.) -rust_library = "cargo.rust_library" # A plain Rust library -rust_binary = "cargo.rust_binary" # A Rust executable -cxx_library = "third_party_rust_cxx_library" # A C++ library (mostly for Rust -> C dependencies) -prebuilt_cxx_library = "third_party_rust_prebuilt_cxx_library" # A prebuilt library (mostly for Rust -> C dependencies) -buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources +rust_library = "cargo_rust_library" # A plain Rust library +rust_binary = "cargo.rust_binary" # A Rust executable +cxx_library = "third_party_rust_cxx_library" # A C++ library (mostly for Rust -> C dependencies) +prebuilt_cxx_library = "third_party_rust_prebuilt_cxx_library" # A prebuilt library (mostly for Rust -> C dependencies) +buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources # Load the macros to which the rules above will resolve. buckfile_imports = """ load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") load("@prelude//rust:cargo_package.bzl", "cargo") -load("//third-party/macros:rust_third_party.bzl", "third_party_rust_cxx_library", "third_party_rust_prebuilt_cxx_library") +load("//third-party/macros:rust_third_party.bzl", "cargo_rust_library", "third_party_rust_cxx_library", "third_party_rust_prebuilt_cxx_library") """ # Banner comment for the generated BUCK File. diff --git a/examples/03-complex-fixups/setup.sh b/examples/03-complex-fixups/setup.sh new file mode 100755 index 000000000..917803a76 --- /dev/null +++ b/examples/03-complex-fixups/setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Generate file 'third-party/BUCK'. + +set -e + +(cd ../..; cargo build) + +# Build a BUCK file to build third-party crates. +# +# This will resolve all the dependencies, and create or update +# third-party/Cargo.lock as required. +# +# It will create a template fixup.toml which you can edit as needed. You would +# typically commit these fixups and the generated third-party/BUCK in the same +# commit as above. + +../../target/debug/reindeer --third-party-dir third-party buckify diff --git a/examples/03-complex-fixups/third-party/.gitignore b/examples/03-complex-fixups/third-party/.gitignore new file mode 100644 index 000000000..cd2eec533 --- /dev/null +++ b/examples/03-complex-fixups/third-party/.gitignore @@ -0,0 +1,28 @@ +# Ignore Cargo-related stuff +.cargo +target + +# Various cruft in vendored packages +vendor/*/target +vendor/*/Cargo.lock +vendor/**/.buckconfig +vendor/**/BUCK +vendor/**/OWNERS +vendor/**/*~ +vendor/**/*.bzl +vendor/*/.github/** +vendor/*/.appveyor.yml +vendor/*/.travis.yml +/target/** + +# Bad Windows names - oh for case-insensitive regex matching! +vendor/**/[Aa][Uu][Xx] +vendor/**/[Aa][Uu][Xx].* +vendor/**/[Cc][Oo][Mm][1-9] +vendor/**/[Cc][Oo][Mm][1-9].* +vendor/**/[Cc][Oo][Nn] +vendor/**/[Cc][Oo][Nn].* +vendor/**/[Ll][Pp][Tt][1-9] +vendor/**/[Ll][Pp][Tt][1-9].* +vendor/**/[Nn][Uu][Ll] +vendor/**/[Nn][Uu][Ll].* diff --git a/examples/03-complex-fixups/third-party/Cargo.lock b/examples/03-complex-fixups/third-party/Cargo.lock new file mode 100644 index 000000000..31c8a4e3b --- /dev/null +++ b/examples/03-complex-fixups/third-party/Cargo.lock @@ -0,0 +1,99 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "cc" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "rust-third-party" +version = "0.0.0" +dependencies = [ + "blake3", + "crossbeam-queue", + "libc", + "once_cell", + "typenum", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" diff --git a/examples/03-complex-fixups/third-party/Cargo.toml b/examples/03-complex-fixups/third-party/Cargo.toml new file mode 100644 index 000000000..a6f334d2b --- /dev/null +++ b/examples/03-complex-fixups/third-party/Cargo.toml @@ -0,0 +1,25 @@ +[workspace] + +[package] +# Pseudo-package whose dependencies are imported and buckified +name = "rust-third-party" +version = "0.0.0" +edition = "2021" +publish = false + +# Dummy target to keep Cargo happy +[[bin]] +name = "top" +path = "top/main.rs" + +# List of packages to be imported, with version constraints, features +# and all options Cargo supports. +[dependencies] +blake3 = { version = "1.5.5", features = [] } +once_cell = "1.4" +libc = "0.2.169" +typenum = "1.17.0" +crossbeam-queue = "0.3.12" + +# Local patches - typically git references +[patch.crates-io] diff --git a/example/third-party/fixups/blake3/fixups.toml b/examples/03-complex-fixups/third-party/fixups/blake3/fixups.toml similarity index 100% rename from example/third-party/fixups/blake3/fixups.toml rename to examples/03-complex-fixups/third-party/fixups/blake3/fixups.toml diff --git a/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml b/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml new file mode 100644 index 000000000..46dfe3a3a --- /dev/null +++ b/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml @@ -0,0 +1,4 @@ +cargo_env = true + +[[buildscript]] +[buildscript.rustc_flags] diff --git a/example/third-party/fixups/libc/fixups.toml b/examples/03-complex-fixups/third-party/fixups/libc/fixups.toml similarity index 100% rename from example/third-party/fixups/libc/fixups.toml rename to examples/03-complex-fixups/third-party/fixups/libc/fixups.toml diff --git a/example/third-party/fixups/typenum/fixups.toml b/examples/03-complex-fixups/third-party/fixups/typenum/fixups.toml similarity index 100% rename from example/third-party/fixups/typenum/fixups.toml rename to examples/03-complex-fixups/third-party/fixups/typenum/fixups.toml diff --git a/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl b/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl new file mode 100644 index 000000000..92c0386f7 --- /dev/null +++ b/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl @@ -0,0 +1,21 @@ +load("@prelude//rust:cargo_package.bzl", "cargo") + +def cargo_rust_library(name, rustc_flags = [], **kwargs): + # Always build dependencies with opt-level 1. + # This disables debug-assertions by default, so re-enable those. + # You can also do this in toolchains/BUCK by specifying global rustc_flags + # on toolchains//:rust, but this way leaves our own crates untouched. + # + rustc_flags = rustc_flags + select({ + "root//config/mode:debug": ["-Copt-level=1", "-Cdebug-assertions=true"], + "root//config/mode:release": ["-Copt-level=3"], + }) + cargo.rust_library(name = name, rustc_flags = rustc_flags, **kwargs) + +def third_party_rust_cxx_library(name, **kwargs): + # @lint-ignore BUCKLINT + native.cxx_library(name = name, **kwargs) + +def third_party_rust_prebuilt_cxx_library(name, **kwargs): + # @lint-ignore BUCKLINT + native.prebuilt_cxx_library(name = name, **kwargs) diff --git a/example/third-party/top/main.rs b/examples/03-complex-fixups/third-party/top/main.rs similarity index 100% rename from example/third-party/top/main.rs rename to examples/03-complex-fixups/third-party/top/main.rs diff --git a/examples/03-complex-fixups/toolchains/.buckconfig b/examples/03-complex-fixups/toolchains/.buckconfig new file mode 100644 index 000000000..e69de29bb diff --git a/examples/03-complex-fixups/toolchains/BUCK b/examples/03-complex-fixups/toolchains/BUCK new file mode 100644 index 000000000..c4031ed47 --- /dev/null +++ b/examples/03-complex-fixups/toolchains/BUCK @@ -0,0 +1,2 @@ +load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") +system_demo_toolchains()