TraceMachina · aaronmondal · Dec 14, 2024 · Dec 2, 2024
diff --git a/.bazelignore b/.bazelignore
@@ -14,3 +14,4 @@ bazel-nativelink
 toolchain-examples
 local-remote-execution/generated-cc
 local-remote-execution/generated-java
+local-remote-execution/rust
diff --git a/.bazelrc b/.bazelrc
@@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
 startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
-
-# Global rust toolchain configuration. Deferred to here so that the cc
-# toolchains are resolved before the rust toolchains.
-build --extra_toolchains=@rust_toolchains//:all
+build:windows --extra_toolchains=@rust_toolchains//:all  # Doesn't support LRE.
 
 # Generated by the LRE flake module.
 try-import %workspace%/lre.bazelrc

diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -20,8 +20,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
-    name: Local / ${{ matrix.os }}
+        os: [ubuntu-24.04, macos-14]
+        toolchain: [lre-cc, lre-rs]
+        exclude:
+          - os: macos-14
+            toolchain: lre-cc
+    name: Local / ${{ matrix.toolchain }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -46,19 +50,22 @@ jobs:
         uses: >- # v4
           DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
 
-      - name: Build hello_lre with LRE toolchain.
+      - name: Build example with ${{ matrix.toolchain }} toolchain
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "bazel run \
            --verbose_failures \
-           @local-remote-execution//examples:hello_lre"
+           @local-remote-execution//examples:${TOOLCHAIN}"
 
   remote:
     strategy:
       fail-fast: false
       matrix:
         os: [large-ubuntu-22.04]
-    name: Remote / ${{ matrix.os }}
+        toolchain: [lre-cc, lre-rs]
+    name: Remote / ${{ matrix.toolchain }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -92,12 +99,17 @@ jobs:
           REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }}
           BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
           COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: |
           nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF
           apiVersion: kustomize.config.k8s.io/v1beta1
           kind: Kustomization
           resources:
-          - deploy/kubernetes-example
+          - kubernetes/resources/flux
+          - kubernetes/resources/nativelink-core
+          - kubernetes/resources/gateway-routes
+          - kubernetes/workers/resources/worker-init
+          - kubernetes/workers/resources/${TOOLCHAIN}
           patches:
           - patch: |-
               - op: replace
@@ -112,6 +124,27 @@ jobs:
             target:
               kind: GitRepository
               name: nativelink
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#image
+            target:
+              kind: Kustomization
+              name: nativelink-alert-core
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#nativelink-worker-init
+            target:
+              kind: Kustomization
+              name: nativelink-alert-worker-init
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#nativelink-worker-${TOOLCHAIN}
+            target:
+              kind: Kustomization
+              name: nativelink-alert-${TOOLCHAIN}
           EOF
           kubectl apply -k . &&
           rm kustomization.yaml'
@@ -124,6 +157,8 @@ jobs:
             nativelink-tekton-resources"
 
       - name: Wait for alerts
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
@@ -134,7 +169,7 @@ jobs:
             nativelink-alert-worker-init && \
           flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-alert-lre-cc"
+            nativelink-alert-${TOOLCHAIN}"
 
       - name: Trigger pipelines
         run: |
@@ -181,21 +216,25 @@ jobs:
             nativelink-core"
 
       - name: Wait for Worker Kustomization
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-lre-cc"
+            nativelink-${TOOLCHAIN}"
 
       - name: Wait for NativeLink
         run: >
           nix develop --impure --command
           bash -c "kubectl rollout status deploy/nativelink"
 
       - name: Wait for worker
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
-          bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc"
+          bash -c "kubectl rollout status deploy/nativelink-worker-${TOOLCHAIN}"
 
       - name: Get gateway IPs
         id: gateway-ips
@@ -214,11 +253,13 @@ jobs:
           echo "worker"
           kubectl logs -l app=nativelink-worker
 
-      - name: Build hello_lre with LRE toolchain.
+      - name: Build example with ${{ matrix.toolchain }} toolchain
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "bazel run \
             --remote_cache=grpc://$nativelink_ip \
             --remote_executor=grpc://$nativelink_ip \
             --verbose_failures \
-            @local-remote-execution//examples:hello_lre"
+            @local-remote-execution//examples:${TOOLCHAIN}"
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -159,13 +159,15 @@ jobs:
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
@@ -195,6 +197,7 @@ jobs:
         build-args: |
           OPT_LEVEL=fastbuild
           OS_VERSION=${{ matrix.os_version }}
+          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
         load: true # This brings the build into `docker images` from buildx.
         tags: trace_machina/nativelink:latest
 

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -72,7 +72,9 @@ jobs:
       - name: Run Bazel tests
         run: |
           if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
-            bazel test //... --verbose_failures
+            bazel test //... \
+              --extra_toolchains=@rust_toolchains//:all \
+              --verbose_failures
           elif [ "$RUNNER_OS" == "Windows" ]; then
             bazel \
               --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "rules_cc", version = "0.0.17")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_python", version = "0.36.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
@@ -19,10 +19,50 @@ python.toolchain(
 use_repo(python, python = "python_versions")
 
 bazel_dep(name = "rules_rust", version = "0.54.1")
+archive_override(
+    module_name = "rules_rust",
+    integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
+    patch_strip = 1,
+    patches = ["//tools:rules_rust-musl-platforms.diff"],
+    urls = [
+        "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
+    ],
+)
+
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate.from_cargo(
+    name = "crates",
+    cargo_lockfile = "//:Cargo.lock",
+    manifests = ["//:Cargo.toml"],
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "armv7-unknown-linux-gnueabi",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+    ],
+)
+use_repo(crate, "crates")
 
+# NativeLink uses Local Remote Execution for Rust by default which automatically
+# handles Rust toolchain configuration via Nix.
+#
+# If you build outside of Nix you'll have to register these toolchains
+# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
+# Bazel invocation.
+#
+# WARNING: This configuration exists entirely as a convenience option and
+#          migration and is not a supported way of building production
+#          grade nativelink executables. It may be removed at any point in time.
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
     edition = "2021",
+
+    # These should always follow the versions from LRE.
     rust_analyzer_version = "nightly/2024-11-23",
     rustfmt_version = "nightly/2024-11-23",
     sha256s = {
@@ -44,43 +84,10 @@ rust.toolchain(
         "nightly/2024-11-23",
     ],
 )
-
-rust_host_tools = use_extension(
-    "@rules_rust//rust:extension.bzl",
-    "rust_host_tools",
-)
-rust_host_tools.host_tools(
-    edition = "2021",
-    version = "1.82.0",
-)
-
 use_repo(rust, "rust_toolchains")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
-crate.from_cargo(
-    name = "crates",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = ["//:Cargo.toml"],
-    supported_platform_triples = [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "arm-unknown-linux-gnueabi",
-        "armv7-unknown-linux-gnueabi",
-        "x86_64-apple-darwin",
-        "x86_64-pc-windows-msvc",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(crate, "crates")
-
-rust_analyzer = use_extension(
-    "@rules_rust//tools/rust_analyzer:extension.bzl",
-    "rust_analyzer_dependencies",
-)
-rust_analyzer.rust_analyzer_dependencies()
-
-bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
-bazel_dep(name = "toolchains_protoc", version = "0.3.3")
+bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
+bazel_dep(name = "toolchains_protoc", version = "0.3.4")
 
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
 protoc.toolchain(

diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml
@@ -2,7 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../kubernetes-example
+- ../../kubernetes/resources/flux
+- ../../kubernetes/resources/nativelink-core
+- ../../kubernetes/resources/gateway-routes
+- ../../kubernetes/workers/resources/worker-init
+- ../../kubernetes/workers/resources/lre-cc
+- ../../kubernetes/workers/resources/lre-rs
 
 # Modify this value to change the repository containing the deployment files.
 #
@@ -52,3 +57,10 @@ patches:
   target:
     kind: Kustomization
     name: nativelink-alert-lre-cc
+- patch: |-
+    - op: replace
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+      value: ./src_root#nativelink-worker-lre-rs
+  target:
+    kind: Kustomization
+    name: nativelink-alert-lre-rs
diff --git a/deployment-examples/docker-compose/README.md b/deployment-examples/docker-compose/README.md
@@ -24,11 +24,23 @@ something like:
 
 ```sh
 bazel test //... \
+  --extra_toolchains=@rust_toolchains//:all \
   --remote_cache=grpc://127.0.0.1:50051 \
   --remote_executor=grpc://127.0.0.1:50052 \
   --remote_default_exec_properties=cpu_count=1
 ```
 
+> [!NOTE]
+> The `nativelink` repository doesn't register any toolchains by default. The
+> remote execution container in this example is currently a classic non-LRE
+> container, so you'll have to add `--extra_toolchains=@rust_toolchains//:all`
+> to the invocation to register `rules_rust`'s default toolchains. If you run
+> this example against another `rules_rust` project you'll likely have these
+> toolchains already registered in your `.bazelrc` or `MODULE.bazel` and can
+> omit the flag.
+>
+> See: https://www.nativelink.com/docs/contribute/bazel
+
 ## Instances
 
 All instances use the same Docker image, `trace_machina/nativelink:latest`,

diff --git a/deployment-examples/docker-compose/scheduler.json5 b/deployment-examples/docker-compose/scheduler.json5
@@ -27,7 +27,8 @@
         "supported_platform_properties": {
           "cpu_count": "minimum",
           "OSFamily": "priority",
-          "container-image": "priority"
+          "container-image": "priority",
+          "lre-rs": "priority"
         }
       }
     }

diff --git a/deployment-examples/docker-compose/worker.json5 b/deployment-examples/docker-compose/worker.json5
@@ -59,6 +59,9 @@
         },
         "container-image": {
           "values": [""]
+        },
+        "lre-rs": {
+          "values": [""]
         }
       }
     }