Create docs and examples for classic remote execution (#1498)

These examples should help users test their infrastructure and worker
images against different remote execution scenarios.

.bazelignore

@@ -11,5 +11,6 @@ bazel-remote-nativelink
 bazel-root
 bazel-testlogs
 bazel-nativelink
+toolchain-examples
 local-remote-execution/generated-cc
 local-remote-execution/generated-java

.github/styles/config/vocabularies/TraceMachina/accept.txt

@@ -4,9 +4,11 @@ Astro
 Bazel
 Cloudflare
 ELB
+FFI
 GPUs
 Goma
 [Hh]ermeticity
+JDK
 Kustomization
 LLD
 LLM
@@ -30,7 +32,9 @@ TraceMachina
 Qwik
 Verilator
 Verilog
+Zstandard
 alex
+[Ee]xecution
 autoscaling
 blazingly
 bundler

toolchain-examples/.bazelrc

@@ -0,0 +1,17 @@
+# Don't use the host's default PATH and LD_LIBRARY_PATH.
+build --incompatible_strict_action_env
+
+# Use rules_python's builtin script to emulate a bootstrap python.
+build --@rules_python//python/config_settings:bootstrap_impl=script
+
+# Toolchain to verify remote execution with zig-cc.
+build:zig-cc --platforms @zig_sdk//platform:linux_amd64
+build:zig-cc --extra_toolchains @zig_sdk//toolchain:linux_amd64_gnu.2.38
+
+# Toolchain to verify remote execution with contrib/toolchains_llvm.
+build:llvm --platforms=@toolchains_llvm//platforms:linux-x86_64
+build:llvm --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
+
+# Java runtime to ensure hermeticity on the remote.
+build:java --java_runtime_version=remotejdk_21
+build:java --tool_java_runtime_version=remotejdk_21

toolchain-examples/.bazelversion

@@ -0,0 +1 @@
+7.4.1

toolchain-examples/MODULE.bazel

@@ -0,0 +1,88 @@
+module(
+    name = "toolchain-examples",
+    version = "0.0.0",
+    compatibility_level = 0,
+)
+
+bazel_dep(name = "platforms", version = "0.0.10")
+
+# C++
+bazel_dep(name = "rules_cc", version = "0.0.17")
+
+# Java
+bazel_dep(name = "rules_java", version = "8.5.1")
+
+java = use_extension("//java:extensions.bzl", "toolchains")
+use_repo(java, "local_jdk")
+
+# Python
+bazel_dep(name = "rules_python", version = "0.40.0")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip",
+    python_version = "3.12",
+    requirements_lock = "//:requirements_lock.txt",
+)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    is_default = True,
+    python_version = "3.12",
+)
+
+use_repo(pip, "pip")
+
+# Go
+bazel_dep(name = "rules_go", version = "0.50.1")
+
+# Rust
+bazel_dep(name = "rules_rust", version = "0.54.1")
+
+# C++ toolchain via zig-cc.
+#
+# - Hermetic:      YES
+# - Speed:         SLOW  Optimized for size rather than speed of execution.
+# - Download size: SMALL
+#
+# To test this toolchain, use for use with the config flag `--config=zig-cc`.
+#
+bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1")
+
+zig = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
+use_repo(zig, "zig_sdk")
+
+# C++ toolchain via contrib/toolchains_llvm.
+#
+# - Hermetic: NO            LLVM artifacts are dynamically linked and will
+#                           depend on your host's glibc.
+# - Speed:         FAST     Release build optimized for speed of execution.
+# - Download size: LARGE    Around 1.5 Gb for the linux toolchain.
+#
+# To test this toolchain, use for use with the config flag `--config=llvm`.
+#
+bazel_dep(name = "toolchains_llvm", version = "1.2.0")
+
+llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
+llvm.toolchain(llvm_version = "19.1.0")
+use_repo(llvm, "llvm_toolchain")
+
+# Example projects to test compilation.
+
+# libcurl (C)
+bazel_dep(name = "curl", version = "8.8.0.bcr.1")
+
+# Zstandard (C)
+bazel_dep(name = "zstd", version = "1.5.6")
+
+# Abseil for C++
+bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.1")
+
+# Abseil for python
+bazel_dep(name = "abseil-py", version = "2.1.0")
+
+# GRPC
+bazel_dep(name = "grpc", version = "1.68.0")
+
+# Circl (Go, C++)
+bazel_dep(name = "circl", version = "1.3.8")

toolchain-examples/README.md

@@ -0,0 +1,3 @@
+# Example builds to test remote execution
+
+See: [RBE examples docs](http://nativelink.com/docs/rbe/remote-execution-examples)

toolchain-examples/cpp/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+
+cc_test(
+    name = "cpp",
+    srcs = ["main.cpp"],
+)

toolchain-examples/cpp/main.cpp

@@ -0,0 +1,5 @@
+#include <iostream>
+
+auto main() -> int {
+  std::cout << "Hello, world!\n";
+}

toolchain-examples/go/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "go",
+    srcs = ["main.go"],
+)

toolchain-examples/go/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("hello world")
+}

toolchain-examples/java/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@rules_java//java:defs.bzl", "java_test")
+
+java_test(
+    name = "HelloWorld",
+    srcs = ["HelloWorld.java"],
+)

toolchain-examples/java/HelloWorld.java

@@ -0,0 +1,8 @@
+import org.junit.Test;
+
+public class HelloWorld {
+    @Test
+    public void testHelloWorld() {
+        System.out.println("Hello, World!");
+    }
+}

toolchain-examples/nativelink-config.json

@@ -0,0 +1,146 @@
+{
+  "stores": {
+    "AC_MAIN_STORE": {
+      "filesystem": {
+        "content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
+        "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
+        "eviction_policy": {
+          // 1gb.
+          "max_bytes": 1000000000,
+        }
+      }
+    },
+    "WORKER_FAST_SLOW_STORE": {
+      "fast_slow": {
+        // "fast" must be a "filesystem" store because the worker uses it to make
+        // hardlinks on disk to a directory where the jobs are running.
+        "fast": {
+          "filesystem": {
+            "content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
+            "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
+            "eviction_policy": {
+              // 10gb.
+              "max_bytes": 10000000000,
+            }
+          }
+        },
+        "slow": {
+          /// Discard data.
+          /// This example usage has the CAS and the Worker live in the same place,
+          /// so they share the same underlying CAS. Since workers require a fast_slow
+          /// store, we use the fast store as our primary data store, and the slow store
+          /// is just a noop, since there's no shared storage in this config.
+          "noop": {}
+        }
+      }
+    }
+  },
+  "schedulers": {
+    "MAIN_SCHEDULER": {
+      "simple": {
+        "supported_platform_properties": {
+          "cpu_count": "minimum",
+          "memory_kb": "minimum",
+          "network_kbps": "minimum",
+          "cpu_arch": "exact",
+          "OSFamily": "priority",
+          "container-image": "priority",
+        }
+      }
+    }
+  },
+  "workers": [{
+    "local": {
+      "worker_api_endpoint": {
+        "uri": "grpc://127.0.0.1:50061",
+      },
+      "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
+      "upload_action_result": {
+        "ac_store": "AC_MAIN_STORE",
+      },
+      "work_directory": "/tmp/nativelink/work",
+      "platform_properties": {
+        "cpu_count": {
+          "values": ["16"],
+        },
+        "memory_kb": {
+          "values": ["500000"],
+        },
+        "network_kbps": {
+          "values": ["100000"],
+        },
+        "cpu_arch": {
+          "values": ["x86_64"],
+        },
+        "OSFamily": {
+          "values": [""]
+        },
+        "container-image": {
+          "values": [""]
+        },
+      }
+    }
+  }],
+  "servers": [{
+    "name": "public",
+    "listener": {
+      "http": {
+        "socket_address": "0.0.0.0:50051"
+      }
+    },
+    "services": {
+      "cas": {
+        "": {
+          "cas_store": "WORKER_FAST_SLOW_STORE"
+        }
+      },
+      "ac": {
+        "": {
+          "ac_store": "AC_MAIN_STORE"
+        }
+      },
+      "execution": {
+        "": {
+          "cas_store": "WORKER_FAST_SLOW_STORE",
+          "scheduler": "MAIN_SCHEDULER",
+        }
+      },
+      "capabilities": {
+        "": {
+          "remote_execution": {
+            "scheduler": "MAIN_SCHEDULER",
+          }
+        }
+      },
+      "bytestream": {
+        "cas_stores": {
+          "": "WORKER_FAST_SLOW_STORE",
+        }
+      }
+    }
+  }, {
+    "name": "private_workers_servers",
+    "listener": {
+      "http": {
+        "socket_address": "0.0.0.0:50061"
+      }
+    },
+    "services": {
+      "experimental_prometheus": {
+        "path": "/metrics"
+      },
+      // Note: This should be served on a different port, because it has
+      // a different permission set than the other services.
+      // In other words, this service is a backend api. The ones above
+      // are a frontend api.
+      "worker_api": {
+        "scheduler": "MAIN_SCHEDULER",
+      },
+      "admin": {},
+      "health": {},
+    }
+  }],
+  "global": {
+    "max_open_files": 512
+  }
+}

toolchain-examples/python/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "python",
+    srcs = ["python.py"],
+)

toolchain-examples/python/python.py

@@ -0,0 +1,3 @@
+import sys
+
+print(f"{sys.version_info.major}.{sys.version_info.minor}")

toolchain-examples/rust/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@rules_rust//rust:defs.bzl", "rust_test")
+
+rust_test(
+    name = "rust",
+    srcs = ["main.rs"],
+)

toolchain-examples/rust/main.rs

@@ -0,0 +1,17 @@
+fn main() {
+    println!("Hello, World!");
+}
+
+fn get_greeting() -> String {
+    String::from("Hello, World!")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_greeting() {
+        assert_eq!(get_greeting(), "Hello, World!");
+    }
+}