From a8246c9b04e4a088b487288437a47fbb46068c8f Mon Sep 17 00:00:00 2001
From: Daniel Noland <daniel.noland@dnoland.cloud>
Date: Thu, 26 Sep 2024 12:02:48 -0600
Subject: [PATCH] Address some of @qmonnet comments
---
.../workflows/build-and-push-container.yml | 36 +++++
dpdk-sys-old/src/main.rs | 16 +-
flake.lock | 6 +-
nix/dev-env.nix | 153 ++++++++++++++++++
runner.sh | 23 ++-
sysroot-helper/src/lib.rs | 4 +-
6 files changed, 220 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/build-and-push-container.yml b/.github/workflows/build-and-push-container.yml
index e69de29bb..bb8a75102 100644
--- a/.github/workflows/build-and-push-container.yml
+++ b/.github/workflows/build-and-push-container.yml
@@ -0,0 +1,36 @@
+name: test suite
+on: [ push, pull_request ]
+
+jobs:
+ build:
+ name: cargo build
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Login to ghcr.io
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Fetch dev env container
+ run: |
+ declare -rx PROFILE=debug
+ docker pull "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}" || true
+ # container may not exist yet
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ with:
+ driver: remote
+ driver-opts: network=host
+ buildkitd-flags: '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host'
+ - name: Build sysroot
+ run: |
+ declare -rx PROFILE=debug
+ pushd dpdk-sys
+ ./gen-sysroot/gen-sysroot.sh "${PROFILE}"
+ - name: Build in debug container
+ run: |
+ declare -rx PROFILE=debug
+ docker tag "dpdk-sysroot-${PROFILE}" "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}"
+ docker push "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}"
diff --git a/dpdk-sys-old/src/main.rs b/dpdk-sys-old/src/main.rs
index 84b212987..da6711067 100644
--- a/dpdk-sys-old/src/main.rs
+++ b/dpdk-sys-old/src/main.rs
@@ -124,21 +124,21 @@ fn main() {
"--huge-worker-stack=8192",
"--socket-mem=4096,4096,4096,4096",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool_ring.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool_ring.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool_stack.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool_stack.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_bus_pci.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_bus_pci.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_net_mlx5.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_net_mlx5.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_common_mlx5.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_common_mlx5.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_regex_mlx5.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_regex_mlx5.so",
// "-d",
- // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_vdpa_mlx5.so",
+ // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_vdpa_mlx5.so",
];
info!("DPDK arguments: {args:?}");
let eal = Eal::new(args);
diff --git a/flake.lock b/flake.lock
index 935f22637..6e2aab74f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
- "lastModified": 1726871744,
- "narHash": "sha256-V5LpfdHyQkUF7RfOaDPrZDP+oqz88lTJrMT1+stXNwo=",
+ "lastModified": 1727140925,
+ "narHash": "sha256-ZHSasdLwEEjSOD/WTW1o7dr3/EjuYsdwYB4NSgICZ2I=",
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "a1d92660c6b3b7c26fb883500a80ea9d33321be2",
+ "rev": "189e5f171b163feb7791a9118afa778d9a1db81f",
"type": "github"
},
"original": {
diff --git a/nix/dev-env.nix b/nix/dev-env.nix
index e69de29bb..69b8132ab 100644
--- a/nix/dev-env.nix
+++ b/nix/dev-env.nix
@@ -0,0 +1,153 @@
+rec {
+ nixpkgsFn = import <nixpkgs>;
+
+ debugFlags = rec {
+ AR="llvm-ar";
+ CC="clang";
+ CFLAGS="-Og -ggdb3";
+ CXX="clang++";
+ CXXFLAGS=CFLAGS;
+ LD="mold";
+ LDFLAGS="-fuse-ld=mold";
+ NM="llvm-nm";
+ RANLIB="llvm-ranlib";
+ };
+
+ releaseFlags = rec {
+ AR="llvm-ar";
+ CC="clang";
+ CFLAGS="-O3 -ggdb3 -march=x86-64-v4 -mtune=znver4 -flto=thin -Werror=odr -Werror=strict-aliasing -fstack-protector-strong";
+ CXX="clang++";
+ CXXFLAGS=CFLAGS;
+ LD="mold";
+ LDFLAGS="-fuse-ld=mold -Wl,-O3 -Wl,--gc-sections -Wl,-z,relro,-z,now";
+ NM="llvm-nm";
+ RANLIB="llvm-ranlib";
+ };
+
+ buildWithFlags = flags: pkg: (pkg.overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} ${flags.CFLAGS}";
+ CXXFLAGS = "${orig.CXXFLAGS or ""} ${flags.CXXFLAGS}";
+ LDFLAGS = "${orig.LDFLAGS or ""} ${flags.LDFLAGS}";
+ }));
+
+ overlays = {
+ disableAppArmor = self: super: { libapparmor = null; };
+ disableSystemd = self: super: { systemd = null; }; # git rekt
+ disableDoxygen = self: super: { doxygen = null; };
+ useLatestLlvm = self: super: { llvmPackages = super.llvmPackages_latest; };
+ rdma-core = self: super: {
+ rdma-core = super.rdma-core.overrideAttrs (orig: {
+ nativeBuildInputs = with super; [ cmake pkg-config python3 mold ];
+ buildInputs = with super; [ libnl ethtool iproute2 ];
+ outputs = super.lib.lists.remove "man" orig.outputs;
+ cmakeFlags = orig.cmakeFlags ++ [
+ "-DENABLE_STATIC=1"
+ "-DNO_MAN_PAGES=1"
+ "-DNO_PY_VERBS=1"
+ ];
+ });
+ };
+
+ buildSomeThingsWithLLVM = self: super: let
+ cc = super.llvmPackages.clangUseLLVM;
+ stdenv = with super; overrideCC llvmPackages.stdenv cc;
+ buildWithLLVM = (pkg: pkg.override { inherit stdenv; });
+ includeMold = (pkg: pkg.overrideAttrs (orig: { nativeBuildInputs = (orig.nativeBuildInputs or []) ++ [ super.mold ]; }));
+ in {
+ ethtool = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.ethtool))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ iproute2 = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.iproute2))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ iptables = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.iptables))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ libmnl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libmnl))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ libnl = includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnl));
+ libnetfilter_conntrack = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnetfilter_conntrack))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ libnftnl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnftnl))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ libpcap = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libpcap))).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+# numactl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.numactl))).overrideAttrs (orig: {
+## outputs = super.lib.lists.remove "man" orig.outputs;
+# configurePhase = ''
+# set -euxo pipefail;
+# ./configure --prefix=$out --libdir=$out/lib --includedir=$out/include --enable-static --enable-shared;
+# '';
+# CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+# });
+ rdma-core = (buildWithFlags releaseFlags (buildWithLLVM super.rdma-core)).overrideAttrs (orig: {
+ CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+ });
+ udev = includeMold (buildWithFlags releaseFlags (buildWithLLVM super.udev));
+
+ dpdk = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.dpdk))).overrideAttrs (orig: rec {
+ outputs = super.lib.lists.remove "doc" orig.outputs;
+ version = "24.07";
+ src = super.fetchurl {
+ url = "https://fast.dpdk.org/rel/dpdk-${version}.tar.xz";
+ sha256 = "sha256-mUT35fJo56ybQZPizVTvbZj24dfd3JZ8d65PZhbW+70=";
+ };
+ nativeBuildInputs = (orig.nativeBuildInputs or []) ++ [ super.llvmPackages.bintools super.numactl.dev ];
+ buildInputs = (orig.buildInputs or []) ++ [ super.numactl.dev ];
+ LIBRARY_PATH = "${super.numactl}/lib:${orig.LIBRARY_PATH or ""}";
+# configurePhase = ''
+# cp -r ${super.numactl} numactl;
+# cp -r ${super.numactl.dev} numactl.dev;
+# ls numactl/lib
+# CFLAGS="${orig.CFLAGS} -DRTE_HAS_LIBNUMA=1" LDFLAGS="${orig.LDFLAGS} -lnuma" meson setup build ${pkgs.lib.concatStringsSep " " mesonFlags} -Dc_args="${orig.CFLAGS} -Inumactl.dev/include" -Dc_link_args="${orig.LDFLAGS} -Lnumactl/lib"
+# '';
+ mesonFlags = [
+ "-Dauto_features=auto"
+ "-Db_colorout=never"
+ "-Db_coverage=false"
+ "-Db_lto=true"
+ "-Db_lundef=true"
+ "-Db_pch=true"
+ "-Db_pgo=off"
+ "-Db_pie=true"
+ "-Db_sanitize=none"
+ "-Dbackend=ninja"
+ "-Ddefault_library=static"
+ "-Denable_docs=false"
+ "-Denable_driver_sdk=false"
+ "-Dibverbs_link=static"
+ "-Dmax_numa_nodes=1"
+ "-Dprefer_static=true"
+ "-Dstrip=false"
+ "-Dtests=false"
+ "-Duse_hpet=false"
+ "-Db_lto_mode=thin"
+ "-Doptimization=3"
+ # ''-Ddisabled_drivers=${pkgs.lib.concatStringsSep "," dpdkDrivers.disabled}''
+ # ''-Denable_drivers=${pkgs.lib.concatStringsSep "," dpdkDrivers.enabled}''
+ ];
+# buildPhase = ''
+# cd build;
+# ninja install;
+# '';
+ });
+
+
+ };
+ };
+
+ defaultOverlays = builtins.attrValues overlays;
+
+ pkgs = (nixpkgsFn {
+ overlays = defaultOverlays;
+ });
+
+ rdma-core = pkgs.rdma-core;
+ numactl = pkgs.numactl;
+
+}
diff --git a/runner.sh b/runner.sh
index b8646c6f8..2af7570f7 100755
--- a/runner.sh
+++ b/runner.sh
@@ -1,11 +1,22 @@
#!/usr/bin/env bash
-# This is a wrapper for cargo which sets up environment variables as needed
+set -euo pipefail
-PATH="$(pwd)/debug/sysroot/usr/bin:$PATH"
-delcare -x PATH
+declare script_dir
+script_dir="$(dirname "$(readlink --canonicalize-existing "$(dirname "${BASH_SOURCE[0]}")")")"
+declare -r script_dir
-LD_LIBRARY_PATH="$(pwd)/debug/sysroot/usr/lib:$LD_LIBRARY_PATH"
-declare -x LD_LIBRARY_PATH
+declare -r project_dir="${script_dir}"
-echo exec "${0}" "${@}"
+exec docker run \
+ --rm \
+ -it \
+ --privileged \
+ --network=host \
+ --name dataplane-runner \
+ -v "${project_dir}:${project_dir}" \
+ -v "/etc/passwd:/etc/passwd:ro" \
+ --user "$(id -u):$(id -g)" \
+ -w "${project_dir}" \
+ ghcr.io/githedgehog/dataplane/development-environment:debug \
+ "${@}"
diff --git a/sysroot-helper/src/lib.rs b/sysroot-helper/src/lib.rs
index 1df023906..7861d3924 100644
--- a/sysroot-helper/src/lib.rs
+++ b/sysroot-helper/src/lib.rs
@@ -21,7 +21,9 @@ pub fn get_project_root() -> String {
pub fn get_sysroot() -> String {
let project_root = get_project_root();
let profile_name = get_build_profile_name();
- format!("{project_root}/../sysroot/{profile_name}")
+ let path_components = project_root.split("/").collect::<Vec<_>>();
+ let repo_root = path_components[..path_components.len() - 1].join("/");
+ format!("{repo_root}/sysroot/{profile_name}")
}
pub fn get_lld_path() -> String {