Address some of @qmonnet comments

.github/workflows/build-and-push-container.yml

@@ -0,0 +1,36 @@
+name: test suite
+on: [ push, pull_request ]
+
+jobs:
+  build:
+    name: cargo build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Fetch dev env container
+        run: |
+          declare -rx PROFILE=debug
+          docker pull "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}" || true
+          # container may not exist yet
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          driver-opts: network=host
+          buildkitd-flags: '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host'
+      - name: Build sysroot
+        run: |
+          declare -rx PROFILE=debug
+          pushd dpdk-sys
+          ./gen-sysroot/gen-sysroot.sh "${PROFILE}"
+      - name: Build in debug container
+        run: |
+          declare -rx PROFILE=debug
+          docker tag "dpdk-sysroot-${PROFILE}" "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}"
+          docker push "ghcr.io/githedgehog/dataplane/development-environment:${PROFILE}"

dpdk-sys-old/src/main.rs

@@ -124,21 +124,21 @@ fn main() {
         "--huge-worker-stack=8192",
         "--socket-mem=4096,4096,4096,4096",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool_ring.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool_ring.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/librte_mempool_stack.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/librte_mempool_stack.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_bus_pci.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_bus_pci.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_net_mlx5.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_net_mlx5.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_common_mlx5.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_common_mlx5.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_regex_mlx5.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_regex_mlx5.so",
         // "-d",
-        // "/mnt/dpdk-arch-gen-sysroot/usr/lib/dpdk/pmds-24.2/librte_vdpa_mlx5.so",
+        // "/mnt/dpdk-arch-gen-dev-env/usr/lib/dpdk/pmds-24.2/librte_vdpa_mlx5.so",
     ];
     info!("DPDK arguments: {args:?}");
     let eal = Eal::new(args);

nix/dev-env.nix

@@ -0,0 +1,153 @@
+rec {
+  nixpkgsFn = import <nixpkgs>;
+
+  debugFlags = rec {
+    AR="llvm-ar";
+    CC="clang";
+    CFLAGS="-Og -ggdb3";
+    CXX="clang++";
+    CXXFLAGS=CFLAGS;
+    LD="mold";
+    LDFLAGS="-fuse-ld=mold";
+    NM="llvm-nm";
+    RANLIB="llvm-ranlib";
+  };
+
+  releaseFlags = rec {
+    AR="llvm-ar";
+    CC="clang";
+    CFLAGS="-O3 -ggdb3 -march=x86-64-v4 -mtune=znver4 -flto=thin -Werror=odr -Werror=strict-aliasing -fstack-protector-strong";
+    CXX="clang++";
+    CXXFLAGS=CFLAGS;
+    LD="mold";
+    LDFLAGS="-fuse-ld=mold -Wl,-O3 -Wl,--gc-sections -Wl,-z,relro,-z,now";
+    NM="llvm-nm";
+    RANLIB="llvm-ranlib";
+  };
+
+  buildWithFlags = flags: pkg: (pkg.overrideAttrs (orig: {
+    CFLAGS = "${orig.CFLAGS or ""} ${flags.CFLAGS}";
+    CXXFLAGS = "${orig.CXXFLAGS or ""} ${flags.CXXFLAGS}";
+    LDFLAGS = "${orig.LDFLAGS or ""} ${flags.LDFLAGS}";
+  }));
+
+  overlays = {
+    disableAppArmor = self: super: { libapparmor = null; };
+    disableSystemd = self: super: { systemd = null; }; # git rekt
+    disableDoxygen = self: super: { doxygen = null; };
+    useLatestLlvm = self: super: { llvmPackages = super.llvmPackages_latest; };
+    rdma-core = self: super: {
+      rdma-core = super.rdma-core.overrideAttrs (orig: {
+          nativeBuildInputs = with super; [ cmake pkg-config python3 mold ];
+          buildInputs = with super; [ libnl ethtool iproute2 ];
+          outputs = super.lib.lists.remove "man" orig.outputs;
+          cmakeFlags = orig.cmakeFlags ++ [
+            "-DENABLE_STATIC=1"
+            "-DNO_MAN_PAGES=1"
+            "-DNO_PY_VERBS=1"
+          ];
+      });
+    };
+
+    buildSomeThingsWithLLVM = self: super: let
+        cc = super.llvmPackages.clangUseLLVM;
+        stdenv = with super; overrideCC llvmPackages.stdenv cc;
+        buildWithLLVM = (pkg: pkg.override { inherit stdenv; });
+        includeMold = (pkg: pkg.overrideAttrs (orig: { nativeBuildInputs = (orig.nativeBuildInputs or []) ++ [ super.mold ]; }));
+        in {
+          ethtool = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.ethtool))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          iproute2 = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.iproute2))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          iptables = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.iptables))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          libmnl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libmnl))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          libnl = includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnl));
+          libnetfilter_conntrack = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnetfilter_conntrack))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          libnftnl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libnftnl))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          libpcap = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.libpcap))).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+#          numactl = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.numactl))).overrideAttrs (orig: {
+##            outputs = super.lib.lists.remove "man" orig.outputs;
+#            configurePhase = ''
+#              set -euxo pipefail;
+#              ./configure --prefix=$out --libdir=$out/lib --includedir=$out/include --enable-static --enable-shared;
+#            '';
+#            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+#          });
+          rdma-core = (buildWithFlags releaseFlags (buildWithLLVM super.rdma-core)).overrideAttrs (orig: {
+            CFLAGS = "${orig.CFLAGS or ""} -ffat-lto-objects";
+          });
+          udev = includeMold (buildWithFlags releaseFlags (buildWithLLVM super.udev));
+
+          dpdk = (includeMold (buildWithFlags releaseFlags (buildWithLLVM super.dpdk))).overrideAttrs (orig: rec {
+            outputs = super.lib.lists.remove "doc" orig.outputs;
+            version = "24.07";
+            src = super.fetchurl {
+              url = "https://fast.dpdk.org/rel/dpdk-${version}.tar.xz";
+              sha256 = "sha256-mUT35fJo56ybQZPizVTvbZj24dfd3JZ8d65PZhbW+70=";
+            };
+            nativeBuildInputs = (orig.nativeBuildInputs or []) ++ [ super.llvmPackages.bintools super.numactl.dev ];
+            buildInputs = (orig.buildInputs or []) ++ [ super.numactl.dev ];
+            LIBRARY_PATH = "${super.numactl}/lib:${orig.LIBRARY_PATH or ""}";
+#            configurePhase = ''
+#              cp -r ${super.numactl} numactl;
+#              cp -r ${super.numactl.dev} numactl.dev;
+#              ls numactl/lib
+#              CFLAGS="${orig.CFLAGS} -DRTE_HAS_LIBNUMA=1" LDFLAGS="${orig.LDFLAGS} -lnuma" meson setup build ${pkgs.lib.concatStringsSep " " mesonFlags} -Dc_args="${orig.CFLAGS} -Inumactl.dev/include" -Dc_link_args="${orig.LDFLAGS} -Lnumactl/lib"
+#            '';
+            mesonFlags = [
+              "-Dauto_features=auto"
+              "-Db_colorout=never"
+              "-Db_coverage=false"
+              "-Db_lto=true"
+              "-Db_lundef=true"
+              "-Db_pch=true"
+              "-Db_pgo=off"
+              "-Db_pie=true"
+              "-Db_sanitize=none"
+              "-Dbackend=ninja"
+              "-Ddefault_library=static"
+              "-Denable_docs=false"
+              "-Denable_driver_sdk=false"
+              "-Dibverbs_link=static"
+              "-Dmax_numa_nodes=1"
+              "-Dprefer_static=true"
+              "-Dstrip=false"
+              "-Dtests=false"
+              "-Duse_hpet=false"
+              "-Db_lto_mode=thin"
+              "-Doptimization=3"
+              # ''-Ddisabled_drivers=${pkgs.lib.concatStringsSep "," dpdkDrivers.disabled}''
+              # ''-Denable_drivers=${pkgs.lib.concatStringsSep "," dpdkDrivers.enabled}''
+            ];
+#            buildPhase = ''
+#               cd build;
+#               ninja install;
+#            '';
+          });
+
+
+        };
+  };
+
+  defaultOverlays = builtins.attrValues overlays;
+
+  pkgs = (nixpkgsFn {
+    overlays = defaultOverlays;
+  });
+
+  rdma-core = pkgs.rdma-core;
+  numactl = pkgs.numactl;
+
+}

runner.sh

@@ -1,11 +1,22 @@
 #!/usr/bin/env bash
 
-# This is a wrapper for cargo which sets up environment variables as needed
+set -euo pipefail
 
-PATH="$(pwd)/debug/sysroot/usr/bin:$PATH"
-delcare -x PATH
+declare script_dir
+script_dir="$(dirname "$(readlink --canonicalize-existing "$(dirname "${BASH_SOURCE[0]}")")")"
+declare -r script_dir
 
-LD_LIBRARY_PATH="$(pwd)/debug/sysroot/usr/lib:$LD_LIBRARY_PATH"
-declare -x LD_LIBRARY_PATH
+declare -r project_dir="${script_dir}"
 
-echo exec "${0}" "${@}"
+exec docker run \
+  --rm \
+  -it \
+  --privileged \
+  --network=host \
+  --name dataplane-runner \
+  -v "${project_dir}:${project_dir}" \
+  -v "/etc/passwd:/etc/passwd:ro" \
+  --user "$(id -u):$(id -g)" \
+  -w "${project_dir}" \
+  ghcr.io/githedgehog/dataplane/development-environment:debug \
+  "${@}"

sysroot-helper/src/lib.rs

@@ -21,7 +21,9 @@ pub fn get_project_root() -> String {
 pub fn get_sysroot() -> String {
     let project_root = get_project_root();
     let profile_name = get_build_profile_name();
-    format!("{project_root}/../sysroot/{profile_name}")
+    let path_components = project_root.split("/").collect::<Vec<_>>();
+    let repo_root = path_components[..path_components.len() - 1].join("/");
+    format!("{repo_root}/sysroot/{profile_name}")
 }
 
 pub fn get_lld_path() -> String {