Tweaks

benchmate/benchmate/warden.py

@@ -71,9 +71,8 @@ def _default():
     return []
 
 
-
 def _rocm_parse_processes():
-    cmd = ["rocm-smi", f"--showpids", "--json", "--loglevel", "error"]
+    cmd = ["rocm-smi", "--loglevel", "error", "--showpids", "--json"]
     output = subprocess.check_output(cmd, text=True)
 
     info = []
@@ -84,7 +83,14 @@ def _rocm_parse_processes():
         return info
 
     for key, data in data.get("system", {}).items():
-        process_name, ngpu, vram, sdma, cu_occupancy = data.split(",")
+        cols = data.split(",")
+
+        process_name = cols[0]
+        ngpu = cols[1]
+        vram = cols[2]
+        sdma = cols[3]
+        cu_occupancy = cols[4]
+
         pid = key[3:]
         info.append(ProcessInfo(pid=pid, process_name=process_name, used_memory=vram))
     return info
@@ -167,27 +173,30 @@ def children_warden(enabled=True):
         yield
         return
 
+    pid = os.getpid()
 
     def get_children():
         with open(f"/proc/{pid}/task/{pid}/children", "r") as f:
             return [int(c) for c in f.read().strip().split()]
 
     prev = set(get_children())
 
+    yield
+
     def get_children():
         with open(f"/proc/{pid}/task/{pid}/children", "r") as f:
             return set([int(c) for c in f.read().strip().split()]) - prev
 
-    yield
-
-    pid = os.getpid()
-
     def wait_for_children():
         children = get_children()
         start_time = time.time()
 
         while children and time.time()  - start_time < 15:
             children = get_children()
+
+            for child in children:
+                os.waitpid(child, os.WNOHANG)
+
             time.sleep(0)
 
         return children

pyproject.toml

@@ -10,7 +10,7 @@ authors = [
 license = "MIT"
 
 [tool.poetry.dependencies]
-voir =  ">=0.2.19"
+voir =  ">=0.2.20"
 benchmate =  {path = "benchmate", develop = false}
 python = ">=3.10,<4.0"
 giving = "^0.4.0"

scripts/article/run_rocm.sh

@@ -155,16 +155,17 @@ fi
 
 
 
-# (
-#     . $MILABENCH_WORDIR/env/bin/activate
-#     pip install git+https://github.com/breuleux/voir.git
-
-#     (
-#         . $BENCHMARK_VENV/bin/activate
-#         pip install git+https://github.com/breuleux/voir.git
-#     )
+(
+    . $MILABENCH_WORDIR/env/bin/activate
+    pip install git+https://github.com/breuleux/voir.git
+    pip install -e /home/testroot/milabench/benchmate
+    (
+        . $BENCHMARK_VENV/bin/activate
+        pip install git+https://github.com/breuleux/voir.git
+        pip install -e /home/testroot/milabench/benchmate
+    )
 
-# )
+)
 
 # (
 #     # . $BENCHMARK_VENV/bin/activate