Merge commit '3d3e444618ca1b757061c607fe7a399f750ca0e3' into upgrade-…

…pydantic

Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04
+FROM nvcr.io/nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04
 LABEL maintainer="Learning@home"
 LABEL repository="hivemind"
 
@@ -21,7 +21,7 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -
   bash install_miniconda.sh -b -p /opt/conda && rm install_miniconda.sh
 ENV PATH="/opt/conda/bin:${PATH}"
 
-RUN conda install python~=3.8 pip && \
+RUN conda install python~=3.11.0 pip && \
     pip install --no-cache-dir torch torchvision torchaudio && \
     conda clean --all
 

tests/conftest.py

@@ -1,6 +1,5 @@
 import asyncio
 import gc
-from contextlib import suppress
 
 import psutil
 import pytest
@@ -40,13 +39,12 @@ def cleanup_children():
 
     children = psutil.Process().children(recursive=True)
     if children:
-        logger.info(f"Cleaning up {len(children)} leftover child processes")
-        for child in children:
-            with suppress(psutil.NoSuchProcess):
-                child.terminate()
-        psutil.wait_procs(children, timeout=1)
-        for child in children:
-            with suppress(psutil.NoSuchProcess):
-                child.kill()
+        gone, alive = psutil.wait_procs(children, timeout=0.1)
+        logger.debug(f"Cleaning up {len(alive)} leftover child processes")
+        for child in alive:
+            child.terminate()
+        gone, alive = psutil.wait_procs(alive, timeout=1)
+        for child in alive:
+            child.kill()
 
     MPFuture.reset_backend()

tests/test_allreduce_fault_tolerance.py

@@ -209,3 +209,4 @@ def _make_tensors():
 
     for averager in averagers:
         averager.shutdown()
+    dht.shutdown()

tests/test_cli_scripts.py

@@ -1,3 +1,4 @@
+import os
 import re
 from subprocess import PIPE, Popen
 from time import sleep
@@ -8,11 +9,16 @@
 def test_dht_connection_successful():
     dht_refresh_period = 1
 
+    cloned_env = os.environ.copy()
+    # overriding the loglevel to prevent debug print statements
+    cloned_env["HIVEMIND_LOGLEVEL"] = "INFO"
+
     dht_proc = Popen(
         ["hivemind-dht", "--host_maddrs", "/ip4/127.0.0.1/tcp/0", "--refresh_period", str(dht_refresh_period)],
         stderr=PIPE,
         text=True,
         encoding="utf-8",
+        env=cloned_env,
     )
 
     first_line = dht_proc.stderr.readline()
@@ -28,6 +34,7 @@ def test_dht_connection_successful():
         stderr=PIPE,
         text=True,
         encoding="utf-8",
+        env=cloned_env,
     )
 
     # skip first two lines with connectivity info

tests/test_dht_experts.py

@@ -82,6 +82,10 @@ def test_beam_search(
         assert all(isinstance(e, hivemind.RemoteExpert) for experts in batch_experts for e in experts)
         assert all(len(experts) == beam_size for experts in batch_experts)
 
+    you.shutdown()
+    for dht_instance in dht_instances:
+        dht_instance.shutdown()
+
 
 @pytest.mark.forked
 def test_dht_single_node():
@@ -119,6 +123,8 @@ def test_dht_single_node():
     with pytest.raises(AssertionError):
         beam_search.get_active_successors(["e.1.2.", "e.2", "e.4.5."])
 
+    node.shutdown()
+
 
 def test_uid_patterns():
     valid_experts = [