Merge branch 'main' into feature/optimization

src/miv_simulator/interface/h5_types.py

@@ -12,7 +12,7 @@ class Config(BaseModel):
 
         cell_distributions: config.CellDistributions = Field("???")
         projections: config.SynapticProjections = Field("???")
-        mpi: Optional[str] = None
+        mpi_args: str = "-n 1"
         nodes: str = "1"
 
     def config_from_file(self, filename: str) -> Dict:
@@ -36,6 +36,6 @@ def __call__(self) -> None:
 
     def compute_context(self):
         context = super().compute_context()
-        del context["config"]["mpi"]
+        del context["config"]["mpi_args"]
         del context["config"]["nodes"]
         return context

src/miv_simulator/interface/network.py

@@ -27,7 +27,10 @@ def launch(self):
                     "cell_distributions": config.cell_distributions,
                 },
             ],
-        ).launch()
+        )
+        # eager execute to avoid scheduling overheads
+        if not self.h5_types.cached():
+            self.h5_types.commit().dispatch()
 
         self.architecture = get(
             "miv_simulator.interface.architecture",

src/miv_simulator/interface/neuroh5_graph.py

@@ -6,7 +6,7 @@
 
 class NeuroH5Graph(Component):
     class Config:
-        mpi: Optional[str] = None
+        mpi_args: str = "-n 1"
         nodes: str = "1"
 
     def __init__(self, *args, **kwargs):
@@ -102,7 +102,7 @@ def files(self) -> Dict[str, str]:
 
     def compute_context(self):
         context = super().compute_context()
-        del context["config"]["mpi"]
+        del context["config"]["mpi_args"]
         del context["config"]["nodes"]
         context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
         return context

src/miv_simulator/interface/synapse_forest.py

@@ -12,7 +12,7 @@ class Config(BaseModel):
         filepath: str = Field("???")
         population: config.PopulationName = Field("???")
         morphology: config.SWCFilePath = Field("???")
-        mpi: Optional[str] = None
+        mpi_args: Optional[str] = "-n 1"
         nodes: str = "1"
 
     @property
@@ -23,20 +23,46 @@ def tree_output_filepath(self) -> str:
     def output_filepath(self) -> str:
         return self.local_directory("forest.h5")
 
-    def __call__(self) -> None:
+    def __call__(self, runner=None) -> None:
+        kwargs = {}
+        if runner is not None:
+            kwargs["_run"] = runner
         simulator.generate_synapse_forest(
             filepath=self.config.filepath,
             tree_output_filepath=self.tree_output_filepath,
             output_filepath=self.output_filepath,
             population=self.config.population,
             morphology=self.config.morphology,
+            **kwargs,
         )
-        print("generate_synapse_forest() completed")
+
+    def dispatch_code(
+        self,
+        inline: bool = True,
+        project_directory: Optional[str] = None,
+        python: Optional[str] = None,
+    ) -> Optional[str]:
+        # since generate_synapse_forest just subprocesses
+        # the calls, we can return the code directly to
+        # avoid overheads
+
+        cmds = []
+        self.__call__(cmds.append)
+        lines = [" ".join(cmd) for cmd in cmds]
+
+        if isinstance(python, str):
+            runner, _, python = python.rpartition(" ")
+            if runner:
+                lines = [runner + " " + line for line in lines]
+
+        cache_marker = self.local_directory("cached")
+        lines.append(f'echo "finished" > {cache_marker}')
+
+        return "\n".join(lines)
 
     def compute_context(self):
-        # remove filepath in favor of uses
         context = super().compute_context()
-        del context["config"]["mpi"]
+        del context["config"]["mpi_args"]
         del context["config"]["nodes"]
         del context["config"]["filepath"]
         context["config"]["morphology"] = file_hash(

src/miv_simulator/simulator/generate_synapse_forest.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import shlex
 import subprocess
 
 import h5py
@@ -11,7 +12,10 @@ def _bin_check(bin: str) -> None:
         raise FileNotFoundError(f"{bin} not found. Did you add it to the PATH?")
 
 
-def _run(cmd):
+def _sh(cmd, spawn_process=True):
+    if not spawn_process:
+        return os.system(" ".join([shlex.quote(c) for c in cmd]))
+
     try:
         subprocess.check_output(
             cmd,
@@ -33,6 +37,7 @@ def generate_synapse_forest(
     output_filepath: str,
     population: config.PopulationName,
     morphology: config.SWCFilePath,
+    _run=_sh,
 ) -> None:
     # create tree
     if not os.path.isfile(tree_output_filepath):

src/miv_simulator/utils/io.py

@@ -3,6 +3,7 @@
 import gc
 import pathlib
 import os
+import shlex
 import subprocess
 from collections import defaultdict
 from miv_simulator import config
@@ -1133,10 +1134,12 @@ def query_cell_attributes(input_file, population_names, namespace_ids=None):
     return namespace_id_lst, attr_info_dict
 
 
-def _run(commands):
+def _run(commands, spawn_process=False):
     cmd = " ".join(commands)
     print(cmd)
-    subprocess.check_output(commands)
+    if not spawn_process:
+        return os.system(" ".join([shlex.quote(cmd) for cmd in commands]))
+    return subprocess.check_output(commands)
 
 
 def copy_dataset(f_src: h5py.File, f_dst: h5py.File, dset_path: str) -> None: