From 53633fe4422d88aa2cc7889e6f84dd9321d93539 Mon Sep 17 00:00:00 2001 From: TorecLuik Date: Tue, 3 Dec 2024 14:41:54 +0000 Subject: [PATCH] deploy: 4b54777fcc4ee0ee2b305c5e29723d470e61a3d1 --- _modules/biomero/slurm_client.html | 470 +++++++++++++++++++++------ biomero.html | 492 ++++++++++++++++------------- genindex.html | 30 +- objects.inv | Bin 989 -> 1094 bytes readme_link.html | 6 +- searchindex.js | 2 +- tutorial_link.html | 12 +- 7 files changed, 689 insertions(+), 323 deletions(-) diff --git a/_modules/biomero/slurm_client.html b/_modules/biomero/slurm_client.html index 6dbc28d..2ff0a1c 100644 --- a/_modules/biomero/slurm_client.html +++ b/_modules/biomero/slurm_client.html @@ -113,6 +113,7 @@

Source code for biomero.slurm_client

 # See the License for the specific language governing permissions and
 # limitations under the License.
 from typing import Dict, List, Optional, Tuple, Any
+from uuid import UUID
 from fabric import Connection, Result
 from fabric.transfer import Result as TransferResult
 from invoke.exceptions import UnexpectedExit
@@ -129,6 +130,12 @@ 
Source code for biomero.slurm_client
 from importlib_resources import files
 import io
 import os
+from biomero.eventsourcing import WorkflowTracker, NoOpWorkflowTracker
+from biomero.views import JobAccounting, JobProgress, WorkflowAnalytics, WorkflowProgress
+from biomero.database import EngineManager, JobProgressView, JobView, TaskExecution, WorkflowProgressView
+from eventsourcing.system import System, SingleThreadedRunner
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.sql import text
 
 logger = logging.getLogger(__name__)
 
@@ -145,19 +152,20 @@ 
Source code for biomero.slurm_client
         submit_result (Result): The result of submitting the job.
         ok (bool): Indicates whether the job submission was successful.
         job_state (str): The current state of the Slurm job.
-        error_message (str): The error message, if any.
-
-    Args:
-        submit_result (Result): The result of submitting the job.
-        job_id (int): The Slurm job ID.
+        progress (str): The progress of the Slurm job.
+        error_message (str): The error message, if any, encountered during job submission.
+        wf_id (UUID): The workflow ID associated with the job.
+        task_id (UUID): The task ID within the workflow.
+        slurm_polling_interval (int): The polling interval (in seconds) for checking the job status.
 
     Example:
         # Submit some job with the SlurmClient
-        submit_result, job_id = slurmClient.run_workflow(
-            workflow_name, workflow_version, input_data, email, time, **kwargs)
+        submit_result, job_id, wf_id, task_id = slurmClient.run_workflow(
+            workflow_name, workflow_version, input_data, email, time, wf_id,
+            **kwargs)
             
         # Create a SlurmJob instance
-        slurmJob = SlurmJob(submit_result, job_id)
+        slurmJob = SlurmJob(submit_result, job_id, wf_id, task_id)
 
         if not slurmJob.ok:
             logger.warning(f"Error with job: {slurmJob.get_error()}")
@@ -173,21 +181,33 @@ 
Source code for biomero.slurm_client
                 raise e
 
     """
+    SLURM_POLLING_INTERVAL = 10  # seconds
     
     def __init__(self,
                  submit_result: Result,
-                 job_id: int):
+                 job_id: int,
+                 wf_id: UUID, 
+                 task_id: UUID,
+                 slurm_polling_interval: int = SLURM_POLLING_INTERVAL):
         """
         Initialize a SlurmJob instance.
 
         Args:
             submit_result (Result): The result of submitting the job.
             job_id (int): The Slurm job ID.
+            wf_id (UUID): The workflow ID associated with this job.
+            task_id (UUID): The task ID within the workflow.
+            slurm_polling_interval (int, optional): The interval in seconds for 
+                polling the job status. Defaults to SLURM_POLLING_INTERVAL.
         """
         self.job_id = job_id
+        self.wf_id = wf_id
+        self.task_id = task_id
+        self.slurm_polling_interval = slurm_polling_interval
         self.submit_result = submit_result
         self.ok = self.submit_result.ok
         self.job_state = None
+        self.progress = None
         self.error_message = self.submit_result.stderr if hasattr(self.submit_result, 'stderr') else ''
 
 
[docs]    def wait_for_completion(self, slurmClient, omeroConn) -> str:
@@ -211,6 +231,7 @@ 
Source code for biomero.slurm_client
                                      "TIMEOUT+"):
             job_status_dict, poll_result = slurmClient.check_job_status(
                 [self.job_id])
+            self.progress = slurmClient.get_active_job_progress(self.job_id)
             if not poll_result.ok:
                 logger.warning(
                     f"Error checking job status:{poll_result.stderr}")
@@ -219,7 +240,11 @@ 
Source code for biomero.slurm_client
             self.job_state = job_status_dict[self.job_id]
             # wait for 10 seconds before checking again
             omeroConn.keepAlive()  # keep the OMERO connection alive
-            timesleep.sleep(10)
+            slurmClient.workflowTracker.update_task_status(self.task_id, 
+                                                           self.job_state)
+            slurmClient.workflowTracker.update_task_progress(
+                self.task_id, self.progress)
+            timesleep.sleep(self.slurm_polling_interval)
         logger.info(f"Job {self.job_id} finished: {self.job_state}")
         logger.info(
             f"You can get the logfile using `Slurm Get Update` on job {self.job_id}")
@@ -301,6 +326,7 @@ 
Source code for biomero.slurm_client
             containing the Slurm job submission scripts. Optional.
 
     Example:
+        
         # Create a SlurmClient object as contextmanager
 
         with SlurmClient.from_config() as client:
@@ -319,6 +345,7 @@ 
Source code for biomero.slurm_client
             print(result.stdout)
 
     Example 2:
+        
         # Create a SlurmClient and setup Slurm (download containers etc.)
 
         with SlurmClient.from_config(init_slurm=True) as client:
@@ -376,82 +403,100 @@ 
Source code for biomero.slurm_client
                  slurm_script_path: str = _DEFAULT_SLURM_GIT_SCRIPT_PATH,
                  slurm_script_repo: str = None,
                  init_slurm: bool = False,
-                 ):
-        """Initializes a new instance of the SlurmClient class.
+                 track_workflows: bool = True,
+                 enable_job_accounting: bool = True,
+                 enable_job_progress: bool = True,
+                 enable_workflow_analytics: bool = True,
+                 sqlalchemy_url: str = None):
+        """
+        Initializes a new instance of the SlurmClient class.
 
-        It is preferable to use #from_config(...) method to initialize
-        parameters from a config file.
+        It is preferable to use the `#from_config(...)` method to initialize 
+        parameters from a configuration file.
 
         Args:
-            host (str, optional): The hostname or IP address of the remote
-                server. Defaults to _DEFAULT_HOST.
+            host (str, optional): The hostname or IP address of the remote 
+                server. Defaults to `_DEFAULT_HOST`.
             user (str, optional): The username to use when connecting to 
-                the remote server. Defaults to None, which defaults 
-                to config.user.
+                the remote server. Defaults to None, which falls back to 
+                `config.user`.
             port (int, optional): The SSH port to use when connecting.
-                Defaults to None, which defaults to config.port.
+                Defaults to None, which falls back to `config.port`.
             config (str, optional): Path to the SSH config file.
-                Defaults to None, which defaults to your SSH config file.
+                Defaults to None, which falls back to your SSH config file.
             gateway (Connection, optional): An optional gateway for connecting 
                 through a jump host. Defaults to None.
             forward_agent (bool, optional): Whether to forward the local SSH 
                 agent to the remote server. Defaults to None, which 
-                defaults to config.forward_agent.
+                falls back to `config.forward_agent`.
             connect_timeout (int, optional): Timeout for establishing the SSH 
-                connection. Defaults to None, which defaults 
-                to config.timeouts.connect.
+                connection. Defaults to None, which falls back to 
+                `config.timeouts.connect`.
             connect_kwargs (dict, optional): Additional keyword arguments for 
-                the underlying SSH connection. Handed verbatim to 
+                the underlying SSH connection. These are passed verbatim to 
                 `SSHClient.connect <paramiko.client.SSHClient.connect>`. 
                 Defaults to None. 
-            inline_ssh_env (bool, optional): Whether to use inline SSH
-                environment. This is necessary if the remote server has 
-                a restricted ``AcceptEnv`` setting (which is the common 
-                default). Defaults to _DEFAULT_INLINE_SSH_ENV.
-            slurm_data_path (str, optional): The path to the directory
-                containing the data files for Slurm jobs.
-                Defaults to _DEFAULT_SLURM_DATA_PATH.
-            slurm_images_path (str, optional): The path to the directory
-                containing the Singularity images for Slurm jobs.
-                Defaults to _DEFAULT_SLURM_IMAGES_PATH.
-            slurm_converters_path (str, optional): The path to the directory
-                containing the Singularity images for file converters.
-                Defaults to _DEFAULT_SLURM_CONVERTERS_PATH.
+            inline_ssh_env (bool, optional): Whether to use inline SSH 
+                environment variables. This is necessary if the remote server 
+                has a restricted `AcceptEnv` setting (the common default). 
+                Defaults to `_DEFAULT_INLINE_SSH_ENV`.
+            slurm_data_path (str, optional): The path to the directory 
+                containing the data files for Slurm jobs. 
+                Defaults to `_DEFAULT_SLURM_DATA_PATH`.
+            slurm_images_path (str, optional): The path to the directory 
+                containing the Singularity images for Slurm jobs. 
+                Defaults to `_DEFAULT_SLURM_IMAGES_PATH`.
+            slurm_converters_path (str, optional): The path to the directory 
+                containing the Singularity images for file converters. 
+                Defaults to `_DEFAULT_SLURM_CONVERTERS_PATH`.
             slurm_model_paths (dict, optional): A dictionary containing the 
-                paths to the Singularity images for specific Slurm job models.
+                paths to the Singularity images for specific Slurm job models. 
                 Defaults to None.
             slurm_model_repos (dict, optional): A dictionary containing the 
-                git repositories of Singularity images for specific Slurm 
-                job models.
-                Defaults to None.
+                Git repositories of Singularity images for specific Slurm 
+                job models. Defaults to None.
             slurm_model_images (dict, optional): A dictionary containing the 
-                dockerhub of the Singularity images for specific Slurm 
-                job models. Will fill automatically from the data in the git 
-                repository if you set init_slurm.
+                DockerHub images of the Singularity images for specific 
+                Slurm job models. Will be filled automatically from the 
+                data in the Git repository if `init_slurm` is set to True. 
                 Defaults to None.
-            converter_images (dict, optional): A dictionairy containing the
-                dockerhub of the Singularity images for converters. 
-                Will default to building converter available in this package
-                on Slurm instead if not configured.
+            converter_images (dict, optional): A dictionary containing the 
+                DockerHub images of the Singularity images for file converters. 
+                Will default to building the converter available in this package 
+                on Slurm instead if not configured. 
                 Defaults to None.
-            slurm_model_jobs (dict, optional): A dictionary containing
-                information about specific Slurm job models.
+            slurm_model_jobs (dict, optional): A dictionary containing 
+                information about specific Slurm job models. 
                 Defaults to None.
-            slurm_model_jobs_params (dict, optional): A dictionary containing
-                parameters for specific Slurm job models.
+            slurm_model_jobs_params (dict, optional): A dictionary containing 
+                parameters for specific Slurm job models. 
                 Defaults to None.
-            slurm_script_path (str, optional): The path to the directory
-                containing the Slurm job submission scripts on Slurm.
-                Defaults to _DEFAULT_SLURM_GIT_SCRIPT_PATH.
-            slurm_script_repo (str, optional): The git https URL for cloning
-                the repo containing the Slurm job submission scripts.
+            slurm_script_path (str, optional): The path to the directory 
+                containing the Slurm job submission scripts on Slurm. 
+                Defaults to `_DEFAULT_SLURM_GIT_SCRIPT_PATH`.
+            slurm_script_repo (str, optional): The Git HTTPS URL for cloning 
+                the repository containing the Slurm job submission scripts. 
                 Defaults to None.
-            init_slurm (bool): Whether to set up the required structures 
+            init_slurm (bool, optional): Whether to set up the required structures 
                 on Slurm after initiating this client. This includes creating 
-                missing folders, downloading container images, cloning git,etc.
-                This will take a while at first but will validate your setup.
-                Defaults to False to save time.
+                missing folders, downloading container images, cloning Git, etc. 
+                This process will take some time initially but will validate 
+                your setup. Defaults to False to save time.
+            track_workflows (bool, optional): Whether to track workflows. 
+                Defaults to True.
+            enable_job_accounting (bool, optional): Whether to enable job 
+                accounting. Defaults to True.
+            enable_job_progress (bool, optional): Whether to track job 
+                progress. Defaults to True.
+            enable_workflow_analytics (bool, optional): Whether to enable 
+                workflow analytics. Defaults to True.
+            sqlalchemy_url (str, optional): URL for eventsourcing database 
+                connection. Defaults to None, which falls back to the
+                `SQLALCHEMY_URL` environment variable. Note that it will
+                always be overridden with the environment variable 
+                `SQLALCHEMY_URL`, if that is set.
         """
+
         super(SlurmClient, self).__init__(host,
                                           user,
                                           port,
@@ -480,6 +525,159 @@ 
Source code for biomero.slurm_client
 
         self.init_workflows()
         self.validate(validate_slurm_setup=init_slurm)
+        
+        # Setup workflow tracking and accounting
+        # Initialize the analytics settings
+        self.track_workflows = track_workflows
+        self.enable_job_accounting = enable_job_accounting
+        self.enable_job_progress = enable_job_progress
+        self.enable_workflow_analytics = enable_workflow_analytics
+        
+        # Initialize the analytics system
+        self.sqlalchemy_url = sqlalchemy_url
+        self.initialize_analytics_system(reset_tables=init_slurm)
+    
+
[docs]    def initialize_analytics_system(self, reset_tables=False):
+        """
+        Initialize the analytics system based on the analytics configuration
+        passed to the constructor.
+        
+        Args:
+            reset_tables (bool): If True, drops and recreates all views.
+        """
+        # Get persistence settings, prioritize environment variables
+        persistence_module = os.getenv("PERSISTENCE_MODULE", "eventsourcing_sqlalchemy")
+        if persistence_module != "eventsourcing_sqlalchemy": 
+            raise NotImplementedError(f"Can't handle {persistence_module}. Currently only supports 'eventsourcing_sqlalchemy' as PERSISTENCE_MODULE")
+        
+        sqlalchemy_url = os.getenv("SQLALCHEMY_URL", self.sqlalchemy_url)
+        if not sqlalchemy_url:
+            raise ValueError("SQLALCHEMY_URL must be set either in init, config ('sqlalchemy_url') or as an environment variable.")
+        if sqlalchemy_url != self.sqlalchemy_url:
+            logger.info("Overriding configured SQLALCHEMY_URL with env var SQLALCHEMY_URL.")
+
+        # Build the system based on the analytics configuration
+        pipes = []
+        runner = None
+        if self.track_workflows:
+            # Add JobAccounting to the pipeline if enabled
+            if self.enable_job_accounting:
+                pipes.append([WorkflowTracker, JobAccounting])
+
+            # Add JobProgress to the pipeline if enabled
+            if self.enable_job_progress:
+                pipes.append([WorkflowTracker, JobProgress])
+                pipes.append([WorkflowTracker, WorkflowProgress])
+            
+            # Add WorkflowAnalytics to the pipeline if enabled
+            if self.enable_workflow_analytics:
+                pipes.append([WorkflowTracker, WorkflowAnalytics])
+
+            # Add onlys WorkflowTracker if no listeners are enabled
+            if not pipes:
+                pipes = [[WorkflowTracker]]
+                 
+            system = System(pipes=pipes)        
+            scoped_session_topic = EngineManager.create_scoped_session(
+                sqlalchemy_url=sqlalchemy_url)
+            runner = SingleThreadedRunner(system, env={
+                'SQLALCHEMY_SCOPED_SESSION_TOPIC': scoped_session_topic,
+                'PERSISTENCE_MODULE': persistence_module})
+            runner.start()
+            self.workflowTracker = runner.get(WorkflowTracker)  
+        else:  # turn off persistence, override
+            logger.warning("Tracking workflows is disabled. No-op WorkflowTracker will be used.")        
+            self.workflowTracker = NoOpWorkflowTracker()
+            
+        self.setup_listeners(runner, reset_tables)

+
+
[docs]    def setup_listeners(self, runner, reset_tables):
+        # Only when people run init script, we just drop and rebuild.
+        self.get_listeners(runner)
+            
+        # Optionally drop and recreate tables
+        if reset_tables:
+            logger.info("Resetting view tables.")
+            tables = [] 
+            # gather the listener tables
+            listeners = [self.jobAccounting, 
+                         self.jobProgress,
+                         self.wfProgress, 
+                         self.workflowAnalytics]
+            for listener in listeners:
+                if not isinstance(listener, NoOpWorkflowTracker):
+                    tables.append(listener.recorder.tracking_table_name)
+                    tables.append(listener.recorder.events_table_name)
+            runner.stop()
+            # gather the view tables
+            tables.append(TaskExecution.__tablename__)
+            tables.append(JobProgressView.__tablename__)
+            tables.append(WorkflowProgressView.__tablename__)
+            tables.append(JobView.__tablename__) 
+            with EngineManager.get_session() as session:
+                try:
+                    # Begin a transaction
+                    for table in tables:
+                        # Drop the table if it exists
+                        logger.info(f"Dropping table {table}")
+                        drop_table_sql = text(f'DROP TABLE IF EXISTS {table}')
+                        session.execute(drop_table_sql)
+                    # Only when people run init script, we just drop and rebuild.
+                    session.commit()
+                    logger.info("Dropped view tables successfully")
+                except IntegrityError as e:
+                    logger.error(e)
+                    session.rollback()
+                    raise Exception(f"Error trying to reset the view tables: {e}") 
+                
+            EngineManager.close_engine() # close current sql session          
+            # restart runner, listeners and recreate views
+            self.initialize_analytics_system(reset_tables=False)
+            # Update the view tables again
+            listeners = [self.jobAccounting, 
+                         self.jobProgress,
+                         self.wfProgress, 
+                         self.workflowAnalytics]
+            for listener in listeners:
+                if listener:
+                    self.bring_listener_uptodate(listener)

+            
+
[docs]    def get_listeners(self, runner):
+        if self.track_workflows and self.enable_job_accounting:
+            self.jobAccounting = runner.get(JobAccounting)   
+        else:
+            self.jobAccounting = NoOpWorkflowTracker()
+        
+        if self.track_workflows and self.enable_job_progress:
+            self.jobProgress = runner.get(JobProgress)
+            self.wfProgress = runner.get(WorkflowProgress)
+        else:
+            self.jobProgress = NoOpWorkflowTracker()
+            self.wfProgress = NoOpWorkflowTracker()
+        
+        if self.track_workflows and self.enable_workflow_analytics:
+            self.workflowAnalytics = runner.get(WorkflowAnalytics)
+        else:
+            self.workflowAnalytics = NoOpWorkflowTracker()

+
+
[docs]    def bring_listener_uptodate(self, listener, start=1):
+        with EngineManager.get_session() as session:
+            try:
+                # Begin a transaction
+                listener.pull_and_process(leader_name=WorkflowTracker.__name__, start=start)
+                session.commit()
+                logger.info("Updated listener successfully")
+            except IntegrityError as e:
+                logger.error(e)
+                session.rollback()

+            
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Ensure to call the parent class's __exit__ 
+        # to clean up Connection resources
+        super().__exit__(exc_type, exc_val, exc_tb)
+        # Cleanup resources specific to SlurmClient
+        EngineManager.close_engine()
+        # If we have any other resources to close or cleanup, do it here
 
 
[docs]    def init_workflows(self, force_update: bool = False):
         """
@@ -768,17 +966,16 @@ 
Source code for biomero.slurm_client
             - /etc/slurm-config.ini
             - ~/slurm-config.ini
 
-        Note that this is only for the SLURM specific values that we added.
+        Note that this is only for the SLURM-specific values that we added.
         Most configuration values are set via configuration mechanisms from
-        Fabric library,
-        like SSH settings being loaded from SSH config, /etc/fabric.yml or
-        environment variables.
+        Fabric library, like SSH settings being loaded from SSH config, 
+        /etc/fabric.yml or environment variables.
         See Fabric's documentation for more info on configuration if needed.
 
         Args:
             configfile (str): The path to your configuration file. Optional.
             init_slurm (bool): Initiate / validate slurm setup. Optional
-                Might take some time the first time with downloading etc.
+                Might take some time the first time with downloading, etc.
 
         Returns:
             SlurmClient: A new SlurmClient object.
@@ -789,6 +986,7 @@ 
Source code for biomero.slurm_client
         configs.read([cls._DEFAULT_CONFIG_PATH_1,
                      cls._DEFAULT_CONFIG_PATH_2,
                      configfile])
+        
         # Read the required parameters from the configuration file,
         # fallback to defaults
         host = configs.get("SSH", "host", fallback=cls._DEFAULT_HOST)
@@ -820,12 +1018,11 @@ 
Source code for biomero.slurm_client
                 slurm_model_jobs[k[:-len(suffix_job)]] = v
                 slurm_model_jobs_params[k[:-len(suffix_job)]] = []
             elif job_param_match:
-                print(f"Match: {slurm_model_jobs_params}")
                 slurm_model_jobs_params[job_param_match.group(1)].append(
                     f" --{job_param_match.group(2)}={v}")
-                print(f"Added: {slurm_model_jobs_params}")
             else:
                 slurm_model_paths[k] = v
+        logger.info(f"Using job params: {slurm_model_jobs_params}")
 
         slurm_script_path = configs.get(
             "SLURM", "slurm_script_path",
@@ -836,7 +1033,6 @@ 
Source code for biomero.slurm_client
         )
         
         # Parse converters, if available
-        # Should be key=value where key is a name and value a docker image
         try:
             converter_items = configs.items("CONVERTERS")
             if converter_items:
@@ -844,7 +1040,22 @@ 
Source code for biomero.slurm_client
             else:
                 converter_images = None  # Section exists but is empty
         except configparser.NoSectionError:
-            converter_images = None  # Section does not exist       
+            converter_images = None  # Section does not exist    
+            
+        # Read the analytics section, if available
+        try:
+            track_workflows = configs.getboolean('ANALYTICS', 'track_workflows', fallback=True)
+            enable_job_accounting = configs.getboolean('ANALYTICS', 'enable_job_accounting', fallback=True)
+            enable_job_progress = configs.getboolean('ANALYTICS', 'enable_job_progress', fallback=True)
+            enable_workflow_analytics = configs.getboolean('ANALYTICS', 'enable_workflow_analytics', fallback=True)
+            sqlalchemy_url = configs.get('ANALYTICS', 'sqlalchemy_url', fallback=None)
+        except configparser.NoSectionError:
+            # If the ANALYTICS section is missing, fallback to default values
+            track_workflows = True
+            enable_job_accounting = True
+            enable_job_progress = True
+            enable_workflow_analytics = True
+            sqlalchemy_url = None
         
         # Create the SlurmClient object with the parameters read from
         # the config file
@@ -861,7 +1072,13 @@ 
Source code for biomero.slurm_client
                    slurm_model_jobs_params=slurm_model_jobs_params,
                    slurm_script_path=slurm_script_path,
                    slurm_script_repo=slurm_script_repo,
-                   init_slurm=init_slurm)

+                   init_slurm=init_slurm,
+                   # Pass analytics settings to the constructor
+                   track_workflows=track_workflows,
+                   enable_job_accounting=enable_job_accounting,
+                   enable_job_progress=enable_job_progress,
+                   enable_workflow_analytics=enable_workflow_analytics,
+                   sqlalchemy_url=sqlalchemy_url)

 
 
[docs]    def cleanup_tmp_files(self,
                           slurm_job_id: str,
@@ -971,7 +1188,7 @@ 
Source code for biomero.slurm_client
 
[docs]    def get_active_job_progress(self,
                                 slurm_job_id: str,
                                 pattern: str = r"\d+%",
-                                env: Optional[Dict[str, str]] = None) -> str:
+                                env: Optional[Dict[str, str]] = None) -> Any:
         """
         Get the progress of an active Slurm job from its logfiles.
 
@@ -984,7 +1201,7 @@ 
Source code for biomero.slurm_client
                 to set when running the command. Defaults to None.
 
         Returns:
-            str: The progress of the Slurm job.
+            Any: The progress of the Slurm job according to the pattern, or None.
         """
         cmdlist = []
         cmd = self.get_recent_log_command(
@@ -997,13 +1214,14 @@ 
Source code for biomero.slurm_client
         except Exception as e:
             logger.error(f"Issue with run command: {e}")
         # Match the specified pattern in the result's stdout
+        latest_progress = None
         try:
             latest_progress = re.findall(
                 pattern, result.stdout)[-1]
         except Exception as e:
-            logger.error(f"Issue with extracting progress: {e}")
+            logger.warning(f"Issue with extracting progress: {e}")
 
-        return f"Progress: {latest_progress}\n"

+        return latest_progress

 
 
[docs]    def run_commands(self, cmdlist: List[str],
                      env: Optional[Dict[str, str]] = None,
@@ -1338,8 +1556,9 @@ 
Source code for biomero.slurm_client
                      input_data: str,
                      email: Optional[str] = None,
                      time: Optional[str] = None,
+                     wf_id: Optional[UUID] = None,
                      **kwargs
-                     ) -> Tuple[Result, int]:
+                     ) -> Tuple[Result, int, UUID, UUID]:
         """
         Run a specified workflow on Slurm using the given parameters.
 
@@ -1352,24 +1571,48 @@ 
Source code for biomero.slurm_client
             email (str, optional): Email address for Slurm job notifications.
             time (str, optional): Time limit for the Slurm job in the 
                 format HH:MM:SS.
+            wf_id (UUID, optional): Workflow ID for tracking purposes. If not provided, a new one is created.
             **kwargs: Additional keyword arguments for the workflow.
 
         Returns:
-            Tuple[Result, int]:
-                A tuple containing the result of starting the workflow job and
-                the Slurm job ID, or -1 if the job ID could not be extracted.
+            Tuple[Result, int, UUID, UUID]:
+                A tuple containing the result of starting the workflow job, 
+                the Slurm job ID, the workflow ID, and the task ID. 
+                If the Slurm job ID could not be extracted, it returns -1 for the job ID.
 
         Note:
-            The Slurm job ID is extracted from the result of the 
-            `run_commands` method.
+            The Slurm job ID is extracted from the result of the `run_commands` method. 
+            If `track_workflows` is enabled, workflow and task tracking is performed.
         """
+        if not wf_id:
+            wf_id = self.workflowTracker.initiate_workflow(
+                workflow_name,
+                workflow_version,
+                -1,
+                -1
+            )
+        task_id = self.workflowTracker.add_task_to_workflow(
+            wf_id,
+            workflow_name, 
+            workflow_version,
+            input_data,
+            kwargs)
+        logger.debug(f"Added new task {task_id} to workflow {wf_id}")
+            
         sbatch_cmd, sbatch_env = self.get_workflow_command(
             workflow_name, workflow_version, input_data, email, time, **kwargs)
         print(f"Running {workflow_name} job on {input_data} on Slurm:\
             {sbatch_cmd} w/ {sbatch_env}")
         logger.info(f"Running {workflow_name} job on {input_data} on Slurm")
         res = self.run_commands([sbatch_cmd], sbatch_env)
-        return res, self.extract_job_id(res)

+        slurm_job_id = self.extract_job_id(res)
+        
+        if task_id:
+            self.workflowTracker.start_task(task_id)
+            self.workflowTracker.add_job_id(task_id, slurm_job_id)
+            self.workflowTracker.add_result(task_id, res)
+            
+        return res, slurm_job_id, wf_id, task_id

 
 
[docs]    def run_workflow_job(self,
                          workflow_name: str,
@@ -1377,6 +1620,7 @@ 
Source code for biomero.slurm_client
                          input_data: str,
                          email: Optional[str] = None,
                          time: Optional[str] = None,
+                         wf_id: Optional[UUID] = None,
                          **kwargs
                          ) -> SlurmJob:
         """
@@ -1388,19 +1632,23 @@ 
Source code for biomero.slurm_client
             input_data (str): Name of the input data folder containing input image files.
             email (str, optional): Email address for Slurm job notifications.
             time (str, optional): Time limit for the Slurm job in the format HH:MM:SS.
+            wf_id (UUID, optional): Workflow ID for tracking purposes. If not provided, a new one is created.
             **kwargs: Additional keyword arguments for the workflow.
 
         Returns:
             SlurmJob: A SlurmJob instance representing the started workflow job.
         """
-        result, job_id = self.run_workflow(
-            workflow_name, workflow_version, input_data, email, time, **kwargs)
-        return SlurmJob(result, job_id)

+        result, job_id, wf_id, task_id = self.run_workflow(
+            workflow_name, workflow_version, input_data, email, time, wf_id, 
+            **kwargs)
+        return SlurmJob(result, job_id, wf_id, task_id)

 
-
[docs]    def run_conversion_workflow_job(self, folder_name: str,
+
[docs]    def run_conversion_workflow_job(self, 
+                                    folder_name: str,
                                     source_format: str = 'zarr',
-                                    target_format: str = 'tiff'
-                                    ) -> Tuple[Result, int]:
+                                    target_format: str = 'tiff',
+                                    wf_id: UUID = None
+                                    ) -> SlurmJob:
         """
         Run the data conversion workflow on Slurm using the given data folder.
 
@@ -1410,9 +1658,8 @@ 
Source code for biomero.slurm_client
             target_format (str): Target data format after conversion (default is 'tiff').
 
         Returns:
-            Tuple[Result, int]:
-                A tuple containing the result of starting the conversion job and
-                the Slurm job ID, or -1 if the job ID could not be extracted.
+            SlurmJob:
+                the conversion job
 
         Warning:
             The default implementation only supports conversion from 'zarr' to 'tiff'.
@@ -1424,7 +1671,7 @@ 
Source code for biomero.slurm_client
 
         # Construct all commands to run consecutively
         data_path = f"{self.slurm_data_path}/{folder_name}"
-        conversion_cmd, sbatch_env = self.get_conversion_command(
+        conversion_cmd, sbatch_env, chosen_converter, version = self.get_conversion_command(
             data_path, config_file, source_format, target_format)
         commands = [
             f"find \"{data_path}/data/in\" -name \"*.{source_format}\" | awk '{{print NR, $0}}' > \"{config_file}\"",
@@ -1432,11 +1679,34 @@ 
Source code for biomero.slurm_client
             f"echo \"Number of .{source_format} files: $N\"",
             conversion_cmd
         ]
+        logger.debug(f"wf_id: {wf_id}")
+        if not wf_id:
+            wf_id = self.workflowTracker.initiate_workflow(
+                "conversion",
+                -1,
+                -1,
+                -1
+            )
+        logger.debug(f"wf_id: {wf_id}")
+        task_id = self.workflowTracker.add_task_to_workflow(
+            wf_id,
+            f"convert_{source_format}_to_{target_format}".upper(),
+            version,
+            data_path,
+            sbatch_env
+        )
 
         # Run all commands consecutively
         res = self.run_commands(commands, sbatch_env)
-
-        return SlurmJob(res, self.extract_job_id(res))

+        
+        slurm_job_id = self.extract_job_id(res)
+        
+        if task_id:
+            self.workflowTracker.start_task(task_id)
+            self.workflowTracker.add_job_id(task_id, slurm_job_id)
+            self.workflowTracker.add_result(task_id, res)
+        
+        return SlurmJob(res, slurm_job_id, wf_id, task_id)

 
 
[docs]    def extract_job_id(self, result: Result) -> int:
         """
@@ -1861,7 +2131,7 @@ 
Source code for biomero.slurm_client
 
[docs]    def get_conversion_command(self, data_path: str,
                                config_file: str,
                                source_format: str = 'zarr',
-                               target_format: str = 'tiff') -> Tuple[str, Dict]:
+                               target_format: str = 'tiff') -> Tuple[str, Dict, str, str]:
         """
         Generate Slurm conversion command and environment variables for data conversion.
 
@@ -1872,9 +2142,9 @@ 
Source code for biomero.slurm_client
             target_format (str): Target data format (default is 'tiff').
 
         Returns:
-            Tuple[str, Dict]:
+            Tuple[str, Dict, str, str]:
                 A tuple containing the Slurm conversion command and
-                the environment variables.
+                the environment variables, followed by the converter image name and version.
 
         Warning:
             The default implementation only supports conversion from 'zarr' to 'tiff'.
@@ -1890,11 +2160,13 @@ 
Source code for biomero.slurm_client
                 f"Conversion from {source_format} to {target_format} is not supported by default!")
 
         chosen_converter = f"convert_{source_format}_to_{target_format}_latest.sif"
+        version = None
         if self.converter_images:
             image = self.converter_images[f"{source_format}_to_{target_format}"]  
             version, image = self.parse_docker_image_version(image)
             if version:
-                chosen_converter = f"convert_{source_format}_to_{target_format}_{version}.sif"    
+                chosen_converter = f"convert_{source_format}_to_{target_format}_{version}.sif"
+        version = version or "latest"
         
         logger.info(f"Converting with {chosen_converter}")
         sbatch_env = {
@@ -1908,7 +2180,7 @@ 
Source code for biomero.slurm_client
         conversion_cmd = "sbatch --job-name=conversion --export=ALL,CONFIG_PATH=\"$PWD/$CONFIG_FILE\" --array=1-$N \"$SCRIPT_PATH/convert_job_array.sh\""
         # conversion_cmd_waiting = "sbatch --job-name=conversion --export=ALL,CONFIG_PATH=\"$PWD/$CONFIG_FILE\" --array=1-$N --wait $SCRIPT_PATH/convert_job_array.sh"
 
-        return conversion_cmd, sbatch_env

+        return conversion_cmd, sbatch_env, chosen_converter, version

 
 
[docs]    def workflow_params_to_envvars(self, **kwargs) -> Dict:
         """
diff --git a/biomero.html b/biomero.html
index 0b5231c..ff6888a 100644
--- a/biomero.html
+++ b/biomero.html
@@ -111,7 +111,7 @@ 
biomero package
biomero.slurm_client module

 

 

-class biomero.slurm_client.SlurmClient(host='slurm', user=None, port=None, config=None, gateway=None, forward_agent=None, connect_timeout=None, connect_kwargs=None, inline_ssh_env=True, slurm_data_path: str = 'my-scratch/data', slurm_images_path: str = 'my-scratch/singularity_images/workflows', slurm_converters_path: str = 'my-scratch/singularity_images/converters', slurm_model_paths: Optional[dict] = None, slurm_model_repos: Optional[dict] = None, slurm_model_images: Optional[dict] = None, converter_images: Optional[dict] = None, slurm_model_jobs: Optional[dict] = None, slurm_model_jobs_params: Optional[dict] = None, slurm_script_path: str = 'slurm-scripts', slurm_script_repo: Optional[str] = None, init_slurm: bool = False)[source]

+class biomero.slurm_client.SlurmClient(host='slurm', user=None, port=None, config=None, gateway=None, forward_agent=None, connect_timeout=None, connect_kwargs=None, inline_ssh_env=True, slurm_data_path: str = 'my-scratch/data', slurm_images_path: str = 'my-scratch/singularity_images/workflows', slurm_converters_path: str = 'my-scratch/singularity_images/converters', slurm_model_paths: Optional[dict] = None, slurm_model_repos: Optional[dict] = None, slurm_model_images: Optional[dict] = None, converter_images: Optional[dict] = None, slurm_model_jobs: Optional[dict] = None, slurm_model_jobs_params: Optional[dict] = None, slurm_script_path: str = 'slurm-scripts', slurm_script_repo: Optional[str] = None, init_slurm: bool = False, track_workflows: bool = True, enable_job_accounting: bool = True, enable_job_progress: bool = True, enable_workflow_analytics: bool = True, sqlalchemy_url: Optional[str] = None)[source]
 
Bases: fabric.connection.Connection

 
A client for connecting to and interacting with a Slurm cluster over
 SSH.

@@ -128,7 +128,7 @@ 
biomero package
 
Type

-
str

+
str

 

 

 
@@ -140,7 +140,7 @@ 
biomero package
 
Type

-
str

+
str

 

 
 
@@ -152,7 +152,7 @@ 
biomero package
 
Type

-
str

+
str

 

 
 
@@ -164,7 +164,7 @@ 
biomero package
 
Type

-
dict

+
dict

 

 
 
@@ -176,7 +176,7 @@ 
biomero package
 
Type

-
dict

+
dict

 

 
 
@@ -190,7 +190,7 @@ 
biomero package
 
Type

-
dict

+
dict

 

 
 
@@ -202,7 +202,7 @@ 
biomero package
 
Type

-
str

+
str

 

 
 
@@ -214,7 +214,7 @@ 
biomero package
 
Type

-
str

+
str

 

 
 
@@ -233,17 +233,22 @@ 
biomero package
+

+bring_listener_uptodate(listener, start=1)[source]

+

+
 

 

-check_job_status(slurm_job_ids: List[int], env: Optional[Dict[str, str]] = None) → Tuple[Dict[int, str], fabric.runners.Result][source]

+check_job_status(slurm_job_ids: List[int], env: Optional[Dict[str, str]] = None) → Tuple[Dict[int, str], fabric.runners.Result][source]
 
Check the status of Slurm jobs with the given job IDs.

 
Note: This doesn’t return job arrays individually.
 It takes the last value returned for those sub ids
@@ -251,8 +256,8 @@ 
biomero package
 
Parameters

 
    
-
  • slurm_job_ids (List[int]) – The job IDs of the Slurm jobs to check.
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables to
+
  • slurm_job_ids (List[int]) – The job IDs of the Slurm jobs to check.
    • 
+
  • env (Dict[str, str], optional) – Optional environment variables to
 set when running the command. Defaults to None.
    • 
 

 

@@ -261,7 +266,7 @@ 
biomero packageReturn type
-
Tuple[Dict[int, str], Result]

+
Tuple[Dict[int, str], Result]

 

 
Raises

 
SSHException – If the command execution fails or no response is
@@ -272,16 +277,16 @@ 
biomero package
 

-cleanup_tmp_files(slurm_job_id: str, filename: Optional[str] = None, data_location: Optional[str] = None, logfile: Optional[str] = None) → fabric.runners.Result[source]

+cleanup_tmp_files(slurm_job_id: str, filename: Optional[str] = None, data_location: Optional[str] = None, logfile: Optional[str] = None) → fabric.runners.Result[source]
 
Cleanup zip and unzipped files/folders associated with a Slurm job.

 

 
Parameters

 
    
-
  • slurm_job_id (str) – The job ID of the Slurm script.
    • 
-
  • filename (str) – The zip filename on Slurm.
    • 
-
  • data_location (str, optional) – The location of data files on Slurm.
+
  • slurm_job_id (str) – The job ID of the Slurm script.
    • 
+
  • filename (str) – The zip filename on Slurm.
    • 
+
  • data_location (str, optional) – The location of data files on Slurm.
 If not provided, it will be extracted from the log file.
    • 
-
  • logfile (str, optional) – The log file of the Slurm job.
+
  • logfile (str, optional) – The log file of the Slurm job.
 If not provided, a default log file will be used.
    • 
 

 

@@ -304,7 +309,7 @@ 
biomero package
 

-convert_cytype_to_omtype(cytype: str, _default, *args, **kwargs) → Any[source]

+convert_cytype_to_omtype(cytype: str, _default, *args, **kwargs) → Any[source]
 
Convert a Cytomine type to an OMERO type and instantiates it
 with args/kwargs.

 
Note that Cytomine has a Python Client, and some conversion methods
@@ -315,7 +320,7 @@ 
biomero package
 
Parameters

 
    
-
  • cytype (str) – The Cytomine type to convert.
    • 
+
  • cytype (str) – The Cytomine type to convert.
    • 
 
  • _default – The default value. Required to distinguish between float
 and int.
    • 
 
  • *args – Additional positional arguments.
    • 
@@ -334,28 +339,28 @@ 
    biomero package
 
    
-convert_url(input_url: str) → str[source]
    
+convert_url(input_url: str) → str[source]
 
    Convert the input GitHub URL to an output URL that retrieves
 the ‘descriptor.json’ file in raw format.
    
 
    
 
    Parameters
    
-
    input_url (str) – The input GitHub URL.
    
+
    input_url (str) – The input GitHub URL.
    
 
    
 
    Returns
    
 
    The output URL to the ‘descriptor.json’ file.
    
 
    
 
    Return type
    
-
    str
    
+
    str
    
 
    
 
    Raises
    
-
    ValueError – If the input URL is not a valid GitHub URL.
    
+
    ValueError – If the input URL is not a valid GitHub URL.
    
 
    
 
    
 

 
 

 

-copy_zip_locally(local_tmp_storage: str, filename: str) → fabric.transfer.Result[source]

+copy_zip_locally(local_tmp_storage: str, filename: str) → fabric.transfer.Result[source]
 
Copy a zip file from Slurm to the local server.

 
Note about (Transfer)Result:

 
Unlike similar classes such as invoke.runners.Result or
@@ -382,21 +387,21 @@ 
biomero package
 

-extract_data_location_from_log(slurm_job_id: Optional[str] = None, logfile: Optional[str] = None) → str[source]

+extract_data_location_from_log(slurm_job_id: Optional[str] = None, logfile: Optional[str] = None) → str[source]
 
Read SLURM job logfile to find location of the data.

 
One of the parameters is required, either id or file.

 

 
Parameters

 
    
-
  • slurm_job_id (str) – Id of the slurm job
    • 
-
  • logfile (str) – Path to the logfile
    • 
+
  • slurm_job_id (str) – Id of the slurm job
    • 
+
  • logfile (str) – Path to the logfile
    • 
 

 

 
Returns

 
Data location according to the log

 

 
Return type

-
str

+
str

 

 
Raises

 
SSHException – If there is an issue with the command execution.

@@ -406,7 +411,7 @@ 
biomero package
 

-extract_job_id(result: fabric.runners.Result) → int[source]

+extract_job_id(result: fabric.runners.Result) → int[source]
 
Extract the Slurm job ID from the result of a command.

 

 
Parameters

@@ -417,35 +422,35 @@ 
biomero packageReturn type
-
int

+
int

 

 

 

 
 

 

-extract_parts_from_url(input_url: str) → Tuple[List[str], str][source]

+extract_parts_from_url(input_url: str) → Tuple[List[str], str][source]
 
Extract the repository and branch information from the input URL.

 

 
Parameters

-
input_url (str) – The input GitHub URL.

+
input_url (str) – The input GitHub URL.

 

 
Returns

 
The list of url parts and the branch/version.
 If no branch is found, it will return “master”

 

 
Return type

-
Tuple[List[str], str]

+
Tuple[List[str], str]

 

 
Raises

-
ValueError – If the input URL is not a valid GitHub URL.

+
ValueError – If the input URL is not a valid GitHub URL.

 

 

 

 
 

 

-classmethod from_config(configfile: str = '', init_slurm: bool = False) → biomero.slurm_client.SlurmClient[source]

+classmethod from_config(configfile: str = '', init_slurm: bool = False) → biomero.slurm_client.SlurmClient[source]
 
Creates a new SlurmClient object using the parameters read from a
 configuration file (.ini).

 

@@ -455,18 +460,17 @@ 
biomero package
 
Parameters

 
    
-
  • configfile (str) – The path to your configuration file. Optional.
    • 
-
  • init_slurm (bool) – Initiate / validate slurm setup. Optional
-Might take some time the first time with downloading etc.
    • 
+
  • configfile (str) – The path to your configuration file. Optional.
    • 
+
  • init_slurm (bool) – Initiate / validate slurm setup. Optional
+Might take some time the first time with downloading, etc.
    • 
 

 

 
Returns

@@ -480,15 +484,15 @@ 
biomero package
 

-generate_slurm_job_for_workflow(workflow: str, substitutes: Dict[str, str], template: str = 'job_template.sh') → str[source]

+generate_slurm_job_for_workflow(workflow: str, substitutes: Dict[str, str], template: str = 'job_template.sh') → str[source]
 
Generate a Slurm job script for a specific workflow.

 

 
Parameters

 
    
-
  • workflow (str) – The name of the workflow.
    • 
-
  • substitutes (Dict[str, str]) – A dictionary containing key-value
+
  • workflow (str) – The name of the workflow.
    • 
+
  • substitutes (Dict[str, str]) – A dictionary containing key-value
 pairs for substituting placeholders in the job template.
    • 
-
  • template (str, optional) – The filename of the job template.
+
  • template (str, optional) – The filename of the job template.
 Defaults to “job_template.sh”.
    • 
 

 

@@ -496,37 +500,37 @@ 
biomero package
The generated Slurm job script as a string.

 

 
Return type

-
str

+
str

 

 

 

 
 

 

-get_active_job_progress(slurm_job_id: str, pattern: str = '\\d+%', env: Optional[Dict[str, str]] = None) → str[source]

+get_active_job_progress(slurm_job_id: str, pattern: str = '\\d+%', env: Optional[Dict[str, str]] = None) → Any[source]
 
Get the progress of an active Slurm job from its logfiles.

 

 
Parameters

 
    
-
  • slurm_job_id (str) – The ID of the Slurm job.
    • 
-
  • pattern (str) – The pattern to match in the job log to extract
+
  • slurm_job_id (str) – The ID of the Slurm job.
    • 
+
  • pattern (str) – The pattern to match in the job log to extract
 the progress (default: r”d+%”).
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables
+
  • env (Dict[str, str], optional) – Optional environment variables
 to set when running the command. Defaults to None.
    • 
 

 

 
Returns

-
The progress of the Slurm job.

+
The progress of the Slurm job according to the pattern, or None.

 

 
Return type

-
str

+
Any

 

 

 

 
 

 

-get_all_image_versions_and_data_files() → Tuple[Dict[str, List[str]], List[str]][source]

+get_all_image_versions_and_data_files() → Tuple[Dict[str, List[str]], List[str]][source]
 
Retrieve all available image versions and data files from
 the Slurm cluster.

 

@@ -536,35 +540,35 @@ 
biomero packageReturn type
-
Tuple[Dict[str, List[str]], List[str]]

+
Tuple[Dict[str, List[str]], List[str]]

 

 

 

 
 

 

-get_cellpose_command(image_version: str, input_data: str, cp_model: str, nuc_channel: int, prob_threshold: float, cell_diameter: float, email: Optional[str] = None, time: Optional[str] = None, use_gpu: bool = True, model: str = 'cellpose') → Tuple[str, Dict][source]

+get_cellpose_command(image_version: str, input_data: str, cp_model: str, nuc_channel: int, prob_threshold: float, cell_diameter: float, email: Optional[str] = None, time: Optional[str] = None, use_gpu: bool = True, model: str = 'cellpose') → Tuple[str, Dict][source]
 
Return the command and environment dictionary to run a CellPose job
 on the Slurm workload manager.
 A specific example of using the generic ‘get_workflow_command’.

 

 
Parameters

 
    
-
  • image_version (str) – The version of the Singularity image to use.
    • 
-
  • input_data (str) – The name of the input data folder on the shared
+
  • image_version (str) – The version of the Singularity image to use.
    • 
+
  • input_data (str) – The name of the input data folder on the shared
 file system.
    • 
-
  • cp_model (str) – The name of the CellPose model to use.
    • 
-
  • nuc_channel (int) – The index of the nuclear channel.
    • 
-
  • prob_threshold (float) – The probability threshold for
+
  • cp_model (str) – The name of the CellPose model to use.
    • 
+
  • nuc_channel (int) – The index of the nuclear channel.
    • 
+
  • prob_threshold (float) – The probability threshold for
 nuclei detection.
    • 
-
  • cell_diameter (float) – The expected cell diameter in pixels.
    • 
-
  • email (Optional[str]) – The email address to send notifications to.
+
  • cell_diameter (float) – The expected cell diameter in pixels.
    • 
+
  • email (Optional[str]) – The email address to send notifications to.
 Defaults to None.
    • 
-
  • time (Optional[str]) – The maximum time for the job to run.
+
  • time (Optional[str]) – The maximum time for the job to run.
 Defaults to None.
    • 
-
  • use_gpu (bool) – Whether to use GPU for the CellPose job.
+
  • use_gpu (bool) – Whether to use GPU for the CellPose job.
 Defaults to True.
    • 
-
  • model (str) – The name of the folder of the Docker image to use.
+
  • model (str) – The name of the folder of the Docker image to use.
 Defaults to “cellpose”.
    • 
 

 

@@ -573,30 +577,30 @@ 
biomero packageReturn type
-
Tuple[str, dict]

+
Tuple[str, dict]

 

 

 

 
 

 

-get_conversion_command(data_path: str, config_file: str, source_format: str = 'zarr', target_format: str = 'tiff') → Tuple[str, Dict][source]

+get_conversion_command(data_path: str, config_file: str, source_format: str = 'zarr', target_format: str = 'tiff') → Tuple[str, Dict, str, str][source]
 
Generate Slurm conversion command and environment variables for data conversion.

 

 
Parameters

 
    
-
  • data_path (str) – Path to the data folder.
    • 
-
  • config_file (str) – Path to the configuration file.
    • 
-
  • source_format (str) – Source data format (default is ‘zarr’).
    • 
-
  • target_format (str) – Target data format (default is ‘tiff’).
    • 
+
  • data_path (str) – Path to the data folder.
    • 
+
  • config_file (str) – Path to the configuration file.
    • 
+
  • source_format (str) – Source data format (default is ‘zarr’).
    • 
+
  • target_format (str) – Target data format (default is ‘tiff’).
    • 
 

 

 
Returns

 
A tuple containing the Slurm conversion command and
-the environment variables.

+the environment variables, followed by the converter image name and version.

 

 
Return type

-
Tuple[str, Dict]

+
Tuple[str, Dict, str, str]

 

 

 

@@ -609,12 +613,12 @@ 
biomero package
 

-get_image_versions_and_data_files(model: str) → Tuple[List[str], List[str]][source]

+get_image_versions_and_data_files(model: str) → Tuple[List[str], List[str]][source]
 
Retrieve the available image versions and input data files for a
 given model.

 

 
Parameters

-
model (str) – The name of the model to query for.

+
model (str) – The name of the model to query for.

 

 
Returns

 
A tuple containing two lists:
@@ -628,10 +632,10 @@ 
biomero packageReturn type
-
Tuple[List[str], List[str]]

+
Tuple[List[str], List[str]]

 

 
Raises

-
ValueError – If the provided model is not found in the
+
ValueError – If the provided model is not found in the
     SlurmClient’s known model paths.

 

 

@@ -639,25 +643,25 @@ 
biomero package
 

-get_job_status_command(slurm_job_ids: List[int]) → str[source]

+get_job_status_command(slurm_job_ids: List[int]) → str[source]
 
Return the Slurm command to get the status of jobs with the given
 job IDs.

 

 
Parameters

-
slurm_job_ids (List[int]) – The job IDs of the jobs to check.

+
slurm_job_ids (List[int]) – The job IDs of the jobs to check.

 

 
Returns

 
The Slurm command to get the status of the jobs.

 

 
Return type

-
str

+
str

 

 

 

 
 

 

-get_jobs_info_command(start_time: str = '2023-01-01', end_time: str = 'now', columns: str = 'JobId', states: str = 'r,cd,f,to,rs,dl,nf') → str[source]

+get_jobs_info_command(start_time: str = '2023-01-01', end_time: str = 'now', columns: str = 'JobId', states: str = 'r,cd,f,to,rs,dl,nf') → str[source]
 
Return the Slurm command to retrieve information about old jobs.

 
The command will be formatted with the specified start time, which is
 expected to be in the ISO format “YYYY-MM-DD”.
@@ -668,13 +672,13 @@ 
biomero package
 
Parameters

 
    
-
  • start_time (str) – The start time from which to retrieve job
+
  • start_time (str) – The start time from which to retrieve job
 information. Defaults to “2023-01-01”.
    • 
-
  • end_time (str) – The end time until which to retrieve job
+
  • end_time (str) – The end time until which to retrieve job
 information. Defaults to “now”.
    • 
-
  • columns (str) – The columns to retrieve from the job information.
+
  • columns (str) – The columns to retrieve from the job information.
 Defaults to “JobId”. It is comma separated, e.g. “JobId,State”.
    • 
-
  • states (str) – The job states to include in the query.
+
  • states (str) – The job states to include in the query.
 Defaults to “r,cd,f,to,rs,dl,nf”.
    • 
 

 

@@ -683,14 +687,19 @@ 
biomero packageReturn type
-
str

+
str

 

 

 

 
+

+

+get_listeners(runner)[source]

+

+
 

 

-get_logfile_from_slurm(slurm_job_id: str, local_tmp_storage: str = '/tmp/', logfile: Optional[str] = None) → Tuple[str, str, fabric.transfer.Result][source]

+get_logfile_from_slurm(slurm_job_id: str, local_tmp_storage: str = '/tmp/', logfile: Optional[str] = None) → Tuple[str, str, fabric.transfer.Result][source]
 
Copy the logfile of the given SLURM job to the local server.

 
Note about (Transfer)Result:

 
Unlike similar classes such as invoke.runners.Result
@@ -702,10 +711,10 @@ 
biomero package
 
Parameters

 
    
-
  • slurm_job_id (str) – The ID of the SLURM job.
    • 
-
  • local_tmp_storage (str, optional) – Path to store the logfile
+
  • slurm_job_id (str) – The ID of the SLURM job.
    • 
+
  • local_tmp_storage (str, optional) – Path to store the logfile
 locally. Defaults to “/tmp/”.
    • 
-
  • logfile (str, optional) – Path to the logfile on the SLURM server.
+
  • logfile (str, optional) – Path to the logfile on the SLURM server.
 Defaults to None.
    • 
 

 

@@ -725,14 +734,14 @@ 
biomero package
 

-get_recent_log_command(log_file: str, n: int = 10) → str[source]

+get_recent_log_command(log_file: str, n: int = 10) → str[source]
 
Get the command to retrieve the recent log entries from a
 specified log file.

 

 
Parameters

 
    
-
  • log_file (str) – The path to the log file.
    • 
-
  • n (int, optional) – The number of recent log entries to retrieve.
+
  • log_file (str) – The path to the log file.
    • 
+
  • n (int, optional) – The number of recent log entries to retrieve.
 Defaults to 10.
    • 
 

 

@@ -740,22 +749,22 @@ 
biomero package
The command to retrieve the recent log entries.

 

 
Return type

-
str

+
str

 

 

 

 
 

 

-get_unzip_command(zipfile: str, filter_filetypes: str = '*.zarr *.tiff *.tif') → str[source]

+get_unzip_command(zipfile: str, filter_filetypes: str = '*.zarr *.tiff *.tif') → str[source]
 
Generate a command string for unzipping a data archive and creating
 required directories for Slurm jobs.

 

 
Parameters

 
    
-
  • zipfile (str) – The name of the zip archive file to extract.
+
  • zipfile (str) – The name of the zip archive file to extract.
 Without extension.
    • 
-
  • filter_filetypes (str, optional) – A space-separated string
+
  • filter_filetypes (str, optional) – A space-separated string
 containing the file extensions to extract from the zip file.
 E.g. defaults to “*.zarr *.tiff *.tif”.
 Setting this argument to None will omit the file
@@ -767,14 +776,14 @@ 
    biomero packageReturn type
-
    str
    
+
    str
    
 
    
 

 

 
 

 

-get_update_slurm_scripts_command() → str[source]

+get_update_slurm_scripts_command() → str[source]
 
Generate the command to update the Git repository containing
 the Slurm scripts, if necessary.

 

@@ -783,25 +792,25 @@ 
biomero packageReturn type
-
str

+
str

 

 

 

 
 

 

-get_workflow_command(workflow: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, **kwargs) → Tuple[str, Dict][source]

+get_workflow_command(workflow: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, **kwargs) → Tuple[str, Dict][source]
 
Generate the Slurm workflow command and environment variables.

 

 
Parameters

 
    
-
  • workflow (str) – The name of the workflow.
    • 
-
  • workflow_version (str) – The version of the workflow.
    • 
-
  • input_data (str) – The name of the input data folder containing
+
  • workflow (str) – The name of the workflow.
    • 
+
  • workflow_version (str) – The version of the workflow.
    • 
+
  • input_data (str) – The name of the input data folder containing
 the input image files.
    • 
-
  • email (str, optional) – The email address for job notifications.
+
  • email (str, optional) – The email address for job notifications.
 Defaults to None, which defaults to what is in the job script.
    • 
-
  • time (str, optional) – The time limit for the job in the
+
  • time (str, optional) – The time limit for the job in the
 format HH:MM:SS. Defaults to None, which defaults to what
 is in the job script.
    • 
 
  • **kwargs – Additional keyword arguments for the workflow.
    • 
@@ -812,27 +821,27 @@ 
    biomero packageReturn type
-
    Tuple[str, Dict]
    
+
    Tuple[str, Dict]
    
 
    
 

 

 
 

 

-get_workflow_parameters(workflow: str) → Dict[str, Dict[str, Any]][source]

+get_workflow_parameters(workflow: str) → Dict[str, Dict[str, Any]][source]
 
Retrieve the parameters of a workflow.

 

 
Parameters

-
workflow (str) – The workflow for which to retrieve the parameters.

+
workflow (str) – The workflow for which to retrieve the parameters.

 

 
Returns

 
A dictionary containing the workflow parameters.

 

 
Return type

-
Dict[str, Dict[str, Any]]

+
Dict[str, Dict[str, Any]]

 

 
Raises

-
ValueError – If an error occurs while retrieving the workflow
+
ValueError – If an error occurs while retrieving the workflow
     parameters.

 

 

@@ -840,13 +849,13 @@ 
biomero package
 

-get_zip_command(data_location: str, filename: str) → str[source]

+get_zip_command(data_location: str, filename: str) → str[source]
 
Generate a command string for zipping the data on Slurm.

 

 
Parameters

 
    
-
  • data_location (str) – The folder to be zipped.
    • 
-
  • filename (str) – The name of the zip archive file to extract.
+
  • data_location (str) – The folder to be zipped.
    • 
+
  • filename (str) – The name of the zip archive file to extract.
 Without extension.
    • 
 

 

@@ -854,63 +863,75 @@ 
biomero package
The command to create the zip file.

 

 
Return type

-
str

+
str

 

 

 
 
 

 

-init_workflows(force_update: bool = False)[source]

+init_workflows(force_update: bool = False)[source]
 
Retrieves the required info for the configured workflows from github.
 It will fill slurm_model_images with dockerhub links.

 

 
Parameters

-
force_update (bool) – Will overwrite already given paths
+
force_update (bool) – Will overwrite already given paths
 in slurm_model_images

 

 

 

 
+

+

+initialize_analytics_system(reset_tables=False)[source]

+
Initialize the analytics system based on the analytics configuration
+passed to the constructor.

+

+
Parameters

+
reset_tables (bool) – If True, drops and recreates all views.

+

+

+

+
 

 

-list_active_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]

+list_active_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]
 
Get a list of active jobs from SLURM.

 

 
Parameters

-
env (Dict[str, str], optional) – Optional environment variables to
+
env (Dict[str, str], optional) – Optional environment variables to
 set when running the command. Defaults to None.

 

 
Returns

 
A list of job IDs.

 

 
Return type

-
List[str]

+
List[str]

 

 

 

 
 

 

-list_all_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]

+list_all_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]
 
Get a list of all jobs from SLURM.

 

 
Parameters

-
env (Dict[str, str], optional) – Optional environment variables
+
env (Dict[str, str], optional) – Optional environment variables
 to set when running the command. Defaults to None.

 

 
Returns

 
A list of job IDs.

 

 
Return type

-
List[str]

+
List[str]

 

 

 

 
 

 

-list_available_converter_versions() → Dict[source]

+list_available_converter_versions() → Dict[source]
 
Note, assumes you use versioned converters.
 Will return a dict with a version of each converter on your Slurm.
 However, doesn’t work properly with unversioned sif.

@@ -918,29 +939,29 @@ 
biomero package
 

-list_completed_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]

+list_completed_jobs(env: Optional[Dict[str, str]] = None) → List[str][source]
 
Get a list of completed jobs from SLURM.

 

 
Parameters

-
env (Dict[str, str], optional) – Optional environment variables to
+
env (Dict[str, str], optional) – Optional environment variables to
 set when running the command. Defaults to None.

 

 
Returns

 
A list of job IDs.

 

 
Return type

-
List[str]

+
List[str]

 

 

 

 
 

 

-parse_docker_image_version(image: str) → Tuple[str, str][source]

+parse_docker_image_version(image: str) → Tuple[str, str][source]
 
Parses the Docker image string to extract the image name and version tag.

 

 
Parameters

-
image (str) – The Docker image string in the format ‘image_name:version’.

+
image (str) – The Docker image string in the format ‘image_name:version’.

 

 
Returns

 

@@ -953,18 +974,18 @@ 
biomero packageReturn type
-
tuple

+
tuple

 

 

 

 
 

 

-pull_descriptor_from_github(workflow: str) → Dict[source]

+pull_descriptor_from_github(workflow: str) → Dict[source]
 
Pull the workflow descriptor from GitHub.

 

 
Parameters

-
workflow (str) – The workflow for which to pull the descriptor.

+
workflow (str) – The workflow for which to pull the descriptor.

 

 
Returns

 
The JSON descriptor.

@@ -973,14 +994,14 @@ 
biomero package
Dict

 

 
Raises

-
ValueError – If an error occurs while pulling the descriptor file.

+
ValueError – If an error occurs while pulling the descriptor file.

 

 

 

 
 

 

-run_commands(cmdlist: List[str], env: Optional[Dict[str, str]] = None, sep: str = ' && ', **kwargs) → fabric.runners.Result[source]

+run_commands(cmdlist: List[str], env: Optional[Dict[str, str]] = None, sep: str = ' && ', **kwargs) → fabric.runners.Result[source]
 
Run a list of shell commands consecutively on the Slurm cluster,
 ensuring the success of each before proceeding to the next.

 
The environment variables can be set using the env argument.
@@ -989,10 +1010,10 @@ 
biomero package
 
Parameters

 
    
-
  • cmdlist (List[str]) – A list of shell commands to run on Slurm.
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables to
+
  • cmdlist (List[str]) – A list of shell commands to run on Slurm.
    • 
+
  • env (Dict[str, str], optional) – Optional environment variables to
 set when running the command. Defaults to None.
    • 
-
  • sep (str, optional) – The separator used to concatenate the
+
  • sep (str, optional) – The separator used to concatenate the
 commands. Defaults to ‘ && ‘.
    • 
 
  • **kwargs – Additional keyword arguments.
    • 
 

@@ -1008,7 +1029,7 @@ 
biomero package
 

-run_commands_split_out(cmdlist: List[str], env: Optional[Dict[str, str]] = None) → List[str][source]

+run_commands_split_out(cmdlist: List[str], env: Optional[Dict[str, str]] = None) → List[str][source]
 
Run a list of shell commands consecutively and split the output
 of each command.

 
Each command in the list is executed with a separator in between
@@ -1019,8 +1040,8 @@ 
biomero package
 
Parameters

 
    
-
  • cmdlist (List[str]) – A list of shell commands to run.
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables
+
  • cmdlist (List[str]) – A list of shell commands to run.
    • 
+
  • env (Dict[str, str], optional) – Optional environment variables
 to set when running the command. Defaults to None.
    • 
 

 

@@ -1030,7 +1051,7 @@ 
biomero packageReturn type
-
List[str]

+
List[str]

 

 
Raises

 
SSHException – If any of the commands fail to execute successfully.

@@ -1040,22 +1061,21 @@ 
biomero package
 

-run_conversion_workflow_job(folder_name: str, source_format: str = 'zarr', target_format: str = 'tiff') → Tuple[fabric.runners.Result, int][source]

+run_conversion_workflow_job(folder_name: str, source_format: str = 'zarr', target_format: str = 'tiff', wf_id: Optional[uuid.UUID] = None) → biomero.slurm_client.SlurmJob[source]
 
Run the data conversion workflow on Slurm using the given data folder.

 

 
Parameters

 
    
-
  • folder_name (str) – The name of the data folder containing source format files.
    • 
-
  • source_format (str) – Source data format for conversion (default is ‘zarr’).
    • 
-
  • target_format (str) – Target data format after conversion (default is ‘tiff’).
    • 
+
  • folder_name (str) – The name of the data folder containing source format files.
    • 
+
  • source_format (str) – Source data format for conversion (default is ‘zarr’).
    • 
+
  • target_format (str) – Target data format after conversion (default is ‘tiff’).
    • 
 

 

 
Returns

-
A tuple containing the result of starting the conversion job and
-the Slurm job ID, or -1 if the job ID could not be extracted.

+
the conversion job

 

 
Return type

-
Tuple[Result, int]

+
SlurmJob

 

 

 

@@ -1068,49 +1088,52 @@ 
biomero package
 

-run_workflow(workflow_name: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, **kwargs) → Tuple[fabric.runners.Result, int][source]

+run_workflow(workflow_name: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, wf_id: Optional[uuid.UUID] = None, **kwargs) → Tuple[fabric.runners.Result, int, uuid.UUID, uuid.UUID][source]
 
Run a specified workflow on Slurm using the given parameters.

 

 
Parameters

 
    
-
  • workflow_name (str) – Name of the workflow to execute.
    • 
-
  • workflow_version (str) – Version of the workflow (image version
+
  • workflow_name (str) – Name of the workflow to execute.
    • 
+
  • workflow_version (str) – Version of the workflow (image version
 on Slurm).
    • 
-
  • input_data (str) – Name of the input data folder containing input
+
  • input_data (str) – Name of the input data folder containing input
 image files.
    • 
-
  • email (str, optional) – Email address for Slurm job notifications.
    • 
-
  • time (str, optional) – Time limit for the Slurm job in the
+
  • email (str, optional) – Email address for Slurm job notifications.
    • 
+
  • time (str, optional) – Time limit for the Slurm job in the
 format HH:MM:SS.
    • 
+
  • wf_id (UUID, optional) – Workflow ID for tracking purposes. If not provided, a new one is created.
    • 
 
  • **kwargs – Additional keyword arguments for the workflow.
    • 
 

 

 
Returns

-
A tuple containing the result of starting the workflow job and
-the Slurm job ID, or -1 if the job ID could not be extracted.

+
A tuple containing the result of starting the workflow job,
+the Slurm job ID, the workflow ID, and the task ID.
+If the Slurm job ID could not be extracted, it returns -1 for the job ID.

 

 
Return type

-
Tuple[Result, int]

+
Tuple[Result, int, UUID, UUID]

 

 

 

 
Note

-
The Slurm job ID is extracted from the result of the
-run_commands method.

+
The Slurm job ID is extracted from the result of the run_commands method.
+If track_workflows is enabled, workflow and task tracking is performed.

 

 

 
 

 

-run_workflow_job(workflow_name: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, **kwargs) → biomero.slurm_client.SlurmJob[source]

+run_workflow_job(workflow_name: str, workflow_version: str, input_data: str, email: Optional[str] = None, time: Optional[str] = None, wf_id: Optional[uuid.UUID] = None, **kwargs) → biomero.slurm_client.SlurmJob[source]
 
Run a specified workflow on Slurm using the given parameters and return a SlurmJob instance.

 

 
Parameters

 
    
-
  • workflow_name (str) – Name of the workflow to execute.
    • 
-
  • workflow_version (str) – Version of the workflow (image version on Slurm).
    • 
-
  • input_data (str) – Name of the input data folder containing input image files.
    • 
-
  • email (str, optional) – Email address for Slurm job notifications.
    • 
-
  • time (str, optional) – Time limit for the Slurm job in the format HH:MM:SS.
    • 
+
  • workflow_name (str) – Name of the workflow to execute.
    • 
+
  • workflow_version (str) – Version of the workflow (image version on Slurm).
    • 
+
  • input_data (str) – Name of the input data folder containing input image files.
    • 
+
  • email (str, optional) – Email address for Slurm job notifications.
    • 
+
  • time (str, optional) – Time limit for the Slurm job in the format HH:MM:SS.
    • 
+
  • wf_id (UUID, optional) – Workflow ID for tracking purposes. If not provided, a new one is created.
    • 
 
  • **kwargs – Additional keyword arguments for the workflow.
    • 
 

 

@@ -1178,6 +1201,11 @@ 
biomero package
+

+setup_listeners(runner, reset_tables)[source]

+

+
 

 

 setup_slurm()[source]

@@ -1191,13 +1219,13 @@ 
biomero package
 

-str_to_class(module_name: str, class_name: str, *args, **kwargs)[source]

+str_to_class(module_name: str, class_name: str, *args, **kwargs)[source]
 
Return a class instance from a string reference.

 

 
Parameters

 
    
-
  • module_name (str) – The name of the module.
    • 
-
  • class_name (str) – The name of the class.
    • 
+
  • module_name (str) – The name of the module.
    • 
+
  • class_name (str) – The name of the class.
    • 
 
  • *args – Additional positional arguments for the class constructor.
    • 
 
  • **kwargs – Additional keyword arguments for the class constructor.
    • 
 

@@ -1210,19 +1238,19 @@ 
biomero packageReturn type
-
object

+
object

 

 

 

 
 

 

-transfer_data(local_path: str) → fabric.runners.Result[source]

+transfer_data(local_path: str) → fabric.runners.Result[source]
 
Transfers a file or directory from the local machine to the remote
 Slurm cluster.

 

 
Parameters

-
local_path (str) – The local path to the file or directory to
+
local_path (str) – The local path to the file or directory to
 transfer.

 

 
Returns

@@ -1236,13 +1264,13 @@ 
biomero package
 

-unpack_data(zipfile: str, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]

+unpack_data(zipfile: str, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]
 
Unpacks a zipped file on the remote Slurm cluster.

 

 
Parameters

 
    
-
  • zipfile (str) – The name of the zipped file to be unpacked.
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables
+
  • zipfile (str) – The name of the zipped file to be unpacked.
    • 
+
  • env (Dict[str, str], optional) – Optional environment variables
 to set when running the command. Defaults to None.
    • 
 

 

@@ -1257,7 +1285,7 @@ 
biomero package
 

-update_slurm_scripts(generate_jobs: bool = False, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]

+update_slurm_scripts(generate_jobs: bool = False, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]
 
Updates the local copy of the Slurm job submission scripts.

 
This function pulls the latest version of the scripts from the Git
 repository and copies them to the slurm_script_path directory.
@@ -1267,10 +1295,10 @@ 
biomero package
 
Parameters

 
    
-
  • generate_jobs (bool) – Whether to generate new slurm job scripts
+
  • generate_jobs (bool) – Whether to generate new slurm job scripts
 INSTEAD (of pulling from git). Defaults to False, except
 if no slurm_script_repo is configured.
    • 
-
  • env (Dict[str, str], optional) – Optional environment variables
+
  • env (Dict[str, str], optional) – Optional environment variables
 to set when running the command. Defaults to None.
    • 
 

 

@@ -1285,12 +1313,12 @@ 
biomero package
 

-validate(validate_slurm_setup: bool = False)[source]

+validate(validate_slurm_setup: bool = False)[source]
 
Validate the connection to the Slurm cluster by running
 a simple command.

 

 
Parameters

-
validate_slurm_setup (bool) – Whether to also check
+
validate_slurm_setup (bool) – Whether to also check
 and fix the Slurm setup (folders, images, etc.)

 

 
Returns

@@ -1298,14 +1326,14 @@ 
biomero packageReturn type
-
bool

+
bool

 

 

 

 
 

 

-workflow_params_to_envvars(**kwargs) → Dict[source]

+workflow_params_to_envvars(**kwargs) → Dict[source]
 
Convert workflow parameters to environment variables.

 

 
Parameters

@@ -1322,7 +1350,7 @@ 
biomero package
 

-workflow_params_to_subs(params) → Dict[str, str][source]

+workflow_params_to_subs(params) → Dict[str, str][source]
 
Convert workflow parameters to substitution dictionary for job script.

 

 
Parameters

@@ -1336,14 +1364,14 @@ 
biomero packageReturn type
-
Dict[str, str]

+
Dict[str, str]

 

 

 

 
 

 

-zip_data_on_slurm_server(data_location: str, filename: str, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]

+zip_data_on_slurm_server(data_location: str, filename: str, env: Optional[Dict[str, str]] = None) → fabric.runners.Result[source]
 
Zip the output folder of a job on Slurm

 

 
Parameters

@@ -1351,7 +1379,7 @@ 
biomero packagestr, str], optional) – Optional environment variables to
+
  • env (Dict[str, str], optional) – Optional environment variables to
 set when running the command. Defaults to None.
    • 
 
 
    
@@ -1368,8 +1396,8 @@ 
    biomero package
 
    
-class biomero.slurm_client.SlurmJob(submit_result: fabric.runners.Result, job_id: int)[source]
    
-
    Bases: object
    
+class biomero.slurm_client.SlurmJob(submit_result: fabric.runners.Result, job_id: int, wf_id: uuid.UUID, task_id: uuid.UUID, slurm_polling_interval: int = 10)[source]
+
    Bases: object
    
 
    Represents a job submitted to a Slurm cluster.
    
 
    This class encapsulates information and methods related to managing a job
 submitted to a Slurm cluster. It provides functionality to monitor the
@@ -1380,7 +1408,7 @@ 
    biomero package
 
    Type
    
-
    int
    
+
    int
    
 
    
 
    
 
    
@@ -1402,7 +1430,7 @@ 
    biomero package
 
    Type
    
-
    bool
    
+
    bool
    
 
    
 
    
 
    
@@ -1413,7 +1441,18 @@ 
    biomero package
 
    Type
    
-
    str
    
+
    str
    
+
    
+
    
+
    
+
+
    
+
    
+progress
    
+
    The progress of the Slurm job.
    
+
    
+
    Type
    
+
    str
    
 
    
 
    
 
    
@@ -1421,30 +1460,56 @@ 
    biomero package
 
    
 error_message
    
-
    The error message, if any.
    
+
    The error message, if any, encountered during job submission.
    
 
    
 
    Type
    
-
    str
    
+
    str
    
 
    
 
    
 
    
 
+
    
+
    
+wf_id
    
+
    The workflow ID associated with the job.
    
 
    
-
    Parameters
    
-
      
-
    • submit_result (Result) – The result of submitting the job.
      • 
-
    • job_id (int) – The Slurm job ID.
      • 
-
    
+
    Type
    
+
    UUID
    
 
    
 
    
+
    
+
+
    
+
    
+task_id
    
+
    The task ID within the workflow.
    
+
    
+
    Type
    
+
    UUID
    
+
    
+
    
+
    
+
+
    
+
    
+slurm_polling_interval
    
+
    The polling interval (in seconds) for checking the job status.
    
+
    
+
    Type
    
+
    int
    
+
    
+
    
+
    
+
 
    Example
    
 
    # Submit some job with the SlurmClient
-submit_result, job_id = slurmClient.run_workflow(
    
+submit_result, job_id, wf_id, task_id = slurmClient.run_workflow(
    
 
    
-
    workflow_name, workflow_version, input_data, email, time, **kwargs)
    
+
    workflow_name, workflow_version, input_data, email, time, wf_id,
+**kwargs)
    
 
    
 
    # Create a SlurmJob instance
-slurmJob = SlurmJob(submit_result, job_id)
    
+slurmJob = SlurmJob(submit_result, job_id, wf_id, task_id)
    
 
    
 
    if not slurmJob.ok:
    logger.warning(f”Error with job: {slurmJob.get_error()}”)
    
 
    
@@ -1465,6 +1530,11 @@ 
    biomero package
+
    
+SLURM_POLLING_INTERVAL = 10
    
+
    
+
 
    
 
    
 cleanup(slurmClient) → fabric.runners.Result[source]
    
@@ -1491,28 +1561,28 @@ 
    biomero package
    True if the job has completed; False otherwise.
    
 
 
    Return type
    
-
    bool
    
+
    bool
    
 
    
 
    
 
 
 
    
 
    
-get_error() → str[source]
    
+get_error() → str[source]
 
    Get the error message associated with the Slurm job submission.
    
 
    
 
    Returns
    
 
    The error message, or an empty string if no error occurred.
    
 
    
 
    Return type
    
-
    str
    
+
    str
    
 
    
 
    
 
    
 
 
    
 
    
-wait_for_completion(slurmClient, omeroConn) → str[source]
    
+wait_for_completion(slurmClient, omeroConn) → str[source]
 
    Wait for the Slurm job to reach completion, cancellation, failure, or timeout.
    
 
    
 
    Parameters
    
@@ -1525,7 +1595,7 @@ 
    biomero package
    The final state of the Slurm job.
    
 
    
 
    Return type
    
-
    str
    
+
    str
    
 
    
 
    
 
diff --git a/genindex.html b/genindex.html
index df14fa7..cd3493f 100644
--- a/genindex.html
+++ b/genindex.html
@@ -131,6 +131,10 @@ 
    B
    
 
       
   
+  
 
 
 
    C
    
@@ -197,10 +201,12 @@ 
    G
    
       
  • get_image_versions_and_data_files() (biomero.slurm_client.SlurmClient method)
 
    • 
       
  • get_job_status_command() (biomero.slurm_client.SlurmClient method)
+
    • 
+      
  • get_jobs_info_command() (biomero.slurm_client.SlurmClient method)
 
    • 
   
   
   
@@ -318,6 +330,8 @@ 
    S
    
       
  • setup_directories() (biomero.slurm_client.SlurmClient method)
 
    • 
       
  • setup_job_scripts() (biomero.slurm_client.SlurmClient method)
+
    • 
+      
  • setup_listeners() (biomero.slurm_client.SlurmClient method)
 
    • 
       
  • setup_slurm() (biomero.slurm_client.SlurmClient method)
 
    • 
@@ -327,13 +341,17 @@ 
    S
    
 
       
  • slurm_images_path (biomero.slurm_client.SlurmClient attribute)
 
    • 
-  
-  
+  
      
       
    • slurm_model_paths (biomero.slurm_client.SlurmClient attribute)
 
      • 
       
    • slurm_model_repos (biomero.slurm_client.SlurmClient attribute)
+
      • 
+      
    • SLURM_POLLING_INTERVAL (biomero.slurm_client.SlurmJob attribute)
+
      • 
+      
    • slurm_polling_interval (biomero.slurm_client.SlurmJob attribute)
 
      • 
       
    • slurm_script_path (biomero.slurm_client.SlurmClient attribute)
 
      • 
@@ -352,6 +370,10 @@ 
      S
      
 
 
      T
      
 
+  
      
   
        
       
      • transfer_data() (biomero.slurm_client.SlurmClient method)
 
        • 
@@ -382,6 +404,8 @@ 
        W
        
 
        
   
   
          
diff --git a/objects.inv b/objects.inv
index 8e613f9f623ae108449de0b3a3cd2ee4eecbbd6e..4feb6ddfe7b4df2d367fe0724a138f2bf2999cfa 100644
GIT binary patch
delta 990
zcmV<410np~2gV4Hc7K`8jvF@&fbV{aDblNr=9*g*1ci&;>|!@*kAgfy+ln=kAWGg$
zPJNBOUZ14oXl##>1=_5bn@2yND2k*gdSiSK(XY_v*sInWaH$Eyo8EWXLi0v3X|V>XAR*oguB&h|TM3#{ZeT+C<}a${I}IAb+YHlJ9haQ8Gw--@S<1
z?x8)X&wisY>6Eb+XF3G1~vV(4(TrgpD8b`o+ARYvfZ
zw?3*T9}ipWpMR?9Nzvxb5k_)|eis2zf@-ZYz21Q$14TM~RN8gL9wZ!!*J#1o;1NjE
zeNS`=){gh1cfFjD&w|Zka-IF5}T$l!ul&7JB}29l4rFM*xhy}?K7#B
zynzl)03D&FpDebZLbKzLEyxSqCDvdYO-Pb9b&6U)0)I*ITN6vhxa72IoVf^z0x+Yk
z`3e+7v>lRZQQ<&NRcrk0nX{V!MR0Tlh_Q~^SiP|@MQA$bb3m2g(_sfo*3(HyOr}Ve
zQs>(PNqH6gYr&aR+CgzlidAFmE0L;_9P|0YLqU|nV9EY{P9o6r-F)_XIxPXRP&PG3
z2}p*JGJl65m$Y#wOF}7VaUvO9XXth#gET_ojR{S;eMERtc}Jm1BpXx5`55T>N_%GN
zl%m;WdJn%GJ0_Qsz+!o#$liBgXC^JjWGRJ_5$KsAg)D|Gs>}(=VhSKrDRi+#RJF@QTBb#>r+28_8n&*3b2j5Ty4
zPuhe?0qv#O-^-Qxg3=<*Ts_`=dbm~hcQ-fd55KAPhmY?c{`mE#!XFqq)d4GZv^uAj
zH4uoA^1Ys3qZBYJ_rwpJkyj(orS?`K$*FKryCZ+71k4Lx6`0Z*ytQ27HjZv~iVU^T
zynpEp2Y1XiRkD)CL;bNmX_AMlkdtXwJZwhX_B%vsFm-R}Y1J&R$DBSIz9q;<7CUgp
zb?{}JKu<@oJmu6{=B)YW;xcUF&PMzRmNp4Z^3B`z-R=8_yMO-r8ygy!d)mZU*8XWh
zS;8}Gg<*N>D6re|$4gnY{NIA}b*HAQ{Z}kzl>flr52Sy0y-Eb#^*{-*JIq&_HJ7>5
z=9*>m*7!ANWzAS&K!c=?X6#%b=-BSBr6_F$@>u%sS^X*VdqCT7mcHsJ^WJeCH+nAT
M#dH7SUm}U=>Nbz?v;Y7A

delta 884
zcmV-)1B?8|2;B#ec7K??Zrd;r$9F#ki*|L;t{GEwNP!?hGjt?q>1?y6NP-_u)Tyu0
z*Xxt?PLyQ{HP9fuY!W|yJRXmK9=Xx3gW#^B$)S^t(O{GC{dMQsY(RVzo3X(@b>@93
zU4Ibk)#gISt4I3zJV6?xVicd2jF)u2YWC0^0FAi#Q7YS(c;MJjG)4o)cozXZ-E~OkKn@#<>}7<$wI9x^$7nP{
zqttfm7@4^BY0hPr4J{d8F{`v>qBgU
z_K7nNPD=9@NYq?~EZi6jR6eh2tItN(nr3vIYb*~A#ebEJbyV7@je$}7a5W~W;^5T+U%PN&|q
zH*vKX$V_?49K~Q+gp^4vY*N~uA}Mk~%O>K%wHnVL3`osn-jFJkGlGT3afpu;XBH3t^bh2*gkc~nLdv=XyS5+f}G*yB-E3+n}N%+;9*WuJrA
z;2Ao469Nj_Yq4L;A@_<0G3q#Mro|eXie`!uFVrs83BNyvso70aVslwG??Z9f=
z!t>ApZyaE#%c->Vxuwt5x!K5_wfGJO4Fr*#xVgQ*`||Dn&#&K6V8h(ue1><%cPl)q
zE^UnE`}L`!!0yVMbs4ohRmb_bW6@$hs~P1#_=g^SXP2V{P;Cd&XnUBCG%GHB$H|Gc
zxY6#GQUjr^GJVt$-64j`RRk59{nCn3Rv?e1{(;<|GNqBq{4?~TqV)Sr4{O2ETTPhi
KSAPKsY~;Sr(YCDs

diff --git a/readme_link.html b/readme_link.html
index f30c6dd..19809b5 100644
--- a/readme_link.html
+++ b/readme_link.html
@@ -127,7 +127,7 @@
              
   
          
 
          BIOMERO - BioImage analysis in OMERO
          
-
          License DOI PyPI - Version PyPI - Python Versions Slurm OMERO fair-software.eu OpenSSF Best Practices Sphinx build pages-build-deployment python-package build python-publish build
          
+
          License DOI PyPI - Version PyPI - Python Versions Slurm OMERO fair-software.eu OpenSSF Best Practices Sphinx build pages-build-deployment python-package build python-publish build Coverage Status
          
 
          The BIOMERO framework, for BioImage analysis in OMERO, allows you to run (FAIR) bioimage analysis workflows directly from OMERO on a high-performance compute (HPC) cluster, remotely through SSH.
          
 
          The BIOMERO framework consists of this Python library biomero, together with the BIOMERO scripts that can be run directly from the OMERO web interface.
          
 
          The package includes the SlurmClient class, which provides SSH-based connectivity and interaction with a Slurm (high-performance compute) cluster. The package enables users to submit jobs, monitor job status, retrieve job output, and perform other Slurm-related tasks. Additionally, the package offers functionality for configuring and managing paths to Slurm data and Singularity images (think Docker containers…), as well as specific FAIR image analysis workflows and their associated repositories.
          
@@ -200,7 +200,7 @@ 
          OMERO Requirements
 
        • SSH client and access to the Slurm cluster (w/ private key / headless)
          • 
 
        • SCP access to the Slurm cluster
          • 
-
        • Python3.7+
          • 
+
        • Python3.8+
          • 
 
        • This library installed
          
 
            
 
          • Latest release on PyPI python3 -m pip install biomero
            • 
@@ -546,7 +546,7 @@ 
            How to add your new custom workflow
 
          • Rewrite your script to be headless / to be executable on the commandline. This requires handling of commandline parameters as input.
            
             
 
            • 
 
          • Describe these commandline parameters in a descriptor.json (see previous chapter). E.g. like this.
            • 
diff --git a/searchindex.js b/searchindex.js
index 8f61cae..1e51250 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["biomero","index","modules","readme_link","tutorial_link"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["biomero.rst","index.rst","modules.rst","readme_link.md","tutorial_link.md"],objects:{"biomero.slurm_client":[[0,1,1,"","SlurmClient"],[0,1,1,"","SlurmJob"]],"biomero.slurm_client.SlurmClient":[[0,2,1,"","check_job_status"],[0,2,1,"","cleanup_tmp_files"],[0,2,1,"","convert_cytype_to_omtype"],[0,2,1,"","convert_url"],[0,2,1,"","copy_zip_locally"],[0,2,1,"","extract_data_location_from_log"],[0,2,1,"","extract_job_id"],[0,2,1,"","extract_parts_from_url"],[0,2,1,"","from_config"],[0,2,1,"","generate_slurm_job_for_workflow"],[0,2,1,"","get_active_job_progress"],[0,2,1,"","get_all_image_versions_and_data_files"],[0,2,1,"","get_cellpose_command"],[0,2,1,"","get_conversion_command"],[0,2,1,"","get_image_versions_and_data_files"],[0,2,1,"","get_job_status_command"],[0,2,1,"","get_jobs_info_command"],[0,2,1,"","get_logfile_from_slurm"],[0,2,1,"","get_or_create_github_session"],[0,2,1,"","get_recent_log_command"],[0,2,1,"","get_unzip_command"],[0,2,1,"","get_update_slurm_scripts_command"],[0,2,1,"","get_workflow_command"],[0,2,1,"","get_workflow_parameters"],[0,2,1,"","get_zip_command"],[0,2,1,"","init_workflows"],[0,2,1,"","list_active_jobs"],[0,2,1,"","list_all_jobs"],[0,2,1,"","list_available_converter_versions"],[0,2,1,"","list_completed_jobs"],[0,2,1,"","parse_docker_image_version"],[0,2,1,"","pull_descriptor_from_github"],[0,2,1,"","run_commands"],[0,2,1,"","run_commands_split_out"],[0,2,1,"","run_conversion_workflow_job"],[0,2,1,"","run_workflow"],[0,2,1,"","run_workflow_job"],[0,2,1,"","setup_container_images"],[0,2,1,"","setup_converters"],[0,2,1,"","setup_directories"],[0,2,1,"","setup_job_scripts"],[0,2,1,"","setup_slurm"],[0,3,1,"","slurm_converters_path"],[0,3,1,"","slurm_data_path"],[0,3,1,"","slurm_images_path"],[0,3,1,"","slurm_model_images"],[0,3,1,"","slurm_model_paths"],[0,3,1,"","slurm_model_repos"],[0,3,1,"","slurm_script_path"],[0,3,1,"","slurm_script_repo"],[0,2,1,"","str_to_class"],[0,2,1,"","transfer_data"],[0,2,1,"","unpack_data"],[0,2,1,"","update_slurm_scripts"],[0,2,1,"","validate"],[0,2,1,"","workflow_params_to_envvars"],[0,2,1,"","workflow_params_to_subs"],[0,2,1,"","zip_data_on_slurm_server"]],"biomero.slurm_client.SlurmJob":[[0,2,1,"","cleanup"],[0,2,1,"","completed"],[0,3,1,"","error_message"],[0,2,1,"","get_error"],[0,3,1,"","job_id"],[0,3,1,"","job_state"],[0,3,1,"","ok"],[0,3,1,"","submit_result"],[0,2,1,"","wait_for_completion"]],biomero:[[0,0,0,"-","slurm_client"]]},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:attribute"},terms:{"0":[1,3],"00":[3,4],"001":4,"01":[0,4],"02":4,"03":4,"04":4,"05":3,"06":4,"08":[3,4],"09":3,"1":[0,1,3],"10":[0,3,4],"100":[3,4],"1000":3,"1006116864":4,"1006305280":4,"10802":4,"10gb":[3,4],"10x":3,"117":4,"12":4,"12345":0,"126":4,"12m":4,"13":4,"1336":4,"13t17":4,"15":[3,4],"15gb":[3,4],"16":4,"17":4,"172":4,"18":4,"18gb":4,"1g":[3,4],"2":[0,1,3],"2014":4,"2017":4,"2020":3,"2022":4,"2023":[0,4],"2024":4,"21":[3,4],"22":[3,4],"2215916":4,"2222":4,"224":3,"224x224":3,"2269097984":4,"23":4,"23t08":4,"23t09":4,"24":4,"25":[3,4],"256x256":4,"27":4,"28":4,"2a":4,"2aoamjs0":4,"2b":4,"2c":4,"3":[1,3],"30":[3,4],"300":4,"31":4,"3184":4,"32":[3,4],"33":4,"3358":4,"34":4,"3406":4,"3500":4,"36":4,"37":4,"3a":4,"3b":1,"3c":4,"3m":4,"4":[1,3],"40":4,"4064":4,"4080":4,"4096":4,"41":4,"42":4,"43":4,"44":4,"45":[3,4],"45min":3,"46":4,"4672820":4,"4784967680":4,"48":4,"4j":3,"4x":4,"5":[1,3],"50":3,"5120":4,"52":4,"54":4,"5gb":4,"6":[1,3],"60":3,"600":4,"64":3,"6817":4,"695":4,"7":[3,4],"700":4,"777":4,"7zip":[1,3],"8":[3,4],"9":[3,4],"900":4,"982536":4,"982720":4,"997":3,"case":[3,4],"class":[0,1],"default":[0,3,4],"do":[3,4],"export":3,"final":[0,4],"float":0,"function":[0,3,4],"import":[1,3],"int":0,"long":[3,4],"new":[0,1,4],"null":4,"public":[3,4],"return":[0,3,4],"short":[3,4],"static":4,"true":[0,3,4],"try":[0,1,3],"var":4,"while":[0,3,4],A:[0,3],And:[3,4],As:4,At:[3,4],But:[3,4],By:[3,4],For:[3,4],IS:4,IT:4,If:[0,3,4],In:[3,4],It:[0,3,4],NOT:4,No:4,ONE:3,Of:4,On:4,One:[0,4],Or:[3,4],THAT:4,That:4,The:[0,3,4],Then:[3,4],There:4,These:[0,3,4],To:[3,4],WITH:0,Will:[0,4],With:3,_:3,__:4,_______:4,_______________________________________:4,__init__:3,__main__:[3,4],__name__:[3,4],_aggreg:4,_cellexpans:4,_cp_mask:3,_cpmask:4,_default:0,_imag:4,_job:3,_job_:3,_job_cpu:4,_mask_c1:4,_out_sep:0,_repo:[3,4],abl:4,about:[0,4],abov:[3,4],accept:[0,4],access:[1,3],accomplish:3,accord:[0,3,4],account:[0,1,3],acct:4,action:4,activ:[0,4],actual:[3,4],ad:[0,3,4],adapt:4,add:1,addendum:1,addit:[0,3],addition:3,address:[0,4],adjust:4,admin:3,advanc:4,after:[0,3,4],afterward:[3,4],ag:4,again:[3,4],aggreg:4,algorithm:4,alia:[3,4],aliv:4,all:[0,3,4],allow:[3,4],along:4,alreadi:[0,3,4],also:[0,3,4],altern:0,alwai:[3,4],am:4,amc:4,amount:4,amsterdam:[3,4],an:[0,1,4],analysi:[1,4],analyz:4,ani:[0,3,4],annot:4,anoth:[3,4],anyon:4,anyth:4,anywai:[0,4],anywher:4,apach:4,api:4,app:4,appar:4,append:3,appli:[3,4],applic:4,approach:4,appropri:[0,4],apptain:[1,3],apt:4,ar:[0,3,4],archiv:0,arg:0,argument:[0,4],argv:[3,4],around:4,arrai:0,art:4,asctim:3,ask:[3,4],assign:4,associ:[0,3,4],assum:[0,3,4],astyp:4,attach:[3,4],attack:4,attempt:0,attribut:[0,3],aug:4,authent:[3,4],authorized_kei:4,autom:[3,4],automat:[0,3,4],avail:[0,3,4],azur:1,azureadmin:4,azureus:4,b:3,back:[3,4],base:[0,3,4],base_path:4,bash:4,bashrc:4,basic:3,basicconfig:3,batch:1,becam:4,becaus:[3,4],becom:4,been:[3,4],befor:[0,4],behavior:0,behaviour:4,being:[0,4],below:[0,3,4],benefici:3,best:3,better:[3,4],between:[0,3,4],bfimg:3,bi:1,biaflow:[3,4],biaflowsjob:[3,4],big:4,biggest:4,bimg:3,bin:[3,4],binari:4,bioformats2raw:3,bioimag:[1,4],biomerostorag:4,bit:4,bitbucket:3,bj:[3,4],blob:[0,4],blog:4,blosc:3,bonu:4,bool:[0,4],borrow:4,both:[3,4],boutiqu:3,box:4,branch:[0,3,4],brief:3,bring:4,browser:4,bsst265:4,bugfix:4,build:[0,3,4],builddir:4,built:[3,4],bump:4,button:4,c1:4,c2:4,cacert:4,calcul:1,call:[3,4],can:[0,3,4],cancel:0,candl:4,cannot:0,cat:4,caus:4,cc:4,cd:[0,3,4],cell:[0,3,4],cell_diamet:0,cellexpans:1,cellexpansion_job:4,cellexpansion_repo:4,cellnumgranul:4,cellpos:[0,1,3],cellpose_imag:4,cellpose_job:[3,4],cellpose_job_cpu:4,cellpose_job_gr:[3,4],cellpose_job_mem:[3,4],cellpose_job_tim:3,cellpose_mask:4,cellpose_repo:[3,4],cellpose_v1:4,cellposemask:4,cellprob_threshold:[3,4],cellprofil:[1,3],cellprofiler_job:4,cellprofiler_repo:4,cellprofiler_result:4,cellprofiler_spot:4,cellprofiler_spot_job:4,cellprofiler_spot_repo:4,cellprofiler_v1:4,cells_aggreg:4,cells_cellexpans:4,cellsgranuleslabel:4,cellsnucleilabel:4,cellular:[3,4],cellularimagingcf:3,centroid:4,certif:4,cetera:3,cf:4,challeng:4,chan:[3,4],chang:[3,4],channel:[0,3,4],chapter:[3,4],cheap:4,cheaper:4,check:[0,3,4],check_job_statu:0,checkout:4,chmod:4,choos:4,chosen:[3,4],chown:4,chunk:3,ci:4,class_nam:0,class_objseg:4,classmethod:0,clean:4,cleanup:0,cleanup_tmp_fil:0,cli:3,click:4,client:[0,1,3],clone:[0,3,4],close:4,closesess:4,cloud:[1,3],cluster:[0,3],cluster_kei:4,cmd:[3,4],cmdlist:0,code:[1,4],coffe:[3,4],color:4,column:[0,4],com:[0,3,4],combin:4,come:[3,4],comma:0,command:[0,3],commandlin:[3,4],comment:[3,4],commerci:4,commit:4,commmand:4,common:4,commonwl:3,commun:4,compat:3,complet:[0,4],compon:4,compos:4,comput:[3,4],concaten:0,concept:0,conf:4,config:[0,1,4],config_fil:0,config_omero_:3,config_omero_logging_level:3,config_omero_master_host:3,configfil:[0,3,4],configur:[0,3,4],congratul:[3,4],conn:0,connect:[0,3,4],connect_kwarg:0,connect_timeout:0,connectionlostexcept:4,consecut:0,consist:3,consol:[3,4],constant:4,constructor:0,contain:[0,1],container:3,container_nam:[3,4],content:[3,4],context:4,contextmanag:0,continu:4,contributor:4,control:4,conveni:3,convers:0,convert:[0,3,4],convert_cytype_to_omtyp:0,convert_job_arrai:4,convert_url:0,converter_imag:0,copi:[0,3,4],copy_zip_loc:[0,3],copyleft:4,copyright:4,core:[3,4],correct:4,correctli:[3,4],correspond:[0,3],cost:[3,4],could:[0,3,4],count:4,count_nonzero:4,cours:[3,4],cover:4,cow:4,cp:[3,4],cp_model:[0,3],cppipe:4,cpproj:4,cpu:[3,4],creat:[0,3],creating_bia_workflow_and_adding_to_biaflows_inst:4,creation:0,credenti:4,credit:4,creditcard:4,crop:3,crt:4,csv:4,ctld:4,ctrl:4,current:[0,3,4],custom:1,cv2:4,cwd:3,cyclecloud:4,cython:4,cytomin:[0,3,4],cytomine_:3,cytomine_host:4,cytomine_id_project:4,cytomine_id_softwar:4,cytomine_job:0,cytomine_private_kei:4,cytomine_public_kei:4,cytoplasm:4,cytyp:0,d:[0,3],da:4,daemon:4,danger:4,data:[0,1],data_loc:0,data_path:[0,3],databas:0,datadisk:3,datafil:4,datafram:4,dataset:[3,4],date:[3,4],db:1,dbd:4,dbformysql:4,dd:0,de:4,debug:[3,4],deepcel:[3,4],deepcell_job:4,deepcell_repo:4,def:4,defin:[3,4],definit:[0,4],deleg:4,delet:[3,4],demand:4,deni:4,depend:[0,3],depends_on:4,deploi:[3,4],deploy:4,depth:3,deriv:4,descend:0,describ:[3,4],descript:[3,4],descriptor:[0,1],desir:3,desktop:4,destin:4,destroi:4,detail:[3,4],detect:0,dev:[3,4],devel:3,develop:4,diamet:[0,3,4],dict:0,dictionari:0,did:4,didn:4,differ:[3,4],digicert:4,digicertglobalrootca:4,dimens:3,dir:[3,4],directli:[3,4],directori:[0,3,4],dirti:4,discuss:3,distinguish:0,distribut:4,dl:[0,4],dn:4,doc:4,docker:[0,1],dockerfil:3,dockerfile_slurmctld:4,dockerhub:[0,3],dockerhub_token:4,dockerhub_usernam:4,docstr:3,document:[0,4],doe:[0,4],doesn:[0,3,4],don:[3,4],done:[3,4],dot:4,down:[3,4],download:[0,3,4],drastic:4,drive:[3,4],drwxr:4,drwxrwxr:4,ds:4,dtype:4,dunde:4,dyn:4,dynam:4,e:[0,3],each:[0,3,4],earlier:3,easi:[3,4],easier:[3,4],easiest:[0,3,4],easili:4,eavesdrop:4,echo:[3,4],edit:4,editor:4,effect:4,effici:3,either:[0,3,4],elaps:4,els:[0,3],email:[0,3,4],embarrassingli:3,empti:[0,4],en:4,enabl:3,encapsul:0,encourag:[3,4],end:[0,3,4],end_tim:0,endtim:4,enforc:3,enhanc:4,enjoi:4,enough:[3,4],ensur:[0,3,4],enter:4,entri:[0,4],entrypoint:[3,4],env:[0,4],environ:[0,3,4],equal:3,equip:3,err_desc:3,error:[0,3,4],error_messag:0,escap:4,especi:[3,4],essenti:4,estim:4,et:3,etc:[0,3,4],etc_mung:4,etc_slurm:4,etric:4,euro:4,europ:4,even:[3,4],everi:[3,4],everybodi:4,ex:4,exampl:[0,3,4],excel:4,except:[0,4],exec:4,execut:[0,3,4],exercis:4,exist:[0,1,4],exit:[3,4],expect:[0,3,4],expens:[3,4],expire_tim:4,explain:4,explan:3,explicitli:3,explor:[3,4],expos:4,ext:4,extend:[0,3],extens:[0,4],extern:[3,4],extra:[1,3],extract:[0,1],extract_data_location_from_log:0,extract_job_id:0,extract_parts_from_url:0,ezomero:3,f:[0,3,4],fabric:[0,3,4],facil:3,fact:3,fail:[0,3,4],failur:0,fair:[1,3],fals:[0,4],faster:[3,4],favorit:4,featur:3,few:4,field:3,figur:3,fiji:3,file:[0,3],filenam:[0,3],filename_no_extens:3,filepath:4,fileserv:4,filetyp:0,fill:[0,4],filler:4,filter:[0,4],filter_filetyp:0,find:[0,3,4],fine:4,finish:3,fire:4,firewal:4,first:[0,3,4],fit:4,fix:[0,4],flag:[0,3,4],flexibl:[3,4],flexibleserv:4,fluorescens:4,folder:[0,3,4],folder_nam:0,folk:4,follow:[3,4],forc:0,force_upd:0,forget:4,form:[3,4],format:[0,3,4],forward:[3,4],forward_ag:0,found:[0,3,4],framework:3,free:[3,4],friendli:3,from:[0,1,3],from_cli:[3,4],from_config:[0,3,4],full:[0,4],fulli:4,fun:[3,4],funni:4,futur:[0,3,4],fw:4,g:[0,3,4],gain:4,gatewai:[0,4],gb:[3,4],gcloud:4,gcslurm:4,gener:[0,1,4],generate_job:[0,3],generate_slurm_job_for_workflow:0,get:[0,4],get_active_job_progress:0,get_all_image_versions_and_data_fil:[0,4],get_cellpose_command:0,get_conversion_command:0,get_error:0,get_image_versions_and_data_fil:0,get_job_status_command:0,get_jobs_info_command:0,get_logfile_from_slurm:[0,3],get_or_create_github_sess:0,get_recent_log_command:0,get_unzip_command:0,get_update_slurm_scripts_command:0,get_workflow_command:0,get_workflow_paramet:0,get_zip_command:0,getenv:4,getinput:4,getlogg:4,ghpc:4,giant:4,gid:3,giovtorr:4,git:[0,3,4],github:[0,3],give:[0,3,4],given:[0,3,4],global:3,gnu:4,go:4,godlovedc:4,good:4,googl:1,gosu:4,got:4,gpu:[0,1,4],grab:[1,3],granuleslabel:4,granulesstat:4,granulesstatsincelllabel:4,granulesstatsnp:4,grayscal:3,gre:4,great:4,grei:3,grid:4,ground:3,group:4,groupadd:4,gt:4,gt_img:[3,4],gt_path:[3,4],gtfolder:[3,4],guid:[3,4],gz:4,ha:[0,3,4],had:4,half:4,hand:4,handl:[3,4],handout:4,hardcod:[3,4],hardwar:3,have:[0,3,4],headbash:4,header:0,headless:3,heard:4,hello:3,help:3,helper:4,here:[3,4],hh:[0,3],hidden:4,high:3,higher:3,highli:4,hoc:4,hold:4,home:[3,4],hope:4,hopefulli:[3,4],host:[0,3,4],hostnam:[3,4],hour:3,how:1,howev:[0,3,4],hpc:[3,4],hpcsmall:4,htc:4,html:[3,4],htop:4,http:[0,3,4],hub:4,human:4,i:[1,4],iap:4,ic:4,id:[0,3,4],id_rsa:[3,4],idea:4,ideal:4,identif:4,identityfil:4,idkei:4,idl:4,ignor:[3,4],ij:3,ij_min_threshold:3,ij_radiu:3,ijm:3,imag:[0,3],image_nam:0,image_path:3,image_tag:4,image_vers:[0,3],imageio:3,imagej:[3,4],imagej_job:4,imagej_repo:4,imcellscelllabel:4,imcellsgranuleslabel:4,img:3,implement:[0,3,4],improv:[3,4],imread:[3,4],imread_anydepth:4,imwrit:3,in_img:[3,4],in_path:[3,4],inbound:4,incl:3,includ:[0,3,4],incur:3,indefinit:3,index:[0,1],indic:0,individu:0,infinit:4,info:[0,3,4],infold:[3,4],inform:[0,3,4],ingredi:4,ingress:4,ini:[0,1,4],init:[3,4],init_environ:4,init_slurm:[0,3,4],init_workflow:0,initi:[0,4],initialis:4,initscript:4,inject:3,inline_ssh_env:0,innodb_lock_wait_timeout:4,input:[0,3,4],input_data:0,input_fold:3,input_url:0,insert:4,insid:4,insight:[3,4],instal:[1,3],instanc:[0,3,4],instanti:[0,4],instead:[0,3,4],insuffici:4,int16:4,integr:[3,4],interact:[0,3,4],interfac:3,intern:4,internet:4,interoper:4,introduct:1,invest:4,invok:0,involv:0,io:[3,4],ip:4,is_2d:[3,4],iso:0,isol:3,issu:[0,3,4],its:[0,3,4],jar:3,java:3,job:[0,1,4],job_id:0,job_stat:0,job_templ:[0,3,4],jobid:[0,4],jobnam:4,jobscript:[3,4],join:[3,4],json:[0,1],jump:4,just:[3,4],keep:[3,4],kei:[0,3,4],keygen:4,keyword:0,kickstart:4,kind:4,know:4,knowledg:4,known:[0,4],known_host:3,kwarg:0,la:4,last:[0,4],later:[0,4],latest:[0,3,4],launch:[3,4],law:4,lead:4,learn:4,least:3,leav:4,left:[3,4],less:[3,4],let:4,lethal:3,level:3,levelnam:3,lib:3,libgeo:4,librari:[0,3,4],licens:[1,3],lightweight:4,like:[0,3,4],limit:[0,1,4],line:[0,3,4],linear:4,link:[0,4],linux:4,lisbon:4,list:[0,4],list_active_job:0,list_all_job:0,list_available_converter_vers:0,list_completed_job:0,liter:[3,4],ll:4,load:[0,4],local:[0,1,3],local_path:0,local_tmp_storag:0,localhost:[3,4],localslurm:4,locat:[0,3,4],log:[0,1,4],log_fil:0,logfil:[0,3,4],logger:[0,4],logic:4,login:[3,4],logout:4,lolcow:4,longer:[3,4],look:[0,3,4],lose:3,lot:[3,4],lower:4,ls:4,lt:4,luckili:4,luik:4,m:[3,4],mac:4,machin:[0,3,4],macro:3,made:[3,4],mage:3,magic:4,mai:4,mail:[3,4],main:[3,4],mainli:[3,4],major:4,make:[3,4],man:4,manag:[0,1,3],mandatori:[3,4],manipul:4,manual:3,map:[0,4],mar:4,march:4,mask:[1,3],master:[0,3,4],match:[0,3,4],matter:4,max:[3,4],maximum:[0,3],maxshap:3,maxvm:4,mayb:[3,4],mconfig:4,md:4,me:4,mean:4,meant:4,measur:4,mechan:0,medium:3,mem:4,memori:3,mention:0,merchant:4,messag:[0,3,4],metadata:1,method:[0,3,4],metric:[3,4],microscopi:4,microsoft:1,middl:4,might:[0,3,4],min:[3,4],min_threshold:3,mine:4,minim:[3,4],minshap:3,minut:[3,4],miss:[3,4],mkdir:4,mlflow:3,mm:[0,3],mnt:3,mod_pipelin:4,mode:4,model:[0,3,4],model_path:3,modern:4,modifi:4,modul:[1,2,3,4],module_nam:0,moment:4,monei:4,monitor:[0,3],month:4,more:[0,3,4],most:[0,4],mount:[3,4],move:3,movi:4,ms:4,much:4,multipl:[0,1,4],mung:4,must:[0,4],my:[0,3,4],myjob:0,mysql:4,n:[0,4],name:[0,3,4],namespac:4,nasti:4,ndarrai:4,ndata:4,necessari:[0,3],need:[0,3,4],net:4,network:3,neubia:[3,4],newest:4,newli:4,next:[0,3,4],nextflow:4,nf:0,ngc:4,nice:[3,4],nicer:3,nj:3,nl:[3,4],nmc:4,no_npi:[3,4],node:[3,4],nodelist:4,nodenam:4,nologin:4,non:[3,4],none:[0,3],normal:4,note2:3,note:[0,3,4],noth:0,notif:0,now:[0,3,4],np:4,nsdynam:4,nuc:[3,4],nuc_channel:[0,3,4],nuclear:0,nuclei:[0,4],nucleilabel:4,nucleu:4,num:4,number:[0,4],numcel:4,o:[0,1,4],object:[0,3],oblig:4,occur:0,off:4,offer:3,offload:4,ok:[0,4],old:[0,4],older:4,om:3,omero:[0,1],omero_worker_nam:3,omeroconn:0,omeroinsight:4,omeroserv:3,omeroweb:4,omerowork:4,omit:[0,3],omput:4,onc:[3,4],one:[0,3,4],ones:[0,4],onli:[0,3],onlin:4,onto:4,oo:4,op:4,open:4,openssh:4,oper:[0,3,4],opt:[3,4],option:[0,1,3],orchestr:4,order:0,origin:[0,3,4],original_fil:4,os:[3,4],osc:3,oserror:0,other:[0,3,4],otherwis:[0,3,4],our:[3,4],out:[0,3],out_path:[3,4],outfold:[3,4],output:[0,3,4],outsid:4,over:[0,3,4],overal:3,overhead:[3,4],overlap:1,overrid:[3,4],overview:1,overwrit:[0,4],own:[1,4],ownership:4,p7zip:4,p:4,packag:[2,3],package_upgrad:4,page:4,paid:4,pair:[0,4],parallel:[3,4],param:[0,3],param_nam:4,paramet:[0,1],paramiko:0,pars:[0,4],parse_cellprofiler_paramet:4,parse_docker_image_vers:0,parsecpparam:4,part:[0,4],particularli:[0,4],partit:4,password:4,passwordless:3,past:4,path:[0,3,4],pattern:0,pd:4,peer:4,pem:4,pend:0,peopl:4,per:[0,3,4],perform:[0,3,4],perhap:[0,3,4],permiss:4,pick:4,pickup:4,pin:3,pip:[3,4],pipelin:[1,3],pitfal:3,pixel:0,pla:4,pla_data:4,place:3,placehold:[0,4],plai:4,plate:4,platform:3,pleas:[3,4],plugin:[3,4],png:4,podman:[3,4],point:[3,4],poll:3,port:[0,3,4],portal:4,posit:0,possibl:[3,4],potenti:3,power:4,ppa:4,practic:[3,4],predict:3,prefix:4,prepar:[3,4],prepare_data:[3,4],preprocess:3,prerequisit:1,present:0,press:4,pretrained_model:[3,4],pretti:4,previou:3,price:4,primit:3,print:[0,3,4],prioriti:4,privat:[3,4],prob:3,prob_thresh:3,prob_threshold:[0,3,4],probabl:[0,4],problem:3,problem_cl:[3,4],proceed:0,process:[0,3,4],processor:[3,4],profil:4,program:[3,4],programmat:3,progress:[0,3,4],project:4,promot:4,promptless:4,proper:4,properli:[0,4],properti:4,protein:4,protocol:4,provid:[0,3,4],pub:[3,4],pubkei:3,publish:3,pull:[0,3,4],pull_descriptor_from_github:0,pull_imag:4,purpos:[3,4],push:4,put:[3,4],pw:4,py:[0,1],pypi:3,pytest:3,python3:[3,4],python:[0,1,4],quantifi:4,queri:0,queue:4,quick:[3,4],quicker:4,quickstart:1,quit:4,r:[0,4],radiu:3,rais:[0,3],ran:4,rang:4,raw:[0,4],re:4,reach:[0,4],read:[0,3],readabl:4,reader:4,readi:4,readm:4,realli:4,realmemori:4,reason:4,rebuild:4,receiv:0,recent:[0,4],recogn:4,recommend:3,recreat:4,reduc:4,refer:[0,3,4],referenc:3,refresh:[3,4],region:4,regionprops_t:4,registr:4,rel:4,relat:[0,3,4],releas:[3,4],remain:[0,4],remot:[0,3,4],remov:[0,4],renam:4,replac:[3,4],repo:[0,3,4],repositori:[0,3],repres:0,reproduc:[3,4],request:[3,4],requir:[0,1],reserv:4,resiz:3,resourc:[3,4],respons:0,rest:4,restart:4,restrict:4,result:[0,3,4],retain:[0,4],retri:0,retriev:[0,3,4],return_cod:3,returncod:3,reus:4,review:3,rewrit:3,rewrot:4,rgb:3,rich:4,right:4,rm:[3,4],rocki:4,rockylinux:4,role:4,root:4,rootless:4,round:4,row:4,rs:0,rstring:4,rtype:4,rule:4,run:[0,1,3],run_command:0,run_commands_split_out:0,run_conversion_workflow_job:0,run_workflow:[0,4],run_workflow_job:0,runcmd:4,runner:[0,4],runscript:[3,4],runtim:3,rw:4,rwxr:4,s:[0,3,4],sacct:[0,4],sadli:4,safe:4,sai:[3,4],same:[0,3,4],satisfi:4,save:4,save_tif:[3,4],sbatch:[0,3,4],sbin:4,sc:3,scale:4,scenario:3,schedmd:3,schedul:[3,4],schema:[1,4],scheme:3,scp:[0,3,4],scratch:[0,4],screen:[3,4],script:[0,1],scroll:4,second:[0,4],secret:[3,4],section:[3,4],secur:4,see:[0,1,4],seem:4,segment:[3,4],select:[3,4],send:[0,3],sep:0,separ:[0,4],seriou:4,server:[0,3,4],servic:4,session:0,set:[0,1,3],setoutput:4,setup:[0,1,3],setup_container_imag:0,setup_convert:0,setup_directori:0,setup_job_script:0,setup_slurm:0,sever:[3,4],sftp:3,sh:[0,3,4],shakespear:4,shame:4,shape:3,share:[0,3,4],sharg:4,shell:[0,3,4],shenanigan:4,shorter:3,should:[1,3],shoulder:4,show:[3,4],shown:4,showtim:1,shutil:3,side:4,sif:[0,3,4],sign:4,similar:[0,4],simpl:[0,3,4],simpli:3,simplifi:[3,4],sinc:4,sing:4,singl:0,singular:[0,1,3],singularity_imag:[0,3,4],singularity_vers:4,size:4,sleep:4,slow:3,slurm:[0,1],slurm_cellpose_segment:3,slurm_client:[1,2],slurm_config:4,slurm_converters_path:0,slurm_data_path:[0,3,4],slurm_images_path:[0,3,4],slurm_job_id:0,slurm_jobdir:4,slurm_model_imag:0,slurm_model_job:0,slurm_model_jobs_param:0,slurm_model_path:0,slurm_model_repo:0,slurm_run_workflow_batch:4,slurm_script_path:[0,3,4],slurm_script_repo:[0,3,4],slurmclient:[0,1,4],slurmctld:4,slurmdbd:4,slurmjob:0,small:4,smaller:[3,4],smart:[3,4],snakemak:4,so:[3,4],softwar:[3,4],solut:4,some:[0,3,4],someon:4,someth:[3,4],sort:0,sound:4,sourc:[0,3,4],source_format:0,south:4,space:[0,4],specif:[0,3,4],specifi:[0,3],speckl:4,speed:[3,4],speedup:4,spent:4,spin:[3,4],split:[0,4],spot:4,spotcount:4,spun:4,squeue:4,ss:[0,3],ssh:[0,1],sshd:4,sshdocker:4,sshexcept:0,sshgroup:4,sshuser:4,ssl:4,st:4,stai:[3,4],stain:4,stand:4,standard:3,standard_dc2s_v3:4,standard_dc4s_v3:4,standard_nd96amsr_a100_v4:4,stardist:[3,4],stardist_job:4,stardist_repo:4,start:[0,4],start_tim:0,starttim:4,state:[0,4],statu:[0,3,4],statuscom:[3,4],stdout:[0,3,4],step:[3,4],stick:4,still:[0,4],storag:[0,4],storageaccount:4,store:[0,3,4],str:[0,4],str_to_class:0,stream:3,stricthostkeycheck:4,strictli:3,string:[0,3,4],strong:3,strongli:4,stuff:4,style:4,sub:0,subdirectori:4,subfold:[0,4],subject:4,submiss:[0,3,4],submit:[0,3,4],submit_result:0,submitit:3,subprocess:[3,4],subscript:4,substitut:0,succeed:0,success:[0,4],successfulli:0,sudo:4,suffix:4,suit:4,superus:4,suppli:4,support:[0,3,4],sure:[3,4],sweet:4,sy:3,system:[0,4],systemctld:4,t:[0,3,4],t_nucleisegment:4,t_t_luik:4,t_t_luik_amsterdamumc_nl:4,tab:[3,4],tabl:4,tag:[0,3,4],tail:[3,4],take:[0,3,4],tar:4,target:[0,3],target_format:0,task:[3,4],tbc:3,tcp:4,tell:4,templat:[0,3,4],temporari:[0,4],term:4,termin:[3,4],test:1,testproject:4,text:[3,4],than:[3,4],thei:[3,4],them:[0,3,4],themselv:[0,4],theoret:3,theori:[3,4],thi:[0,1,3],thing:4,think:[3,4],third:4,those:[0,3,4],though:4,thought:1,threshold:[0,3,4],through:[3,4],thrown:4,tick:4,tif:[0,3,4],tiff:[0,3,4],tifffil:3,time:[0,1,4],timelimit:4,timeout:[0,3],tischi:4,tmp:[0,3],tmp_path:[3,4],to_numpi:4,togeth:[3,4],token:4,too:1,took:4,tool:0,top:4,torec:4,torecluik:[3,4],total:4,touch:4,toward:3,track:4,trailer:0,transfer:[0,1],transfer_data:[0,3],transferresult:0,tree:[3,4],trick:4,troubl:4,trust:[3,4],truth:3,truthi:0,tue:4,tune:4,tupl:0,turn:[3,4],twice:4,two:0,txt:[3,4],type:[0,4],u:4,ubuntu:4,ui:[1,3],uid:3,uint16:4,uk:4,ulieg:[0,4],umc:[3,4],uncheck:[3,4],under:[3,4],understand:4,uniform:3,uniqu:[0,3,4],univers:4,unknown:4,unless:[3,4],unlik:0,unpack:[0,3],unpack_data:[0,3],until:[0,3],unvers:0,unwrap:4,unzip:[0,3],up:[0,1,3],updat:[0,3],update_slurm_script:[0,3],upgrad:4,upload:[1,3],url:[0,4],us:[0,1],use_gpu:[0,3,4],user:[0,3,4],useradd:4,userknownhostsfil:4,usern:3,usernam:[3,4],usr:[3,4],utf:4,util:[3,4],util_script:4,v0:4,v1:[3,4],v2:4,v3:4,v:4,valid:[0,3,4],validate_slurm_setup:[0,3,4],valu:[0,3,4],valueerror:[0,3],var_log_slurm:4,variabl:[0,3,4],variou:3,ve:4,venic:4,venv:[3,4],venvtest:3,veri:[3,4],verifi:4,version:[0,1,4],vi:4,via:[0,1,4],video:4,view:4,vm:4,vnet:4,volum:[3,4],vs:4,w:[3,4],w_cellexpans:4,w_cellexpansion_v2:4,w_countmaskoverlap:4,w_countmaskoverlap_v1:4,w_nucleisegment:[3,4],w_spotcount:4,wa:[0,4],wai:[0,3,4],wait:[0,3,4],wait_for_complet:0,walk:4,want:[3,4],warn:[0,3,4],watch:3,we:[0,1,3],web:[1,3],websit:[3,4],wed:4,well:[3,4],were:4,western:4,wf:4,wg5:4,wget:4,what:[0,3,4],whatev:4,when:[0,3,4],whenev:4,where:[0,3,4],whether:0,which:[0,3,4],whole:4,william:4,window:[3,4],within:[0,3,4],without:[0,4],won:4,work:[0,3,4],worker:4,workflow:[0,1],workflow_nam:0,workflow_params_to_envvar:0,workflow_params_to_sub:0,workflow_vers:0,workload:[0,4],world:3,worth:0,would:[3,4],wq:4,wrap:[3,4],wrapper:1,write:[3,4],wrong:4,wrote:4,x:[0,4],xmx6000m:3,xr:4,xvfb:3,xzf:4,y:[3,4],ye:4,yet:[0,3,4],yml:[0,3,4],you:[0,3,4],your:[0,1],yourself:3,yum:[3,4],yyyi:0,zarr:[0,3],zero:4,zip:[0,3,4],zip_data_on_slurm_serv:[0,3],zipfil:[0,4]},titles:["biomero package","Welcome to BIOMERO\u2019s documentation!","biomero","BIOMERO - BioImage analysis in OMERO","Cellprofiler tutorial"],titleterms:{"0":4,"1":4,"2":4,"3":4,"3b":4,"4":4,"5":4,"6":4,"7zip":4,"class":3,"export":4,"import":4,"new":3,"try":4,A:4,access:4,account:4,add:[3,4],addendum:4,altern:4,an:3,analysi:3,apptain:4,azur:4,b:4,basic:4,batch:[3,4],bi:4,bioimag:3,biomero:[0,1,2,3,4],c:4,calcul:4,cellexpans:4,cellpos:4,cellprofil:4,client:4,cloud:4,cluster:4,code:3,command:4,config:3,contain:[3,4],creat:4,custom:3,d:4,data:[3,4],databas:4,db:4,descriptor:[3,4],docker:[3,4],dockerfil:4,dockerhub:4,document:1,dr:4,e:4,exist:3,extra:4,extract:4,fair:4,file:4,from:4,gener:3,get:[1,3],github:4,googl:4,gpu:3,grab:4,headless:4,how:[3,4],i:3,imag:4,indic:1,ini:3,instal:4,introduct:4,job:3,json:[3,4],licens:4,limit:3,local:4,log:3,manag:4,manual:4,mask:4,memori:4,metadata:[3,4],microsoft:4,modul:0,multipl:3,network:4,o:3,omero:[3,4],onli:4,option:4,out:4,overlap:4,overview:3,own:3,packag:[0,1,4],paramet:[3,4],pipelin:4,prerequisit:[3,4],publish:4,py:[3,4],python:3,quickstart:3,repositori:4,requir:[3,4],run:4,s:1,schema:3,script:[3,4],see:3,set:4,setup:4,should:4,showtim:4,singular:4,slurm:[3,4],slurm_client:0,slurmclient:3,ssh:[3,4],start:[1,3],subnet:4,suggest:4,tabl:1,test:[3,4],thi:4,thought:4,time:3,tl:4,too:4,transfer:3,tutori:[1,3,4],ui:4,up:4,updat:4,upload:4,us:[3,4],version:3,via:3,virtual:4,we:4,web:4,welcom:1,workflow:[3,4],wrapper:[3,4],your:[3,4]}})
\ No newline at end of file
+Search.setIndex({docnames:["biomero","index","modules","readme_link","tutorial_link"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["biomero.rst","index.rst","modules.rst","readme_link.md","tutorial_link.md"],objects:{"biomero.slurm_client":[[0,1,1,"","SlurmClient"],[0,1,1,"","SlurmJob"]],"biomero.slurm_client.SlurmClient":[[0,2,1,"","bring_listener_uptodate"],[0,2,1,"","check_job_status"],[0,2,1,"","cleanup_tmp_files"],[0,2,1,"","convert_cytype_to_omtype"],[0,2,1,"","convert_url"],[0,2,1,"","copy_zip_locally"],[0,2,1,"","extract_data_location_from_log"],[0,2,1,"","extract_job_id"],[0,2,1,"","extract_parts_from_url"],[0,2,1,"","from_config"],[0,2,1,"","generate_slurm_job_for_workflow"],[0,2,1,"","get_active_job_progress"],[0,2,1,"","get_all_image_versions_and_data_files"],[0,2,1,"","get_cellpose_command"],[0,2,1,"","get_conversion_command"],[0,2,1,"","get_image_versions_and_data_files"],[0,2,1,"","get_job_status_command"],[0,2,1,"","get_jobs_info_command"],[0,2,1,"","get_listeners"],[0,2,1,"","get_logfile_from_slurm"],[0,2,1,"","get_or_create_github_session"],[0,2,1,"","get_recent_log_command"],[0,2,1,"","get_unzip_command"],[0,2,1,"","get_update_slurm_scripts_command"],[0,2,1,"","get_workflow_command"],[0,2,1,"","get_workflow_parameters"],[0,2,1,"","get_zip_command"],[0,2,1,"","init_workflows"],[0,2,1,"","initialize_analytics_system"],[0,2,1,"","list_active_jobs"],[0,2,1,"","list_all_jobs"],[0,2,1,"","list_available_converter_versions"],[0,2,1,"","list_completed_jobs"],[0,2,1,"","parse_docker_image_version"],[0,2,1,"","pull_descriptor_from_github"],[0,2,1,"","run_commands"],[0,2,1,"","run_commands_split_out"],[0,2,1,"","run_conversion_workflow_job"],[0,2,1,"","run_workflow"],[0,2,1,"","run_workflow_job"],[0,2,1,"","setup_container_images"],[0,2,1,"","setup_converters"],[0,2,1,"","setup_directories"],[0,2,1,"","setup_job_scripts"],[0,2,1,"","setup_listeners"],[0,2,1,"","setup_slurm"],[0,3,1,"","slurm_converters_path"],[0,3,1,"","slurm_data_path"],[0,3,1,"","slurm_images_path"],[0,3,1,"","slurm_model_images"],[0,3,1,"","slurm_model_paths"],[0,3,1,"","slurm_model_repos"],[0,3,1,"","slurm_script_path"],[0,3,1,"","slurm_script_repo"],[0,2,1,"","str_to_class"],[0,2,1,"","transfer_data"],[0,2,1,"","unpack_data"],[0,2,1,"","update_slurm_scripts"],[0,2,1,"","validate"],[0,2,1,"","workflow_params_to_envvars"],[0,2,1,"","workflow_params_to_subs"],[0,2,1,"","zip_data_on_slurm_server"]],"biomero.slurm_client.SlurmJob":[[0,3,1,"","SLURM_POLLING_INTERVAL"],[0,2,1,"","cleanup"],[0,2,1,"","completed"],[0,3,1,"","error_message"],[0,2,1,"","get_error"],[0,3,1,"","job_id"],[0,3,1,"","job_state"],[0,3,1,"","ok"],[0,3,1,"","progress"],[0,3,1,"","slurm_polling_interval"],[0,3,1,"","submit_result"],[0,3,1,"","task_id"],[0,2,1,"","wait_for_completion"],[0,3,1,"","wf_id"]],biomero:[[0,0,0,"-","slurm_client"]]},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:attribute"},terms:{"0":[1,3],"00":[3,4],"001":4,"01":[0,4],"02":4,"03":4,"04":4,"05":3,"06":4,"08":[3,4],"09":3,"1":[0,1,3],"10":[0,3,4],"100":[3,4],"1000":3,"1006116864":4,"1006305280":4,"10802":4,"10gb":[3,4],"10x":3,"117":4,"12":4,"12345":0,"126":4,"12m":4,"13":4,"1336":4,"13t17":4,"15":[3,4],"15gb":[3,4],"16":4,"17":4,"172":4,"18":4,"18gb":4,"1g":[3,4],"2":[0,1,3],"2014":4,"2017":4,"2020":3,"2022":4,"2023":[0,4],"2024":4,"21":[3,4],"22":[3,4],"2215916":4,"2222":4,"224":3,"224x224":3,"2269097984":4,"23":4,"23t08":4,"23t09":4,"24":4,"25":[3,4],"256x256":4,"27":4,"28":4,"2a":4,"2aoamjs0":4,"2b":4,"2c":4,"3":[1,3],"30":[3,4],"300":4,"31":4,"3184":4,"32":[3,4],"33":4,"3358":4,"34":4,"3406":4,"3500":4,"36":4,"37":4,"3a":4,"3b":1,"3c":4,"3m":4,"4":[1,3],"40":4,"4064":4,"4080":4,"4096":4,"41":4,"42":4,"43":4,"44":4,"45":[3,4],"45min":3,"46":4,"4672820":4,"4784967680":4,"48":4,"4j":3,"4x":4,"5":[1,3],"50":3,"5120":4,"52":4,"54":4,"5gb":4,"6":[1,3],"60":3,"600":4,"64":3,"6817":4,"695":4,"7":[3,4],"700":4,"777":4,"7zip":[1,3],"8":[3,4],"9":[3,4],"900":4,"982536":4,"982720":4,"997":3,"case":[3,4],"class":[0,1],"default":[0,3,4],"do":[3,4],"export":3,"final":[0,4],"float":0,"function":[0,3,4],"import":[1,3],"int":0,"long":[3,4],"new":[0,1,4],"null":4,"public":[3,4],"return":[0,3,4],"short":[3,4],"static":4,"true":[0,3,4],"try":[0,1,3],"var":4,"while":[0,3,4],A:[0,3],And:[3,4],As:4,At:[3,4],But:[3,4],By:[3,4],For:[3,4],IS:4,IT:4,If:[0,3,4],In:[3,4],It:[0,3,4],NOT:4,No:4,ONE:3,Of:4,On:4,One:[0,4],Or:[3,4],THAT:4,That:4,The:[0,3,4],Then:[3,4],There:4,These:[0,3,4],To:[3,4],WITH:0,Will:[0,4],With:3,_:3,__:4,_______:4,_______________________________________:4,__init__:3,__main__:[3,4],__name__:[3,4],_aggreg:4,_cellexpans:4,_cp_mask:3,_cpmask:4,_default:0,_imag:4,_job:3,_job_:3,_job_cpu:4,_mask_c1:4,_out_sep:0,_repo:[3,4],abl:4,about:[0,4],abov:[3,4],accept:[0,4],access:[1,3],accomplish:3,accord:[0,3,4],account:[0,1,3],acct:4,action:4,activ:[0,4],actual:[3,4],ad:[0,3,4],adapt:4,add:1,addendum:1,addit:[0,3],addition:3,address:[0,4],adjust:4,admin:3,advanc:4,after:[0,3,4],afterward:[3,4],ag:4,again:[3,4],aggreg:4,algorithm:4,alia:[3,4],aliv:4,all:[0,3,4],allow:[3,4],along:4,alreadi:[0,3,4],also:[0,3,4],altern:0,alwai:[3,4],am:4,amc:4,amount:4,amsterdam:[3,4],an:[0,1,4],analysi:[1,4],analyt:0,analyz:4,ani:[0,3,4],annot:4,anoth:[3,4],anyon:4,anyth:4,anywai:[0,4],anywher:4,apach:4,api:4,app:4,appar:4,append:3,appli:[3,4],applic:4,approach:4,appropri:[0,4],apptain:[1,3],apt:4,ar:[0,3,4],archiv:0,arg:0,argument:[0,4],argv:[3,4],around:4,arrai:0,art:4,asctim:3,ask:[3,4],assign:4,associ:[0,3,4],assum:[0,3,4],astyp:4,attach:[3,4],attack:4,attempt:0,attribut:[0,3],aug:4,authent:[3,4],authorized_kei:4,autom:[3,4],automat:[0,3,4],avail:[0,3,4],azur:1,azureadmin:4,azureus:4,b:3,back:[3,4],base:[0,3,4],base_path:4,bash:4,bashrc:4,basic:3,basicconfig:3,batch:1,becam:4,becaus:[3,4],becom:4,been:[3,4],befor:[0,4],behavior:0,behaviour:4,being:[0,4],below:[0,3,4],benefici:3,best:3,better:[3,4],between:[0,3,4],bfimg:3,bi:1,biaflow:[3,4],biaflowsjob:[3,4],big:4,biggest:4,bimg:3,bin:[3,4],binari:4,bioformats2raw:3,bioimag:[1,4],biomerostorag:4,biomerowork:4,bit:4,bitbucket:3,bj:[3,4],blob:[0,4],blog:4,blosc:3,bonu:4,bool:[0,4],borrow:4,both:[3,4],boutiqu:3,box:4,branch:[0,3,4],brief:3,bring:4,bring_listener_uptod:0,browser:4,bsst265:4,bugfix:4,build:[0,3,4],builddir:4,built:[3,4],bump:4,button:4,c1:4,c2:4,cacert:4,calcul:1,call:[3,4],can:[0,3,4],cancel:0,candl:4,cannot:0,cat:4,caus:4,cc:4,cd:[0,3,4],cell:[0,3,4],cell_diamet:0,cellexpans:1,cellexpansion_job:4,cellexpansion_repo:4,cellnumgranul:4,cellpos:[0,1,3],cellpose_imag:4,cellpose_job:[3,4],cellpose_job_cpu:4,cellpose_job_gr:[3,4],cellpose_job_mem:[3,4],cellpose_job_tim:3,cellpose_mask:4,cellpose_repo:[3,4],cellpose_v1:4,cellposemask:4,cellprob_threshold:[3,4],cellprofil:[1,3],cellprofiler_job:4,cellprofiler_repo:4,cellprofiler_result:4,cellprofiler_spot:4,cellprofiler_spot_job:4,cellprofiler_spot_repo:4,cellprofiler_v1:4,cells_aggreg:4,cells_cellexpans:4,cellsgranuleslabel:4,cellsnucleilabel:4,cellular:[3,4],cellularimagingcf:3,centroid:4,certif:4,cetera:3,cf:4,challeng:4,chan:[3,4],chang:[3,4],channel:[0,3,4],chapter:[3,4],cheap:4,cheaper:4,check:[0,3,4],check_job_statu:0,checkout:4,chmod:4,choos:4,chosen:[3,4],chown:4,chunk:3,ci:4,class_nam:0,class_objseg:4,classmethod:0,clean:4,cleanup:0,cleanup_tmp_fil:0,cli:3,click:4,client:[0,1,3],clone:[0,3,4],close:4,closesess:4,cloud:[1,3],cluster:[0,3],cluster_kei:4,cmd:[3,4],cmdlist:0,code:[1,4],coffe:[3,4],color:4,column:[0,4],com:[0,3,4],combin:4,come:[3,4],comma:0,command:[0,3],commandlin:[3,4],comment:[3,4],commerci:4,commit:4,commmand:4,common:4,commonwl:3,commun:4,compat:3,complet:[0,4],compon:4,compos:4,comput:[3,4],concaten:0,concept:0,conf:4,config:[0,1,4],config_fil:0,config_omero_:3,config_omero_logging_level:3,config_omero_master_host:3,configfil:[0,3,4],configur:[0,3,4],congratul:[3,4],conn:0,connect:[0,3,4],connect_kwarg:0,connect_timeout:0,connectionlostexcept:4,consecut:0,consist:3,consol:[3,4],constant:4,constructor:0,contain:[0,1],container:3,container_nam:[3,4],content:[3,4],context:4,contextmanag:0,continu:4,contributor:4,control:4,conveni:3,convers:0,convert:[0,3,4],convert_cytype_to_omtyp:0,convert_job_arrai:4,convert_url:0,converter_imag:0,copi:[0,3,4],copy_zip_loc:[0,3],copyleft:4,copyright:4,core:[3,4],correct:4,correctli:[3,4],correspond:[0,3],cost:[3,4],could:[0,3,4],count:4,count_nonzero:4,cours:[3,4],cover:4,cow:4,cp:[3,4],cp_model:[0,3],cppipe:4,cpproj:4,cpu:[3,4],creat:[0,3],creating_bia_workflow_and_adding_to_biaflows_inst:4,creation:0,credenti:4,credit:4,creditcard:4,crop:3,crt:4,csv:4,ctld:4,ctrl:4,current:[0,3,4],custom:1,cv2:4,cwd:3,cyclecloud:4,cython:4,cytomin:[0,3,4],cytomine_:3,cytomine_host:4,cytomine_id_project:4,cytomine_id_softwar:4,cytomine_job:0,cytomine_private_kei:4,cytomine_public_kei:4,cytoplasm:4,cytyp:0,d:[0,3],da:4,daemon:4,danger:4,data:[0,1],data_loc:0,data_path:[0,3],databas:0,datadisk:3,datafil:4,datafram:4,dataset:[3,4],date:[3,4],db:1,dbd:4,dbformysql:4,dd:0,de:4,debug:[3,4],deepcel:[3,4],deepcell_job:4,deepcell_repo:4,def:4,defin:[3,4],definit:[0,4],deleg:4,delet:[3,4],demand:4,deni:4,depend:[0,3],depends_on:4,deploi:[3,4],deploy:4,depth:3,deriv:4,descend:0,describ:[3,4],descript:[3,4],descriptor:[0,1],desir:3,desktop:4,destin:4,destroi:4,detail:[3,4],detect:0,dev:[3,4],devel:3,develop:4,diamet:[0,3,4],dict:0,dictionari:0,did:4,didn:4,differ:[3,4],digicert:4,digicertglobalrootca:4,dimens:3,dir:[3,4],directli:[3,4],directori:[0,3,4],dirti:4,discuss:3,distinguish:0,distribut:4,dl:[0,4],dn:4,doc:4,docker:[0,1],dockerfil:3,dockerfile_slurmctld:4,dockerhub:[0,3],dockerhub_token:4,dockerhub_usernam:4,docstr:3,document:[0,4],doe:[0,4],doesn:[0,3,4],don:[3,4],done:[3,4],dot:4,down:[3,4],download:[0,3,4],drastic:4,drive:[3,4],drop:0,drwxr:4,drwxrwxr:4,ds:4,dtype:4,dunde:4,dure:0,dyn:4,dynam:4,e:[0,3],each:[0,3,4],earlier:3,easi:[3,4],easier:[3,4],easiest:[0,3,4],easili:4,eavesdrop:4,echo:[3,4],edit:4,editor:4,effect:4,effici:3,either:[0,3,4],elaps:4,els:[0,3],email:[0,3,4],embarrassingli:3,empti:[0,4],en:4,enabl:[0,3],enable_job_account:0,enable_job_progress:0,enable_workflow_analyt:0,encapsul:0,encount:0,encourag:[3,4],end:[0,3,4],end_tim:0,endtim:4,enforc:3,enhanc:4,enjoi:4,enough:[3,4],ensur:[0,3,4],enter:4,entri:[0,4],entrypoint:[3,4],env:[0,4],environ:[0,3,4],equal:3,equip:3,err_desc:3,error:[0,3,4],error_messag:0,escap:4,especi:[3,4],essenti:4,estim:4,et:3,etc:[0,3,4],etc_mung:4,etc_slurm:4,etric:4,euro:4,europ:4,even:[3,4],everi:[3,4],everybodi:4,ex:4,exampl:[0,3,4],excel:4,except:[0,4],exec:4,execut:[0,3,4],exercis:4,exist:[0,1,4],exit:[3,4],expect:[0,3,4],expens:[3,4],expire_tim:4,explain:4,explan:3,explicitli:3,explor:[3,4],expos:4,ext:4,extend:[0,3],extens:[0,4],extern:[3,4],extra:[1,3],extract:[0,1],extract_data_location_from_log:0,extract_job_id:0,extract_parts_from_url:0,ezomero:3,f:[0,3,4],fabric:[0,3,4],facil:3,fact:3,fail:[0,3,4],failur:0,fair:[1,3],fals:[0,4],faster:[3,4],favorit:4,featur:3,few:4,field:3,figur:3,fiji:3,file:[0,3],filenam:[0,3],filename_no_extens:3,filepath:4,fileserv:4,filetyp:0,fill:[0,4],filler:4,filter:[0,4],filter_filetyp:0,find:[0,3,4],fine:4,finish:3,fire:4,firewal:4,first:[0,3,4],fit:4,fix:[0,4],flag:[0,3,4],flexibl:[3,4],flexibleserv:4,fluorescens:4,folder:[0,3,4],folder_nam:0,folk:4,follow:[0,3,4],forc:0,force_upd:0,forget:4,form:[3,4],format:[0,3,4],forward:[3,4],forward_ag:0,found:[0,3,4],framework:3,free:[3,4],friendli:3,from:[0,1,3],from_cli:[3,4],from_config:[0,3,4],full:[0,4],fulli:4,fun:[3,4],funni:4,futur:[0,3,4],fw:4,g:[0,3,4],gain:4,gatewai:[0,4],gb:[3,4],gcloud:4,gcslurm:4,gener:[0,1,4],generate_job:[0,3],generate_slurm_job_for_workflow:0,get:[0,4],get_active_job_progress:0,get_all_image_versions_and_data_fil:[0,4],get_cellpose_command:0,get_conversion_command:0,get_error:0,get_image_versions_and_data_fil:0,get_job_status_command:0,get_jobs_info_command:0,get_listen:0,get_logfile_from_slurm:[0,3],get_or_create_github_sess:0,get_recent_log_command:0,get_unzip_command:0,get_update_slurm_scripts_command:0,get_workflow_command:0,get_workflow_paramet:0,get_zip_command:0,getenv:4,getinput:4,getlogg:4,ghpc:4,giant:4,gid:3,giovtorr:4,git:[0,3,4],github:[0,3],give:[0,3,4],given:[0,3,4],global:3,gnu:4,go:4,godlovedc:4,good:4,googl:1,gosu:4,got:4,gpu:[0,1,4],grab:[1,3],granuleslabel:4,granulesstat:4,granulesstatsincelllabel:4,granulesstatsnp:4,grayscal:3,gre:4,great:4,grei:3,grid:4,ground:3,group:4,groupadd:4,gt:4,gt_img:[3,4],gt_path:[3,4],gtfolder:[3,4],guid:[3,4],gz:4,ha:[0,3,4],had:4,half:4,hand:4,handl:[3,4],handout:4,hardcod:[3,4],hardwar:3,have:[0,3,4],headbash:4,header:0,headless:3,heard:4,hello:3,help:3,helper:4,here:[3,4],hh:[0,3],hidden:4,high:3,higher:3,highli:4,hoc:4,hold:4,home:[3,4],hope:4,hopefulli:[3,4],host:[0,3,4],hostnam:[3,4],hour:3,how:1,howev:[0,3,4],hpc:[3,4],hpcsmall:4,htc:4,html:[3,4],htop:4,http:[0,3,4],hub:4,human:4,i:[1,4],iap:4,ic:4,id:[0,3,4],id_rsa:[3,4],idea:4,ideal:4,identif:4,identityfil:4,idkei:4,idl:4,ignor:[3,4],ij:3,ij_min_threshold:3,ij_radiu:3,ijm:3,imag:[0,3],image_nam:0,image_path:3,image_tag:4,image_vers:[0,3],imageio:3,imagej:[3,4],imagej_job:4,imagej_repo:4,imcellscelllabel:4,imcellsgranuleslabel:4,img:3,implement:[0,3,4],improv:[3,4],imread:[3,4],imread_anydepth:4,imwrit:3,in_img:[3,4],in_path:[3,4],inbound:4,incl:3,includ:[0,3,4],incur:3,indefinit:3,index:[0,1],indic:0,individu:0,infinit:4,info:[0,3,4],infold:[3,4],inform:[0,3,4],ingredi:4,ingress:4,ini:[0,1,4],init:[3,4],init_environ:4,init_slurm:[0,3,4],init_workflow:0,initi:[0,4],initialis:4,initialize_analytics_system:0,initscript:4,inject:3,inline_ssh_env:0,innodb_lock_wait_timeout:4,input:[0,3,4],input_data:0,input_fold:3,input_url:0,insert:4,insid:4,insight:[3,4],instal:[1,3],instanc:[0,3,4],instanti:[0,4],instead:[0,3,4],insuffici:4,int16:4,integr:[3,4],interact:[0,3,4],interfac:3,intern:4,internet:4,interoper:4,interv:0,introduct:1,invest:4,invok:0,involv:0,io:[3,4],ip:4,is_2d:[3,4],iso:0,isol:3,issu:[0,3,4],its:[0,3,4],jar:3,java:3,job:[0,1,4],job_id:0,job_stat:0,job_templ:[0,3,4],jobid:[0,4],jobnam:4,jobscript:[3,4],join:[3,4],json:[0,1],jump:4,just:[3,4],keep:[3,4],kei:[0,3,4],keygen:4,keyword:0,kickstart:4,kind:4,know:4,knowledg:4,known:[0,4],known_host:3,kwarg:0,la:4,last:[0,4],later:[0,4],latest:[0,3,4],launch:[3,4],law:4,lead:4,learn:4,least:3,leav:4,left:[3,4],less:[3,4],let:4,lethal:3,level:3,levelnam:3,lib:3,libgeo:4,librari:[0,3,4],licens:[1,3],lightweight:4,like:[0,3,4],limit:[0,1,4],line:[0,3,4],linear:4,link:[0,4],linux:4,lisbon:4,list:[0,4],list_active_job:0,list_all_job:0,list_available_converter_vers:0,list_completed_job:0,listen:0,liter:[3,4],ll:4,load:[0,4],local:[0,1,3],local_path:0,local_tmp_storag:0,localhost:[3,4],localslurm:4,locat:[0,3,4],log:[0,1,4],log_fil:0,logfil:[0,3,4],logger:[0,4],logic:4,login:[3,4],logout:4,lolcow:4,longer:[3,4],look:[0,3,4],lose:3,lot:[3,4],lower:4,ls:4,lt:4,luckili:4,luik:4,m:[3,4],mac:4,machin:[0,3,4],macro:3,made:[3,4],mage:3,magic:4,mai:4,mail:[3,4],main:[3,4],mainli:[3,4],major:4,make:[3,4],man:4,manag:[0,1,3],mandatori:[3,4],manipul:4,manual:3,map:[0,4],mar:4,march:4,mask:[1,3],master:[0,3,4],match:[0,3,4],matter:4,max:[3,4],maximum:[0,3],maxshap:3,maxvm:4,mayb:[3,4],mconfig:4,md:4,me:4,mean:4,meant:4,measur:4,mechan:0,medium:3,mem:4,memori:3,mention:0,merchant:4,messag:[0,3,4],metadata:1,method:[0,3,4],metric:[3,4],microscopi:4,microsoft:1,middl:4,might:[0,3,4],min:[3,4],min_threshold:3,mine:4,minim:[3,4],minshap:3,minut:[3,4],miss:[3,4],mkdir:4,mlflow:3,mm:[0,3],mnt:3,mod_pipelin:4,mode:4,model:[0,3,4],model_path:3,modern:4,modifi:4,modul:[1,2,3,4],module_nam:0,moment:4,monei:4,monitor:[0,3],month:4,more:[0,3,4],most:[0,4],mount:[3,4],move:3,movi:4,ms:4,much:4,multipl:[0,1,4],mung:4,must:[0,4],my:[0,3,4],myjob:0,mysql:4,n:[0,4],name:[0,3,4],namespac:4,nasti:4,ndarrai:4,ndata:4,necessari:[0,3],need:[0,3,4],net:4,network:3,neubia:[3,4],newest:4,newli:4,next:[0,3,4],nextflow:4,nf:0,ngc:4,nice:[3,4],nicer:3,nj:3,nl:[3,4],nmc:4,no_npi:[3,4],node:[3,4],nodelist:4,nodenam:4,nologin:4,non:[3,4],none:[0,3],normal:4,note2:3,note:[0,3,4],noth:0,notif:0,now:[0,3,4],np:4,nsdynam:4,nuc:[3,4],nuc_channel:[0,3,4],nuclear:0,nuclei:[0,4],nucleilabel:4,nucleu:4,num:4,number:[0,4],numcel:4,o:[0,1,4],object:[0,3],oblig:4,occur:0,off:4,offer:3,offload:4,ok:[0,4],old:[0,4],older:4,om:3,omero:[0,1],omero_worker_nam:3,omeroconn:0,omeroinsight:4,omeroserv:3,omeroweb:4,omerowork:4,omit:[0,3],omput:4,onc:[3,4],one:[0,3,4],ones:[0,4],onli:[0,3],onlin:4,onto:4,oo:4,op:4,open:4,openssh:4,oper:[0,3,4],opt:[3,4],option:[0,1,3],orchestr:4,order:0,origin:[0,3,4],original_fil:4,os:[3,4],osc:3,oserror:0,other:[0,3,4],otherwis:[0,3,4],our:[3,4],out:[0,3],out_path:[3,4],outfold:[3,4],output:[0,3,4],outsid:4,over:[0,3,4],overal:3,overhead:[3,4],overlap:1,overrid:[3,4],overview:1,overwrit:[0,4],own:[1,4],ownership:4,p7zip:4,p:4,packag:[2,3],package_upgrad:4,page:4,paid:4,pair:[0,4],parallel:[3,4],param:[0,3],param_nam:4,paramet:[0,1],paramiko:0,pars:[0,4],parse_cellprofiler_paramet:4,parse_docker_image_vers:0,parsecpparam:4,part:[0,4],particularli:[0,4],partit:4,pass:0,password:4,passwordless:3,past:4,path:[0,3,4],pattern:0,pd:4,peer:4,pem:4,pend:0,peopl:4,per:[0,3,4],perform:[0,3,4],perhap:[0,3,4],permiss:4,pick:4,pickup:4,pin:3,pip:[3,4],pipelin:[1,3],pitfal:3,pixel:0,pla:4,pla_data:4,place:3,placehold:[0,4],plai:4,plate:4,platform:3,pleas:[3,4],plugin:[3,4],png:4,podman:[3,4],point:[3,4],poll:[0,3],port:[0,3,4],portal:4,posit:0,possibl:[3,4],potenti:3,power:4,ppa:4,practic:[3,4],predict:3,prefix:4,prepar:[3,4],prepare_data:[3,4],preprocess:3,prerequisit:1,present:0,press:4,pretrained_model:[3,4],pretti:4,previou:3,price:4,primit:3,print:[0,3,4],prioriti:4,privat:[3,4],prob:3,prob_thresh:3,prob_threshold:[0,3,4],probabl:[0,4],problem:3,problem_cl:[3,4],proceed:0,process:[0,3,4],processor:[3,4],profil:4,program:[3,4],programmat:3,progress:[0,3,4],project:4,promot:4,promptless:4,proper:4,properli:[0,4],properti:4,protein:4,protocol:4,provid:[0,3,4],pub:[3,4],pubkei:3,publish:3,pull:[0,3,4],pull_descriptor_from_github:0,pull_imag:4,purpos:[0,3,4],push:4,put:[3,4],pw:4,py:[0,1],pypi:3,pytest:3,python3:[3,4],python:[0,1,4],quantifi:4,queri:0,queue:4,quick:[3,4],quicker:4,quickstart:1,quit:4,r:[0,4],radiu:3,rais:[0,3],ran:4,rang:4,raw:[0,4],re:4,reach:[0,4],read:[0,3],readabl:4,reader:4,readi:4,readm:4,realli:4,realmemori:4,reason:4,rebuild:4,receiv:0,recent:[0,4],recogn:4,recommend:3,recreat:[0,4],reduc:4,refer:[0,3,4],referenc:3,refresh:[3,4],region:4,regionprops_t:4,registr:4,rel:4,relat:[0,3,4],releas:[3,4],remain:[0,4],remot:[0,3,4],remov:[0,4],renam:4,replac:[3,4],repo:[0,3,4],repositori:[0,3],repres:0,reproduc:[3,4],request:[3,4],requir:[0,1],reserv:4,reset_t:0,resiz:3,resourc:[3,4],respons:0,rest:4,restart:4,restrict:4,result:[0,3,4],retain:[0,4],retri:0,retriev:[0,3,4],return_cod:3,returncod:3,reus:4,review:3,rewrit:3,rewrot:4,rgb:3,rich:4,right:4,rm:[3,4],rocki:4,rockylinux:4,role:4,root:4,rootless:4,round:4,row:4,rs:0,rstring:4,rtype:4,rule:4,run:[0,1,3],run_command:0,run_commands_split_out:0,run_conversion_workflow_job:0,run_workflow:[0,4],run_workflow_job:0,runcmd:4,runner:[0,4],runscript:[3,4],runtim:3,rw:4,rwxr:4,s:[0,3,4],sacct:[0,4],sadli:4,safe:4,sai:[3,4],same:[0,3,4],satisfi:4,save:4,save_tif:[3,4],sbatch:[0,3,4],sbin:4,sc:3,scale:4,scenario:3,schedmd:3,schedul:[3,4],schema:[1,4],scheme:3,scp:[0,3,4],scratch:[0,4],screen:[3,4],script:[0,1],scroll:4,second:[0,4],secret:[3,4],section:[3,4],secur:4,see:[0,1,4],seem:4,segment:[3,4],select:[3,4],send:[0,3],sep:0,separ:[0,4],seriou:4,server:[0,3,4],servic:4,session:0,set:[0,1,3],setoutput:4,setup:[0,1,3],setup_container_imag:0,setup_convert:0,setup_directori:0,setup_job_script:0,setup_listen:0,setup_slurm:0,sever:[3,4],sftp:3,sh:[0,3,4],shakespear:4,shame:4,shape:3,share:[0,3,4],sharg:4,shell:[0,3,4],shenanigan:4,shorter:3,should:[1,3],shoulder:4,show:[3,4],shown:4,showtim:1,shutil:3,side:4,sif:[0,3,4],sign:4,similar:[0,4],simpl:[0,3,4],simpli:3,simplifi:[3,4],sinc:4,sing:4,singl:0,singular:[0,1,3],singularity_imag:[0,3,4],singularity_vers:4,size:4,sleep:4,slow:3,slurm:[0,1],slurm_cellpose_segment:3,slurm_client:[1,2],slurm_config:4,slurm_converters_path:0,slurm_data_path:[0,3,4],slurm_images_path:[0,3,4],slurm_job_id:0,slurm_jobdir:4,slurm_model_imag:0,slurm_model_job:0,slurm_model_jobs_param:0,slurm_model_path:0,slurm_model_repo:0,slurm_polling_interv:0,slurm_run_workflow_batch:4,slurm_script_path:[0,3,4],slurm_script_repo:[0,3,4],slurmclient:[0,1,4],slurmctld:4,slurmdbd:4,slurmjob:0,small:4,smaller:[3,4],smart:[3,4],snakemak:4,so:[3,4],softwar:[3,4],solut:4,some:[0,3,4],someon:4,someth:[3,4],sort:0,sound:4,sourc:[0,3,4],source_format:0,south:4,space:[0,4],specif:[0,3,4],specifi:[0,3],speckl:4,speed:[3,4],speedup:4,spent:4,spin:[3,4],split:[0,4],spot:4,spotcount:4,spun:4,sqlalchemy_url:0,squeue:4,ss:[0,3],ssh:[0,1],sshd:4,sshdocker:4,sshexcept:0,sshgroup:4,sshuser:4,ssl:4,st:4,stai:[3,4],stain:4,stand:4,standard:3,standard_dc2s_v3:4,standard_dc4s_v3:4,standard_nd96amsr_a100_v4:4,stardist:[3,4],stardist_job:4,stardist_repo:4,start:[0,4],start_tim:0,starttim:4,state:[0,4],statu:[0,3,4],statuscom:[3,4],stdout:[0,3,4],step:[3,4],stick:4,still:[0,4],storag:[0,4],storageaccount:4,store:[0,3,4],str:[0,4],str_to_class:0,stream:3,stricthostkeycheck:4,strictli:3,string:[0,3,4],strong:3,strongli:4,stuff:4,style:4,sub:0,subdirectori:4,subfold:[0,4],subject:4,submiss:[0,3,4],submit:[0,3,4],submit_result:0,submitit:3,subprocess:[3,4],subscript:4,substitut:0,succeed:0,success:[0,4],successfulli:0,sudo:4,suffix:4,suit:4,superus:4,suppli:4,support:[0,3,4],sure:[3,4],sweet:4,sy:3,system:[0,4],systemctld:4,t:[0,3,4],t_nucleisegment:4,t_t_luik:4,t_t_luik_amsterdamumc_nl:4,tab:[3,4],tabl:4,tag:[0,3,4],tail:[3,4],take:[0,3,4],tar:4,target:[0,3],target_format:0,task:[0,3,4],task_id:0,tbc:3,tcp:4,tell:4,templat:[0,3,4],temporari:[0,4],term:4,termin:[3,4],test:1,testproject:4,text:[3,4],than:[3,4],thei:[3,4],them:[0,3,4],themselv:[0,4],theoret:3,theori:[3,4],thi:[0,1,3],thing:4,think:[3,4],third:4,those:[0,3,4],though:4,thought:1,threshold:[0,3,4],through:[3,4],thrown:4,tick:4,tif:[0,3,4],tiff:[0,3,4],tifffil:3,time:[0,1,4],timelimit:4,timeout:[0,3],tischi:4,tmp:[0,3],tmp_path:[3,4],to_numpi:4,togeth:[3,4],token:4,too:1,took:4,tool:0,top:4,torec:4,torecluik:[3,4],total:4,touch:4,toward:3,track:[0,4],track_workflow:0,trailer:0,transfer:[0,1],transfer_data:[0,3],transferresult:0,tree:[3,4],trick:4,troubl:4,trust:[3,4],truth:3,truthi:0,tue:4,tune:4,tupl:0,turn:[3,4],twice:4,two:0,txt:[3,4],type:[0,4],u:4,ubuntu:4,ui:[1,3],uid:3,uint16:4,uk:4,ulieg:[0,4],umc:[3,4],uncheck:[3,4],under:[3,4],understand:4,uniform:3,uniqu:[0,3,4],univers:4,unknown:4,unless:[3,4],unlik:0,unpack:[0,3],unpack_data:[0,3],until:[0,3],unvers:0,unwrap:4,unzip:[0,3],up:[0,1,3],updat:[0,3],update_slurm_script:[0,3],upgrad:4,upload:[1,3],url:[0,4],us:[0,1],use_gpu:[0,3,4],user:[0,3,4],useradd:4,userknownhostsfil:4,usern:3,usernam:[3,4],usr:[3,4],utf:4,util:[3,4],util_script:4,uuid:0,v0:4,v1:[3,4],v2:4,v3:4,v:4,valid:[0,3,4],validate_slurm_setup:[0,3,4],valu:[0,3,4],valueerror:[0,3],var_log_slurm:4,variabl:[0,3,4],variou:3,ve:4,venic:4,venv:[3,4],venvtest:3,veri:[3,4],verifi:4,version:[0,1,4],vi:4,via:[0,1,4],video:4,view:[0,4],vm:4,vnet:4,volum:[3,4],vs:4,w:[3,4],w_cellexpans:4,w_cellexpansion_v2:4,w_countmaskoverlap:4,w_countmaskoverlap_v1:4,w_nucleisegment:[3,4],w_spotcount:4,wa:[0,4],wai:[0,3,4],wait:[0,3,4],wait_for_complet:0,walk:4,want:[3,4],warn:[0,3,4],watch:3,we:[0,1,3],web:[1,3],websit:[3,4],wed:4,well:[3,4],were:4,western:4,wf:4,wf_id:0,wg5:4,wget:4,what:[0,3,4],whatev:4,when:[0,3,4],whenev:4,where:[0,3,4],whether:0,which:[0,3,4],whole:4,william:4,window:[3,4],within:[0,3,4],without:[0,4],won:4,work:[0,3,4],worker:4,workflow:[0,1],workflow_nam:0,workflow_params_to_envvar:0,workflow_params_to_sub:0,workflow_vers:0,workload:[0,4],world:3,worth:0,would:[3,4],wq:4,wrap:[3,4],wrapper:1,write:[3,4],wrong:4,wrote:4,x:[0,4],xmx6000m:3,xr:4,xvfb:3,xzf:4,y:[3,4],ye:4,yet:[0,3,4],yml:[0,3,4],you:[0,3,4],your:[0,1],yourself:3,yum:[3,4],yyyi:0,zarr:[0,3],zero:4,zip:[0,3,4],zip_data_on_slurm_serv:[0,3],zipfil:[0,4]},titles:["biomero package","Welcome to BIOMERO\u2019s documentation!","biomero","BIOMERO - BioImage analysis in OMERO","Cellprofiler tutorial"],titleterms:{"0":4,"1":4,"2":4,"3":4,"3b":4,"4":4,"5":4,"6":4,"7zip":4,"class":3,"export":4,"import":4,"new":3,"try":4,A:4,access:4,account:4,add:[3,4],addendum:4,altern:4,an:3,analysi:3,apptain:4,azur:4,b:4,basic:4,batch:[3,4],bi:4,bioimag:3,biomero:[0,1,2,3,4],c:4,calcul:4,cellexpans:4,cellpos:4,cellprofil:4,client:4,cloud:4,cluster:4,code:3,command:4,config:3,contain:[3,4],creat:4,custom:3,d:4,data:[3,4],databas:4,db:4,descriptor:[3,4],docker:[3,4],dockerfil:4,dockerhub:4,document:1,dr:4,e:4,exist:3,extra:4,extract:4,fair:4,file:4,from:4,gener:3,get:[1,3],github:4,googl:4,gpu:3,grab:4,headless:4,how:[3,4],i:3,imag:4,indic:1,ini:3,instal:4,introduct:4,job:3,json:[3,4],licens:4,limit:3,local:4,log:3,manag:4,manual:4,mask:4,memori:4,metadata:[3,4],microsoft:4,modul:0,multipl:3,network:4,o:3,omero:[3,4],onli:4,option:4,out:4,overlap:4,overview:3,own:3,packag:[0,1,4],paramet:[3,4],pipelin:4,prerequisit:[3,4],publish:4,py:[3,4],python:3,quickstart:3,repositori:4,requir:[3,4],run:4,s:1,schema:3,script:[3,4],see:3,set:4,setup:4,should:4,showtim:4,singular:4,slurm:[3,4],slurm_client:0,slurmclient:3,ssh:[3,4],start:[1,3],subnet:4,suggest:4,tabl:1,test:[3,4],thi:4,thought:4,time:3,tl:4,too:4,transfer:3,tutori:[1,3,4],ui:4,up:4,updat:4,upload:4,us:[3,4],version:3,via:3,virtual:4,we:4,web:4,welcom:1,workflow:[3,4],wrapper:[3,4],your:[3,4]}})
\ No newline at end of file
diff --git a/tutorial_link.html b/tutorial_link.html
index b26498e..477b721 100644
--- a/tutorial_link.html
+++ b/tutorial_link.html
@@ -1990,10 +1990,10 @@ 
            5. Setting up (BI)OMERO in Azure too (Optional)ssh localslurm (or whatever you called the alias)
            
 
          
 
            
-
          1. Let’s edit the BIOMERO configuration slurm-config.ini, located in the worker-processor node
            2. 
+
          2. Let’s edit the BIOMERO configuration slurm-config.ini, located in the biomeroworker node
            3. 
 
          
 
            
-
          • vi ~/NL-BIOMERO/worker-processor/slurm-config.ini
            • 
+
          • vi ~/NL-BIOMERO/biomeroworker/slurm-config.ini
            • 
 
          • Change the host if you did not use the localslurm alias in the config above.
            • 
 
          • Change ALL the [SLURM] paths to match our new slurm setup:
            • 
 
          
@@ -2040,7 +2040,7 @@ 
          5. Setting up (BI)OMERO in Azure too (Optional)cd NL-BIOMERO
          • 
 
        • docker compose down
          • 
 
        • docker compose up -d --build
          • 
-
        • Now docker logs -f nl-biomero-omeroworker-processor-1 should show some good logs leading to: Starting node omeroworker-processor.
          • 
+
        • Now docker logs -f nl-biomero-biomeroworker-1 should show some good logs leading to: Starting node biomeroworker.
          •