From 737de290225ca6caf83565dc08a2b6ee8a97959a Mon Sep 17 00:00:00 2001 From: Soline Date: Tue, 7 Nov 2023 16:21:49 -0500 Subject: [PATCH] Update the Slurm version retrieving in Slurm parsers --- slurm_state/helpers/clusters_helper.py | 1 + slurm_state/mongo_update.py | 8 +++--- slurm_state/parsers/job_parser.py | 8 +++--- slurm_state/parsers/node_parser.py | 6 ++--- slurm_state/parsers/slurm_parser.py | 36 +++++++++++++++++++------- test_config.toml | 2 ++ 6 files changed, 41 insertions(+), 20 deletions(-) diff --git a/slurm_state/helpers/clusters_helper.py b/slurm_state/helpers/clusters_helper.py index ce65d219..c17a8293 100644 --- a/slurm_state/helpers/clusters_helper.py +++ b/slurm_state/helpers/clusters_helper.py @@ -65,6 +65,7 @@ def _load_clusters_from_config(): clusters_valid.add_field("sacct_path", optional_string) clusters_valid.add_field("sinfo_path", optional_string) + clusters_valid.add_field("slurm_version", optional_string, default=None) # Load the clusters from the configuration file, asserting that it uses the # predefined format diff --git a/slurm_state/mongo_update.py b/slurm_state/mongo_update.py index 8d60e6f8..2e9d0471 100644 --- a/slurm_state/mongo_update.py +++ b/slurm_state/mongo_update.py @@ -139,6 +139,10 @@ def main_read_report_and_update_collection( # Initialize the time of this operation's beginning timestamp_start = time.time() + # Retrieve clusters data from the configuration file + clusters = get_all_clusters() + assert cluster_name in clusters + # Check the input parameters assert entity in ["jobs", "nodes"] @@ -164,10 +168,6 @@ def main_read_report_and_update_collection( f'Incorrect value for entity in main_read_sacct_and_update_collection: "{entity}" when it should be "jobs" or "nodes".' ) - # Retrieve clusters data from the configuration file - clusters = get_all_clusters() - assert cluster_name in clusters - ## Retrieve entities ## # Generate a report file if required diff --git a/slurm_state/parsers/job_parser.py b/slurm_state/parsers/job_parser.py index f780d2da..d186992d 100644 --- a/slurm_state/parsers/job_parser.py +++ b/slurm_state/parsers/job_parser.py @@ -22,8 +22,8 @@ class JobParser(SlurmParser): """ """ - def __init__(self, cluster_name): - super().__init__("jobs", "sacct", cluster_name) + def __init__(self, cluster_name, slurm_version=None): + super().__init__("jobs", "sacct", cluster_name, slurm_version=slurm_version) def generate_report(self, file_name): @@ -58,9 +58,9 @@ def generate_report(self, file_name): def parser(self, f): """ """ - if re.search(r"^slurm 22\..*$", self.slurm_version): + if re.search(r"^22\..*$", self.slurm_version): return self.parser_v22_and_23(f) - elif re.search(r"^slurm 23\..*$", self.slurm_version): + elif re.search(r"^23\..*$", self.slurm_version): return self.parser_v22_and_23(f) else: raise Exception( diff --git a/slurm_state/parsers/node_parser.py b/slurm_state/parsers/node_parser.py index 02686a8d..74e4ee36 100644 --- a/slurm_state/parsers/node_parser.py +++ b/slurm_state/parsers/node_parser.py @@ -16,8 +16,8 @@ class NodeParser(SlurmParser): """ """ - def __init__(self, cluster_name): - super().__init__("nodes", "sinfo", cluster_name) + def __init__(self, cluster_name, slurm_version=None): + super().__init__("nodes", "sinfo", cluster_name, slurm_version=slurm_version) def generate_report(self, file_name): # The command to be launched through SSH is "sinfo --json" @@ -27,7 +27,7 @@ def generate_report(self, file_name): def parser(self, f): """ """ - if re.search("^slurm 22\..*$", self.slurm_version): + if re.search(r"^22\..*$", self.slurm_version): return self.parser_v22(f) else: raise Exception( diff --git a/slurm_state/parsers/slurm_parser.py b/slurm_state/parsers/slurm_parser.py index edf3868a..56c9bb62 100644 --- a/slurm_state/parsers/slurm_parser.py +++ b/slurm_state/parsers/slurm_parser.py @@ -3,7 +3,7 @@ from slurm_state.helpers.clusters_helper import get_all_clusters # Common imports -import os +import os, re class SlurmParser: @@ -11,7 +11,7 @@ class SlurmParser: A parser for Slurm entities """ - def __init__(self, entity, slurm_command, cluster_name): + def __init__(self, entity, slurm_command, cluster_name, slurm_version=None): self.entity = entity assert entity in ["jobs", "nodes"] @@ -31,18 +31,36 @@ def __init__(self, entity, slurm_command, cluster_name): self.slurm_command ), f"Error. The {self.slurm_command}_path configuration needs to end with '{self.slurm_command}'. It is currently {self.slurm_command_path} ." - # Retrieve the version of Slurm installed on the current cluster - self.slurm_version = self.get_slurm_version() + if slurm_version is not None: + self.slurm_version = slurm_version + else: + # If no Slurm version is provided, retrieve the version of Slurm installed on the current cluster + self.slurm_version = self.get_slurm_version() def get_slurm_version(self): """ Get the Slurm version """ - # Launch the sacct or sinfo command to get its version - remote_command = f"{self.slurm_command_path} -V" - response = self.launch_slurm_command(remote_command) - assert len(response) == 1 - return response[0] + if ( + "slurm_version" in self.cluster + and self.cluster["slurm_version"] is not None + ): + # If the Slurm version has been added to the configuration file, + # return the value of the configuration + return self.cluster["slurm_version"] + else: + print("3") + # Launch the sacct or sinfo command to get its version + remote_command = f"{self.slurm_command_path} -V" + response = self.launch_slurm_command(remote_command) + assert len(response) == 1 + version_regex = re.compile(r"^slurm (\d+\.\d+\.\d+)$") + if m := version_regex.match(response): + return m.group(1) + # If the version has not been identified, raise an error + raise Exception( + f'The version "{response}" has not been recognized as a Slurm version.' + ) def launch_slurm_command(self, remote_command): """ """ diff --git a/test_config.toml b/test_config.toml index 551ddd70..3d3b2e91 100644 --- a/test_config.toml +++ b/test_config.toml @@ -53,6 +53,7 @@ remote_hostname="clockwork-stats" sacct_enabled=true sacct_path="/opt/slurm/bin/sacct" sinfo_path="/opt/slurm/bin/sinfo" +slurm_version="22.05.9" ssh_key_filename="id_clockwork" ssh_port=22 @@ -110,6 +111,7 @@ remote_hostname="cedar.computecanada.ca" sacct_enabled=false sacct_path="/opt/software/slurm/bin/sacct" sinfo_path="/opt/software/slurm/bin/sinfo" +slurm_version="23.02.6" ssh_key_filename="id_clockwork" ssh_port=22