Merge remote-tracking branch 'upstream/master' into cylc-remove

.github/workflows/shortlog.yml

@@ -17,6 +17,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check CONTRIBUTING.md
         uses: cylc/release-actions/check-shortlog@v1

.github/workflows/test_fast.yml

@@ -150,7 +150,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ github.workflow }}
           flags: fast-tests

.github/workflows/test_functional.yml

@@ -335,7 +335,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ github.workflow }}
           flags: functional-tests

.mailmap

@@ -57,3 +57,4 @@ Utheri Wagura              <[email protected]>
 github-actions[bot]        <[email protected]>                      <41898282+github-actions[bot]@users.noreply.github.com>
 github-actions[bot]        <[email protected]>  GitHub Action
 Diquan Jabbour             <[email protected]>
+Maxime Rio                 <[email protected]>

CONTRIBUTING.md

@@ -29,7 +29,7 @@ We use [semver](https://semver.org/) to separate riskier changes (e.g. new featu
 (e.g. 8.1, 8.2, 8.3)
 
 **Bugfixes** and minor usability enhancements are made on bugfix branches and
-released as the next maintainance version (e.g. 8.0.1, 8.0.2, 8.0.3). E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to
+released as the next maintenance version (e.g. 8.0.1, 8.0.2, 8.0.3). E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to
 develop your bugfix, then raise the pull request against the `8.0.x` branch. We will later merge the `8.0.x` branch into `master`.
 
 Feel free to ask questions on the issue or
@@ -96,6 +96,8 @@ requests_).
  - Diquan Jabbour
  - Shixian Sheng
  - Utheri Wagura
+ - Maxime Rio
+ - Paul Armstrong
  - Paul Earnshaw
 <!-- end-shortlog -->
 

changes.d/6081.fix.md

@@ -0,0 +1,2 @@
+Fix job submission when a batch of jobs is submitted to a runner that does
+not return a newline with the job ID (did not affect built-in job runners).

changes.d/6440.feat.md

@@ -0,0 +1 @@
+The "cylc dump" command now prints task IDs. Use "--legacy" if you need the old format.

changes.d/6456.feat.md

@@ -0,0 +1 @@
+`cylc lint` now checks for unnecessary continuation characters in the graph section.

changes.d/6475.feat.md

@@ -0,0 +1 @@
+Allow easy definition of multiple install targets in `global.cylc[install][symlink dirs]` using comma separated lists.

changes.d/6476.break.md

@@ -0,0 +1 @@
+Remove support for the EmPy template engine.

changes.d/6491.feat.md

@@ -0,0 +1 @@
+The "cylc show" command now says if the target task is held, queued, or runahead limited.

conda-environment.yml

@@ -21,7 +21,6 @@ dependencies:
   - tomli >=2 # [py<3.11]
 
 # optional dependencies
-  #- empy >=3.3,<3.4
   #- pandas >=1.0,<2
   #- pympler
   #- matplotlib-base

cylc/flow/cfgspec/globalcfg.py

@@ -18,7 +18,7 @@
 import os
 from pathlib import Path
 from sys import stderr
-from textwrap import dedent
+from textwrap import dedent, indent
 from typing import List, Optional, Tuple, Any, Union
 
 from contextlib import suppress
@@ -588,6 +588,47 @@
     '''
 }
 
+
+def comma_sep_section_note(version_changed: str = '') -> str:
+    note_text = "This section can be a comma separated list."
+    if version_changed:
+        note_text = (
+            f".. versionchanged:: {version_changed}\n\n" +
+            indent(note_text, 3 * ' ')
+        )
+
+    example = dedent('''
+
+    .. spoiler:: Example
+
+       For example:
+
+       .. code-block:: cylc
+
+          [a, b]
+              setting = x
+          [a]
+              another_setting = y
+
+       Will become:
+
+       .. code-block:: cylc
+
+          [a]
+              setting = x
+          [b]
+              setting = x
+          [a]
+              another_setting = y
+
+       Which will then be combined according to
+       :ref:`the rules for Cylc config syntax<syntax>`.
+
+    ''')
+
+    return "\n\n.. note::\n\n" + indent(note_text + example, 3 * ' ')
+
+
 # ----------------------------------------------------------------------------
 
 
@@ -1132,9 +1173,12 @@ def default_for(
 
             .. versionadded:: 8.0.0
         """):
-            with Conf('<install target>', desc="""
+            with Conf('<install target>', desc=dedent("""
                 :ref:`Host <Install targets>` on which to create the symlinks.
-            """):
+
+                .. versionadded:: 8.0.0
+
+            """) + comma_sep_section_note(version_changed='8.4.0')):
                 Conf('run', VDR.V_STRING, None, desc="""
                     Alternative location for the run dir.
 
@@ -1176,7 +1220,9 @@ def default_for(
 
         .. versionadded:: 8.0.0
     '''):
-        with Conf('<platform name>', desc='''
+        with Conf(
+            '<platform name>',
+            desc=dedent('''
             Configuration defining a platform.
 
             Many of these settings have replaced those of the same name from
@@ -1220,8 +1266,10 @@ def default_for(
                - :ref:`AdminGuide.PlatformConfigs`, an administrator's guide to
                  platform configurations.
 
-            .. versionadded:: 8.0.0
-        ''') as Platform:
+        ''')
+            + comma_sep_section_note()
+            + ".. versionadded:: 8.0.0",
+        ) as Platform:
             with Conf('meta', desc=PLATFORM_META_DESCR):
                 Conf('<custom metadata>', VDR.V_STRING, '', desc='''
                     Any user-defined metadata item.
@@ -1987,6 +2035,7 @@ def load(self) -> None:
         self.dense.clear()
         LOG.debug("Loading site/user config files")
         conf_path_str = os.getenv("CYLC_CONF_PATH")
+
         if conf_path_str:
             # Explicit config file override.
             fname = os.path.join(conf_path_str, self.CONF_BASENAME)
@@ -1999,7 +2048,7 @@ def load(self) -> None:
 
         # Expand platforms needs to be performed first because it
         # manipulates the sparse config.
-        self._expand_platforms()
+        self._expand_commas()
 
         # Flesh out with defaults
         self.expand()
@@ -2043,8 +2092,8 @@ def _no_platform_group_name_overlap(self):
                     msg += f'\n * {name}'
                 raise GlobalConfigError(msg)
 
-    def _expand_platforms(self):
-        """Expand comma separated platform names.
+    def _expand_commas(self):
+        """Expand comma separated section headers.
 
         E.G. turn [platforms][foo, bar] into [platforms][foo] and
         platforms[bar].
@@ -2053,6 +2102,12 @@ def _expand_platforms(self):
             self.sparse['platforms'] = expand_many_section(
                 self.sparse['platforms']
             )
+        if (
+            self.sparse.get("install", {}).get("symlink dirs")
+        ):
+            self.sparse["install"]["symlink dirs"] = expand_many_section(
+                self.sparse["install"]["symlink dirs"]
+            )
 
     def platform_dump(
         self,

cylc/flow/config.py

@@ -80,7 +80,6 @@
     get_cylc_run_dir,
     is_relative_to,
 )
-from cylc.flow.print_tree import print_tree
 from cylc.flow.task_qualifiers import ALT_QUALIFIERS
 from cylc.flow.simulation import configure_sim_modes
 from cylc.flow.subprocctx import SubFuncContext
@@ -1604,18 +1603,6 @@ def get_linearized_ancestors(self):
     def get_first_parent_descendants(self):
         return self.runtime['first-parent descendants']
 
-    @staticmethod
-    def define_inheritance_tree(tree, hierarchy):
-        """Combine inheritance hierarchies into a tree structure."""
-        for rt_ in hierarchy:
-            hier = copy(hierarchy[rt_])
-            hier.reverse()
-            cur_tree = tree
-            for item in hier:
-                if item not in cur_tree:
-                    cur_tree[item] = {}
-                cur_tree = cur_tree[item]
-
     def add_tree_titles(self, tree):
         for key, val in tree.items():
             if val == {}:
@@ -1651,34 +1638,6 @@ def get_mro(self, ns):
             mro = ["no such namespace: " + ns]
         return mro
 
-    def print_first_parent_tree(self, pretty=False, titles=False):
-        # find task namespaces (no descendants)
-        tasks = []
-        for ns in self.cfg['runtime']:
-            if ns not in self.runtime['descendants']:
-                tasks.append(ns)
-
-        pruned_ancestors = self.get_first_parent_ancestors(pruned=True)
-        tree = {}
-        self.define_inheritance_tree(tree, pruned_ancestors)
-        padding = ''
-        if titles:
-            self.add_tree_titles(tree)
-            # compute pre-title padding
-            maxlen = 0
-            for namespace in pruned_ancestors:
-                items = copy(pruned_ancestors[namespace])
-                items.reverse()
-                for itt, item in enumerate(items):
-                    tmp = 2 * itt + 1 + len(item)
-                    if itt == 0:
-                        tmp -= 1
-                    if tmp > maxlen:
-                        maxlen = tmp
-            padding = maxlen * ' '
-
-        print_tree(tree, padding=padding, use_unicode=pretty)
-
     def process_workflow_env(self):
         """Export Workflow context to the local environment.
 

cylc/flow/cycling/__init__.py

@@ -100,7 +100,7 @@ def standardise(self, allow_truncated: bool = True) -> 'PointBase':
                missing off the front) will be tollerated, if False, truncated
                points will cause an exception to be raised.
         """
-        return self
+        raise NotImplementedError
 
     @abstractmethod
     def sub(self, other):
@@ -254,9 +254,6 @@ def sub(self, other):
         """Subtract other (interval) from self; return an interval."""
         pass
 
-    def is_null(self):
-        return (self == self.get_null())
-
     def __str__(self) -> str:
         # Stringify.
         return self.value

cylc/flow/etc/tutorial/consolidation-tutorial/flow.cylc

@@ -1,7 +1,4 @@
 #!jinja2
-[scheduler]
-    UTC mode = True
-
 [scheduling]
     # Start the workflow 7 hours before now ignoring minutes and seconds
     # * previous(T-00) takes the current time ignoring minutes and seconds.

cylc/flow/etc/tutorial/cylc-forecasting-workflow/flow.cylc

@@ -1,6 +1,4 @@
 #!jinja2
-[scheduler]
-    UTC mode = True
 [task parameters]
     # A list of the weather stations we will be fetching observations from.
     station = camborne, heathrow, shetland, aldergrove

cylc/flow/etc/tutorial/retries-tutorial/flow.cylc

@@ -1,6 +1,3 @@
-[scheduler]
-    UTC mode = True # Ignore DST
-
 [scheduling]
     [[graph]]
         R1 = start => roll_doubles => win

cylc/flow/etc/tutorial/runtime-introduction/flow.cylc

@@ -1,6 +1,4 @@
 #!jinja2
-[scheduler]
-    UTC mode = True
 [scheduling]
     initial cycle point = 2000-01-01T00Z
     final cycle point = 2000-01-01T06Z

cylc/flow/etc/tutorial/runtime-tutorial/flow.cylc

@@ -1,6 +1,5 @@
 #!jinja2
 [scheduler]
-    UTC mode = True
     allow implicit tasks = True  # TODO: remove at end of exercise
 
 [scheduling]

cylc/flow/job_runner_handlers/background.py

@@ -89,7 +89,7 @@ def submit(cls, job_file_path, submit_opts):
                 exc.filename = "nohup"
             return (1, None, str(exc))
         else:
-            return (0, "%d\n" % (proc.pid), None)
+            return (0, str(proc.pid), None)
 
 
 JOB_RUNNER_HANDLER = BgCommandHandler()

cylc/flow/job_runner_handlers/documentation.py

@@ -424,8 +424,7 @@ def submit(
             ret_code:
                 Subprocess return code.
             out:
-                Subprocess standard output, note this should be newline
-                terminated.
+                Subprocess standard output.
             err:
                 Subprocess standard error.
 

cylc/flow/job_runner_mgr.py

@@ -210,12 +210,10 @@ def jobs_kill(self, job_log_root, job_log_dirs):
                 self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code))
             # Note: Print STDERR to STDOUT may look a bit strange, but it
             # requires less logic for the workflow to parse the output.
-            if err.strip():
-                for line in err.splitlines(True):
-                    if not line.endswith("\n"):
-                        line += "\n"
-                    sys.stdout.write("%s%s|%s|%s" % (
-                        self.OUT_PREFIX_CMD_ERR, now, job_log_dir, line))
+            for line in err.strip().splitlines():
+                sys.stdout.write(
+                    f"{self.OUT_PREFIX_CMD_ERR}{now}|{job_log_dir}|{line}\n"
+                )
 
     def jobs_poll(self, job_log_root, job_log_dirs):
         """Poll multiple jobs.
@@ -303,13 +301,13 @@ def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False,
             sys.stdout.write("%s%s|%s|%d|%s\n" % (
                 self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code, job_id))
             for key, value in [("STDERR", err), ("STDOUT", out)]:
-                if value is None or not value.strip():
+                if value is None:
                     continue
-                for line in value.splitlines(True):
-                    if not value.endswith("\n"):
-                        value += "\n"
-                    sys.stdout.write("%s%s|%s|[%s] %s" % (
-                        self.OUT_PREFIX_COMMAND, now, job_log_dir, key, line))
+                for line in value.strip().splitlines():
+                    sys.stdout.write(
+                        f"{self.OUT_PREFIX_COMMAND}{now}"
+                        f"|{job_log_dir}|[{key}] {line}\n"
+                    )
 
     def job_kill(self, st_file_path):
         """Ask job runner to terminate the job specified in "st_file_path".