Merge pull request flux-framework#5766 from grondo/output-append

shell: support opening output files with `-o output.mode=append`
wihobbs · Mar 4, 2024 · 6bbe29c · 6bbe29c
2 parents e50ce65 + 656e40d
commit 6bbe29c
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 10 deletions.
diff --git a/doc/man1/common/job-shell-options.rst b/doc/man1/common/job-shell-options.rst
@@ -47,3 +47,7 @@
      - Set KVS output limit to SIZE bytes, where SIZE may be a floating point
        value including optional SI units: k, K, M, G. This value is ignored
        if output is directed to a file with :option:`--output`.
+
+   * - :option:`output.mode`
+     - Set the open mode for output files to either "truncate" or "append".
+       The default is "truncate".
diff --git a/doc/man1/flux-shell.rst b/doc/man1/flux-shell.rst
@@ -330,6 +330,11 @@ plugins include:
 
   Set job stderr/out file output to PATH.
 
+.. option:: output.mode=truncate|append
+
+  Set the mode in which output files are opened to either truncate or
+  append. The default is to truncate.
+
 .. option:: input.stdin.type=TYPE
 
   Set job input for **stdin** to *TYPE*. *TYPE* may be either ``service``

diff --git a/src/shell/output.c b/src/shell/output.c
@@ -78,6 +78,7 @@ struct shell_output_type_file {
     struct shell_output_fd *fdp;
     char *path;
     int label;
+    int flags;
 };
 
 struct shell_output {
@@ -791,12 +792,25 @@ shell_output_setup_type_file (struct shell_output *out,
                               struct shell_output_type_file *ofp_copy)
 {
     const char *path = NULL;
+    const char *mode = "truncate";
 
-    if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s:{s?s}}",
-                                  stream, "path", &path) < 0)
+    if (flux_shell_getopt_unpack (out->shell,
+                                  "output",
+                                  "{s?s s:{s?s s?b}}",
+                                  "mode", &mode,
+                                  stream,
+                                    "path", &path,
+                                    "label", &ofp->label) < 0)
         return -1;
 
+    ofp->flags = O_CREAT | O_WRONLY;
+    if (streq (mode, "append"))
+        ofp->flags |= O_APPEND;
+    else if (streq (mode, "truncate"))
+        ofp->flags |= O_TRUNC;
+    else
+        shell_warn ("ignoring invalid output.mode=%s", mode);
+
     if (path == NULL) {
         shell_log_error ("path for %s file output not specified", stream);
         return -1;
@@ -805,15 +819,11 @@ shell_output_setup_type_file (struct shell_output *out,
     if (!(ofp->path = flux_shell_mustache_render (out->shell, path)))
         return -1;
 
-    if (flux_shell_getopt_unpack (out->shell, "output",
-                                  "{s:{s?b}}",
-                                  stream, "label", &(ofp->label)) < 0)
-        return -1;
-
     if (ofp_copy) {
         if (!(ofp_copy->path = strdup (ofp->path)))
             return -1;
         ofp_copy->label = ofp->label;
+        ofp_copy->flags = ofp->flags;
     }
 
     return 0;
@@ -963,7 +973,6 @@ static int shell_output_type_file_setup (struct shell_output *out,
                                          struct shell_output_type_file *ofp)
 {
     mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
-    int open_flags = O_CREAT | O_TRUNC | O_WRONLY;
     struct shell_output_fd *fdp = NULL;
     int saved_errno, fd = -1;
 
@@ -973,7 +982,7 @@ static int shell_output_type_file_setup (struct shell_output *out,
         return 0;
     }
 
-    if ((fd = open (ofp->path, open_flags, mode)) < 0) {
+    if ((fd = open (ofp->path, ofp->flags, mode)) < 0) {
         shell_log_errno ("error opening output file '%s'", ofp->path);
         goto error;
     }

diff --git a/t/t2606-job-shell-output-redirection.t b/t/t2606-job-shell-output-redirection.t
@@ -366,4 +366,27 @@ test_expect_success LONGTEST 'job-shell: no truncation at 10MB for single-user j
 test_expect_success 'job-shell: invalid output.limit string is rejected' '
 	test_must_fail flux run -o output.limit=foo hostname
 '
+test_expect_success 'job-shell: output.mode=append works' '
+	flux bulksubmit --watch --output=append.out \
+		-o output.mode=append echo {} \
+		::: one two three &&
+	test_debug "cat append.out" &&
+	test $(wc -l < append.out) -eq 3 &&
+	grep one append.out &&
+	grep two append.out &&
+	grep three append.out
+'
+test_expect_success 'job-shell: output.mode=truncate works' '
+	cat <<-EOF >trunc.out &&
+	test text
+	EOF
+	flux run --output=trunc.out hostname &&
+	test $(wc -l <trunc.out) -eq 1 &&
+	test_must_fail grep "test text" trunc.out
+'
+test_expect_success 'job-shell: invalid output.mode emits warning' '
+	flux run --output=inval.out -o output.mode=foo hostname 2>inval.err &&
+	test_debug "cat inval.out" &&
+	grep "ignoring invalid output.mode=foo" inval.err
+'
 test_done