drenv: Improve gather logs

We used to drop gather info logs. Watch the gather process and log the
gather logs as they are written to stderr.

Example logs with this change:

    % drenv gather envs/regional-dr.yaml
    2025-01-15 22:04:20,865 INFO    [rdr] Gathering environment
    2025-01-15 22:04:21,124 INFO    [gather] Using kubeconfig "/Users/nsoffer/.kube/config"
    2025-01-15 22:04:21,126 INFO    [gather] Gathering from all namespaces
    2025-01-15 22:04:21,126 INFO    [gather] Using all addons
    2025-01-15 22:04:21,126 INFO    [gather] Storing data in "gather.20250115220421"
    2025-01-15 22:04:21,126 INFO    [gather] Gathering from cluster "dr1"
    2025-01-15 22:04:21,126 INFO    [gather] Gathering from cluster "dr2"
    2025-01-15 22:04:21,126 INFO    [gather] Gathering from cluster "hub"
    2025-01-15 22:04:23,095 INFO    [gather] Gathered 1535 resources from cluster "hub" in 1.969 seconds
    2025-01-15 22:04:24,395 INFO    [gather] Gathered 1909 resources from cluster "dr1" in 3.269 seconds
    2025-01-15 22:04:24,544 INFO    [gather] Gathered 1920 resources from cluster "dr2" in 3.417 seconds
    2025-01-15 22:04:24,544 INFO    [gather] Gathered 5364 resources from 3 clusters in 3.418 seconds
    2025-01-15 22:04:24,555 INFO    [rdr] Environment gathered in 3.69 seconds

Note: This change requires kubectl-gather 0.6.0. Developers that want to
use the `drenv gather` command will have to upgrade kubectl-gather to
latest version.

Signed-off-by: Nir Soffer <[email protected]>

test/drenv/__main__.py

@@ -285,6 +285,7 @@ def do_gather(args):
         [p["name"] for p in env["profiles"]],
         directory=args.directory,
         namespaces=args.namespaces,
+        name=env["name"],
     )
     logging.info(
         "[%s] Environment gathered in %.2f seconds",

test/drenv/kubectl.py

@@ -1,7 +1,11 @@
 # SPDX-FileCopyrightText: The RamenDR authors
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
+import subprocess
+
 from . import commands
+from . import zap
 
 JSONPATH_NEWLINE = '{"\\n"}'
 
@@ -188,16 +192,32 @@ def watch(
     return commands.watch(*cmd, timeout=timeout)
 
 
-def gather(contexts, namespaces=None, directory=None):
+def gather(contexts, namespaces=None, directory=None, name="gather"):
     """
-    Run kubectl gather plugin.
+    Run kubectl gather plugin, logging gather logs.
     """
-    cmd = ["kubectl", "gather", "--contexts", ",".join(contexts)]
+    cmd = [
+        "kubectl",
+        "gather",
+        "--log-format",
+        "json",
+        "--contexts",
+        ",".join(contexts),
+    ]
     if namespaces:
         cmd.extend(("--namespaces", ",".join(namespaces)))
     if directory:
         cmd.extend(("--directory", directory))
-    commands.run(*cmd)
+
+    # Redirecting stderr to stdout to get the logs. kubectl-gather does not
+    # output anything to stdout.
+    for line in commands.watch(*cmd, stderr=subprocess.STDOUT):
+        try:
+            zap.log_json(line, name=name)
+        except ValueError:
+            # We don't want to crash if kubectl-gather has a logging bug, and
+            # the line may contain useful info.
+            logging.debug("[%s] %s", name, line)
 
 
 def _run(cmd, *args, env=None, context=None):