From 3fedafdc059ec6177a943399af092366d8b393f2 Mon Sep 17 00:00:00 2001
From: Yan Cheng <yanc@nvidia.com>
Date: Tue, 26 Dec 2023 11:18:17 -0500
Subject: [PATCH] reduce heartbeat error log

---
 nvflare/fuel/utils/pipe/cell_pipe.py    | 15 +++++++++++----
 nvflare/fuel/utils/pipe/pipe.py         |  8 ++++++++
 nvflare/fuel/utils/pipe/pipe_handler.py | 10 +---------
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/nvflare/fuel/utils/pipe/cell_pipe.py b/nvflare/fuel/utils/pipe/cell_pipe.py
index 513e48ccc9..d9a75dd51c 100644
--- a/nvflare/fuel/utils/pipe/cell_pipe.py
+++ b/nvflare/fuel/utils/pipe/cell_pipe.py
@@ -28,7 +28,7 @@
 from nvflare.fuel.utils.constants import Mode
 from nvflare.fuel.utils.validation_utils import check_object_type, check_str
 
-from .pipe import Message, Pipe
+from .pipe import Message, Pipe, Topic
 
 SSL_ROOT_CERT = "rootCA.pem"
 _PREFIX = "cell_pipe."
@@ -215,21 +215,28 @@ def send(self, msg: Message, timeout=None) -> bool:
             if self.closed:
                 raise BrokenPipeError("pipe closed")
 
+            optional = False
+            if msg.topic in [Topic.END, Topic.ABORT, Topic.HEARTBEAT]:
+                optional = True
+
             reply = self.cell.send_request(
                 channel=self.channel,
                 topic=msg.topic,
                 target=self.peer_fqcn,
                 request=_to_cell_message(msg),
                 timeout=timeout,
+                optional=optional,
             )
             if reply:
                 rc = reply.get_header(MessageHeaderKey.RETURN_CODE)
                 if rc == ReturnCode.OK:
                     return True
                 else:
-                    self.logger.error(
-                        f"failed to send '{msg.topic}' to '{self.peer_fqcn}' in channel '{self.channel}': {rc}"
-                    )
+                    err = f"failed to send '{msg.topic}' to '{self.peer_fqcn}' in channel '{self.channel}': {rc}"
+                    if optional:
+                        self.logger.debug(err)
+                    else:
+                        self.logger.error(err)
                     return False
             else:
                 return False
diff --git a/nvflare/fuel/utils/pipe/pipe.py b/nvflare/fuel/utils/pipe/pipe.py
index 7b6e745384..5993896e7d 100644
--- a/nvflare/fuel/utils/pipe/pipe.py
+++ b/nvflare/fuel/utils/pipe/pipe.py
@@ -22,6 +22,14 @@
 from nvflare.fuel.utils.validation_utils import check_str
 
 
+class Topic(object):
+
+    ABORT = "_ABORT_"
+    END = "_END_"
+    HEARTBEAT = "_HEARTBEAT_"
+    PEER_GONE = "_PEER_GONE_"
+
+
 class Message:
 
     REQUEST = "REQ"
diff --git a/nvflare/fuel/utils/pipe/pipe_handler.py b/nvflare/fuel/utils/pipe/pipe_handler.py
index 3aee9e2870..e0fa48b612 100644
--- a/nvflare/fuel/utils/pipe/pipe_handler.py
+++ b/nvflare/fuel/utils/pipe/pipe_handler.py
@@ -19,7 +19,7 @@
 from typing import Optional
 
 from nvflare.apis.signal import Signal
-from nvflare.fuel.utils.pipe.pipe import Message, Pipe
+from nvflare.fuel.utils.pipe.pipe import Message, Pipe, Topic
 from nvflare.fuel.utils.validation_utils import (
     check_callable,
     check_non_negative_number,
@@ -29,14 +29,6 @@
 from nvflare.security.logging import secure_format_exception
 
 
-class Topic(object):
-
-    ABORT = "_ABORT_"
-    END = "_END_"
-    HEARTBEAT = "_HEARTBEAT_"
-    PEER_GONE = "_PEER_GONE_"
-
-
 class PipeHandler(object):
     """
     PipeHandler monitors a pipe for messages from the peer. It reads the pipe periodically and puts received data