From 414bdd4cc5995dd0c7aaf49117372b2b9d517463 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Tue, 17 Jan 2023 14:41:13 -0500 Subject: [PATCH] rgw: add conditional backtrace logging after "WARNING: blocking librados call" Signed-off-by: Ali Maredia --- CMakeLists.txt | 1 + src/rgw/CMakeLists.txt | 4 ++++ src/rgw/driver/rados/rgw_tools.cc | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3aab8c8a4537b..0724fcf5075f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -440,6 +440,7 @@ option(WITH_RADOSGW_MOTR "CORTX-Motr backend for Rados Gateway" OFF) option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF) option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON) option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF) +option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF) option(WITH_SYSTEM_ARROW "Use system-provided arrow" OFF) option(WITH_SYSTEM_UTF8PROC "Use system-provided utf8proc" OFF) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 3f792bf8e069b..b02cdd0e6c167 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -3,6 +3,10 @@ if(NOT GPERF) message(FATAL_ERROR "Can't find gperf") endif() +if(WITH_RADOSGW_BACKTRACE_LOGGING) + add_definitions(-D_BACKTRACE_LOGGING) +endif(WITH_RADOSGW_BACKTRACE_LOGGING) + if(WITH_RADOSGW_SELECT_PARQUET) set(ARROW_LIBRARIES Arrow::Arrow Arrow::Parquet) add_definitions(-D_ARROW_EXIST) diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc index 5a8aefaac3d31..22c1b26971c92 100644 --- a/src/rgw/driver/rados/rgw_tools.cc +++ b/src/rgw/driver/rados/rgw_tools.cc @@ -10,6 +10,7 @@ #include "rgw_acl_s3.h" #include "rgw_aio_throttle.h" #include "rgw_compression.h" +#include "common/BackTrace.h" #define dout_subsys ceph_subsys_rgw @@ -182,6 +183,9 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con // work on asio threads should be asynchronous, so warn when they block if (is_asio_thread) { ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; +#ifdef _BACKTRACE_LOGGING + ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl; +#endif } return ioctx.operate(oid, op, nullptr, flags); } @@ -199,6 +203,9 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con } if (is_asio_thread) { ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; +#ifdef _BACKTRACE_LOGGING + ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl; +#endif } return ioctx.operate(oid, op, flags); } @@ -220,6 +227,9 @@ int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, cons } if (is_asio_thread) { ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; +#ifdef _BACKTRACE_LOGGING + ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl; +#endif } return ioctx.notify2(oid, bl, timeout_ms, pbl); }