Skip to content

Commit

Permalink
[fix](be) Fix creating hdfs connection coredump within bthread
Browse files Browse the repository at this point in the history
* https://brpc.apache.org/docs/server/basics/
* According to the brpc doc, `JNI code checks stack layout and cannot be run in bthreads`
  so create a pthread for creating hdfs connection if necessary.
  • Loading branch information
SWJTU-ZhangLei committed Oct 30, 2024
1 parent 1d1c425 commit 52e27dc
Showing 1 changed file with 18 additions and 1 deletion.
19 changes: 18 additions & 1 deletion be/src/io/hdfs_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <gen_cpp/cloud.pb.h>

#include <ostream>
#include <thread>

#include "common/logging.h"
#include "io/fs/err_utils.h"
Expand All @@ -30,7 +31,7 @@
namespace doris::io {
namespace {

Status create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, hdfsFS* fs) {
Status _create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, hdfsFS* fs) {
HDFSCommonBuilder builder;
RETURN_IF_ERROR(create_hdfs_builder(hdfs_params, fs_name, &builder));
hdfsFS hdfs_fs = hdfsBuilderConnect(builder.get());
Expand All @@ -41,6 +42,22 @@ Status create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name
return Status::OK();
}

inline bool is_bthread() {
return (bthread_self() != 0);
}

Status create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, hdfsFS* fs) {
if (is_bthread()) {
Status st;
std::thread t([&] { st = _create_hdfs_fs(hdfs_params, fs_name, fs); });
if (t.joinable()) {
t.join();
}
return st;
}
return _create_hdfs_fs(hdfs_params, fs_name, fs);
}

uint64_t hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name) {
uint64_t hash_code = 0;
// The specified fsname is used first.
Expand Down

0 comments on commit 52e27dc

Please sign in to comment.