Skip to content

Commit

Permalink
[branch-2.1] avoid glog coredump when running with ASAN (apache#37134)
Browse files Browse the repository at this point in the history
## Proposed changes

This is just a workround try avoid coredump like this:
```
#0 0x56414f0e8ed1 in __asan::CheckUnwind() crtstuff.c
    #1 0x56414f1009a2 in __sanitizer::CheckFailed(char const*, int, char const*, unsigned long long, unsigned long long) crtstuff.c
    apache#2 0x56414f0ecbf3 in __asan::AsanThread::GetStackFrameAccessByAddr(unsigned long, __asan::AsanThread::StackFrameAccess*) crtstuff.c
    apache#3 0x56414f050d87 in __asan::AddressDescription::AddressDescription(unsigned long, unsigned long, bool) crtstuff.c
    apache#4 0x56414f052a73 in __asan::ErrorGeneric::ErrorGeneric(unsigned int, unsigned long, unsigned long, unsigned long, unsigned long, bool, unsigned long) crtstuff.c
    apache#5 0x56414f0e6a9e in __asan::ReportGenericError(unsigned long, unsigned long, unsigned long, unsigned long, bool, unsigned long, unsigned int, bool) crtstuff.c
    apache#6 0x56414f066885 in gmtime_r (/mnt/hdd01/ci/branch21-deploy/be/lib/doris_be+0x17ef3885) (BuildId: f58eb5e327529636)
    apache#7 0x564177940521 in google::LogMessage::Init(char const*, int, int, void (google::LogMessage::*)()) crtstuff.c
    apache#8 0x564151de36fc in doris::Status doris::ThriftRpcHelper::rpc(std::__cxx11::basic_string, std::allocator> const&, int, std::function&)>, int) /home/zcp/repo_center/doris_branch-2.1/doris/be/src/util/thrift_rpc_helper.cpp:76:13
    apache#9 0x56417603cda7 in doris::vectorized::VRowDistribution::automatic_create_partition() /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/sink/vrow_distribution.cpp:99:5
    apache#10 0x56417614cffa in doris::vectorized::VTabletWriter::_send_new_partition_batch() /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/sink/writer/vtablet_writer.cpp:1346:9
....
```
  • Loading branch information
morningman authored Jul 2, 2024
1 parent cf86eb8 commit f5d0cde
Showing 1 changed file with 16 additions and 0 deletions.
16 changes: 16 additions & 0 deletions be/src/util/thrift_rpc_helper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,38 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port,
try {
callback(client);
} catch (apache::thrift::transport::TTransportException& e) {
#ifndef ADDRESS_SANITIZER
LOG(WARNING) << "retrying call frontend service after "
<< config::thrift_client_retry_interval_ms << " ms, address=" << address
<< ", reason=" << e.what();
#else
std::cerr << "retrying call frontend service after "
<< config::thrift_client_retry_interval_ms << " ms, address=" << address
<< ", reason=" << e.what() << std::endl;
#endif
std::this_thread::sleep_for(
std::chrono::milliseconds(config::thrift_client_retry_interval_ms));
status = client.reopen(timeout_ms);
if (!status.ok()) {
#ifndef ADDRESS_SANITIZER
LOG(WARNING) << "client reopen failed. address=" << address
<< ", status=" << status;
#else
std::cerr << "client reopen failed. address=" << address << ", status=" << status
<< std::endl;
#endif
return status;
}
callback(client);
}
} catch (apache::thrift::TException& e) {
#ifndef ADDRESS_SANITIZER
LOG(WARNING) << "call frontend service failed, address=" << address
<< ", reason=" << e.what();
#else
std::cerr << "call frontend service failed, address=" << address << ", reason=" << e.what()
<< std::endl;
#endif
std::this_thread::sleep_for(
std::chrono::milliseconds(config::thrift_client_retry_interval_ms * 2));
// just reopen to disable this connection
Expand Down

0 comments on commit f5d0cde

Please sign in to comment.