Busy Loop in cassandra driver thread

[Volker Kleinfeld]

Hello,

we are experiencing busy loops in the cassandra driver threads. 

The Cassandra driver version is 2.15.2, the configuration: 4 datacenters (single rack per datacenter) and 16 cassandra nodes per datacenter.

Here a backtrace of the busy thread:

#0  0x00000000017dd124 in __gnu_cxx::__normal_iterator<datastax::internal::SharedRefPtr<datastax::internal::core::Host> const*, std::vector<datastax::internal::SharedRefPtr<datastax::internal::core::Host>, datastax::internal::Allocator<datastax::internal::SharedRefPtr<datastax::internal::core::Host> > > >::operator--() ()
#1  0x00000000017daf48 in std::reverse_iterator<__gnu_cxx::__normal_iterator<datastax::internal::SharedRefPtr<datastax::internal::core::Host> const*, std::vector<datastax::internal::SharedRefPtr<datastax::internal::core::Host>, datastax::internal::Allocator<datastax::internal::SharedRefPtr<datastax::internal::core::Host> > > > >::operator++() ()
#2  0x00000000017da122 in datastax::internal::core::add_replica(datastax::internal::core::CopyOnWritePtr<datastax::internal::Vector<datastax::internal::SharedRefPtr<datastax::internal::core::Host> > >&, datastax::internal::SharedRefPtr<datastax::internal::core::Host> const&) ()
#3  0x000000000180554e in datastax::internal::core::ReplicationStrategy<datastax::internal::core::Murmur3Partitioner>::build_replicas_network_topology(datastax::internal::Vector<std::pair<long, datastax::internal::core::Host*> > const&, datastax::internal::core::DatacenterMap const&, datastax::internal::Vector<std::pair<long, datastax::internal::core::CopyOnWritePtr<datastax::internal::Vector<datastax::internal::SharedRefPtr<datastax::internal::core::Host> > > > >&) const ()
#4  0x00000000017fce95 in datastax::internal::core::ReplicationStrategy<datastax::internal::core::Murmur3Partitioner>::build_replicas(datastax::internal::Vector<std::pair<long, datastax::internal::core::Host*> > const&, datastax::internal::core::DatacenterMap const&, datastax::internal::Vector<std::pair<long, datastax::internal::core::CopyOnWritePtr<datastax::internal::Vector<datastax::internal::SharedRefPtr<datastax::internal::core::Host> > > > >&) const ()
#5  0x00000000017f8353 in datastax::internal::core::TokenMapImpl<datastax::internal::core::Murmur3Partitioner>::build_replicas() ()
#6  0x00000000017f4d62 in datastax::internal::core::TokenMapImpl<datastax::internal::core::Murmur3Partitioner>::build() ()
#7  0x0000000001785b80 in datastax::internal::core::Cluster::update_token_map(datastax::internal::Map<datastax::internal::core::Address, datastax::internal::SharedRefPtr<datastax::internal::core::Host>, std::less<datastax::internal::core::Address> > const&, std::basic_string<char, std::char_traits<char>, datastax::internal::Allocator<char> > const&, datastax::internal::core::ControlConnectionSchema const&) ()
#8  0x0000000001786120 in datastax::internal::core::Cluster::on_reconnect(datastax::internal::core::ControlConnector*) ()
#9  0x00000000017915f6 in datastax::internal::Callback<void, datastax::internal::core::ControlConnector*>::MemberInvoker<void (datastax::internal::core::Cluster::*)(datastax::internal::core::ControlConnector*), datastax::internal::core::Cluster>::invoke(datastax::internal::core::ControlConnector* const&) const ()
#10 0x00000000017a8588 in datastax::internal::Callback<void, datastax::internal::core::ControlConnector*>::operator()(datastax::internal::core::ControlConnector* const&) const ()
#11 0x00000000017a58e5 in datastax::internal::core::ControlConnector::finish() ()
#12 0x00000000017a5ad9 in datastax::internal::core::ControlConnector::on_success() ()
#13 0x00000000017a7e48 in datastax::internal::core::ControlConnector::handle_query_schema(datastax::internal::core::SchemaConnectorRequestCallback*) ()
#14 0x00000000017a8450 in datastax::internal::core::SchemaConnectorRequestCallback::on_chain_set() ()
#15 0x00000000017c60dc in datastax::internal::core::ChainedRequestCallback::maybe_finish() ()
#16 0x00000000017c5e14 in datastax::internal::core::ChainedRequestCallback::set_chain_responses(datastax::internal::core::ChainedRequestCallback::Map&) ()
#17 0x00000000017c60c3 in datastax::internal::core::ChainedRequestCallback::maybe_finish() ()
#18 0x00000000017c5e14 in datastax::internal::core::ChainedRequestCallback::set_chain_responses(datastax::internal::core::ChainedRequestCallback::Map&) ()
#19 0x00000000017c60c3 in datastax::internal::core::ChainedRequestCallback::maybe_finish() ()
#20 0x00000000017c5e14 in datastax::internal::core::ChainedRequestCallback::set_chain_responses(datastax::internal::core::ChainedRequestCallback::Map&) ()
#21 0x00000000017c60c3 in datastax::internal::core::ChainedRequestCallback::maybe_finish() ()
#22 0x00000000017c5cd0 in datastax::internal::core::ChainedRequestCallback::on_internal_set(datastax::internal::core::ResponseMessage*) ()
#23 0x00000000017c55e6 in datastax::internal::core::SimpleRequestCallback::on_set(datastax::internal::core::ResponseMessage*) ()
#24 0x0000000001841280 in datastax::internal::core::Connection::on_read(char const*, unsigned long) ()
#25 0x00000000018402c6 in datastax::internal::core::ConnectionHandler::on_read(datastax::internal::core::Socket*, long, uv_buf_t const*) ()
#26 0x0000000001867241 in datastax::internal::core::Socket::handle_read(long, uv_buf_t const*) ()
#27 0x000000000186718a in datastax::internal::core::Socket::on_read(uv_stream_s*, long, uv_buf_t const*) ()
#28 0x000000000187c720 in uv.read ()
#29 0x000000000187ce2c in uv.stream_io ()
#30 0x0000000001881620 in uv.io_poll ()
#31 0x0000000001876998 in uv_run ()
#32 0x00000000017ba6ac in datastax::internal::core::EventLoop::handle_run() ()
#33 0x00000000017ba66e in datastax::internal::core::EventLoop::internal_on_run(void*) ()
#34 0x00007f50f7dc4724 in start_thread () from /lib64/libpthread.so.0
#35 0x00007f50f7b03eed in clone () from /lib64/libc.so.6

Any clues what could cause this behavior and how we could mitigate the problem?

BR, Volker