Skip to content

Hung check: DETACH DATABASE #66044

@pamarcos

Description

@pamarcos

Report: https://s3.amazonaws.com/clickhouse-test-reports/65838/389a86ec059461f20521e4dd2c7888e2b3b37623/stress_test__tsan_/hung_check.log

Found hung queries in processlist:
Row 1:
──────
is_initial_query:     0
user:                 default
query_id:             4a990f38-3819-4695-a523-35cac4e5020d
address:              ::ffff:127.0.0.1
port:                 52296
initial_user:         default
initial_query_id:     7a82f8c8-a39f-4515-aba0-0ad21be6babe
initial_address:      ::1
initial_port:         52882
interface:            1
os_user:              
client_hostname:      392cc22b14d1
client_name:          ClickHouse client
client_revision:      54468
client_version_major: 24
client_version_minor: 7
client_version_patch: 1
http_method:          0
http_user_agent:      
http_referer:         
forwarded_for:        
quota_key:            
distributed_depth:    0
elapsed:              846.459834
is_cancelled:         0
is_all_data_sent:     0
read_rows:            33334
read_bytes:           533344
total_rows_approx:    0
written_rows:         0
written_bytes:        0
memory_usage:         2740542
peak_memory_usage:    2740542
query:                SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `test_13`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_5743312683625947501_8717263657696418061` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64')
query_kind:           Select
thread_ids:           [27974,27859,14872,27995,27901,27787,27928,28070,27702,7920,14876,28036,8206,27852,27963,27916,28004,4379,27977,27724,27884,28025,14865,27931,28029,28012,28035]
ProfileEvents:        {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':2,'SelectQueriesWithSubqueries':2,'ReadCompressedBytes':270779,'CompressedReadBufferBlocks':7,'CompressedReadBufferBytes':533566,'IOBufferAllocs':2,'IOBufferAllocBytes':1582147,'NetworkReceiveElapsedMicroseconds':13750,'NetworkSendElapsedMicroseconds':1540,'NetworkReceiveBytes':270954,'NetworkSendBytes':1596,'SelectedParts':9,'SelectedRanges':9,'SelectedMarks':1223,'SelectedRows':66668,'SelectedBytes':1066688,'ContextLock':34,'ContextLockWaitMicroseconds':21,'RWLockAcquiredReadLocks':2,'PartsLockHoldMicroseconds':21,'PartsLockWaitMicroseconds':1,'RealTimeMicroseconds':6014,'UserTimeMicroseconds':177,'SystemTimeMicroseconds':76,'SoftPageFaults':214,'OSCPUVirtualTimeMicroseconds':210,'MergeTreeAllRangesAnnouncementsSent':1,'MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds':136,'LogTest':4,'LogTrace':17,'LogDebug':3,'InterfaceNativeReceiveBytes':270954}
Settings:             {'connect_timeout_with_failover_ms':'2000','connect_timeout_with_failover_secure_ms':'3000','use_hedged_requests':'0','idle_connection_timeout':'36000','s3_check_objects_after_upload':'1','stream_like_engine_allow_direct_select':'1','replication_wait_for_inactive_replica_timeout':'30','allow_nonconst_timezone_arguments':'1','group_by_use_nulls':'1','max_parallel_replicas':'2','cluster_for_parallel_replicas':'test_cluster_one_shard_three_replicas_localhost','allow_experimental_parallel_reading_from_replicas':'2','parallel_replicas_for_non_replicated_merge_tree':'1','parallel_replicas_prefer_local_join':'0','log_queries':'1','insert_quorum_timeout':'60000','table_function_remote_max_addresses':'200','fsync_metadata':'0','join_use_nulls':'1','http_send_timeout':'60','http_receive_timeout':'60','opentelemetry_start_trace_probability':'0.1','max_execution_time':'60','join_algorithm':'full_sorting_merge','max_memory_usage':'10000000000','max_memory_usage_for_user':'20001810432','max_untracked_memory':'1048576','memory_profiler_step':'1048576','log_comment':'02967_parallel_replicas_join_algo_and_analyzer.sh','send_logs_level':'trace','allow_introspection_functions':'1','database_atomic_wait_for_drop_and_detach_synchronously':'1','distributed_ddl_entry_format_version':'6','async_insert_busy_timeout_max_ms':'5000','enable_filesystem_cache':'1','enable_filesystem_cache_on_write_operations':'1','filesystem_cache_segments_batch_size':'10','load_marks_asynchronously':'1','allow_prefetched_read_pool_for_remote_filesystem':'0','allow_prefetched_read_pool_for_local_filesystem':'0','filesystem_prefetch_max_memory_usage':'1073741824','allow_deprecated_database_ordinary':'1','insert_keeper_max_retries':'100','insert_keeper_retry_initial_backoff_ms':'1','insert_keeper_retry_max_backoff_ms':'1','insert_keeper_fault_injection_probability':'0.01','ignore_drop_queries_probability':'0.5'}
current_database:     default
stacktraces:          Thread ID 7920
./build_docker/./src/Common/StackTrace.cpp:349::StackTrace::StackTrace(ucontext_t const&)
./build_docker/./src/Storages/System/StorageSystemStackTrace.cpp:107::DB::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
/usr/bin/clickhouse::__tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*)
/usr/bin/clickhouse::sighandler(int, __sanitizer::__sanitizer_siginfo*, void*)
::
::
::
/usr/bin/clickhouse::__tsan::call_pthread_cancel_with_cleanup(int (*)(void*), void (*)(void*), void*)
/usr/bin/clickhouse::__interceptor_pthread_cond_wait
./base/poco/Foundation/src/Event_POSIX.cpp:94::Poco::EventImpl::waitImpl()
./contrib/llvm-project/libcxx/include/__utility/swap.h:37::DB::PullingAsyncPipelineExecutor::cancel()
./build_docker/./src/Processors/Executors/PullingAsyncPipelineExecutor.cpp:62::DB::PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor()
./contrib/llvm-project/libcxx/include/__mutex_base:143::DB::TCPHandler::processOrdinaryQuery()
./build_docker/./src/Server/TCPHandler.cpp:0::DB::TCPHandler::runImpl()
./build_docker/./src/Server/TCPHandler.cpp:2355::DB::TCPHandler::run()
./build_docker/./base/poco/Net/src/TCPServerConnection.cpp:57::Poco::Net::TCPServerConnection::start()
./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:48::Poco::Net::TCPServerDispatcher::run()
./build_docker/./base/poco/Foundation/src/ThreadPool.cpp:219::Poco::PooledThread::run()
./build_docker/./base/poco/Foundation/src/Thread.cpp:46::Poco::(anonymous namespace)::RunnableHolder::run()
./base/poco/Foundation/include/Poco/SharedPtr.h:231::Poco::ThreadImpl::runnableEntry(void*)
/usr/bin/clickhouse::__tsan_thread_start_func
::
::Thread ID 27995
./build_docker/./src/Common/StackTrace.cpp:349::StackTrace::StackTrace(ucontext_t const&)
./build_docker/./src/Storages/System/StorageSystemStackTrace.cpp:107::DB::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
/usr/bin/clickhouse::__tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*)
/usr/bin/clickhouse::sighandler(int, __sanitizer::__sanitizer_siginfo*, void*)
::
::
::
/usr/bin/clickhouse::__tsan::call_pthread_cancel_with_cleanup(int (*)(void*), void (*)(void*), void*)
/usr/bin/clickhouse::__interceptor_pthread_cond_wait
./build_docker/./contrib/llvm-project/libcxx/src/condition_variable.cpp:47::std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&)
./build_docker/./src/Common/ThreadPool.cpp:301::ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::wait()
./build_docker/./src/Processors/Executors/PipelineExecutor.cpp:405::DB::PipelineExecutor::executeImpl(unsigned long, bool)
./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:274::DB::PipelineExecutor::execute(unsigned long, bool)
./build_docker/./src/Processors/Executors/PullingAsyncPipelineExecutor.cpp:94::void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::PullingAsyncPipelineExecutor::pull(DB::Chunk&, unsigned long)::$_0>(DB::PullingAsyncPipelineExecutor::pull(DB::Chunk&, unsigned long)::$_0&&)::'lambda'(), void ()>>(std::__1::__function::__policy_storage const*)
./base/base/../base/wide_integer_impl.h:817::ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>)
./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:302::void* std::__1::__thread_proxy[abi:v15000]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*)
/usr/bin/clickhouse::__tsan_thread_start_func
::
::

Row 2:
──────
is_initial_query:     1
user:                 default
query_id:             a5e938a6-82eb-4241-bf13-3cfea8be5dc9
address:              ::1
port:                 56124
initial_user:         default
initial_query_id:     a5e938a6-82eb-4241-bf13-3cfea8be5dc9
initial_address:      ::1
initial_port:         56124
interface:            1
os_user:              
client_hostname:      392cc22b14d1
client_name:          ClickHouse client
client_revision:      54468
client_version_major: 24
client_version_minor: 7
client_version_patch: 1
http_method:          0
http_user_agent:      
http_referer:         
forwarded_for:        
quota_key:            
distributed_depth:    0
elapsed:              151.645974
is_cancelled:         0
is_all_data_sent:     0
read_rows:            0
read_bytes:           0
total_rows_approx:    0
written_rows:         0
written_bytes:        0
memory_usage:         0
peak_memory_usage:    0
query:                DETACH DATABASE test_13
query_kind:           Drop
thread_ids:           [27711,27755,14882,27937,28004,4389,4492,27898,28002,4478,8191,27822,27916,27728,27869,28010,28044,27857,4997,27980,14893,27973,28020,7995,27830,28065,27881,27954,27766,27733,8202,27942,14878,4991,54803,27966,28035,28093,27999]
ProfileEvents:        {'Query':1,'InitialQuery':1,'QueriesWithSubqueries':1,'ZooKeeperTransactions':1,'ZooKeeperRemove':1,'ReplicaPartialShutdown':3,'CannotRemoveEphemeralNode':2,'ContextLock':461,'ContextLockWaitMicroseconds':154,'StorageBufferFlush':32,'RWLockAcquiredReadLocks':20,'RWLockReadersWaitMilliseconds':4,'RealTimeMicroseconds':12071,'UserTimeMicroseconds':4390,'SystemTimeMicroseconds':1623,'SoftPageFaults':510,'OSCPUWaitMicroseconds':647,'OSCPUVirtualTimeMicroseconds':5953,'OSWriteBytes':8192,'OSWriteChars':7168,'LogTest':3,'LogTrace':15,'LogDebug':3375,'LogInfo':41}
Settings:             {'connect_timeout_with_failover_ms':'2000','connect_timeout_with_failover_secure_ms':'3000','idle_connection_timeout':'36000','s3_check_objects_after_upload':'1','stream_like_engine_allow_direct_select':'1','replication_wait_for_inactive_replica_timeout':'30','allow_nonconst_timezone_arguments':'1','log_queries':'1','insert_quorum_timeout':'60000','table_function_remote_max_addresses':'200','fsync_metadata':'0','http_send_timeout':'60','http_receive_timeout':'60','opentelemetry_start_trace_probability':'0.1','max_execution_time':'60','max_memory_usage':'10000000000','max_memory_usage_for_user':'0','max_untracked_memory':'1073741824','memory_profiler_step':'1073741824','allow_introspection_functions':'1','database_atomic_wait_for_drop_and_detach_synchronously':'1','distributed_ddl_entry_format_version':'6','async_insert_busy_timeout_max_ms':'5000','enable_filesystem_cache':'1','enable_filesystem_cache_on_write_operations':'1','filesystem_cache_segments_batch_size':'10','load_marks_asynchronously':'1','allow_prefetched_read_pool_for_remote_filesystem':'0','allow_prefetched_read_pool_for_local_filesystem':'0','filesystem_prefetch_max_memory_usage':'1073741824','allow_deprecated_database_ordinary':'1','insert_keeper_max_retries':'100','insert_keeper_retry_initial_backoff_ms':'1','insert_keeper_retry_max_backoff_ms':'1','insert_keeper_fault_injection_probability':'0.01'}
current_database:     default
stacktraces:          Thread ID 54803
./build_docker/./src/Common/StackTrace.cpp:349::StackTrace::StackTrace(ucontext_t const&)
./build_docker/./src/Storages/System/StorageSystemStackTrace.cpp:107::DB::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
/usr/bin/clickhouse::__tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*)
/usr/bin/clickhouse::sighandler(int, __sanitizer::__sanitizer_siginfo*, void*)
::
::
::
/usr/bin/clickhouse::___interceptor_nanosleep
./contrib/llvm-project/libcxx/include/__threading_support:412::std::__1::this_thread::sleep_for(std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>> const&)
./contrib/llvm-project/libcxx/include/thread:0::DB::DatabaseAtomic::waitDetachedTableNotInUse(StrongTypedef<wide::integer<128ul, unsigned int>, DB::UUIDTag> const&)
./contrib/llvm-project/libcxx/include/__iterator/wrap_iter.h:100::DB::InterpreterDropQuery::executeToDatabaseImpl(DB::ASTDropQuery const&, std::__1::shared_ptr<DB::IDatabase>&, std::__1::vector<StrongTypedef<wide::integer<128ul, unsigned int>, DB::UUIDTag>, std::__1::allocator<StrongTypedef<wide::integer<128ul, unsigned int>, DB::UUIDTag>>>&)
./build_docker/./src/Interpreters/InterpreterDropQuery.cpp:0::DB::InterpreterDropQuery::executeToDatabase(DB::ASTDropQuery const&)
./build_docker/./src/Interpreters/InterpreterDropQuery.cpp:93::DB::InterpreterDropQuery::executeSingleDropQuery(std::__1::shared_ptr<DB::IAST> const&)
./build_docker/./src/Interpreters/InterpreterDropQuery.cpp:0::DB::InterpreterDropQuery::execute()
./build_docker/./src/Interpreters/executeQuery.cpp:0::DB::executeQueryImpl(char const*, char const*, std::__1::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum, DB::ReadBuffer*)
./build_docker/./src/Interpreters/executeQuery.cpp:1391::DB::executeQuery(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum)
./contrib/llvm-project/libcxx/include/__memory/shared_ptr.h:612::DB::TCPHandler::runImpl()
./build_docker/./src/Server/TCPHandler.cpp:2355::DB::TCPHandler::run()
./build_docker/./base/poco/Net/src/TCPServerConnection.cpp:57::Poco::Net::TCPServerConnection::start()
./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:48::Poco::Net::TCPServerDispatcher::run()
./build_docker/./base/poco/Foundation/src/ThreadPool.cpp:219::Poco::PooledThread::run()
./build_docker/./base/poco/Foundation/src/Thread.cpp:46::Poco::(anonymous namespace)::RunnableHolder::run()
./base/poco/Foundation/include/Poco/SharedPtr.h:231::Poco::ThreadImpl::runnableEntry(void*)
/usr/bin/clickhouse::__tsan_thread_start_func
::
::
None

Metadata

Metadata

Assignees

No one assigned

    Labels

    fuzzProblem found by one of the fuzzers

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions