Wrong MemoryTracking values prevents merges #71511
Description
Describe the unexpected behaviour
Merges fail with MEMORY_LIMIT_EXCEEDED error. MemoryTracking indicates unrealistic memory usage. Extending memory either via swap or via physical RAM up to 32GB does nothing with it.
How to reproduce
Checked ClickHouse server versions: 24.9.1.3278, 24.9.2.42.
RabbitMQ engine is used with low insert rate (less than 2RPM) into ReplicatedMergeTree and ReplicatedSummingMergeTree materialized views and no queries.
If max_server_memory_usage_to_ram_ratio is set, ClickHouse will stop merges after a few hours due to MEMORY_LIMIT_EXCEEDED error. With disabled memory limit it works as expected, except for MemoryTracking metric. Real memory consumption ("resident" in htop) is less than 550MB. MemoryTracking shows hundreds of gigabytes and continues to grow.
SELECT formatReadableSize(value)
FROM system.metrics
WHERE metric = 'MemoryTracking'
FORMAT Pretty;
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ formatReadableSize(value) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
1. │ 263.32 GiB │
└───────────────────────────┘
SELECT formatReadableSize(sum(t.allocations * t.size) - sum(t.deallocations * t.size))
FROM system.jemalloc_bins t
FORMAT Pretty;
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ formatReadableSize(minus(sum(multiply(allocations, size)), sum(multiply(deallocations, size)))) ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
1. │ 199.22 MiB │
└─────────────────────────────────────────────────────────────────────────────────────────────────┘
Error message and/or stacktrace
<Error> a1be084d-9ee0-4360-b7c1-e24d0f28132a::all_0_6121_2581 (MergeFromLogEntryTask): virtual bool DB::ReplicatedMergeMutateTaskBase::executeStep(): Code: 241. DB::Exception: Memory limit (total) exceeded: would use 73.10 GiB (attempt to allocate chunk of 4819411 bytes), current RSS 212.56 MiB, maximum: 14.03 GiB. OvercommitTracker decision: Memory overcommit isn't used. Waiting time or overcommit denominator are set to zero. (MEMORY_LIMIT_EXCEEDED), Stack trace (when copying this message, always include the lines below):
0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000d66ad9b
1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e75b2c
2. DB::Exception::Exception<char const*, char const*, String, long&, String, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<long&>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, long&, String&&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x000000000d689064
3. MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000d68818d
4. MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000d687cb1
5. MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000d687cb1
6. MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000d687cb1
7. Allocator<false, false>::alloc(unsigned long, unsigned long) @ 0x000000000d63c1b0
8. DB::Memory<Allocator<false, false>>::alloc(unsigned long) @ 0x000000000d6ce995
9. DB::WriteBufferFromFileDescriptor::WriteBufferFromFileDescriptor(int, unsigned long, char*, std::shared_ptr<DB::Throttler>, unsigned long, String, bool, unsigned long) @ 0x000000000d6f4ac1
10. DB::WriteBufferFromFile::WriteBufferFromFile(String const&, unsigned long, int, std::shared_ptr<DB::Throttler>, unsigned int, char*, unsigned long, bool, unsigned long) @ 0x000000000d815b4f
11. DB::DiskLocal::writeFile(String const&, unsigned long, DB::WriteMode, DB::WriteSettings const&) @ 0x00000000111156f3
12. DB::DataPartStorageOnDiskFull::writeFile(String const&, unsigned long, DB::WriteMode, DB::WriteSettings const&) @ 0x0000000012992953
13. DB::MergeTreeDataPartWriterOnDisk::Stream<false>::Stream(String const&, std::shared_ptr<DB::IDataPartStorage> const&, String const&, String const&, String const&, String const&, std::shared_ptr<DB::ICompressionCodec> const&, unsigned long, std::shared_ptr<DB::ICompressionCodec> const&, unsigned long, DB::WriteSettings const&) @ 0x0000000012b5caf6
14. void std::__function::__policy_invoker<void (DB::ISerialization::SubstreamPath const&)>::__call_impl<std::__function::__default_alloc_func<DB::MergeTreeDataPartWriterWide::addStreams(DB::NameAndTypePair const&, COW<DB::IColumn>::immutable_ptr<DB::IColumn> const&, std::shared_ptr<DB::IAST> const&)::$_0, void (DB::ISerialization::SubstreamPath const&)>>(std::__function::__policy_storage const*, DB::ISerialization::SubstreamPath const&) @ 0x0000000012b6c4f7
15. DB::ISerialization::enumerateStreams(DB::ISerialization::EnumerateStreamsSettings&, std::function<void (DB::ISerialization::SubstreamPath const&)> const&, DB::ISerialization::SubstreamData const&) const @ 0x0000000010ebb9d0
16. DB::ISerialization::enumerateStreams(std::function<void (DB::ISerialization::SubstreamPath const&)> const&, std::shared_ptr<DB::IDataType const> const&, COW<DB::IColumn>::immutable_ptr<DB::IColumn> const&) const @ 0x0000000010ebbc88
17. DB::MergeTreeDataPartWriterWide::addStreams(DB::NameAndTypePair const&, COW<DB::IColumn>::immutable_ptr<DB::IColumn> const&, std::shared_ptr<DB::IAST> const&) @ 0x0000000012b6445f
18. DB::MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide(String const&, String const&, std::unordered_map<String, std::shared_ptr<DB::ISerialization const>, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, std::shared_ptr<DB::ISerialization const>>>> const&, std::shared_ptr<DB::IDataPartStorage>, DB::MergeTreeIndexGranularityInfo const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::NamesAndTypesList const&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::shared_ptr<DB::VirtualColumnsDescription const> const&, std::vector<std::shared_ptr<DB::IMergeTreeIndex const>, std::allocator<std::shared_ptr<DB::IMergeTreeIndex const>>> const&, std::vector<std::shared_ptr<DB::ColumnStatistics>, std::allocator<std::shared_ptr<DB::ColumnStatistics>>> const&, String const&, std::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&) @ 0x0000000012b6425f
19. DB::createMergeTreeDataPartWriter(DB::MergeTreeDataPartType, String const&, String const&, std::unordered_map<String, std::shared_ptr<DB::ISerialization const>, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, std::shared_ptr<DB::ISerialization const>>>> const&, std::shared_ptr<DB::IDataPartStorage>, DB::MergeTreeIndexGranularityInfo const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::NamesAndTypesList const&, std::unordered_map<String, unsigned long, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, unsigned long>>> const&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::shared_ptr<DB::VirtualColumnsDescription const> const&, std::vector<std::shared_ptr<DB::IMergeTreeIndex const>, std::allocator<std::shared_ptr<DB::IMergeTreeIndex const>>> const&, std::vector<std::shared_ptr<DB::ColumnStatistics>, std::allocator<std::shared_ptr<DB::ColumnStatistics>>> const&, String const&, std::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&) @ 0x0000000012b5a67b
20. DB::MergedBlockOutputStream::MergedBlockOutputStream(std::shared_ptr<DB::IMergeTreeDataPart> const&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, DB::NamesAndTypesList const&, std::vector<std::shared_ptr<DB::IMergeTreeIndex const>, std::allocator<std::shared_ptr<DB::IMergeTreeIndex const>>> const&, std::vector<std::shared_ptr<DB::ColumnStatistics>, std::allocator<std::shared_ptr<DB::ColumnStatistics>>> const&, std::shared_ptr<DB::ICompressionCodec>, DB::TransactionID, bool, bool, DB::WriteSettings const&, DB::MergeTreeIndexGranularity const&) @ 0x0000000012cada3b
21. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const @ 0x0000000012a4060c
22. DB::MergeTask::ExecuteAndFinalizeHorizontalPart::execute() @ 0x0000000012a472ce
23. DB::MergeTask::execute() @ 0x0000000012a4ff22
24. DB::ReplicatedMergeMutateTaskBase::executeStep() @ 0x0000000012d23bc0
25. DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::threadFunction() @ 0x0000000012a6f99d
26. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false, true>, void*>) @ 0x000000000d740b38
27. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d745c71
28. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d74409f
29. ? @ 0x00007ff1e3ec5ac3
30. ? @ 0x00007ff1e3f57850
(version 24.9.2.42 (official build))