Leaking memory in MemoryTracker during long running ingestion (55.59 GiB in query tracker vs 16GB in global memory tracker) #62924
Description
I try to load s3://clickhouse-datasets/tsbs_devops_42_4000/influx-data-2023-01-01.gz to clickhouse.
schema. Trivial tables. Just parse in MV to simpler explain what happens without MATERIALIZED/EPHEMERAL.
SET format_regexp='^(?<measurement>[^,]+),(?<tags>[^\s]+)\s(?<fields>[^\s]+)\s(?<ts>[\d]+)$';
SET format_regexp_escaping_rule='Raw';
SET format_regexp_skip_unmatched=0;
SET use_variant_as_common_type = 1;
SET allow_experimental_variant_type=1;
create database influx;
use influx;
CREATE TABLE tsbs_raw
(
measurement String,
tags String,
fields String,
ts String
) ENGINE = Null;
CREATE TABLE tsbs_parsed
(
measurement String,
tags Map(String, String),
name String,
value Variant(Bool, Int64, Float64),
t DateTime64(9),
series_id UInt64,
PRIMARY KEY ( measurement, name)
);
CREATE MATERIALIZED VIEW tsbs_parse_mv
TO tsbs_parsed
AS
SELECT
measurement,
extractKeyValuePairs(tags, '=') as tags,
arrayJoin(mapApply( (k,val)->(k, multiIf(val IN ('t', 'T', 'true', 'True', 'TRUE'), true::Boolean, val IN ('f', 'F','false', 'False', 'FALSE'), false::Boolean, position(val, 'i') > 0, toInt64(trim(TRAILING 'i' FROM val)), toFloat64OrNull(val)) as x), extractKeyValuePairs(fields, '='))) as field,
field.1 as name,
field.2 as value,
fromUnixTimestamp64Nano(toUInt64OrNull(ts)) as t,
cityHash64(measurement, tags, name) as series_id
FROM tsbs_raw;
ingestion
INSERT INTO tsbs_raw FROM INFILE 'influx-data-2023-01-01.gz' COMPRESSION 'gzip' FORMAT Regexp;
I have this file locally so s3 is not involved in scenario.
Memory limit (total) exceeded: would use 55.59 GiB (attempt to allocate chunk of 52597096 bytes), maximum: 55.59 GiB.
Metrics history (MemoryResident/MemoryTracking) for period of running
https://pastila.nl/?000136a8/433703803c07fc99635aa3040bd341a3#TC4KyPDyYzV8LM65hHNeyA==
Max memory consumption is much lower
┌─MemoryResident─┬─MemoryTracking─┐
1. │ 16.29 GiB │ 16.29 GiB │
└────────────────┴────────────────┘
Query log:
hostname: ip-172-31-42-161.us-east-2.compute.internal
type: ExceptionWhileProcessing
event_date: 2024-04-24
event_time: 2024-04-24 12:56:44
event_time_microseconds: 2024-04-24 12:56:44.004034
query_start_time: 2024-04-24 12:38:40
query_start_time_microseconds: 2024-04-24 12:38:40.124049
query_duration_ms: 1083879
read_rows: 255800263
read_bytes: 117827923811
written_rows: 744323483
written_bytes: 325688805396
result_rows: 0
result_bytes: 0
memory_usage: 10971845405
current_database: influx
query: INSERT INTO tsbs_raw FROM INFILE 'influx-data-2023-01-01.gz' COMPRESSION 'gzip' FORMAT Regexp;
formatted_query:
normalized_query_hash: 12508204732737156661
query_kind: Insert
databases: ['influx']
tables: ['influx.tsbs_parsed','influx.tsbs_raw']
columns: ['influx.tsbs_raw.fields','influx.tsbs_raw.measurement','influx.tsbs_raw.tags','influx.tsbs_raw.ts']
partitions: []
projections: []
views: ['influx.tsbs_parse_mv']
exception_code: 241
exception: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 55.59 GiB (attempt to allocate chunk of 52597096 bytes), maximum: 55.59 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.: while executing 'ARRAY JOIN mapApply(__lambda_8540545387584550790_4739488458267858872, extractKeyValuePairs(__table1.fields, '='_String)) :: 1 -> arrayJoin(mapApply(__lambda_8540545387584550790_4739488458267858872, extractKeyValuePairs(__table1.fields, '='_String))) Tuple(String, Variant(Bool, Float64, Int64)) : 0': while pushing to view influx.tsbs_parse_mv (f62190f2-231f-4d4f-ad26-3aa0a9fbbfde). (MEMORY_LIMIT_EXCEEDED) (version 24.4.1.1)
stack_trace: 0. ./contrib/llvm-project/libcxx/include/exception:141: Poco::Exception::Exception(String const&, int) @ 0x000000001387d232
1. ./build/./src/Common/Exception.cpp:97: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000b0f5e17
2. ./contrib/llvm-project/libcxx/include/string:1499: DB::Exception::Exception<char const*, char const*, String, long&, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<long&>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, long&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x000000000b10eef7
3. ./build/./src/Common/MemoryTracker.cpp:0: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000b10e2fd
4. ./build/./src/Common/MemoryTracker.cpp:402: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000b10dea3
5. ./build/./src/Common/MemoryTracker.cpp:402: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000b10dea3
6. ./build/./src/Common/MemoryTracker.cpp:402: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x000000000b10dea3
7. ./build/./src/Common/CurrentMemoryTracker.cpp:59: CurrentMemoryTracker::alloc(long) @ 0x000000000b0c9b46
8. ./build/./src/Common/Allocator.cpp:0: Allocator<false, false>::alloc(unsigned long, unsigned long) @ 0x000000000b0c8363
9. ./src/Common/PODArray.h:117: DB::ColumnArray::replicateString(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const @ 0x00000000101e4a2c
10. ./build/./src/Columns/ColumnArray.cpp:1040: DB::ColumnArray::replicate(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const @ 0x00000000101e4610
11. ./contrib/llvm-project/libcxx/include/vector:1457: DB::ColumnArray::replicateTuple(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const @ 0x00000000101e57a2
12. ./build/./src/Columns/ColumnArray.cpp:1040: DB::ColumnArray::replicate(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const @ 0x00000000101e465a
13. ./contrib/boost/boost/smart_ptr/intrusive_ptr.hpp:202: DB::ColumnMap::replicate(DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const&) const @ 0x00000000102dd0bc
14. ./contrib/boost/boost/smart_ptr/intrusive_ptr.hpp:117: DB::ExpressionActions::execute(DB::Block&, unsigned long&, bool, bool) const @ 0x000000000f4eb559
15. ./build/./src/Processors/Transforms/ExpressionTransform.cpp:0: DB::ExpressionTransform::transform(DB::Chunk&) @ 0x00000000112a1076
16. ./contrib/llvm-project/libcxx/include/__utility/swap.h:35: DB::ISimpleTransform::transform(DB::Chunk&, DB::Chunk&) @ 0x000000000cadc5d0
17. ./build/./src/Processors/ISimpleTransform.cpp:99: DB::ISimpleTransform::work() @ 0x000000001106a32c
18. ./build/./src/Processors/Executors/ExecutionThreadContext.cpp:0: DB::ExecutionThreadContext::executeTask() @ 0x000000001108346d
19. ./build/./src/Processors/Executors/PipelineExecutor.cpp:273: DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001107a450
20. ./contrib/llvm-project/libcxx/include/atomic:958: DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001107a009
21. ./build/./src/Processors/Executors/PullingPipelineExecutor.cpp:54: DB::PullingPipelineExecutor::pull(DB::Chunk&) @ 0x0000000011086a88
22. ./build/./src/Processors/Transforms/buildPushingToViewsChain.cpp:749: DB::ExecutingInnerQueryFromViewTransform::onGenerate() @ 0x000000001133126b
23. ./build/./src/Processors/Transforms/ExceptionKeepingTransform.cpp:164: void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x00000000112a04d8
24. ./build/./src/Processors/Transforms/ExceptionKeepingTransform.cpp:102: DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000112a0290
25. ./contrib/llvm-project/libcxx/include/__functional/function.h:818: ? @ 0x000000001129fe68
26. ./build/./src/Processors/Executors/ExecutionThreadContext.cpp:0: DB::ExecutionThreadContext::executeTask() @ 0x000000001108346d
27. ./build/./src/Processors/Executors/PipelineExecutor.cpp:273: DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001107a450
28. ./contrib/llvm-project/libcxx/include/atomic:958: DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001107a009
29. ./build/./src/Processors/Executors/PushingPipelineExecutor.cpp:105: DB::PushingPipelineExecutor::push(DB::Chunk) @ 0x0000000011089618
30. ./contrib/llvm-project/libcxx/include/__memory/shared_ptr.h:701: DB::PushingPipelineExecutor::push(DB::Block) @ 0x00000000110896fa
31. ./contrib/llvm-project/libcxx/include/__hash_table:1473: DB::TCPHandler::processInsertQuery() @ 0x000000001101e6ea
is_initial_query: 1
user: default
query_id: 258e45ec-47d9-4d2b-a240-c40f6e6a5d17
address: ::ffff:127.0.0.1
port: 39138
initial_user: default
initial_query_id: 258e45ec-47d9-4d2b-a240-c40f6e6a5d17
initial_address: ::ffff:127.0.0.1
initial_port: 39138
initial_query_start_time: 2024-04-24 12:38:40
initial_query_start_time_microseconds: 2024-04-24 12:38:40.124049
interface: 1
is_secure: 0
os_user: ubuntu
client_hostname: ip-172-31-42-161.us-east-2.compute.internal
client_name: ClickHouse client
client_revision: 54467
client_version_major: 24
client_version_minor: 4
client_version_patch: 1
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 0
revision: 54485
log_comment:
thread_ids: [500889]
peak_threads_usage: 2
ProfileEvents: {'Query':1,'InsertQuery':1,'InitialQuery':1,'QueriesWithSubqueries':84,'SelectQueriesWithSubqueries':83,'InsertQueriesWithSubqueries':1,'FileOpen':2349,'WriteBufferFromFileDescriptorWrite':17446,'WriteBufferFromFileDescriptorWriteBytes':16196273151,'IOBufferAllocs':8181,'IOBufferAllocBytes':2036624715,'ArenaAllocChunks':166,'ArenaAllocBytes':679936,'FunctionExecute':5593,'DiskWriteElapsedMicroseconds':4970505,'NetworkReceiveElapsedMicroseconds':60214375,'NetworkSendElapsedMicroseconds':89037,'NetworkReceiveBytes':89351239027,'NetworkSendBytes':12305217,'InsertedRows':744323483,'InsertedBytes':325688805396,'SelectedRows':255800263,'SelectedBytes':117827923811,'MergeTreeDataWriterRows':536963483,'MergeTreeDataWriterUncompressedBytes':230047463462,'MergeTreeDataWriterCompressedBytes':16196157903,'MergeTreeDataWriterBlocks':81,'MergeTreeDataWriterSortingBlocksMicroseconds':117041412,'MergeTreeDataWriterMergingBlocksMicroseconds':284,'InsertedWideParts':80,'ContextLock':2907,'RWLockAcquiredReadLocks':169,'PartsLockHoldMicroseconds':4704,'RealTimeMicroseconds':2029801807,'UserTimeMicroseconds':1547085906,'SystemTimeMicroseconds':352296819,'SoftPageFaults':263893028,'OSCPUWaitMicroseconds':18160,'OSCPUVirtualTimeMicroseconds':992101643,'OSWriteBytes':16202440704,'OSReadChars':3220389,'OSWriteChars':16197025210,'QueryProfilerRuns':2075,'QueryMemoryLimitExceeded':1,'LogDebug':14,'InterfaceNativeReceiveBytes':89351239027}
Settings: {'allow_experimental_analyzer':'1','use_variant_as_common_type':'1','allow_experimental_variant_type':'1','format_regexp':'^(?<measurement>[^,]+),(?<tags>[^\\s]+)\\s(?<fields>[^\\s]+)\\s(?<ts>[\\d]+)$','format_regexp_escaping_rule':'Raw','format_regexp_skip_unmatched':'0'}
used_aggregate_functions: []
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['Variant','Bool','UInt8','Float64','String','DateTime64','Int64']
used_dictionaries: []
used_formats: []
used_functions: ['toFloat64OrNull','CAST','position','extractKeyValuePairs','in','concat','mapApply','greater','regexpQuoteMeta','toInt64','tupleElement','replicate','replaceRegexpOne','tuple','fromUnixTimestamp64Nano','toUInt64OrNull','cityHash64','multiIf','arrayJoin']
used_storages: []
used_table_functions: []
used_row_policies: []
transaction_id: (0,0,'00000000-0000-0000-0000-000000000000')
query_cache_usage: None
asynchronous_read_counters: {}