Skip to content

Conversation

@xinyiZzz
Copy link
Contributor

pick #40542 #40914

xinyiZzz and others added 2 commits October 16, 2024 18:41
1. Add a new class MemCounter.
2. MemTracker and MemTrackerLimiter no longer have inheritance and
dependencies.
3. MemTrackerLimiter is used to count process memory, the BE web page
`/mem_tracker` also only displays MemTrackerLimiter.
4. MemTracker is used to count the memory of operators and some data
structures. It is not counted in the process memory and is used for
logic control and profile.

In addition, it seems that the crash is caused by memory abnormality,
not sure whether this PR can fix the problem, but this will help locate
the problem and more elegant.

```
==6641==ERROR: AddressSanitizer: heap-use-after-free on address 0x603000980e70 at pc 0x559be21880be bp 0x7fd7cfc75070 sp 0x7fd7cfc75068
READ of size 8 at 0x603000980e70 thread T1453 (memory_maintena)
    #0 0x559be21880bd in std::__atomic_base<long>::load(std::memory_order) const /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/atomic_base.h:481:9
    #1 0x559be21880bd in doris::MemTracker::MemCounter::current_value() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:139:63
    apache#2 0x559be21880bd in doris::MemTracker::consumption() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:165:56
    apache#3 0x559be3985133 in doris::MemTrackerLimiter::refresh_global_counter() /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker_limiter.cpp:245:59
    apache#4 0x559bdfcd52d5 in doris::Daemon::memory_maintenance_thread() /home/zcp/repo_center/doris_master/doris/be/src/common/daemon.cpp:239:13
    apache#5 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
    apache#6 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
    apache#7 0x7fdf5647584f  misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

0x603000980e74 is located 0 bytes after 20-byte region [0x603000980e60,0x603000980e74)
04:19:48   freed by thread T1491 (Scan_normal [wo) here:
04:19:48       #0 0x559bdfb0dd9d in operator delete(void*) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x33546d9d) (BuildId: f170e92ad3c55512)
04:19:48       #1 0x559bdfb1d42b in __gnu_cxx::new_allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:139:2
04:19:48       apache#2 0x559bdfb1d42b in std::allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:187:27
04:19:48       apache#3 0x559bdfb1d42b in std::allocator_traits<std::allocator<char>>::deallocate(std::allocator<char>&, char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:492:13
04:19:48       apache#4 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_destroy(unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:237:9
04:19:48       apache#5 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_dispose() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:232:4
04:19:48       apache#6 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::~basic_string() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:658:9
04:19:48       apache#7 0x559bdfb22501 in void std::destroy_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:88:15
04:19:48       apache#8 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:138:7
04:19:48       apache#9 0x559bdfb22501 in void std::_Destroy_aux<false>::__destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:152:6
04:19:48       apache#10 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:184:7
04:19:48       apache#11 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:746:7
04:19:48       apache#12 0x559bdfb22501 in std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::~vector() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_vector.h:680:2
04:19:48       apache#13 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9867:1
04:19:48       apache#14 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9866:36
04:19:48       apache#15 0x559be0047ee6 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:372:5
04:19:48       apache#16 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13
04:19:48       apache#17 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28
04:19:48       apache#18 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18
04:19:48       apache#19 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18
04:19:48       apache#20 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5
04:19:48       apache#21 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32
04:19:48       apache#22 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5
04:19:48       apache#23 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21
04:19:48       apache#24 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31
04:19:48       apache#25 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
04:19:48       apache#26 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
04:19:48       apache#27 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
04:19:48       apache#28 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24
04:19:48       apache#29 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
04:19:48       apache#30 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
04:19:48 
04:19:48   previously allocated by thread T1491 (Scan_normal [wo) here:
04:19:48       #0 0x559bdfb0d53d in operator new(unsigned long) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x3354653d) (BuildId: f170e92ad3c55512)
04:19:48       #1 0x559bdfb33f74 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:219:14
04:19:48       apache#2 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct_aux<char*>(char*, char*, std::__false_type) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:247:11
04:19:48       apache#3 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:266:4
04:19:48       apache#4 0x559bdfb32705 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::basic_string(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:451:9
04:19:48       apache#5 0x559bdfc326e6 in decltype(::new((void*)(0)) std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>(std::declval<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>())) std::construct_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:97:39
04:19:48       apache#6 0x559bdfc326e6 in void std::allocator_traits<std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::construct<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:514:4
04:19:48       apache#7 0x559bdfc326e6 in void std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::_M_realloc_insert<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(__gnu_cxx::__normal_iterator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/vector.tcc:449:4
04:19:48       apache#8 0x559be0032206 in doris::DeleteHandler::parse_condition(doris::DeleteSubPredicatePB const&, doris::TCondition*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:300:33
04:19:48       apache#9 0x559be0047aa3 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:355:9
04:19:48       apache#10 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13
04:19:48       apache#11 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28
04:19:48       apache#12 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18
04:19:48       apache#13 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18
04:19:48       apache#14 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5
04:19:48       apache#15 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32
04:19:48       apache#16 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5
04:19:48       apache#17 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21
04:19:48       apache#18 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31
04:19:48       apache#19 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
04:19:48       apache#20 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
04:19:48       apache#21 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
04:19:48       apache#22 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24
04:19:48       apache#23 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
04:19:48       apache#24 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
```
## Proposed changes

bug is introduced by apache#40542

<!--Describe your changes.-->

---------

Co-authored-by: yiguolei <yiguolei@gmail.com>
@doris-robot
Copy link

Thank you for your contribution to Apache Doris.
Don't know what should be done next? See How to process your PR

Since 2024-03-18, the Document has been moved to doris-website.
See Doris Document.

@xinyiZzz
Copy link
Contributor Author

run buildall

@doris-robot
Copy link

TPC-H: Total hot run time: 40569 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 6466e61c75a8966b04f557460521390a128a144d, data reload: false

------ Round 1 ----------------------------------
q1	17937	7587	7366	7366
q2	2498	147	152	147
q3	11274	1119	1188	1119
q4	10303	792	699	699
q5	7769	2839	2827	2827
q6	231	142	152	142
q7	993	638	600	600
q8	9345	1901	1901	1901
q9	6614	6394	6400	6394
q10	7015	2295	2296	2295
q11	439	250	250	250
q12	411	221	216	216
q13	17799	2996	2947	2947
q14	232	213	211	211
q15	563	513	506	506
q16	669	605	608	605
q17	989	614	600	600
q18	7273	6524	6622	6524
q19	3293	973	1034	973
q20	583	281	279	279
q21	3916	3124	2981	2981
q22	1129	1009	987	987
Total cold run time: 111275 ms
Total hot run time: 40569 ms

----- Round 2, with runtime_filter_mode=off -----
q1	7615	7277	7255	7255
q2	323	223	222	222
q3	2869	2703	2671	2671
q4	1912	1634	1698	1634
q5	5380	5419	5386	5386
q6	222	137	139	137
q7	2116	1666	1651	1651
q8	3233	3415	3394	3394
q9	8517	8453	8482	8453
q10	3467	3378	3415	3378
q11	578	471	470	470
q12	779	567	540	540
q13	16904	2948	2966	2948
q14	295	278	261	261
q15	553	499	509	499
q16	704	646	641	641
q17	1784	1580	1560	1560
q18	7657	7573	7436	7436
q19	1647	1472	1498	1472
q20	1975	1794	1768	1768
q21	5283	5089	5115	5089
q22	1125	1007	988	988
Total cold run time: 74938 ms
Total hot run time: 57853 ms

@doris-robot
Copy link

TPC-DS: Total hot run time: 189424 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 6466e61c75a8966b04f557460521390a128a144d, data reload: false

query1	989	372	365	365
query2	6533	2146	2042	2042
query3	6699	211	219	211
query4	33832	23533	23206	23206
query5	4372	459	457	457
query6	255	164	160	160
query7	4621	307	303	303
query8	240	203	211	203
query9	9502	2704	2689	2689
query10	505	268	273	268
query11	18082	15057	15191	15057
query12	150	98	97	97
query13	1698	449	447	447
query14	9425	7098	6675	6675
query15	233	170	169	169
query16	8062	403	474	403
query17	1632	571	566	566
query18	2122	307	314	307
query19	209	152	145	145
query20	119	107	106	106
query21	214	106	103	103
query22	4483	4527	4429	4429
query23	34583	33732	33595	33595
query24	12053	2835	2819	2819
query25	697	405	413	405
query26	1754	164	162	162
query27	2874	301	294	294
query28	8220	2513	2464	2464
query29	1017	449	435	435
query30	323	157	157	157
query31	1034	806	798	798
query32	104	58	56	56
query33	772	299	301	299
query34	922	486	501	486
query35	881	767	720	720
query36	1121	941	955	941
query37	166	88	92	88
query38	3926	3872	3858	3858
query39	1482	1406	1459	1406
query40	300	98	99	98
query41	46	48	45	45
query42	117	97	99	97
query43	533	509	504	504
query44	1276	784	781	781
query45	193	162	168	162
query46	1114	721	726	721
query47	1899	1843	1803	1803
query48	429	331	345	331
query49	1245	401	375	375
query50	794	411	415	411
query51	7050	6979	6915	6915
query52	108	88	96	88
query53	274	189	188	188
query54	1316	472	469	469
query55	79	76	76	76
query56	305	264	255	255
query57	1234	1156	1159	1156
query58	243	234	239	234
query59	3230	3195	2971	2971
query60	323	289	266	266
query61	100	101	124	101
query62	871	677	663	663
query63	208	182	178	178
query64	5238	635	618	618
query65	3250	3185	3148	3148
query66	1412	300	316	300
query67	15814	15565	15381	15381
query68	4669	553	540	540
query69	483	281	281	281
query70	1168	1095	1127	1095
query71	441	271	269	269
query72	7267	3887	3914	3887
query73	767	336	329	329
query74	10210	8864	8845	8845
query75	3481	2636	2615	2615
query76	3257	875	862	862
query77	558	299	305	299
query78	9912	9231	9209	9209
query79	4687	585	591	585
query80	2566	445	429	429
query81	586	235	241	235
query82	846	136	135	135
query83	301	135	136	135
query84	294	85	76	76
query85	2127	298	295	295
query86	458	304	301	301
query87	4344	4246	4299	4246
query88	5423	2425	2420	2420
query89	424	284	283	283
query90	1971	186	188	186
query91	134	104	110	104
query92	66	48	46	46
query93	5214	557	542	542
query94	968	301	276	276
query95	346	252	252	252
query96	629	279	278	278
query97	3376	3163	3188	3163
query98	222	196	199	196
query99	1632	1292	1284	1284
Total cold run time: 313734 ms
Total hot run time: 189424 ms

@doris-robot
Copy link

TPC-H: Total hot run time: 40560 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 6466e61c75a8966b04f557460521390a128a144d, data reload: false

------ Round 1 ----------------------------------
q1	17961	7616	7428	7428
q2	2446	183	150	150
q3	11428	1164	1114	1114
q4	10326	692	814	692
q5	7773	2853	2864	2853
q6	233	149	146	146
q7	1008	650	621	621
q8	9375	1945	1999	1945
q9	6621	6460	6427	6427
q10	7010	2289	2289	2289
q11	454	244	252	244
q12	430	214	220	214
q13	17796	2966	2931	2931
q14	238	208	214	208
q15	562	526	525	525
q16	672	605	595	595
q17	995	485	583	485
q18	7442	6607	6658	6607
q19	4062	897	921	897
q20	579	266	263	263
q21	3955	3035	2980	2980
q22	1103	964	946	946
Total cold run time: 112469 ms
Total hot run time: 40560 ms

----- Round 2, with runtime_filter_mode=off -----
q1	7408	7223	7215	7215
q2	325	231	229	229
q3	2927	2737	2691	2691
q4	1918	1609	1620	1609
q5	5391	5433	5408	5408
q6	225	139	136	136
q7	2105	1643	1764	1643
q8	3174	3359	3388	3359
q9	8547	8538	8566	8538
q10	3457	3405	3415	3405
q11	572	478	470	470
q12	740	572	574	572
q13	16905	2948	2974	2948
q14	283	252	272	252
q15	559	518	506	506
q16	695	659	646	646
q17	1807	1608	1561	1561
q18	7706	7336	7196	7196
q19	1665	1449	1560	1449
q20	2000	1793	1782	1782
q21	5319	5079	5049	5049
q22	1083	946	983	946
Total cold run time: 74811 ms
Total hot run time: 57610 ms

@doris-robot
Copy link

TPC-DS: Total hot run time: 189402 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 6466e61c75a8966b04f557460521390a128a144d, data reload: false

query1	984	369	362	362
query2	6547	2135	2050	2050
query3	6707	210	227	210
query4	34366	23369	23420	23369
query5	4329	465	465	465
query6	262	169	161	161
query7	4647	305	301	301
query8	254	210	212	210
query9	9628	2692	2679	2679
query10	469	273	270	270
query11	17919	15112	15225	15112
query12	148	96	97	96
query13	1685	446	479	446
query14	10444	7145	7230	7145
query15	258	167	172	167
query16	7528	467	451	451
query17	1621	565	544	544
query18	1957	303	312	303
query19	247	152	151	151
query20	115	104	105	104
query21	215	105	109	105
query22	4455	4163	4146	4146
query23	34733	33922	33689	33689
query24	12005	2801	2779	2779
query25	685	407	406	406
query26	1783	158	161	158
query27	2813	299	290	290
query28	8138	2527	2479	2479
query29	973	451	436	436
query30	325	157	151	151
query31	1033	799	800	799
query32	98	55	56	55
query33	768	306	284	284
query34	980	509	493	493
query35	932	724	747	724
query36	1121	947	935	935
query37	232	87	84	84
query38	4010	3913	3872	3872
query39	1480	1411	1442	1411
query40	298	100	99	99
query41	47	45	44	44
query42	116	95	96	95
query43	539	493	494	493
query44	1284	780	769	769
query45	195	167	167	167
query46	1142	708	714	708
query47	1942	1820	1798	1798
query48	444	343	352	343
query49	1261	398	383	383
query50	824	411	409	409
query51	6913	6908	6935	6908
query52	98	94	92	92
query53	265	193	183	183
query54	1311	468	461	461
query55	79	79	74	74
query56	281	256	261	256
query57	1289	1147	1139	1139
query58	248	232	233	232
query59	3232	3013	2816	2816
query60	325	265	260	260
query61	121	101	115	101
query62	854	657	659	657
query63	219	179	176	176
query64	5328	634	630	630
query65	3258	3198	3182	3182
query66	1466	306	298	298
query67	15707	15332	15202	15202
query68	4705	546	551	546
query69	547	284	293	284
query70	1179	1100	1122	1100
query71	386	293	266	266
query72	7358	3850	4049	3850
query73	774	346	335	335
query74	10394	8974	8934	8934
query75	3425	2688	2639	2639
query76	3616	951	906	906
query77	447	291	287	287
query78	9844	9262	9095	9095
query79	5606	598	583	583
query80	1883	444	446	444
query81	582	245	239	239
query82	895	140	134	134
query83	268	138	137	137
query84	296	83	74	74
query85	2184	302	284	284
query86	481	292	305	292
query87	4423	4185	4221	4185
query88	4603	2399	2418	2399
query89	427	297	287	287
query90	2020	182	185	182
query91	154	104	111	104
query92	62	47	46	46
query93	5196	548	536	536
query94	996	289	283	283
query95	351	247	251	247
query96	620	277	284	277
query97	3329	3139	3086	3086
query98	221	197	198	197
query99	1707	1277	1295	1277
Total cold run time: 314959 ms
Total hot run time: 189402 ms

@yiguolei yiguolei merged commit 603823a into apache:branch-3.0 Oct 18, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

3 participants