Skip to content

Conversation

@github-actions
Copy link
Contributor

@github-actions github-actions bot commented Jan 3, 2025

Cherry-picked from #46296

…6296)

### What problem does this PR solve?

```cpp
172.20.50.85 be.out: #8 0x56435579ab5e in lucene::store::BufferedIndexInput::readBytes(unsigned char*, int, bool) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/clucene/src/core/CLucene/store/IndexInput.cpp:192:9
    #9 0x564319f7d95d in doris::segment_v2::CSIndexInput::readInternal(unsigned char*, int) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp:107:11
    #10 0x56435579bb5b in lucene::store::BufferedIndexInput::refill() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/clucene/src/core/CLucene/store/IndexInput.cpp:285:5
    #11 0x56435579b096 in lucene::store::BufferedIndexInput::readBytes(unsigned char*, int, int, bool) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/clucene/src/core/CLucene/store/IndexInput.cpp:217:9
    #12 0x56435579a9f6 in lucene::store::BufferedIndexInput::readBytes(unsigned char*, int) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/clucene/src/core/CLucene/store/IndexInput.cpp:186:9
    #13 0x564319fe8aee in doris::segment_v2::InvertedIndexReader::read_null_bitmap(doris::io::IOContext const*, doris::OlapReaderStatistics*, doris::segment_v2::InvertedIndexQueryCacheHandle*, lucene::store::Directory*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp:144:29
    #14 0x564319febab4 in doris::segment_v2::InvertedIndexReader::handle_searcher_cache(doris::segment_v2::InvertedIndexCacheHandle*, doris::io::IOContext const*, doris::OlapReaderStatistics*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp:194:27
    #15 0x564319ff4b0b in doris::segment_v2::StringTypeInvertedIndexReader::query(doris::io::IOContext const*, doris::OlapReaderStatistics*, doris::RuntimeState*, std::__cxx11::basic_string, std::allocator> const&, void const*, doris::segment_v2::InvertedIndexQueryType, std::shared_ptr&) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp:407:5
    #16 0x56431a001f00 in doris::segment_v2::InvertedIndexIterator::read_from_inverted_index(std::__cxx11::basic_string, std::allocator> const&, void const*, doris::segment_v2::InvertedIndexQueryType, unsigned int, std::shared_ptr&, bool) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp:1137:5
    #17 0x5643191f4a52 in doris::ComparisonPredicateBase<(doris::PrimitiveType)23, (doris::PredicateType)1>::evaluate(std::pair, std::allocator>, std::shared_ptr> const&, doris::segment_v2::InvertedIndexIterator*, unsigned int, roaring::Roaring*) const /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/comparison_predicate.h:107:9
    #18 0x56431a249ef7 in doris::segment_v2::SegmentIterator::_apply_inverted_index_on_column_predicate(doris::ColumnPredicate*, std::vector>&, bool*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:871:28
    #19 0x56431a23ddc0 in doris::segment_v2::SegmentIterator::_apply_inverted_index() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:951:13
    #20 0x56431a22f956 in doris::segment_v2::SegmentIterator::_get_row_ranges_by_column_conditions() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:512:13
    #21 0x56431a22b78f in doris::segment_v2::SegmentIterator::_lazy_init() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:383:5
    #22 0x56431a26c667 in doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2008:9
    #23 0x56431a264fb6 in doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*)::$_0::operator()() const /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1917:9
    #24 0x56431a264fb6 in doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1916:19
    #25 0x56431a08315c in doris::segment_v2::LazyInitSegmentIterator::next_batch(doris::vectorized::Block*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h:44:33
    #26 0x564319b95855 in doris::BetaRowsetReader::next_block(doris::vectorized::Block*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/beta_rowset_reader.cpp:366:29
    #27 0x56434e3a0c10 in doris::vectorized::VCollectIterator::Level0Iterator::_refresh() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vcollect_iterator.h
    #28 0x56434e38ba5b in doris::vectorized::VCollectIterator::Level0Iterator::refresh_current_row() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vcollect_iterator.cpp:509:24
    #29 0x56434e38cd76 in doris::vectorized::VCollectIterator::Level0Iterator::ensure_first_row_ref() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vcollect_iterator.cpp:482:14
    #30 0x56434e399745 in doris::vectorized::VCollectIterator::Level1Iterator::ensure_first_row_ref() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vcollect_iterator.cpp:696:27
    #31 0x56434e37f7f9 in doris::vectorized::VCollectIterator::build_heap(std::vector, std::allocator>>&) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vcollect_iterator.cpp:186:9
    #32 0x56434e312fe2 in doris::vectorized::BlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/block_reader.cpp:140:5
    #33 0x56434e3165df in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/block_reader.cpp:212:19
    #34 0x5643515aba04 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:231:32
    #35 0x564336c138c6 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr, std::shared_ptr) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:247:5
    #36 0x564336c18db0 in doris::vectorized::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_1::operator()() const::'lambda'()::operator()() const::'lambda0'()::operator()() const /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:180:21
    #37 0x564336c18db0 in doris::vectorized::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:179:31
    #38 0x564336c18db0 in void std::__invoke_impl, std::shared_ptr)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
    #39 0x564336c18db0 in std::enable_if, std::shared_ptr)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r, std::shared_ptr)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
    #40 0x564336c18db0 in std::_Function_handler, std::shared_ptr)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
    #41 0x56431b9d94eb in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/threadpool.cpp:543:24
    #42 0x56431b9b18c7 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/thread.cpp:498:5
    #43 0x7f31d3b87ac2 in start_thread nptl/pthread_create.c:442:8
    #44 0x7f31d3c1984f  misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

0x61d000a7f3c8 is located 1352 bytes inside of 2064-byte region [0x61d000a7ee80,0x61d000a7f690)
freed by thread T1698 (Pipe_normal [wo) here:
    #0 0x564317278d9d in operator delete(void*) (/mnt/hdd01/ci/doris-deploy-branch-3.0-cloud/be/lib/doris_be+0x3626bd9d) (BuildId: 9dab40b94b1dc995)
    #1 0x5643515b192c in std::unique_ptr>::reset(doris::TabletReader*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:456:7
    #2 0x5643515b192c in doris::vectorized::NewOlapScanner::close(doris::RuntimeState*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:534:20
    #3 0x564352457463 in doris::vectorized::ScannerDelegate::~ScannerDelegate() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/exec/scan/vscan_node.h:35:31
    #4 0x5643172a8324 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:168:6
    #5 0x56434e7d7870 in std::__shared_ptr::~__shared_ptr() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:1149:31
    #6 0x56434e7d7870 in void std::destroy_at>(std::shared_ptr*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:88:15
    #7 0x56434e7d7870 in void std::allocator_traits>>>::destroy>(std::allocator>>&, std::shared_ptr*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:533:4
    #8 0x56434e7d7870 in std::__cxx11::_List_base, std::allocator>>::_M_clear() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/list.tcc:77:4
    #9 0x564351db11d2 in std::__cxx11::_List_base, std::allocator>>::~_List_base() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_list.h:499:9
    #10 0x564351db11d2 in doris::pipeline::ScanLocalState::close(doris::RuntimeState*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/pipeline/exec/scan_operator.cpp:1261:5
    #11 0x56434e6357e3 in doris::pipeline::OperatorXBase::close(doris::RuntimeState*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/pipeline/exec/operator.cpp:245:28
    #12 0x564352ad00c1 in doris::pipeline::PipelineTask::close(doris::Status) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/pipeline/pipeline_task.cpp:489:28
    #13 0x564352b0701f in doris::pipeline::_close_task(doris::pipeline::PipelineTask*, doris::Status) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/pipeline/task_scheduler.cpp:91:27
    #14 0x564352b095f3 in doris::pipeline::TaskScheduler::_do_work(unsigned long) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/pipeline/task_scheduler.cpp:181:17
    #15 0x56431b9d94eb in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/threadpool.cpp:543:24
    #16 0x56431b9b18c7 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/thread.cpp:498:5
    #17 0x7f31d3b87ac2 in start_thread nptl/pthread_create.c:442:8

```

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
apache/doris-website#1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
@hello-stephen
Copy link
Contributor

Thank you for your contribution to Apache Doris.
Don't know what should be done next? See How to process your PR.

Please clearly describe your PR:

  1. What problem was fixed (it's best to include specific error reporting information). How it was fixed.
  2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be.
  3. What features were added. Why was this function added?
  4. Which code was refactored and why was this part of the code refactored?
  5. Which functions were optimized and what is the difference before and after the optimization?

@dataroaring dataroaring closed this Jan 3, 2025
@dataroaring dataroaring reopened this Jan 3, 2025
@hello-stephen
Copy link
Contributor

run buildall

@doris-robot
Copy link

TPC-H: Total hot run time: 40511 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 6cf8a556ac30994362201bdd4aea8f742cf68ff1, data reload: false

------ Round 1 ----------------------------------
q1	17574	7373	7201	7201
q2	2058	178	171	171
q3	10559	1094	1140	1094
q4	10571	744	716	716
q5	7737	2850	2836	2836
q6	235	147	144	144
q7	959	613	593	593
q8	9339	1946	2010	1946
q9	6554	6435	6389	6389
q10	7004	2261	2299	2261
q11	458	260	261	260
q12	394	208	213	208
q13	17763	2969	2940	2940
q14	249	211	205	205
q15	558	516	507	507
q16	681	603	603	603
q17	961	584	489	489
q18	7182	6634	6498	6498
q19	1395	1100	1115	1100
q20	466	200	195	195
q21	4004	3177	3233	3177
q22	1106	1004	978	978
Total cold run time: 107807 ms
Total hot run time: 40511 ms

----- Round 2, with runtime_filter_mode=off -----
q1	7190	7150	7177	7150
q2	328	233	237	233
q3	2891	2843	2866	2843
q4	2037	1777	1826	1777
q5	5710	5737	5676	5676
q6	222	138	143	138
q7	2263	1843	1835	1835
q8	3361	3518	3503	3503
q9	8726	8858	8823	8823
q10	3575	3508	3518	3508
q11	598	501	512	501
q12	823	605	618	605
q13	9160	3257	3139	3139
q14	303	288	271	271
q15	565	526	512	512
q16	717	670	657	657
q17	1825	1629	1627	1627
q18	8162	7718	7752	7718
q19	1661	1577	1520	1520
q20	2075	1894	1875	1875
q21	5477	5313	5357	5313
q22	1157	1051	1024	1024
Total cold run time: 68826 ms
Total hot run time: 60248 ms

@doris-robot
Copy link

TPC-DS: Total hot run time: 198005 ms
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 6cf8a556ac30994362201bdd4aea8f742cf68ff1, data reload: false

query1	1293	884	885	884
query2	6238	2100	2060	2060
query3	10809	4177	4144	4144
query4	61406	29384	23440	23440
query5	5205	462	470	462
query6	396	171	169	169
query7	5399	308	310	308
query8	306	240	236	236
query9	8744	2701	2695	2695
query10	465	269	266	266
query11	17796	15277	16002	15277
query12	159	110	104	104
query13	1436	449	445	445
query14	10855	7416	7117	7117
query15	215	176	177	176
query16	7173	500	458	458
query17	1209	602	588	588
query18	1851	328	323	323
query19	214	164	158	158
query20	117	110	115	110
query21	56	45	45	45
query22	4708	4655	4466	4466
query23	34808	34571	34581	34571
query24	6233	2953	3009	2953
query25	543	429	426	426
query26	677	174	172	172
query27	1786	307	306	306
query28	4275	2502	2488	2488
query29	733	474	467	467
query30	236	158	157	157
query31	1025	835	854	835
query32	64	55	54	54
query33	408	287	303	287
query34	926	500	497	497
query35	853	735	755	735
query36	1095	993	995	993
query37	129	73	75	73
query38	4064	4013	4043	4013
query39	1515	1469	1488	1469
query40	143	81	79	79
query41	51	49	48	48
query42	113	99	107	99
query43	543	495	499	495
query44	1203	828	835	828
query45	186	166	175	166
query46	1175	773	748	748
query47	2007	1912	1956	1912
query48	484	407	399	399
query49	748	398	389	389
query50	850	422	433	422
query51	7297	7418	7238	7238
query52	104	82	92	82
query53	260	187	184	184
query54	598	461	481	461
query55	78	77	77	77
query56	279	267	265	265
query57	1271	1191	1157	1157
query58	221	211	216	211
query59	3453	3235	3003	3003
query60	281	258	252	252
query61	108	109	106	106
query62	766	664	675	664
query63	213	185	188	185
query64	1426	660	630	630
query65	3259	3176	3245	3176
query66	710	306	313	306
query67	15895	15461	15548	15461
query68	3964	591	573	573
query69	439	261	289	261
query70	1190	1110	1120	1110
query71	362	257	260	257
query72	6455	4090	4005	4005
query73	771	341	348	341
query74	10143	9006	9023	9006
query75	3362	2627	2677	2627
query76	1961	1066	1150	1066
query77	509	286	273	273
query78	10640	9595	9565	9565
query79	1442	600	592	592
query80	891	438	426	426
query81	518	245	241	241
query82	1278	115	114	114
query83	235	147	141	141
query84	284	79	85	79
query85	887	295	283	283
query86	339	300	264	264
query87	4474	4310	4265	4265
query88	3905	2402	2371	2371
query89	419	287	289	287
query90	2052	191	190	190
query91	187	147	150	147
query92	61	47	49	47
query93	1946	542	548	542
query94	764	285	288	285
query95	349	251	256	251
query96	636	278	274	274
query97	3324	3171	3210	3171
query98	221	202	196	196
query99	1576	1306	1285	1285
Total cold run time: 314726 ms
Total hot run time: 198005 ms

@doris-robot
Copy link

ClickBench: Total hot run time: 32.32 s
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 6cf8a556ac30994362201bdd4aea8f742cf68ff1, data reload: false

query1	0.03	0.04	0.03
query2	0.07	0.03	0.03
query3	0.23	0.07	0.06
query4	1.62	0.10	0.10
query5	0.52	0.49	0.52
query6	1.14	0.71	0.73
query7	0.03	0.02	0.02
query8	0.04	0.03	0.03
query9	0.57	0.49	0.51
query10	0.55	0.54	0.56
query11	0.14	0.10	0.10
query12	0.14	0.11	0.12
query13	0.62	0.60	0.59
query14	3.06	2.91	3.02
query15	0.91	0.83	0.83
query16	0.39	0.37	0.37
query17	1.06	1.04	0.98
query18	0.23	0.21	0.22
query19	1.88	1.92	2.04
query20	0.01	0.01	0.01
query21	15.36	0.59	0.57
query22	2.79	2.13	1.66
query23	16.97	1.10	0.76
query24	2.98	1.38	0.76
query25	0.25	0.10	0.04
query26	0.43	0.13	0.12
query27	0.04	0.04	0.04
query28	10.72	1.08	1.06
query29	12.60	3.24	3.27
query30	0.24	0.05	0.06
query31	2.85	0.38	0.38
query32	3.27	0.47	0.46
query33	3.00	3.00	3.05
query34	16.97	4.42	4.45
query35	4.48	4.46	4.49
query36	0.68	0.51	0.47
query37	0.10	0.06	0.06
query38	0.05	0.03	0.03
query39	0.04	0.03	0.02
query40	0.16	0.14	0.12
query41	0.08	0.02	0.02
query42	0.03	0.02	0.02
query43	0.03	0.04	0.03
Total cold run time: 107.36 s
Total hot run time: 32.32 s

@yiguolei yiguolei merged commit f5ed360 into branch-3.0 Jan 3, 2025
19 of 22 checks passed
@github-actions github-actions bot deleted the auto-pick-46296-branch-3.0 branch January 3, 2025 08:31
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

6 participants