Skip to content

Commit 58eb559

Browse files
lesandielesandie
committed
Merge branch 'master'
2 parents a4734e4 + ceeeba1 commit 58eb559

1,057 files changed

Lines changed: 12319 additions & 6252 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.claude/CLAUDE.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,7 @@ grep "test_name" ci/tmp/pytest_parallel.jsonl | python3 -c "import sys,json; [pr
3636
You can build multiple versions of ClickHouse inside `build_*` directories, such as `build`, `build_debug`, `build_asan`, etc.
3737

3838
You can run integration tests as in `tests/integration/README.md` using: `python -m ci.praktika run "integration" --test <selectors>` invoked from the repository root.
39+
40+
When writing tests, do not add "no-*" tags (like "no-parallel") unless strictly necessarily.
41+
42+
When writing tests in tests/queries, prefer adding a new test instead of extending existing ones.

‎.github/workflows/master.yml‎

Lines changed: 91 additions & 11 deletions
Large diffs are not rendered by default.

‎.github/workflows/pull_request.yml‎

Lines changed: 186 additions & 106 deletions
Large diffs are not rendered by default.

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,9 @@
366366
[submodule "contrib/StringZilla"]
367367
path = contrib/StringZilla
368368
url = https://github.com/ClickHouse/StringZilla.git
369+
[submodule "contrib/simdcomp"]
370+
path = contrib/simdcomp
371+
url = https://github.com/fast-pack/simdcomp.git
369372
[submodule "contrib/antlr4-cpp-runtime"]
370373
path = contrib/antlr4-cpp-runtime
371374
url = https://github.com/antlr/antlr4.git

‎ci/defs/job_configs.py‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -514,11 +514,15 @@ class JobConfigs:
514514
runs_on=RunnerLabels.AMD_SMALL,
515515
requires=[ArtifactNames.CH_AMD_DEBUG],
516516
),
517-
Job.ParamSet(
518-
parameter=f"amd_tsan, parallel",
519-
runs_on=RunnerLabels.AMD_LARGE,
520-
requires=[ArtifactNames.CH_AMD_TSAN],
521-
),
517+
*[
518+
Job.ParamSet(
519+
parameter=f"amd_tsan, parallel, {batch}/{total_batches}",
520+
runs_on=RunnerLabels.AMD_LARGE,
521+
requires=[ArtifactNames.CH_AMD_TSAN],
522+
)
523+
for total_batches in (2,)
524+
for batch in range(1, total_batches + 1)
525+
],
522526
*[
523527
Job.ParamSet(
524528
parameter=f"amd_tsan, sequential, {batch}/{total_batches}",
@@ -566,11 +570,15 @@ class JobConfigs:
566570
runs_on=RunnerLabels.AMD_SMALL,
567571
requires=[ArtifactNames.CH_AMD_DEBUG],
568572
),
569-
Job.ParamSet(
570-
parameter="amd_tsan, s3 storage, parallel",
571-
runs_on=RunnerLabels.AMD_MEDIUM,
572-
requires=[ArtifactNames.CH_AMD_TSAN],
573-
),
573+
*[
574+
Job.ParamSet(
575+
parameter=f"amd_tsan, s3 storage, parallel, {batch}/{total_batches}",
576+
runs_on=RunnerLabels.AMD_MEDIUM,
577+
requires=[ArtifactNames.CH_AMD_TSAN],
578+
)
579+
for total_batches in (2,)
580+
for batch in range(1, total_batches + 1)
581+
],
574582
*[
575583
Job.ParamSet(
576584
parameter=f"amd_tsan, s3 storage, sequential, {batch}/{total_batches}",

‎ci/docker/integration/clickhouse_with_unity_catalog/Dockerfile‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
ARG FROM_TAG=latest
33
FROM clickhouse/integration-test:$FROM_TAG
44

5+
# Fix missing GPG key for kitware repository to prevent apt-get update failure
6+
RUN rm -f /etc/apt/sources.list.d/kitware.list || true
7+
58
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get -y install openjdk-17-jdk-headless && update-alternatives --config java && update-alternatives --config javac
69

710
# Clone unity catalog and hack their sample data because it contains inconsistent DeltaLake definition
@@ -17,6 +20,11 @@ RUN curl -fsSL -O https://archive.apache.org/dist/spark/spark-3.5.4/spark-3.5.4-
1720

1821
# download spark and packages
1922
# if you change packages, don't forget to update them in tests/integration/helpers/cluster.py
20-
RUN packages="org.apache.hadoop:hadoop-aws:3.3.4,io.delta:delta-spark_2.12:3.2.1,io.unitycatalog:unitycatalog-spark_2.12:0.2.0" \
21-
/spark-3.5.4-bin-hadoop3/bin/spark-shell --packages "$packages" > /dev/null \
22-
&& find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.5.4-bin-hadoop3/jars/ \;
23+
# Using spark-sql with -e flag (instead of spark-shell) to ensure reliable automatic exit
24+
RUN packages="org.apache.hadoop:hadoop-aws:3.3.4,io.delta:delta-spark_2.12:3.2.1,io.unitycatalog:unitycatalog-spark_2.12:0.2.0" && \
25+
cd /spark-3.5.4-bin-hadoop3 && \
26+
for i in 1 2 3; do \
27+
bin/spark-sql --packages "$packages" -e "SELECT 1;" > /dev/null 2>&1 && \
28+
break || sleep 5; \
29+
done && \
30+
find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.5.4-bin-hadoop3/jars/ \;

‎ci/jobs/buzzhouse_job.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@ def main():
221221
"allow_transactions": allow_transactions,
222222
# Run query oracles sometimes
223223
"allow_query_oracles": random.randint(1, 4) == 1,
224+
# Run for 30 minutes max
225+
"time_to_run": 30,
224226
"remote_servers": ["localhost:9000"],
225227
"remote_secure_servers": ["localhost:9440"],
226228
"http_servers": ["localhost:8123"],

‎ci/jobs/fast_test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ def clone_submodules():
5050
"contrib/morton-nd",
5151
"contrib/xxHash",
5252
"contrib/simdjson",
53+
"contrib/simdcomp",
5354
"contrib/liburing",
5455
"contrib/libfiu",
5556
"contrib/yaml-cpp",

‎ci/jobs/functional_tests.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import argparse
22
import os
33
import random
4-
import re
54
import subprocess
65
from pathlib import Path
76

@@ -521,13 +520,19 @@ def start():
521520
collected_test_results.append(test_case_result)
522521
seen_test_names.add(test_case_result.name)
523522

523+
# Control elapsed time for targeted checks: exit if >30 minutes
524+
stop_by_elapsed_time = False
525+
if is_targeted_check and cnt > 0:
526+
stop_by_elapsed_time = stop_watch_.duration / 60 > 30
527+
524528
# On final run, replace results with collected ones
525-
if is_final_run:
529+
if is_final_run or stop_by_elapsed_time:
526530
test_result.results = collected_test_results
527531
# Set overall status to failed if any collected test cases failed
528532
has_failures = any(not t.is_ok() for t in collected_test_results)
529533
if has_failures and test_result.is_ok():
530534
test_result.set_failed()
535+
break
531536

532537
if not info.is_local_run:
533538
CH.stop_log_exports()

‎ci/jobs/integration_test_job.py‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,13 @@ def main():
386386
failed_tests_files = []
387387

388388
has_error = False
389+
if not is_targeted_check:
390+
session_timeout = 5400
391+
else:
392+
# For targeted jobs, use a shorter session timeout to keep feedback fast.
393+
# If this timeout is exceeded but all completed tests have passed, the
394+
# targeted check will not fail solely because the session timed out.
395+
session_timeout = 1200
389396
error_info = []
390397

391398
module_repeat_cnt = 1
@@ -398,7 +405,7 @@ def main():
398405
for attempt in range(module_repeat_cnt):
399406
log_file = f"{temp_path}/pytest_parallel.log"
400407
test_result_parallel = Result.from_pytest_run(
401-
command=f"{' '.join(parallel_test_modules)} --report-log-exclude-logs-on-passed-tests -n {workers} --dist=loadfile --tb=short {repeat_option} --session-timeout=5400",
408+
command=f"{' '.join(parallel_test_modules)} --report-log-exclude-logs-on-passed-tests -n {workers} --dist=loadfile --tb=short {repeat_option} --session-timeout={session_timeout}",
402409
cwd="./tests/integration/",
403410
env=test_env,
404411
pytest_report_file=f"{temp_path}/pytest_parallel.jsonl",
@@ -416,15 +423,20 @@ def main():
416423
if test_result_parallel.files:
417424
failed_tests_files.extend(test_result_parallel.files)
418425
if test_result_parallel.is_error():
419-
has_error = True
420-
error_info.append(test_result_parallel.info)
426+
if not is_targeted_check:
427+
# In targeted checks we may overload the run with many or heavy tests
428+
# (--count N is used). In this mode, a session-timeout is an expected risk
429+
# rather than an infrastructure problem, so we do not treat such errors as job-level
430+
# failures and avoid setting the error flag for targeted runs.
431+
has_error = True
432+
error_info.append(test_result_parallel.info)
421433

422434
fail_num = len([r for r in test_results if not r.is_ok()])
423435
if sequential_test_modules and fail_num < MAX_FAILS_BEFORE_DROP and not has_error:
424436
for attempt in range(module_repeat_cnt):
425437
log_file = f"{temp_path}/pytest_sequential.log"
426438
test_result_sequential = Result.from_pytest_run(
427-
command=f"{' '.join(sequential_test_modules)} --report-log-exclude-logs-on-passed-tests --tb=short {repeat_option} -n 1 --dist=loadfile --session-timeout=5400",
439+
command=f"{' '.join(sequential_test_modules)} --report-log-exclude-logs-on-passed-tests --tb=short {repeat_option} -n 1 --dist=loadfile --session-timeout={session_timeout}",
428440
env=test_env,
429441
cwd="./tests/integration/",
430442
pytest_report_file=f"{temp_path}/pytest_sequential.jsonl",
@@ -442,8 +454,13 @@ def main():
442454
if test_result_sequential.files:
443455
failed_tests_files.extend(test_result_sequential.files)
444456
if test_result_sequential.is_error():
445-
has_error = True
446-
error_info.append(test_result_sequential.info)
457+
if not is_targeted_check:
458+
# In targeted checks we may overload the run with many or heavy tests
459+
# (--count N is used). In this mode, a session-timeout is an expected risk
460+
# rather than an infrastructure problem, so we do not treat such errors as job-level
461+
# failures and avoid setting the error flag for targeted runs.
462+
has_error = True
463+
error_info.append(test_result_sequential.info)
447464

448465
# Collect logs before re-run
449466
attached_files = []

0 commit comments

Comments
 (0)