Skip to content

Commit 563ffc8

Browse files
scanhex12scanhex12
authored
Merge branch 'ClickHouse:master' into revert-83076-revert-67161-sz_attempt_2
2 parents 3dc29c5 + 1dba8ed commit 563ffc8

536 files changed

Lines changed: 3882 additions & 1428 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/master.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,44 @@ jobs:
205205
python3 -m praktika run 'Build (amd_tidy)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
206206
fi
207207
208+
build_arm_tidy:
209+
runs-on: [self-hosted, builder-aarch64]
210+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
211+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV90aWR5KQ==') }}
212+
name: "Build (arm_tidy)"
213+
outputs:
214+
data: ${{ steps.run.outputs.DATA }}
215+
steps:
216+
- name: Checkout code
217+
uses: actions/checkout@v4
218+
with:
219+
ref: ${{ env.CHECKOUT_REF }}
220+
221+
- name: Prepare env script
222+
run: |
223+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
224+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
225+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
226+
export PYTHONPATH=./ci:.:
227+
cat > ./ci/tmp/workflow_config_masterci.json << 'EOF'
228+
${{ needs.config_workflow.outputs.data }}
229+
EOF
230+
cat > ./ci/tmp/workflow_status.json << 'EOF'
231+
${{ toJson(needs) }}
232+
EOF
233+
ENV_SETUP_SCRIPT_EOF
234+
235+
- name: Run
236+
id: run
237+
run: |
238+
. ./ci/tmp/praktika_setup_env.sh
239+
set -o pipefail
240+
if command -v ts &> /dev/null; then
241+
python3 -m praktika run 'Build (arm_tidy)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
242+
else
243+
python3 -m praktika run 'Build (arm_tidy)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
244+
fi
245+
208246
build_amd_debug:
209247
runs-on: [self-hosted, builder]
210248
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]

‎.github/workflows/pull_request.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ jobs:
285285
286286
build_arm_tidy:
287287
runs-on: [self-hosted, builder-aarch64]
288-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_tidy]
288+
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
289289
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV90aWR5KQ==') }}
290290
name: "Build (arm_tidy)"
291291
outputs:
@@ -323,7 +323,7 @@ jobs:
323323
324324
build_amd_debug:
325325
runs-on: [self-hosted, builder]
326-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
326+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
327327
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
328328
name: "Build (amd_debug)"
329329
outputs:
@@ -361,7 +361,7 @@ jobs:
361361
362362
build_amd_release:
363363
runs-on: [self-hosted, builder]
364-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
364+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
365365
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
366366
name: "Build (amd_release)"
367367
outputs:
@@ -399,7 +399,7 @@ jobs:
399399
400400
build_amd_asan:
401401
runs-on: [self-hosted, builder]
402-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
402+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
403403
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9hc2FuKQ==') }}
404404
name: "Build (amd_asan)"
405405
outputs:
@@ -437,7 +437,7 @@ jobs:
437437
438438
build_amd_tsan:
439439
runs-on: [self-hosted, builder]
440-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
440+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
441441
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF90c2FuKQ==') }}
442442
name: "Build (amd_tsan)"
443443
outputs:
@@ -475,7 +475,7 @@ jobs:
475475
476476
build_amd_msan:
477477
runs-on: [self-hosted, builder]
478-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
478+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
479479
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9tc2FuKQ==') }}
480480
name: "Build (amd_msan)"
481481
outputs:
@@ -513,7 +513,7 @@ jobs:
513513
514514
build_amd_ubsan:
515515
runs-on: [self-hosted, builder]
516-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
516+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
517517
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF91YnNhbik=') }}
518518
name: "Build (amd_ubsan)"
519519
outputs:
@@ -551,7 +551,7 @@ jobs:
551551
552552
build_amd_binary:
553553
runs-on: [self-hosted, builder]
554-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
554+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
555555
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
556556
name: "Build (amd_binary)"
557557
outputs:
@@ -589,7 +589,7 @@ jobs:
589589
590590
build_arm_release:
591591
runs-on: [self-hosted, builder-aarch64]
592-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
592+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
593593
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
594594
name: "Build (arm_release)"
595595
outputs:
@@ -627,7 +627,7 @@ jobs:
627627
628628
build_arm_asan:
629629
runs-on: [self-hosted, builder-aarch64]
630-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
630+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
631631
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }}
632632
name: "Build (arm_asan)"
633633
outputs:
@@ -665,7 +665,7 @@ jobs:
665665
666666
build_arm_coverage:
667667
runs-on: [self-hosted, builder-aarch64]
668-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
668+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
669669
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9jb3ZlcmFnZSk=') }}
670670
name: "Build (arm_coverage)"
671671
outputs:
@@ -703,7 +703,7 @@ jobs:
703703
704704
build_arm_binary:
705705
runs-on: [self-hosted, builder-aarch64]
706-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy]
706+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, style_check, fast_test, build_amd_tidy, build_arm_tidy]
707707
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9iaW5hcnkp') }}
708708
name: "Build (arm_binary)"
709709
outputs:

‎ci/defs/job_configs.py‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -81,44 +81,15 @@ class JobConfigs:
8181
command='python3 ./ci/jobs/build_clickhouse.py --build-type "{PARAMETER}"',
8282
run_in_docker="clickhouse/binary-builder+--network=host",
8383
timeout=3600 * 4,
84-
digest_config=Job.CacheDigestConfig(
85-
include_paths=[
86-
"./src",
87-
"./contrib/",
88-
"./CMakeLists.txt",
89-
"./PreLoad.cmake",
90-
"./cmake",
91-
"./base",
92-
"./programs",
93-
"./rust",
94-
"./ci/jobs/build_clickhouse.py",
95-
],
96-
with_git_submodules=True,
97-
),
98-
).parametrize(
99-
parameter=[
100-
BuildTypes.AMD_TIDY,
101-
],
102-
provides=[[]], # [ArtifactNames.CH_TIDY_BIN],
103-
runs_on=[
104-
RunnerLabels.BUILDER_AMD,
105-
],
106-
)
107-
tidy_arm_build_jobs = Job.Config(
108-
name=JobNames.BUILD,
109-
runs_on=[], # from parametrize()
110-
requires=["Build (amd_tidy)"],
111-
command='python3 ./ci/jobs/build_clickhouse.py --build-type "{PARAMETER}"',
112-
# --network=host required for ec2 metadata http endpoint to work
113-
run_in_docker="clickhouse/binary-builder+--network=host",
114-
timeout=3600 * 4,
115-
allow_merge_on_failure=True,
11684
digest_config=build_digest_config,
11785
).parametrize(
11886
parameter=[
87+
BuildTypes.AMD_TIDY,
11988
BuildTypes.ARM_TIDY,
12089
],
90+
provides=[[], []],
12191
runs_on=[
92+
RunnerLabels.BUILDER_AMD,
12293
RunnerLabels.BUILDER_ARM,
12394
],
12495
)

‎ci/docker/integration/runner/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ paramiko==3.4.0
6565
pika==1.2.0
6666
pip==25.0.1
6767
pluggy==1.5.0
68-
protobuf==4.25.2
68+
protobuf==4.25.8
6969
psycopg-binary==3.2.4
7070
psycopg2-binary==2.9.6
7171
psycopg==3.2.4

‎ci/docker/stateless-test/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ pandas==1.5.3
55
scipy==1.12.0
66
pyarrow==18.0.0
77
grpcio==1.60.0
8-
protobuf==4.25.3
8+
protobuf==4.25.8
99

1010
# performance tests >>
1111
clickhouse_driver

‎ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ Ethereum
308308
ExactEdgeLengthKm
309309
ExactEdgeLengthM
310310
ExactEdgeLengthRads
311+
Excel's
311312
ExecutablePool
312313
ExperimentalBadge
313314
ExtType
@@ -908,6 +909,7 @@ RWLockWaitingWriters
908909
RabbitMQ
909910
Rabl
910911
RangeHashed
912+
Raphson
911913
RawBLOB
912914
RawSum
913915
RawWithNames
@@ -1212,6 +1214,8 @@ WriteBuffer
12121214
WriteBuffers
12131215
XCode
12141216
XHTML
1217+
XIRR
1218+
XNPV
12151219
XORs
12161220
Xeon
12171221
YAML
@@ -1487,6 +1491,8 @@ cardinalities
14871491
cardinality
14881492
cartesian
14891493
caseWithExpression
1494+
cashflow
1495+
cashflows
14901496
cassandra
14911497
catboost
14921498
catboostEvaluate
@@ -1836,6 +1842,10 @@ filesystemFree
18361842
filesystemUnreserved
18371843
filesystems
18381844
finalizeAggregation
1845+
financialInternalRateOfReturn
1846+
financialInternalRateOfReturnExtended
1847+
financialNetPresentValue
1848+
financialNetPresentValueExtended
18391849
fips
18401850
firstLine
18411851
firstSignficantSubdomain
@@ -2084,6 +2094,7 @@ isFinite
20842094
isIPAddressInRange
20852095
isIPv
20862096
isInfinite
2097+
isMergeTreePartCoveredBy
20872098
isNaN
20882099
isNotDistinctFrom
20892100
isNotNull
@@ -2092,7 +2103,6 @@ isNullable
20922103
isValidJSON
20932104
isValidUTF
20942105
isZeroOrNull
2095-
isMergeTreePartCoveredBy
20962106
iteratively
20972107
jaccard
20982108
jaccardIndex
@@ -2271,6 +2281,7 @@ mebibytes
22712281
memtable
22722282
memtables
22732283
mergeTreeIndex
2284+
mergeTreePartInfo
22742285
mergeTreeProjection
22752286
mergeable
22762287
mergetree
@@ -2279,7 +2290,6 @@ metacharacters
22792290
metasymbols
22802291
metrica
22812292
metroHash
2282-
mergeTreePartInfo
22832293
mfedotov
22842294
mflix
22852295
minMap

‎ci/jobs/scripts/check_style/check_cpp.sh‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# and then to run formatter only for the specified files.
1414

1515
LC_ALL="en_US.UTF-8"
16-
ROOT_PATH="."
16+
ROOT_PATH=$(git rev-parse --show-toplevel)
1717
EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|programs/keeper-bench/example.yaml|base/base/openpty.h'
1818
EXCLUDE_DOCS='Settings\.cpp|FormatFactorySettings\.h'
1919

@@ -351,20 +351,10 @@ done
351351
# Forbid using quotes for includes (except for autogenerated files)
352352
QUOTE_EXCLUSIONS=(
353353
--exclude "$ROOT_PATH/utils/memcpy-bench/glibc/*"
354-
# TODO (WIP)
355-
--exclude "$ROOT_PATH/src/Functions/*"
356-
--exclude "$ROOT_PATH/src/Interpreters/*"
357-
--exclude "$ROOT_PATH/src/IO/*"
358-
--exclude "$ROOT_PATH/src/Loggers/*"
359-
--exclude "$ROOT_PATH/src/Parsers/*"
360-
--exclude "$ROOT_PATH/src/Processors/*"
361-
--exclude "$ROOT_PATH/src/Server/*"
362-
--exclude "$ROOT_PATH/src/Storages/*"
363-
--exclude "$ROOT_PATH/src/TableFunctions/*"
364354
)
365355
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | \
366356
xargs grep -P '#include[\s]*(").*(")' "${QUOTE_EXCLUSIONS[@]}" | \
367-
grep -v -F -e \"config.h\" -e \"config_tools.h\" -e \"SQLGrammar.pb.h\" | \
357+
grep -v -F -e \"config.h\" -e \"config_tools.h\" -e \"SQLGrammar.pb.h\" -e \"out.pb.h\" -e \"clickhouse_grpc.grpc.pb.h\" -e \"delta_kernel_ffi.hpp\" | \
368358
xargs -i echo "Found include with quotes in '{}'. Please use <> instead"
369359

370360
# Context.h (and a few similar headers) is included in many parts of the

‎ci/jobs/scripts/check_style/codespell-ignore-words.list‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,11 @@ ubuntu
3333
toolchain
3434
vie
3535
nin
36+
cashflow
37+
cashflows
38+
Excel's
39+
Raphson
40+
xirr
41+
XIRR
42+
xnpv
43+
XNPV

‎ci/jobs/scripts/functional_tests_results.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,13 @@ def _process_test_output(self):
9090
)
9191
continue
9292

93-
test_time = ""
93+
test_time = None
9494
try:
9595
time_token = line.split("]")[1].strip().split()[0]
9696
float(time_token)
9797
test_time = time_token
9898
except:
99-
pass
99+
print(f"ERROR: Failed to parse time str from line [{line}]")
100100

101101
total += 1
102102
if TIMEOUT_SIGN in line:

‎ci/workflows/pull_request.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,12 @@
1919
JobConfigs.docs_job,
2020
JobConfigs.fast_test,
2121
*JobConfigs.tidy_build_jobs,
22-
*JobConfigs.tidy_arm_build_jobs,
2322
*[
2423
job.set_dependency(
2524
[
2625
JobNames.STYLE_CHECK,
2726
JobNames.FAST_TEST,
28-
JobConfigs.tidy_build_jobs[0].name,
27+
*[j.name for j in JobConfigs.tidy_build_jobs],
2928
]
3029
)
3130
for job in JobConfigs.build_jobs

0 commit comments

Comments
 (0)