Skip to content

Commit e4c9b02

Browse files
maxknvmaxknv
authored
Merge pull request #71995 from ClickHouse/stress_with_praktika
CI: Stress test with praktika
2 parents 288edbf + 9c54279 commit e4c9b02

26 files changed

Lines changed: 743 additions & 79 deletions

‎ci/docker/fasttest/Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,5 +102,14 @@ RUN groupadd --system --gid 1000 clickhouse \
102102
&& useradd --system --gid 1000 --uid 1000 -m clickhouse \
103103
&& mkdir -p /.cache/sccache && chmod 777 /.cache/sccache
104104

105+
106+
# TODO move nfpm to docker that will do packaging
107+
ARG TARGETARCH
108+
ARG NFPM_VERSION=2.20.0
109+
RUN arch=${TARGETARCH:-amd64} \
110+
&& curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
111+
&& dpkg -i /tmp/nfpm.deb \
112+
&& rm /tmp/nfpm.deb
113+
105114
ENV PYTHONPATH="/wd"
106115
ENV PYTHONUNBUFFERED=1

‎ci/docker/stateless-test/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ RUN apt-get update -y \
5858
curl \
5959
wget \
6060
xz-utils \
61+
ripgrep \
6162
&& apt-get clean \
6263
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
6364

@@ -114,4 +115,5 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo
114115
RUN npm install -g azurite@3.30.0 \
115116
&& npm install -g tslib && npm install -g node
116117

118+
ENV PYTHONPATH=".:./ci"
117119
USER clickhouse

‎ci/docker/stateless-test/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ requests==2.32.3
44
pandas==1.5.3
55
scipy==1.12.0
66
pyarrow==18.0.0
7+
grpcio==1.47.0

‎ci/jobs/__init__.py‎

Whitespace-only changes.

‎ci/jobs/build_clickhouse.py‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,15 @@
44
from praktika.settings import Settings
55
from praktika.utils import MetaClasses, Shell, Utils
66

7+
from ci.jobs.scripts.clickhouse_version import CHVersion
8+
79

810
class JobStages(metaclass=MetaClasses.WithIter):
911
CHECKOUT_SUBMODULES = "checkout"
1012
CMAKE = "cmake"
13+
UNSHALLOW = "unshallow"
1114
BUILD = "build"
15+
PACKAGE = "package"
1216

1317

1418
def parse_args():
@@ -33,8 +37,7 @@ def parse_args():
3337
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
3438
{AUX_DEFS} \
3539
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \
36-
-DCOMPILER_CACHE={CACHE_TYPE} \
37-
-DENABLE_BUILD_PROFILING=1 {DIR}"""
40+
-DCOMPILER_CACHE={CACHE_TYPE} -DENABLE_BUILD_PROFILING=1 {DIR}"""
3841

3942

4043
def main():
@@ -91,6 +94,27 @@ def main():
9194

9295
res = True
9396
results = []
97+
version = ""
98+
99+
if res and JobStages.UNSHALLOW in stages:
100+
results.append(
101+
Result.create_from_command_execution(
102+
name="Repo Unshallow",
103+
command="git rev-parse --is-shallow-repository | grep -q true && git fetch --depth 10000 --no-tags --filter=tree:0 origin $(git rev-parse --abbrev-ref HEAD)",
104+
with_log=True,
105+
)
106+
)
107+
res = results[-1].is_ok()
108+
if res:
109+
try:
110+
version = CHVersion.get_version()
111+
assert version
112+
print(f"Got version from repo [{version}]")
113+
except Exception as e:
114+
results[-1].set_failed().set_info(
115+
f"Failed to get version from repo, ex [{e}]"
116+
)
117+
res = False
94118

95119
if res and JobStages.CHECKOUT_SUBMODULES in stages:
96120
Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
@@ -127,6 +151,38 @@ def main():
127151
Shell.check(f"ls -l {build_dir}/programs/")
128152
res = results[-1].is_ok()
129153

154+
if res and JobStages.PACKAGE in stages:
155+
if "debug" in build_type:
156+
package_type = "debug"
157+
elif "release" in build_type:
158+
package_type = "release"
159+
elif "asan" in build_type:
160+
package_type = "asan"
161+
else:
162+
assert False, "TODO"
163+
164+
if "amd" in build_type:
165+
deb_arch = "amd64"
166+
else:
167+
deb_arch = "arm64"
168+
169+
output_dir = "/tmp/praktika/output/"
170+
assert Shell.check(f"rm -f {output_dir}/*.deb")
171+
172+
results.append(
173+
Result.create_from_command_execution(
174+
name="Build Packages",
175+
command=[
176+
f"DESTDIR={build_dir}/root ninja programs/install",
177+
f"ln -sf {build_dir}/root {Utils.cwd()}/packages/root",
178+
f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={output_dir} BUILD_TYPE={package_type} VERSION_STRING={version} DEB_ARCH={deb_arch} ./build --deb",
179+
],
180+
workdir=build_dir,
181+
with_log=True,
182+
)
183+
)
184+
res = results[-1].is_ok()
185+
130186
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
131187

132188

‎ci/jobs/functional_stateful_tests.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import argparse
2-
import os
32
import time
43
from pathlib import Path
54

@@ -131,6 +130,10 @@ def main():
131130
)
132131
res = res and CH.start()
133132
res = res and CH.wait_ready()
133+
# TODO: Use --database-replicated optionally
134+
res = res and Shell.check(
135+
f"./ci/jobs/scripts/functional_tests/setup_ch_cluster.sh"
136+
)
134137
if res:
135138
print("ch started")
136139
logs_to_attach += [

‎ci/jobs/functional_stateless_tests.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ def main():
101101
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
102102
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
103103
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
104+
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-disks",
104105
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
105106
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
106107
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
@@ -114,6 +115,7 @@ def main():
114115
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
115116
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
116117
f"clickhouse-server --version",
118+
f"chmod +x /tmp/praktika/input/clickhouse-odbc-bridge",
117119
]
118120
results.append(
119121
Result.create_from_command_execution(
@@ -138,6 +140,7 @@ def main():
138140
res = res and Shell.check(
139141
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
140142
)
143+
res = res and CH.log_cluster_config()
141144
res = res and CH.start()
142145
res = res and CH.wait_ready()
143146
if res:
@@ -170,6 +173,7 @@ def main():
170173
batch_total=total_batches,
171174
test=args.test,
172175
)
176+
CH.log_cluster_stop_replication()
173177
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
174178
results[-1].set_timing(stopwatch=stop_watch_)
175179
res = results[-1].is_ok()

‎ci/jobs/scripts/clickhouse_proc.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,24 @@ def start_minio(self, test_type, log_file_path):
6666
print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
6767
return True
6868

69+
def log_cluster_config(self):
70+
return Shell.check(
71+
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --config-logs-export-cluster /tmp/praktika/etc/clickhouse-server/config.d/system_logs_export.yaml",
72+
verbose=True,
73+
)
74+
75+
def log_cluster_setup_replication(self):
76+
return Shell.check(
77+
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication",
78+
verbose=True,
79+
)
80+
81+
def log_cluster_stop_replication(self):
82+
return Shell.check(
83+
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --stop-log-replication",
84+
verbose=True,
85+
)
86+
6987
def start(self):
7088
print("Starting ClickHouse server")
7189
Shell.check(f"rm {self.pid_file}")

‎ci/jobs/scripts/clickhouse_version.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from pathlib import Path
2+
3+
from praktika.utils import Shell
4+
5+
6+
class CHVersion:
7+
FILE_WITH_VERSION_PATH = "./cmake/autogenerated_versions.txt"
8+
9+
@classmethod
10+
def _get_tweak(cls):
11+
tag = Shell.get_output("git describe --tags --abbrev=0")
12+
assert tag.startswith("v24")
13+
num = Shell.get_output(f"git rev-list --count {tag}..HEAD")
14+
return int(num)
15+
16+
@classmethod
17+
def get_version(cls):
18+
versions = {}
19+
for line in (
20+
Path(cls.FILE_WITH_VERSION_PATH).read_text(encoding="utf-8").splitlines()
21+
):
22+
line = line.strip()
23+
if not line.startswith("SET("):
24+
continue
25+
26+
name, value = line[4:-1].split(maxsplit=1)
27+
name = name.removeprefix("VERSION_").lower()
28+
try:
29+
value = int(value)
30+
except ValueError:
31+
pass
32+
versions[name] = value
33+
34+
version_sha = versions["githash"]
35+
tweak = int(
36+
Shell.get_output(f"git rev-list --count {version_sha}..HEAD", verbose=True)
37+
)
38+
return f"{versions['major']}.{versions['minor']}.{versions['patch']}.{tweak}"

0 commit comments

Comments
 (0)