Skip to content

Commit ba0aa8d

Browse files
alexey-milovidovalexey-milovidov
committed
Merge branch 'master' into fix-bad-tests-01338_long_select_and_alter-alesapin
2 parents 5cb4114 + 5611ef0 commit ba0aa8d

25 files changed

Lines changed: 1168 additions & 494 deletions

‎.github/actions/release/action.yml‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
name: Release
2+
3+
description: Makes patch releases and creates new release branch
4+
5+
inputs:
6+
ref:
7+
description: 'Git reference (branch or commit sha) from which to create the release'
8+
required: true
9+
type: string
10+
type:
11+
description: 'The type of release: "new" for a new release or "patch" for a patch release'
12+
required: true
13+
type: choice
14+
options:
15+
- patch
16+
- new
17+
dry-run:
18+
description: 'Dry run'
19+
required: false
20+
default: true
21+
type: boolean
22+
token:
23+
required: true
24+
type: string
25+
26+
runs:
27+
using: "composite"
28+
steps:
29+
- name: Prepare Release Info
30+
shell: bash
31+
run: |
32+
python3 ./tests/ci/create_release.py --prepare-release-info \
33+
--ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
34+
${{ inputs.dry-run && '--dry-run' || '' }}
35+
echo "::group::Release Info"
36+
python3 -m json.tool /tmp/release_info.json
37+
echo "::endgroup::"
38+
release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
39+
commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
40+
echo "Release Tag: $release_tag"
41+
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
42+
echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
43+
- name: Download All Release Artifacts
44+
if: ${{ inputs.type == 'patch' }}
45+
shell: bash
46+
run: |
47+
python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run && '--dry-run' || '' }}
48+
- name: Push Git Tag for the Release
49+
shell: bash
50+
run: |
51+
python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run && '--dry-run' || '' }}
52+
- name: Push New Release Branch
53+
if: ${{ inputs.type == 'new' }}
54+
shell: bash
55+
run: |
56+
python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run && '--dry-run' || '' }}
57+
- name: Bump CH Version and Update Contributors' List
58+
shell: bash
59+
run: |
60+
python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run && '--dry-run' || '' }}
61+
- name: Bump Docker versions, Changelog, Security
62+
if: ${{ inputs.type == 'patch' }}
63+
shell: bash
64+
run: |
65+
python3 ./tests/ci/create_release.py --set-progress-started --progress "update ChangeLog"
66+
[ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
67+
echo "List versions"
68+
./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
69+
echo "Update docker version"
70+
./utils/list-versions/update-docker-version.sh
71+
echo "Generate ChangeLog"
72+
export CI=1
73+
docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
74+
--volume=".:/ClickHouse" clickhouse/style-test \
75+
/ClickHouse/tests/ci/changelog.py -v --debug-helpers \
76+
--gh-user-or-token=${{ inputs.token }} --jobs=5 \
77+
--output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
78+
git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
79+
echo "Generate Security"
80+
python3 ./utils/security-generator/generate_security.py > SECURITY.md
81+
git diff HEAD
82+
- name: Create ChangeLog PR
83+
if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
84+
uses: peter-evans/create-pull-request@v6
85+
with:
86+
author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
87+
token: ${{ inputs.token }}
88+
committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
89+
commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
90+
branch: auto/${{ env.RELEASE_TAG }}
91+
assignees: ${{ github.event.sender.login }} # assign the PR to the tag pusher
92+
delete-branch: true
93+
title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
94+
labels: do not test
95+
body: |
96+
Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
97+
### Changelog category (leave one):
98+
- Not for changelog (changelog entry is not required)
99+
- name: Reset changes if Dry-run
100+
if: ${{ inputs.dry-run }}
101+
shell: bash
102+
run: |
103+
git reset --hard HEAD
104+
- name: Checkout back to GITHUB_REF
105+
shell: bash
106+
run: |
107+
git checkout "$GITHUB_REF_NAME"
108+
# set current progress to OK
109+
python3 ./tests/ci/create_release.py --set-progress-completed
110+
- name: Create GH Release
111+
shell: bash
112+
if: ${{ inputs.type == 'patch' }}
113+
run: |
114+
python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run && '--dry-run' || '' }}
115+
- name: Export TGZ Packages
116+
if: ${{ inputs.type == 'patch' }}
117+
shell: bash
118+
run: |
119+
python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
120+
- name: Test TGZ Packages
121+
if: ${{ inputs.type == 'patch' }}
122+
shell: bash
123+
run: |
124+
python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
125+
- name: Export RPM Packages
126+
if: ${{ inputs.type == 'patch' }}
127+
shell: bash
128+
run: |
129+
python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
130+
- name: Test RPM Packages
131+
if: ${{ inputs.type == 'patch' }}
132+
shell: bash
133+
run: |
134+
python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
135+
- name: Export Debian Packages
136+
if: ${{ inputs.type == 'patch' }}
137+
shell: bash
138+
run: |
139+
python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run && '--dry-run' || '' }}
140+
- name: Test Debian Packages
141+
if: ${{ inputs.type == 'patch' }}
142+
shell: bash
143+
run: |
144+
python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run && '--dry-run' || '' }}
145+
- name: Docker clickhouse/clickhouse-server building
146+
if: ${{ inputs.type == 'patch' }}
147+
shell: bash
148+
run: |
149+
python3 ./tests/ci/create_release.py --set-progress-started --progress "docker server release"
150+
cd "./tests/ci"
151+
export CHECK_NAME="Docker server image"
152+
python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
153+
python3 ./tests/ci/create_release.py --set-progress-completed
154+
- name: Docker clickhouse/clickhouse-keeper building
155+
if: ${{ inputs.type == 'patch' }}
156+
shell: bash
157+
run: |
158+
python3 ./tests/ci/create_release.py --set-progress-started --progress "docker keeper release"
159+
cd "./tests/ci"
160+
export CHECK_NAME="Docker keeper image"
161+
python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
162+
python3 ./tests/ci/create_release.py --set-progress-completed
163+
- name: Set Release progress completed
164+
shell: bash
165+
run: |
166+
# If we here - set completed status, to post proper Slack OK or FAIL message in the next step
167+
python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
168+
python3 ./tests/ci/create_release.py --set-progress-completed
169+
- name: Post Slack Message
170+
if: ${{ !cancelled() }}
171+
shell: bash
172+
run: |
173+
python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run && '--dry-run' || '' }}

‎.github/workflows/auto_release.yml‎

Lines changed: 82 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,110 @@
11
name: AutoRelease
22

33
env:
4-
# Force the stdout and stderr streams to be unbuffered
54
PYTHONUNBUFFERED: 1
5+
DRY_RUN: true
66

77
concurrency:
8-
group: auto-release
8+
group: release
99
on: # yamllint disable-line rule:truthy
10-
# schedule:
11-
# - cron: '0 10-16 * * 1-5'
10+
# Workflow uses a test bucket for packages and dry run mode (no real releases)
11+
schedule:
12+
- cron: '0 9 * * *'
13+
- cron: '0 15 * * *'
1214
workflow_dispatch:
15+
inputs:
16+
dry-run:
17+
description: 'Dry run'
18+
required: false
19+
default: true
20+
type: boolean
1321

1422
jobs:
15-
CherryPick:
16-
runs-on: [self-hosted, style-checker-aarch64]
23+
AutoRelease:
24+
runs-on: [self-hosted, release-maker]
1725
steps:
26+
- name: DebugInfo
27+
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
1828
- name: Set envs
19-
# https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
2029
run: |
2130
cat >> "$GITHUB_ENV" << 'EOF'
22-
TEMP_PATH=${{runner.temp}}/cherry_pick
2331
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
2432
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
2533
RCSK
26-
REPO_OWNER=ClickHouse
27-
REPO_NAME=ClickHouse
28-
REPO_TEAM=core
2934
EOF
35+
- name: Set DRY_RUN for schedule
36+
if: ${{ github.event_name == 'schedule' }}
37+
run: echo "DRY_RUN=true" >> "$GITHUB_ENV"
38+
- name: Set DRY_RUN for dispatch
39+
if: ${{ github.event_name == 'workflow_dispatch' }}
40+
run: echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_ENV"
3041
- name: Check out repository code
3142
uses: ClickHouse/checkout@v1
3243
with:
33-
clear-repository: true
3444
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
3545
fetch-depth: 0
36-
- name: Auto-release
46+
- name: Auto Release Prepare
3747
run: |
3848
cd "$GITHUB_WORKSPACE/tests/ci"
39-
python3 auto_release.py --release-after-days=3
40-
- name: Cleanup
41-
if: always()
49+
python3 auto_release.py --prepare
50+
echo "::group::Auto Release Info"
51+
python3 -m json.tool /tmp/autorelease_info.json
52+
echo "::endgroup::"
53+
{
54+
echo 'AUTO_RELEASE_PARAMS<<EOF'
55+
cat /tmp/autorelease_info.json
56+
echo 'EOF'
57+
} >> "$GITHUB_ENV"
58+
- name: Post Release Branch statuses
59+
run: |
60+
cd "$GITHUB_WORKSPACE/tests/ci"
61+
python3 auto_release.py --post-status
62+
- name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].release_branch }}
63+
if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[0].ready }}
64+
uses: ./.github/actions/release
65+
with:
66+
ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
67+
type: patch
68+
dry-run: ${{ env.DRY_RUN }}
69+
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
70+
- name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].release_branch }}
71+
if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[1].ready }}
72+
uses: ./.github/actions/release
73+
with:
74+
ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].commit_sha }}
75+
type: patch
76+
dry-run: ${{ env.DRY_RUN }}
77+
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
78+
- name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].release_branch }}
79+
if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2] && fromJson(env.AUTO_RELEASE_PARAMS).releases[2].ready }}
80+
uses: ./.github/actions/release
81+
with:
82+
ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].commit_sha }}
83+
type: patch
84+
dry-run: ${{ env.DRY_RUN }}
85+
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
86+
- name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].release_branch }}
87+
if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3] && fromJson(env.AUTO_RELEASE_PARAMS).releases[3].ready }}
88+
uses: ./.github/actions/release
89+
with:
90+
ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].commit_sha }}
91+
type: patch
92+
dry-run: ${{ env.DRY_RUN }}
93+
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
94+
- name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].release_branch }}
95+
if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4] && fromJson(env.AUTO_RELEASE_PARAMS).releases[4].ready }}
96+
uses: ./.github/actions/release
97+
with:
98+
ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].commit_sha }}
99+
type: patch
100+
dry-run: ${{ env.DRY_RUN }}
101+
token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
102+
- name: Post Slack Message
103+
if: ${{ !cancelled() }}
104+
run: |
105+
cd "$GITHUB_WORKSPACE/tests/ci"
106+
python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
107+
- name: Clean up
42108
run: |
43109
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
44110
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:

0 commit comments

Comments
 (0)