Skip to content

Commit 3804528

Browse files
lizanlizan
authored
ci: refactor azp into one pipeline (#10564)
Description: Merge all platforms into one pipelines to - guard mac and windows CI by format check - guard additional checks (asan, tsan, tidy..) by release to reduce CI pressure. - not guard docker image build by sanitizers. - refactor out bazel steps to another yaml file Risk Level: Low (CI only) Testing: CI Docs Changes: Release Notes: Signed-off-by: Lizan Zhou <lizan@tetrate.io>
1 parent 7097ef0 commit 3804528

5 files changed

Lines changed: 200 additions & 197 deletions

File tree

‎.azure-pipelines/bazel.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
parameters:
2+
- name: ciTarget
3+
displayName: "CI target"
4+
type: string
5+
default: bazel.release
6+
7+
steps:
8+
- task: CacheBeta@1
9+
inputs:
10+
key: '"${{ parameters.ciTarget }}" | ./WORKSPACE | **/*.bzl'
11+
path: $(Build.StagingDirectory)/repository_cache
12+
13+
- bash: |
14+
echo "disk space at beginning of build:"
15+
df -h
16+
displayName: "Check disk space at beginning"
17+
18+
- bash: |
19+
sudo mkdir -p /etc/docker
20+
echo '{
21+
"ipv6": true,
22+
"fixed-cidr-v6": "2001:db8:1::/64"
23+
}' | sudo tee /etc/docker/daemon.json
24+
sudo service docker restart
25+
displayName: "Enable IPv6"
26+
27+
- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
28+
workingDirectory: $(Build.SourcesDirectory)
29+
env:
30+
ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
31+
ENVOY_RBE: "true"
32+
BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) --curses=no"
33+
BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
34+
BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
35+
GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
36+
displayName: "Run CI script"
37+
38+
- bash: |
39+
echo "disk space at end of build:"
40+
df -h
41+
displayName: "Check disk space at end"
42+
condition: always()
43+
44+
- task: PublishTestResults@2
45+
inputs:
46+
testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
47+
testRunTitle: "${{ parameters.ciTarget }}"
48+
searchFolder: $(Build.StagingDirectory)/tmp
49+
condition: always()
50+
51+
- task: PublishBuildArtifacts@1
52+
inputs:
53+
pathtoPublish: "$(Build.StagingDirectory)/envoy"
54+
artifactName: ${{ parameters.ciTarget }}
55+
condition: always()

‎.azure-pipelines/linux.yml‎

Lines changed: 0 additions & 146 deletions
This file was deleted.

‎.azure-pipelines/linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pipelines.yml

‎.azure-pipelines/macos.yml‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.azure-pipelines/pipelines.yml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
trigger:
2+
branches:
3+
include:
4+
- "master"
5+
tags:
6+
include:
7+
- "v*"
8+
9+
# PR build config is manually overridden in Azure pipelines UI with different secrets
10+
pr: none
11+
12+
jobs:
13+
- job: format
14+
dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
15+
pool:
16+
vmImage: "ubuntu-16.04"
17+
steps:
18+
- task: CacheBeta@1
19+
inputs:
20+
key: "format | ./WORKSPACE | **/*.bzl"
21+
path: $(Build.StagingDirectory)/repository_cache
22+
23+
- script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh'
24+
workingDirectory: $(Build.SourcesDirectory)
25+
env:
26+
ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
27+
BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
28+
BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
29+
GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
30+
displayName: "Run check format scripts"
31+
32+
- task: PublishBuildArtifacts@1
33+
inputs:
34+
pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff"
35+
artifactName: format
36+
condition: failed()
37+
38+
- job: release
39+
displayName: "Linux-x64 release"
40+
dependsOn: ["format"]
41+
# For master builds, continue even if format fails
42+
condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
43+
pool:
44+
vmImage: "ubuntu-16.04"
45+
steps:
46+
- template: bazel.yml
47+
parameters:
48+
ciTarget: bazel.release
49+
50+
- job: bazel
51+
displayName: "Linux-x64"
52+
dependsOn: ["release"]
53+
# For master builds, continue even if format fails
54+
condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
55+
strategy:
56+
maxParallel: 3
57+
matrix:
58+
gcc:
59+
CI_TARGET: "bazel.gcc"
60+
clang_tidy:
61+
CI_TARGET: "bazel.clang_tidy"
62+
asan:
63+
CI_TARGET: "bazel.asan"
64+
tsan:
65+
CI_TARGET: "bazel.tsan"
66+
compile_time_options:
67+
CI_TARGET: "bazel.compile_time_options"
68+
timeoutInMinutes: 360
69+
pool:
70+
vmImage: "Ubuntu 16.04"
71+
steps:
72+
- template: bazel.yml
73+
parameters:
74+
ciTarget: $(CI_TARGET)
75+
76+
- job: docker
77+
displayName: "Linux-x64 docker"
78+
dependsOn: ["release"]
79+
condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
80+
pool:
81+
vmImage: "ubuntu-16.04"
82+
steps:
83+
- task: DownloadBuildArtifacts@0
84+
inputs:
85+
buildType: current
86+
artifactName: "bazel.release"
87+
itemPattern: "bazel.release/envoy_binary.tar.gz"
88+
downloadType: single
89+
targetPath: $(Build.StagingDirectory)
90+
91+
- bash: |
92+
set -e
93+
tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
94+
ci/docker_build.sh
95+
ci/docker_push.sh
96+
ci/docker_tag.sh
97+
workingDirectory: $(Build.SourcesDirectory)
98+
env:
99+
AZP_BRANCH: $(Build.SourceBranch)
100+
CIRCLE_SHA1: $(Build.SourceVersion)
101+
DOCKERHUB_USERNAME: $(DockerUsername)
102+
DOCKERHUB_PASSWORD: $(DockerPassword)
103+
104+
- job: macOS
105+
dependsOn: ["format"]
106+
timeoutInMinutes: 360
107+
pool:
108+
vmImage: "macos-latest"
109+
steps:
110+
- script: ./ci/mac_ci_setup.sh
111+
displayName: "Install dependencies"
112+
113+
- script: ./ci/mac_ci_steps.sh
114+
displayName: "Run Mac CI"
115+
env:
116+
BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
117+
BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
118+
GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
119+
120+
- task: PublishTestResults@2
121+
inputs:
122+
testResultsFiles: "**/bazel-testlogs/**/test.xml"
123+
testRunTitle: "macOS"
124+
condition: always()
125+
126+
- job: Windows
127+
dependsOn: ["format"]
128+
timeoutInMinutes: 360
129+
pool:
130+
vmImage: "windows-latest"
131+
steps:
132+
- powershell: |
133+
.\ci\windows_ci_setup.ps1
134+
Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR"
135+
displayName: "Install dependencies"
136+
env:
137+
TOOLS_BIN_DIR: $(Pipeline.Workspace)\bin
138+
139+
- bash: ci/windows_ci_steps.sh
140+
displayName: "Run Windows CI"
141+
env:
142+
TMPDIR: $(Agent.TempDirectory)
143+
BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC"
144+
BAZEL_SH: "C:\\Program Files\\Git\\bin\\bash.exe"

0 commit comments

Comments
 (0)