Skip to content

Commit e811416

Browse files
Pavel Zorinmergify[bot]
Pavel Zorin
authored and
mergify[bot]
committed
[CI] Fixed container images packaging (#6972)
* [CI] Fixed container images packaging * Packaging Ubuntu matrix * Improve matrix representation * Removed matrix packaging if favour to unique step keys * Cleanup * Added binfmt * Pinned binfmt to v8.1.5 * Correct step names * Changed packaging ARM machine type * pinned qemu-v9.2.0-51 * Remove qemu * Removed deb and rpm integration packaging * Reordered packaging steps * [CI] Build FIPS ARM Linux on ARM machine. Added missing retries * Fixed FIPS * [CI] Corrected integration tests packaging dependencies * [CI] single buildkite download invocation * Added missing retries (cherry picked from commit 14943f0) # Conflicts: # .buildkite/integration.pipeline.yml
1 parent 8621e5c commit e811416

3 files changed

Lines changed: 98 additions & 27 deletions

File tree

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ steps:
2727
- integration-ess
2828
steps:
2929
- label: "Win2022:sudo:{{matrix}}"
30+
depends_on:
31+
- packaging-windows
3032
command: |
3133
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
3234
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
@@ -50,6 +52,8 @@ steps:
5052
- install-uninstall
5153

5254
- label: "Win2022:non-sudo:{{matrix}}"
55+
depends_on:
56+
- packaging-windows
5357
command: |
5458
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
5559
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
@@ -67,6 +71,8 @@ steps:
6771
- default
6872

6973
- label: "Win2025:sudo:{{matrix}}"
74+
depends_on:
75+
- packaging-windows
7076
command: |
7177
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
7278
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
@@ -77,6 +83,9 @@ steps:
7783
provider: "gcp"
7884
machineType: "n1-standard-8"
7985
image: "family/platform-ingest-elastic-agent-windows-2025"
86+
retry:
87+
automatic:
88+
limit: 1
8089
matrix:
8190
- default
8291
- fleet
@@ -88,12 +97,17 @@ steps:
8897
- install-uninstall
8998

9099
- label: "Win2025:non-sudo:{{matrix}}"
100+
depends_on:
101+
- packaging-windows
91102
command: |
92103
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
93104
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
94105
artifact_paths:
95106
- build/**
96107
- build/diagnostics/**
108+
retry:
109+
automatic:
110+
limit: 1
97111
agents:
98112
provider: "gcp"
99113
machineType: "n1-standard-8"
@@ -107,7 +121,7 @@ steps:
107121
- integration-ess
108122
steps:
109123
- label: "x86_64:non-sudo: {{matrix}}"
110-
# only packaging-ubuntu-x86-64 artifact dependency is required
124+
depends_on: packaging-ubuntu-x86-64
111125
command: |
112126
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
113127
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -125,9 +139,11 @@ steps:
125139
- default
126140

127141
- label: "x86_64:sudo: {{matrix}}"
142+
depends_on:
143+
- packaging-ubuntu-x86-64
128144
# due to deb group present in matrix tar.gz and deb packages artifacts are required
129145
command: |
130-
buildkite-agent artifact download build/distributions/** .
146+
buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
131147
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
132148
artifact_paths:
133149
- build/**
@@ -155,6 +171,8 @@ steps:
155171
- container
156172

157173
- label: "arm:sudo: {{matrix}}"
174+
depends_on:
175+
- packaging-ubuntu-arm64
158176
command: |
159177
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
160178
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -186,6 +204,8 @@ steps:
186204

187205
- label: "arm:non-sudo: {{matrix}}"
188206
skip: true
207+
depends_on:
208+
- packaging-ubuntu-arm64
189209
command: |
190210
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
191211
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -208,8 +228,10 @@ steps:
208228
- integration-ess
209229
steps:
210230
- label: "x86_64:sudo:rpm"
231+
depends_on:
232+
- packaging-ubuntu-x86-64
211233
command: |
212-
buildkite-agent artifact download build/distributions/** .
234+
buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
213235
.buildkite/scripts/steps/integration_tests_tf.sh rpm true
214236
artifact_paths:
215237
- build/**

‎.buildkite/integration.pipeline.yml‎

Lines changed: 72 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,33 @@ steps:
88
- group: "Integration tests: packaging"
99
key: "int-packaging"
1010
steps:
11-
- label: "Packaging: Ubuntu x86_64"
12-
key: "packaging-ubuntu-x86-64"
11+
# Build matrix is not used for packaging in favor to unique step keys
12+
# Packaging linux/amd64
13+
- label: "Packaging: linux/amd64 rpm"
14+
key: packaging-ubuntu-x86-64
1315
env:
14-
PACKAGES: "tar.gz"
1516
PLATFORMS: "linux/amd64"
17+
PACKAGES: "tar.gz,rpm,deb"
1618
command: ".buildkite/scripts/steps/integration-package.sh"
1719
artifact_paths:
18-
- build/distributions/**
20+
- build/distributions/**
1921
retry:
2022
automatic:
2123
limit: 1
2224
agents:
2325
provider: "gcp"
2426
machineType: "n2-standard-8"
2527

28+
<<<<<<< HEAD
2629
- label: "Packaging: Ubuntu arm64"
2730
key: "packaging-ubuntu-arm64"
31+
=======
32+
- label: "Packaging: Ubuntu x86_64 FIPS"
33+
key: "packaging-ubuntu-x86-64-fips"
2834
env:
2935
PACKAGES: "tar.gz"
30-
PLATFORMS: "linux/arm64"
36+
PLATFORMS: "linux/amd64"
37+
FIPS: "true"
3138
command: ".buildkite/scripts/steps/integration-package.sh"
3239
artifact_paths:
3340
- build/distributions/**
@@ -36,10 +43,50 @@ steps:
3643
limit: 1
3744
agents:
3845
provider: "gcp"
39-
machineType: "n2-standard-8"
46+
machineType: "n2-standard-4"
4047

48+
# Packaging linux/arm64
49+
- label: "Packaging: linux/arm64 tar.gz"
50+
key: packaging-ubuntu-arm64
51+
>>>>>>> 14943f0fc ([CI] Fixed container images packaging (#6972))
52+
env:
53+
PLATFORMS: "linux/arm64"
54+
PACKAGES: "tar.gz"
55+
command: ".buildkite/scripts/steps/integration-package.sh"
56+
artifact_paths:
57+
- build/distributions/**
58+
retry:
59+
automatic:
60+
limit: 1
61+
agents:
62+
provider: "aws"
63+
instanceType: "c6g.4xlarge"
64+
imagePrefix: "core-ubuntu-2204-aarch64"
65+
66+
<<<<<<< HEAD
4167
- label: "Packaging: Windows"
4268
key: "packaging-windows"
69+
=======
70+
- label: "Packaging: Ubuntu arm64 FIPS"
71+
key: "packaging-ubuntu-arm64-fips"
72+
env:
73+
PACKAGES: "tar.gz"
74+
PLATFORMS: "linux/arm64"
75+
FIPS: "true"
76+
command: ".buildkite/scripts/steps/integration-package.sh"
77+
artifact_paths:
78+
- build/distributions/**
79+
retry:
80+
automatic:
81+
limit: 1
82+
agents:
83+
provider: "aws"
84+
instanceType: "c6g.4xlarge"
85+
imagePrefix: "core-ubuntu-2204-aarch64"
86+
87+
- label: "Packaging: windows/amd64 zip"
88+
key: packaging-windows
89+
>>>>>>> 14943f0fc ([CI] Fixed container images packaging (#6972))
4390
env:
4491
PACKAGES: "zip"
4592
PLATFORMS: "windows/amd64"
@@ -53,28 +100,31 @@ steps:
53100
provider: "gcp"
54101
machineType: "n2-standard-8"
55102

56-
- label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
57-
key: "packaging-containers"
103+
- label: "Packaging: Containers linux/amd64"
104+
key: packaging-containers-x86-64
58105
env:
59-
PACKAGES: "{{matrix.ext}}"
60-
PLATFORMS: "{{matrix.arch}}"
106+
PACKAGES: "docker"
107+
PLATFORMS: "linux/amd64"
61108
command: ".buildkite/scripts/steps/integration-package.sh"
62109
artifact_paths:
63110
- build/distributions/**
64-
retry:
65-
automatic:
66-
limit: 1
67111
agents:
68112
provider: "gcp"
69113
machineType: "n2-standard-8"
70-
matrix:
71-
setup:
72-
arch:
73-
- linux/amd64
74-
- linux/arm64
75-
ext:
76-
- rpm
77-
- deb
114+
115+
- label: "Packaging: Containers linux/arm64"
116+
key: packaging-containers-arm64
117+
env:
118+
PACKAGES: "docker"
119+
PLATFORMS: "linux/arm64"
120+
command: |
121+
.buildkite/scripts/steps/integration-package.sh
122+
artifact_paths:
123+
- build/distributions/**
124+
agents:
125+
provider: "aws"
126+
instanceType: "c6g.4xlarge"
127+
imagePrefix: "core-ubuntu-2204-aarch64"
78128

79129
- label: "Serverless integration test"
80130
key: "serverless-integration-tests"
@@ -131,9 +181,8 @@ steps:
131181
command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
132182

133183
- label: "Serverless Beats Tests"
134-
# To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
135184
depends_on:
136-
- packaging-ubuntu-x86-64
185+
- int-packaging
137186
key: "serverless-beats-integration-tests"
138187
concurrency_group: elastic-agent-extended-testing/beats-integration
139188
concurrency: 8

‎dev-tools/mage/crossbuild.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ func CrossBuildImage(platform string) (string, error) {
229229
case platform == "darwin/arm64" || platform == "darwin/universal":
230230
tagSuffix = "darwin-arm64-debian11"
231231
case platform == "linux/arm64":
232-
tagSuffix = "arm"
232+
tagSuffix = "base-arm-debian9"
233233
case platform == "linux/armv5" || platform == "linux/armv6":
234234
tagSuffix = "armel"
235235
case platform == "linux/armv7":

0 commit comments

Comments
 (0)