Skip to content

Commit c22981e

Browse files
belimawrbelimawr
committed
Merge branch 'main' of github.com:elastic/elastic-agent into 44040-journalctl-on-docker-iamges
2 parents 9634981 + 5908ed2 commit c22981e

46 files changed

Lines changed: 13354 additions & 7444 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
2+
3+
env:
4+
DOCKER_REGISTRY: "docker.elastic.co"
5+
ASDF_MAGE_VERSION: 1.14.0
6+
7+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749258065"
8+
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1748955449"
9+
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1748955449"
10+
11+
steps:
12+
- label: Build and push custom elastic-agent image
13+
depends_on:
14+
- 'packaging-containers-x86-64-fips' # Reuse artifacts produced in .buildkite/integration.pipeline.yml
15+
key: integration-fips-cloud-image
16+
env:
17+
FIPS: "true"
18+
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
19+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
20+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
21+
command: |
22+
buildkite-agent artifact download build/distributions/elastic-agent-fips-cloud-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
23+
mage cloud:load
24+
mage cloud:push
25+
agents:
26+
provider: "gcp"
27+
machineType: "n1-standard-8"
28+
image: "${IMAGE_UBUNTU_2404_X86_64}"
29+
30+
- label: Start ESS stack for FIPS integration tests
31+
key: integration-fips-ess
32+
depends_on:
33+
- integration-fips-cloud-image
34+
env:
35+
ASDF_TERRAFORM_VERSION: 1.9.2
36+
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
37+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
38+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
39+
command: |
40+
source .buildkite/scripts/steps/ess_start.sh
41+
artifact_paths:
42+
- test_infra/ess/*.tfstate
43+
- test_infra/ess/*.lock.hcl
44+
agents:
45+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
46+
useCustomGlobalHooks: true
47+
48+
- group: "fips:Stateful:Ubuntu"
49+
key: integration-tests-ubuntu-fips
50+
depends_on:
51+
- integration-fips-ess
52+
steps:
53+
- label: "fips:x86_64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
54+
depends_on:
55+
- packaging-ubuntu-x86-64-fips # Reuse artifacts produced in .buildkite/integration.pipeline.yml
56+
env:
57+
FIPS: "true"
58+
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
59+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
60+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
61+
command: |
62+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
63+
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
64+
artifact_paths:
65+
- build/**
66+
- build/diagnostics/**
67+
retry:
68+
automatic:
69+
limit: 1
70+
agents:
71+
provider: "aws"
72+
image: "${IMAGE_UBUNTU_X86_64_FIPS}"
73+
instanceType: "m5.2xlarge"
74+
matrix:
75+
setup:
76+
sudo:
77+
- "false"
78+
- "true"
79+
groups:
80+
- fleet # currently there is only a single test in the fleet group, add more tests once they have been defined
81+
82+
- label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
83+
depends_on:
84+
- packaging-ubuntu-arm64-fips
85+
env:
86+
FIPS: "true"
87+
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
88+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
89+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
90+
command: |
91+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
92+
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
93+
artifact_paths:
94+
- build/**
95+
- build/diagnostics/**
96+
retry:
97+
automatic:
98+
limit: 1
99+
agents:
100+
provider: "aws"
101+
image: "${IMAGE_UBUNTU_ARM64_FIPS}"
102+
instanceType: "m6g.2xlarge"
103+
matrix:
104+
setup:
105+
sudo:
106+
- "false"
107+
- "true"
108+
groups:
109+
- fleet
110+
111+
- label: ESS FIPS stack cleanup
112+
depends_on:
113+
- integration-tests-ubuntu-fips
114+
allow_dependency_failure: true
115+
command: |
116+
buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess"
117+
ls -lah test_infra/ess
118+
.buildkite/scripts/steps/ess_down.sh
119+
agents:
120+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
121+
useCustomGlobalHooks: true
122+
123+
- label: Aggregate test reports
124+
depends_on:
125+
- integration-tests-ubuntu-fips
126+
allow_dependency_failure: true
127+
command: |
128+
buildkite-agent artifact download "build/*.xml" .
129+
agents:
130+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
131+
useCustomGlobalHooks: true
132+
soft_fail:
133+
- exit_status: "*"
134+
plugins:
135+
- elastic/vault-secrets#v0.1.0:
136+
path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
137+
field: "token"
138+
env_var: "BUILDKITE_ANALYTICS_TOKEN"
139+
- test-collector#v1.11.0:
140+
files: "build/*.xml"
141+
format: "junit"
142+
branches: "main"
143+
debug: true

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ env:
55
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
66
ASDF_MAGE_VERSION: 1.14.0
77

8-
IMAGE_UBUNTU_2404_X86_64: "family/platform-ingest-elastic-agent-ubuntu-2404"
9-
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64"
10-
IMAGE_RHEL_8: "family/platform-ingest-elastic-agent-rhel-8"
11-
IMAGE_DEBIAN_12: "family/platform-ingest-elastic-agent-debian-12"
12-
IMAGE_WIN_2022: "family/platform-ingest-elastic-agent-windows-2022"
13-
IMAGE_WIN_2025: "family/platform-ingest-elastic-agent-windows-2025"
8+
# The following images are defined here and their values will be updated by updatecli
9+
# Please do not change them manually.
10+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749258065"
11+
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1749258065"
12+
IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1749258065"
13+
IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1749258065"
14+
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749258065"
15+
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749258065"
1416

1517
steps:
1618
- label: Start ESS stack for integration tests
@@ -46,7 +48,7 @@ steps:
4648
- build/diagnostics/**
4749
agents:
4850
provider: "gcp"
49-
machineType: "n1-standard-8"
51+
machineType: "n2-standard-8"
5052
image: "${IMAGE_WIN_2022}"
5153
retry:
5254
automatic:
@@ -67,7 +69,7 @@ steps:
6769
limit: 1
6870
agents:
6971
provider: "gcp"
70-
machineType: "n1-standard-8"
72+
machineType: "n2-standard-8"
7173
image: "${IMAGE_WIN_2025}"
7274
- label: "Ubuntu:2404:amd64:sudo"
7375
depends_on: packaging-ubuntu-x86-64
@@ -84,7 +86,7 @@ steps:
8486
limit: 1
8587
agents:
8688
provider: "gcp"
87-
machineType: "n1-standard-8"
89+
machineType: "n2-standard-8"
8890
image: "${IMAGE_UBUNTU_2404_X86_64}"
8991

9092
- group: "Stateful: Windows"
@@ -103,7 +105,7 @@ steps:
103105
- build/diagnostics/**
104106
agents:
105107
provider: "gcp"
106-
machineType: "n1-standard-8"
108+
machineType: "n2-standard-8"
107109
image: "${IMAGE_WIN_2022}"
108110
retry:
109111
automatic:
@@ -129,7 +131,7 @@ steps:
129131
- build/diagnostics/**
130132
agents:
131133
provider: "gcp"
132-
machineType: "n1-standard-8"
134+
machineType: "n2-standard-8"
133135
image: "${IMAGE_WIN_2022}"
134136
retry:
135137
automatic:
@@ -148,7 +150,7 @@ steps:
148150
- build/diagnostics/**
149151
agents:
150152
provider: "gcp"
151-
machineType: "n1-standard-8"
153+
machineType: "n2-standard-8"
152154
image: "${IMAGE_WIN_2025}"
153155
retry:
154156
automatic:
@@ -177,7 +179,7 @@ steps:
177179
limit: 1
178180
agents:
179181
provider: "gcp"
180-
machineType: "n1-standard-8"
182+
machineType: "n2-standard-8"
181183
image: "${IMAGE_WIN_2025}"
182184
matrix:
183185
- default
@@ -200,7 +202,7 @@ steps:
200202
limit: 1
201203
agents:
202204
provider: "gcp"
203-
machineType: "n1-standard-8"
205+
machineType: "n2-standard-8"
204206
image: "${IMAGE_UBUNTU_2404_X86_64}"
205207
matrix:
206208
- default
@@ -220,7 +222,7 @@ steps:
220222
limit: 1
221223
agents:
222224
provider: "gcp"
223-
machineType: "n1-standard-8"
225+
machineType: "n2-standard-8"
224226
image: "${IMAGE_UBUNTU_2404_X86_64}"
225227
matrix:
226228
- default
@@ -249,7 +251,7 @@ steps:
249251
- build/diagnostics/**
250252
agents:
251253
provider: "aws"
252-
imagePrefix: "${IMAGE_UBUNTU_2404_ARM_64}"
254+
image: "${IMAGE_UBUNTU_2404_ARM_64}"
253255
instanceType: "m6g.2xlarge"
254256
retry:
255257
automatic:
@@ -286,7 +288,7 @@ steps:
286288
limit: 1
287289
agents:
288290
provider: "aws"
289-
imagePrefix: "${IMAGE_UBUNTU_2404_ARM_64}"
291+
image: "${IMAGE_UBUNTU_2404_ARM_64}"
290292
instanceType: "m6g.xlarge"
291293
matrix:
292294
- default
@@ -309,7 +311,7 @@ steps:
309311
limit: 1
310312
agents:
311313
provider: "gcp"
312-
machineType: "n1-standard-8"
314+
machineType: "n2-standard-8"
313315
image: "${IMAGE_DEBIAN_12}"
314316
matrix:
315317
- default
@@ -329,7 +331,7 @@ steps:
329331
limit: 1
330332
agents:
331333
provider: "gcp"
332-
machineType: "n1-standard-8"
334+
machineType: "n2-standard-8"
333335
image: "${IMAGE_DEBIAN_12}"
334336
matrix:
335337
- default
@@ -367,7 +369,7 @@ steps:
367369
limit: 1
368370
agents:
369371
provider: "gcp"
370-
machineType: "n1-standard-8"
372+
machineType: "n2-standard-8"
371373
image: "${IMAGE_RHEL_8}"
372374

373375
- group: "Kubernetes"
@@ -394,7 +396,7 @@ steps:
394396
limit: 1
395397
agents:
396398
provider: "gcp"
397-
machineType: "n1-standard-4"
399+
machineType: "n2-standard-4"
398400
image: "${IMAGE_UBUNTU_2404_X86_64}"
399401
diskSizeGb: 80
400402
matrix:
@@ -449,7 +451,11 @@ steps:
449451
soft_fail:
450452
- exit_status: "*"
451453
plugins:
452-
- test-collector#v1.10.1:
454+
- elastic/vault-secrets#v0.1.0:
455+
path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
456+
field: "token"
457+
env_var: "BUILDKITE_ANALYTICS_TOKEN"
458+
- test-collector#v1.11.0:
453459
files: "build/*.xml"
454460
format: "junit"
455461
branches: "main"

‎.buildkite/hooks/pre-command‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -58,18 +58,6 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
5858
export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
5959
echo ${API_KEY_TOKEN} > ./apiKey
6060
export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
61-
62-
# BK analytics
63-
echo "--- Prepare BK test analytics token :vault:"
64-
BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
65-
export BUILDKITE_ANALYTICS_TOKEN
66-
fi
67-
68-
if [[ "$BUILDKITE_STEP_KEY" == *"aggregate-reports"* ]]; then
69-
# BK analytics
70-
echo "--- Prepare BK test analytics token :vault:"
71-
BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
72-
export BUILDKITE_ANALYTICS_TOKEN
7361
fi
7462

7563
if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
@@ -84,3 +72,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
8472
release_manager_login
8573
fi
8674
fi
75+
76+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-fips-cloud-image" ]]; then
77+
docker_login
78+
fi

‎.buildkite/integration.pipeline.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,3 +177,8 @@ steps:
177177
depends_on:
178178
- int-packaging
179179
command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
180+
181+
- label: "Triggering custom FIPS integration tests"
182+
depends_on:
183+
- int-packaging
184+
command: "buildkite-agent pipeline upload .buildkite/bk.integration-fips.pipeline.yml"

‎.buildkite/pipeline.elastic-agent-helm-charts.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ steps:
1616
project-number: "911195782929"
1717
agents:
1818
provider: "gcp"
19-
machineType: "n1-standard-8"
19+
machineType: "n2-standard-8"

‎.buildkite/pipeline.integration-test-matrix.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ steps:
1212
- build/distributions/**
1313
agents:
1414
provider: "gcp"
15-
machineType: "n1-standard-8"
15+
machineType: "n2-standard-8"
1616

1717
- label: "Integration test matrix"
1818
key: "integration-tests-matrix"
@@ -26,4 +26,4 @@ steps:
2626
- "build/diagnostics/*"
2727
agents:
2828
provider: "gcp"
29-
machineType: "n1-standard-8"
29+
machineType: "n2-standard-8"

‎.buildkite/pipeline.yml‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ env:
44
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
55
DOCKER_REGISTRY: "docker.elastic.co"
66

7-
IMAGE_UBUNTU_2204_X86_64: "family/platform-ingest-elastic-agent-ubuntu-2204"
8-
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64"
9-
IMAGE_WIN_2016: "family/platform-ingest-elastic-agent-windows-2016"
10-
IMAGE_WIN_2022: "family/platform-ingest-elastic-agent-windows-2022"
11-
IMAGE_WIN_10: "family/platform-ingest-elastic-agent-windows-10"
12-
IMAGE_WIN_11: "family/platform-ingest-elastic-agent-windows-11"
7+
# The following images are defined here and their values will be updated by updatecli
8+
# Please do not change them manually.
9+
IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1749258065"
10+
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1749258065"
11+
IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1749258065"
12+
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749258065"
13+
IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1749258065"
14+
IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1749258065"
1315

1416
steps:
1517
- label: "check-ci"
@@ -91,7 +93,7 @@ steps:
9193
- "coverage-*.out"
9294
agents:
9395
provider: "aws"
94-
imagePrefix: "${IMAGE_UBUNTU_2204_ARM_64}"
96+
image: "${IMAGE_UBUNTU_2204_ARM_64}"
9597
diskSizeGb: 200
9698
instanceType: "m6g.xlarge"
9799
retry:

0 commit comments

Comments
 (0)