Skip to content

Commit 6e88789

Browse files
author
kaiyan-sheng
authored
Merge branch 'main' into vpce_endpoint
2 parents ac2a021 + 093ab5e commit 6e88789

52 files changed

Lines changed: 1591 additions & 98 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/hooks/pre-command‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
set -euo pipefail
44

55
AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
6+
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
67

78
retry() {
89
local retries=$1
@@ -33,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
3334
fi
3435
fi
3536

36-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
37+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
3738
source .buildkite/scripts/setenv.sh
3839
if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
3940
echo "Skipped pre-command when running the Upload pipeline"
@@ -47,3 +48,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP
4748
BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
4849
export BEATS_AWS_ACCESS_KEY
4950
fi
51+
52+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
53+
if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
54+
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
55+
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
56+
fi
57+
fi

‎.buildkite/pull-requests.json‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,86 @@
223223
"skip_target_branches": [ ],
224224
"skip_ci_on_only_changed": [ ],
225225
"always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
226+
},
227+
{
228+
"enabled": true,
229+
"pipelineSlug": "beats-xpack-auditbeat",
230+
"allow_org_users": true,
231+
"allowed_repo_permissions": ["admin", "write"],
232+
"allowed_list": [ ],
233+
"set_commit_status": true,
234+
"build_on_commit": true,
235+
"build_on_comment": true,
236+
"trigger_comment_regex": "^/test x-pack/auditbeat$",
237+
"always_trigger_comment_regex": "^/test x-pack/auditbeat$",
238+
"skip_ci_labels": [ ],
239+
"skip_target_branches": [ ],
240+
"skip_ci_on_only_changed": [ ],
241+
"always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
242+
},
243+
{
244+
"enabled": true,
245+
"pipelineSlug": "beats-xpack-dockerlogbeat",
246+
"allow_org_users": true,
247+
"allowed_repo_permissions": ["admin", "write"],
248+
"allowed_list": [ ],
249+
"set_commit_status": true,
250+
"build_on_commit": true,
251+
"build_on_comment": true,
252+
"trigger_comment_regex": "^/test x-pack/dockerlogbeat$",
253+
"always_trigger_comment_regex": "^/test x-pack/dockerlogbeat$",
254+
"skip_ci_labels": [ ],
255+
"skip_target_branches": [ ],
256+
"skip_ci_on_only_changed": [ ],
257+
"always_require_ci_on_changed": ["^x-pack/dockerlogbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
258+
},
259+
{
260+
"enabled": true,
261+
"pipelineSlug": "beats-xpack-filebeat",
262+
"allow_org_users": true,
263+
"allowed_repo_permissions": ["admin", "write"],
264+
"allowed_list": [ ],
265+
"set_commit_status": true,
266+
"build_on_commit": true,
267+
"build_on_comment": true,
268+
"trigger_comment_regex": "^/test x-pack/filebeat$",
269+
"always_trigger_comment_regex": "^/test x-pack/filebeat$",
270+
"skip_ci_labels": [ ],
271+
"skip_target_branches": [ ],
272+
"skip_ci_on_only_changed": [ ],
273+
"always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
274+
},
275+
{
276+
"enabled": true,
277+
"pipelineSlug": "beats-xpack-heartbeat",
278+
"allow_org_users": true,
279+
"allowed_repo_permissions": ["admin", "write"],
280+
"allowed_list": [ ],
281+
"set_commit_status": true,
282+
"build_on_commit": true,
283+
"build_on_comment": true,
284+
"trigger_comment_regex": "^/test x-pack/heartbeat$",
285+
"always_trigger_comment_regex": "^/test x-pack/heartbeat$",
286+
"skip_ci_labels": [ ],
287+
"skip_target_branches": [ ],
288+
"skip_ci_on_only_changed": [ ],
289+
"always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
290+
},
291+
{
292+
"enabled": true,
293+
"pipelineSlug": "beats-xpack-osquerybeat",
294+
"allow_org_users": true,
295+
"allowed_repo_permissions": ["admin", "write"],
296+
"allowed_list": [ ],
297+
"set_commit_status": true,
298+
"build_on_commit": true,
299+
"build_on_comment": true,
300+
"trigger_comment_regex": "^/test x-pack/osquerybeat$",
301+
"always_trigger_comment_regex": "^/test x-pack/osquerybeat$",
302+
"skip_ci_labels": [ ],
303+
"skip_target_branches": [ ],
304+
"skip_ci_on_only_changed": [ ],
305+
"always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
226306
}
227307
]
228308
}

‎.buildkite/scripts/common.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
2222
[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
2323
[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
2424
[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
25+
[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"
26+
[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
2527

2628
metricbeat_changeset=(
2729
"^metricbeat/.*"
@@ -105,6 +107,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
105107
"beats-xpack-metricbeat")
106108
BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
107109
;;
110+
"beats-xpack-packetbeat")
111+
BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]}
112+
;;
108113
*)
109114
echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
110115
;;

‎.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ steps:
3131
provider: "gcp"
3232
image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
3333
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
34-
disk_size: 100
35-
disk_type: "pd-ssd"
3634
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
3735
3836
- label: ":python: Python Integration Tests"
@@ -42,8 +40,6 @@ steps:
4240
provider: "gcp"
4341
image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
4442
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
45-
disk_size: 100
46-
disk_type: "pd-ssd"
4743
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
4844
4945
- label: ":windows: Windows Unit Tests - {{matrix.image}}"
@@ -106,8 +102,7 @@ else
106102
fi
107103

108104
#TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved
109-
if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then
110-
# if [[ are_conditions_met_macos_tests ]]; then
105+
if are_conditions_met_aws_tests || are_conditions_met_macos_tests ; then
111106
cat >> $pipelineName <<- YAML
112107
113108
- group: "Extended Tests"
@@ -140,8 +135,6 @@ if are_conditions_met_aws_tests; then
140135
provider: "gcp"
141136
image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
142137
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
143-
disk_size: 100
144-
disk_type: "pd-ssd"
145138
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
146139
147140
YAML
@@ -166,6 +159,8 @@ if are_conditions_met_packaging; then
166159
provider: "gcp"
167160
image: "${IMAGE_UBUNTU_X86_64}"
168161
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
162+
disk_size: 100
163+
disk_type: "pd-ssd"
169164
env:
170165
PLATFORMS: "${PACKAGING_PLATFORMS}"
171166

‎.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#!/usr/bin/env bash
2+
3+
source .buildkite/scripts/common.sh
4+
5+
set -euo pipefail
6+
7+
pipelineName="pipeline.xpack-packetbeat-dynamic.yml"
8+
9+
echo "Add the mandatory and extended tests without additional conditions into the pipeline"
10+
if are_conditions_met_mandatory_tests; then
11+
cat > $pipelineName <<- YAML
12+
13+
steps:
14+
15+
- group: "Mandatory Tests"
16+
key: "mandatory-tests"
17+
steps:
18+
- label: ":linux: Ubuntu Unit Tests"
19+
key: "mandatory-linux-unit-test"
20+
command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
21+
agents:
22+
provider: "gcp"
23+
image: "${IMAGE_UBUNTU_X86_64}"
24+
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
25+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
26+
27+
- label: ":linux: Ubuntu System Tests"
28+
key: "mandatory-linux-system-test"
29+
command: "cd $BEATS_PROJECT_NAME && mage systemTest"
30+
agents:
31+
provider: "gcp"
32+
image: "${IMAGE_UBUNTU_X86_64}"
33+
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
34+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
35+
36+
- label: ":rhel: RHEL-9 Unit Tests"
37+
key: "mandatory-rhel9-unit-test"
38+
command: ".buildkite/scripts/unit_tests.sh"
39+
agents:
40+
provider: "gcp"
41+
image: "${IMAGE_RHEL9_X86_64}"
42+
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
43+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
44+
45+
46+
- label: ":windows: Windows Unit Tests - {{matrix.image}}"
47+
command: ".buildkite/scripts/win_unit_tests.ps1"
48+
key: "mandatory-win-unit-tests"
49+
agents:
50+
provider: "gcp"
51+
image: "{{matrix.image}}"
52+
machineType: "${GCP_WIN_MACHINE_TYPE}"
53+
disk_size: 100
54+
disk_type: "pd-ssd"
55+
matrix:
56+
setup:
57+
image:
58+
- "${IMAGE_WIN_2016}"
59+
- "${IMAGE_WIN_2022}"
60+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
61+
62+
## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
63+
# - label: ":windows: Windows 2022 System Tests"
64+
# key: "mandatory-win-2022-system-tests"
65+
# command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
66+
# agents:
67+
# provider: "gcp"
68+
# image: "${IMAGE_WIN_2022}"
69+
# machineType: "${GCP_WIN_MACHINE_TYPE}"
70+
# disk_size: 100
71+
# disk_type: "pd-ssd"
72+
# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
73+
74+
## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h
75+
76+
- group: "Extended Windows Tests"
77+
key: "extended-win-tests"
78+
steps:
79+
80+
- label: ":windows: Windows Unit Tests - {{matrix.image}}"
81+
command: ".buildkite/scripts/win_unit_tests.ps1"
82+
key: "extended-win-unit-tests"
83+
agents:
84+
provider: "gcp"
85+
image: "{{matrix.image}}"
86+
machineType: "${GCP_WIN_MACHINE_TYPE}"
87+
disk_size: 100
88+
disk_type: "pd-ssd"
89+
matrix:
90+
setup:
91+
image:
92+
- "${IMAGE_WIN_10}"
93+
- "${IMAGE_WIN_11}"
94+
- "${IMAGE_WIN_2019}"
95+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
96+
97+
## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
98+
# - label: ":windows: Windows 10 System Tests"
99+
# key: "extended-win-10-system-tests"
100+
# command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
101+
# agents:
102+
# provider: "gcp"
103+
# image: "${IMAGE_WIN_10}"
104+
# machineType: "${GCP_WIN_MACHINE_TYPE}"
105+
# disk_size: 100
106+
# disk_type: "pd-ssd"
107+
# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
108+
109+
YAML
110+
else
111+
echo "The conditions don't match to requirements for generating pipeline steps."
112+
exit 0
113+
fi
114+
115+
if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then
116+
cat >> $pipelineName <<- YAML
117+
118+
- group: "Extended Tests"
119+
key: "extended-tests"
120+
steps:
121+
122+
YAML
123+
fi
124+
125+
if are_conditions_met_macos_tests; then
126+
cat >> $pipelineName <<- YAML
127+
128+
- label: ":mac: MacOS Unit Tests"
129+
key: "extended-macos-unit-tests"
130+
command: ".buildkite/scripts/unit_tests.sh"
131+
agents:
132+
provider: "orka"
133+
imagePrefix: "${IMAGE_MACOS_X86_64}"
134+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
135+
136+
YAML
137+
fi
138+
139+
if are_conditions_met_arm_tests; then
140+
cat >> $pipelineName <<- YAML
141+
- label: ":linux: ARM Ubuntu Unit Tests"
142+
key: "extended-arm64-unit-test"
143+
command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
144+
agents:
145+
provider: "aws"
146+
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
147+
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
148+
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
149+
150+
YAML
151+
fi
152+
153+
echo "Check and add the Packaging into the pipeline"
154+
if are_conditions_met_packaging; then
155+
cat >> $pipelineName <<- YAML
156+
157+
- wait: ~
158+
depends_on:
159+
- step: "mandatory-tests"
160+
allow_failure: false
161+
162+
- group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
163+
key: "packaging"
164+
steps:
165+
- label: ":linux: Packaging Linux"
166+
key: "packaging-linux"
167+
command: "cd $BEATS_PROJECT_NAME && mage package"
168+
agents:
169+
provider: "gcp"
170+
image: "${IMAGE_UBUNTU_X86_64}"
171+
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
172+
disk_size: 100
173+
disk_type: "pd-ssd"
174+
env:
175+
PLATFORMS: "${PACKAGING_PLATFORMS}"
176+
177+
- label: ":linux: Packaging ARM"
178+
key: "packaging-arm"
179+
command: "cd $BEATS_PROJECT_NAME && mage package"
180+
agents:
181+
provider: "aws"
182+
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
183+
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
184+
env:
185+
PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
186+
PACKAGES: "docker"
187+
188+
YAML
189+
fi
190+
191+
echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public
192+
cat $pipelineName
193+
194+
echo "--- Loading dynamic steps"
195+
buildkite-agent pipeline upload $pipelineName

‎.buildkite/scripts/setenv.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ SETUP_WIN_PYTHON_VERSION="3.11.0"
1111
NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
1212
GO_VERSION=$(cat .go-version)
1313
ASDF_MAGE_VERSION="1.15.0"
14-
ASDF_TERRAFORM_VERSION="1.0.2"
1514
PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
1615
PACKAGING_ARM_PLATFORMS="linux/arm64"
16+
ASDF_TERRAFORM_VERSION="1.0.2"
1717
AWS_REGION="eu-central-1"
1818

1919
export SETUP_GVM_VERSION

0 commit comments

Comments
 (0)