Skip to content

Commit ff3e919

Browse files
Merge branch 'main' into system-journald-sles15
2 parents 557f950 + 9f744b6 commit ff3e919

File tree

2,634 files changed

+133930
-142762
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,634 files changed

+133930
-142762
lines changed

.buildkite/hooks/pre-command

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,6 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
3131
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3232
export REPO_BUILD_TAG
3333

34-
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
35-
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
36-
3734
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
3835

3936
EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
@@ -43,6 +40,8 @@ EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
4340
export ENVIRONMENT="ci"
4441
export REPO="${REPO_NAME}"
4542

43+
export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
44+
4645
branch_name_label() {
4746
local branch="$1"
4847

@@ -107,32 +106,13 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
107106
fi
108107

109108
if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
110-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
111-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
112-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
113-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
114-
115-
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
116-
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
117-
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
118-
119-
# Environment variables required by the service deployer
120-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
121-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
122-
123109
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
124110
export BUILDKITE_API_TOKEN
125111
fi
126112
fi
127113

128114
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
129115
if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
130-
# Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
131-
# add the AWS credentials as in the integrations pipeline.
132-
133-
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
134-
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
135-
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
136116

137117
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
138118
export BUILDKITE_API_TOKEN

.buildkite/hooks/pre-exit

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ set -euo pipefail
77
if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
88
# FIXME: update condition depending on the pipeline steps triggered
99
if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
10-
unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
11-
unset ELASTIC_PACKAGE_AWS_SECRET_KEY
12-
unset AWS_ACCESS_KEY_ID
13-
unset AWS_SECRET_ACCESS_KEY
1410

1511
# Ensure that kind cluster is deleted
1612
delete_kind_cluster
@@ -25,10 +21,6 @@ fi
2521

2622
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
2723
if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
28-
unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
29-
unset ELASTIC_PACKAGE_AWS_SECRET_KEY
30-
unset AWS_ACCESS_KEY_ID
31-
unset AWS_SECRET_ACCESS_KEY
3224

3325
# Ensure that kind cluster is deleted
3426
delete_kind_cluster
@@ -44,8 +36,6 @@ fi
4436
unset_secrets
4537
cleanup
4638

47-
google_cloud_logout_active_account
48-
4939
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
5040
cd "${WORKSPACE}"
5141
git config remote.origin.url "https://github.com/elastic/integrations.git"

.buildkite/pipeline.backport.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ name: "integrations-backport"
44

55
env:
66
YQ_VERSION: 'v4.35.2'
7+
# Agent images used in pipeline steps
8+
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
79

810
steps:
911

@@ -49,6 +51,8 @@ steps:
4951
- label: "Creating the backport branch"
5052
key: "create-backport-branch"
5153
command: ".buildkite/scripts/backport_branch.sh"
54+
agents:
55+
image: "${LINUX_AGENT_IMAGE}"
5256
depends_on:
5357
- step: "input-variables"
5458
allow_failure: false

.buildkite/pipeline.serverless.yml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ env:
66
DOCKER_COMPOSE_VERSION: "v2.24.1"
77
DOCKER_VERSION: "false" # not required to set since system tests are not running yet
88
KIND_VERSION: 'v0.27.0'
9-
K8S_VERSION: 'v1.32.0'
9+
K8S_VERSION: 'v1.33.0'
1010
YQ_VERSION: 'v4.35.2'
1111
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
1212
GH_CLI_VERSION: "2.29.0"
@@ -67,6 +67,17 @@ steps:
6767
agents:
6868
provider: "gcp"
6969
image: "${IMAGE_UBUNTU_X86_64}"
70+
plugins:
71+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
72+
# This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
73+
- elastic/oblt-aws-auth#v0.1.0:
74+
duration: 10800 # seconds
75+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
76+
# This plugin authenticates to Google Cloud using the OIDC token.
77+
- elastic/oblt-google-auth#v1.3.0:
78+
lifetime: 10800 # seconds
79+
project-id: "elastic-observability-ci"
80+
project-number: "911195782929"
7081
artifact_paths:
7182
- "build/test-results/*.xml"
7283
- "build/elastic-stack-dump/*/logs/*.log"

.buildkite/pipeline.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ env:
44
DOCKER_COMPOSE_VERSION: "v2.24.1"
55
DOCKER_VERSION: "26.1.2"
66
KIND_VERSION: 'v0.27.0'
7-
K8S_VERSION: 'v1.32.0'
7+
K8S_VERSION: 'v1.33.0'
88
YQ_VERSION: 'v4.35.2'
99
JQ_VERSION: '1.7'
1010
GH_CLI_VERSION: "2.29.0"

.buildkite/scripts/backport_branch.sh

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,26 @@ updateBackportBranchContents() {
130130
rm -rf "$JENKINS_FOLDER_PATH"
131131
fi
132132

133+
# Update scripts used by mage
134+
local MAGEFILE_SCRIPTS_FOLDER="dev/citools"
135+
local TESTSREPORTER_SCRIPTS_FOLDER="dev/testsreporter"
136+
local COVERAGE_SCRIPTS_FOLDER="dev/coverage"
137+
if git ls-tree -d --name-only main:${MAGEFILE_SCRIPTS_FOLDER} > /dev/null 2>&1 ; then
138+
echo "Copying $MAGEFILE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
139+
git checkout "$SOURCE_BRANCH" -- "${MAGEFILE_SCRIPTS_FOLDER}"
140+
echo "Copying $TESTSREPORTER_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
141+
git checkout "$SOURCE_BRANCH" -- "${TESTSREPORTER_SCRIPTS_FOLDER}"
142+
echo "Copying $COVERAGE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
143+
git checkout "$SOURCE_BRANCH" -- "${COVERAGE_SCRIPTS_FOLDER}"
144+
echo "Copying magefile.go from $SOURCE_BRANCH..."
145+
git checkout "$SOURCE_BRANCH" -- "magefile.go"
146+
# Run go mod tidy to update just the dependencies related to magefile and dev scripts
147+
go mod tidy
148+
fi
149+
133150
if [ "${REMOVE_OTHER_PACKAGES}" == "true" ]; then
134151
echo "Removing all packages from $PACKAGES_FOLDER_PATH folder"
135-
removeOtherPackages "$PACKAGES_FOLDER_PATH"
152+
removeOtherPackages "${PACKAGES_FOLDER_PATH}"
136153
ls -la $PACKAGES_FOLDER_PATH
137154
fi
138155

@@ -142,7 +159,12 @@ updateBackportBranchContents() {
142159
echo "Commiting"
143160
git add $BUILDKITE_FOLDER_PATH
144161
if [ -d "${JENKINS_FOLDER_PATH}" ]; then
145-
git add $JENKINS_FOLDER_PATH
162+
git add "${JENKINS_FOLDER_PATH}"
163+
fi
164+
if [ -d "${MAGEFILE_SCRIPTS_FOLDER}" ] ; then
165+
git add ${MAGEFILE_SCRIPTS_FOLDER}
166+
git add ${TESTSREPORTER_SCRIPTS_FOLDER}
167+
git add go.mod go.sum
146168
fi
147169
git add $PACKAGES_FOLDER_PATH/
148170
git status
@@ -156,7 +178,8 @@ updateBackportBranchContents() {
156178

157179
if [ "$DRY_RUN" == "true" ];then
158180
echo "DRY_RUN mode, nothing will be pushed."
159-
git diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME
181+
# Show just the relevant files diff (go.mod, go.sum, .buildkite, dev and package to be backported)
182+
git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME go.mod go.sum .buildkite/ dev/ "packages/${PACKAGE_NAME}"
160183
else
161184
echo "Pushing..."
162185
git push origin $BACKPORT_BRANCH_NAME

.buildkite/scripts/common.sh

Lines changed: 23 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ platform_type_lowercase="${platform_type,,}"
1010

1111
SCRIPTS_BUILDKITE_PATH="${WORKSPACE}/.buildkite/scripts"
1212

13-
GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
1413
export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
1514

1615
API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
@@ -255,34 +254,6 @@ with_github_cli() {
255254
gh version
256255
}
257256

258-
## Logging and logout from Google Cloud
259-
google_cloud_auth_safe_logs() {
260-
local gsUtilLocation
261-
gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
262-
local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
263-
264-
echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
265-
266-
gcloud auth activate-service-account --key-file "${secretFileLocation}" 2> /dev/null
267-
export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
268-
}
269-
270-
google_cloud_logout_active_account() {
271-
local active_account
272-
active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null || true)
273-
if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
274-
echo "Logging out from GCP for active account"
275-
gcloud auth revoke "$active_account" > /dev/null 2>&1
276-
else
277-
echo "No active GCP accounts found."
278-
fi
279-
280-
if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
281-
rm -rf "${GOOGLE_APPLICATION_CREDENTIALS}"
282-
unset GOOGLE_APPLICATION_CREDENTIALS
283-
fi
284-
}
285-
286257
## Helpers for integrations pipelines
287258
check_git_diff() {
288259
cd "${WORKSPACE}"
@@ -301,7 +272,9 @@ use_elastic_package() {
301272
is_already_published() {
302273
local packageZip=$1
303274

304-
if curl -s --head "https://package-storage.elastic.co/artifacts/packages/${packageZip}" | grep -q "HTTP/2 200" ; then
275+
# Avoid using "-q" in grep in this pipe, it could cause some weird behavior in some scenarios due to SIGPIPE errors when "set -o pipefail"
276+
# https://tldp.org/LDP/lpg/node20.html
277+
if curl -s --head "https://package-storage.elastic.co/artifacts/packages/${packageZip}" | grep "HTTP/2 200" > /dev/null; then
305278
echo "- Already published ${packageZip}"
306279
return 0
307280
fi
@@ -407,7 +380,9 @@ is_package_excluded_in_config() {
407380
local package_name=""
408381
package_name=$(package_name_manifest)
409382

410-
if echo "${excluded_packages}" | grep -q -E "\"${package_name}\""; then
383+
# Avoid using "-q" in grep in this pipe, it could cause some weird behavior in some scenarios due to SIGPIPE errors when "set -o pipefail"
384+
# https://tldp.org/LDP/lpg/node20.html
385+
if echo "${excluded_packages}" | grep -E "\"${package_name}\"" > /dev/null; then
411386
return 0
412387
fi
413388
return 1
@@ -438,7 +413,9 @@ is_supported_capability() {
438413
fi
439414

440415
for cap in ${capabilities}; do
441-
if ! echo "${cap}" | grep -q -E "${capabilities_kibana_grep}"; then
416+
# Avoid using "-q" in grep in this pipe, it could cause some weird behavior in some scenarios due to SIGPIPE errors when "set -o pipefail"
417+
# https://tldp.org/LDP/lpg/node20.html
418+
if ! echo "${cap}" | grep -E "${capabilities_kibana_grep}" > /dev/null; then
442419
return 1
443420
fi
444421
done
@@ -768,12 +745,20 @@ is_pr_affected() {
768745

769746
commit_merge=$(git merge-base "${from}" "${to}")
770747
echoerr "[${package}] git-diff: check non-package files (${commit_merge}..${to})"
771-
if git diff --name-only "${commit_merge}" "${to}" | grep -q -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
748+
# Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
749+
# Example:
750+
# https://buildkite.com/elastic/integrations/builds/25606
751+
# https://github.com/elastic/integrations/pull/13810
752+
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' > /dev/null; then
772753
echo "[${package}] PR is affected: found non-package files"
773754
return 0
774755
fi
775756
echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})"
776-
if git diff --name-only "${commit_merge}" "${to}" | grep -q -E "^packages/${package}/" ; then
757+
# Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
758+
# Example:
759+
# https://buildkite.com/elastic/integrations/builds/25606
760+
# https://github.com/elastic/integrations/pull/13810
761+
if git diff --name-only "${commit_merge}" "${to}" | grep -E "^packages/${package}/" > /dev/null ; then
777762
echo "[${package}] PR is affected: found package files"
778763
return 0
779764
fi
@@ -969,16 +954,16 @@ upload_safe_logs() {
969954
local source="$2"
970955
local target="$3"
971956

957+
echo "--- Uploading safe logs to GCP bucket ${bucket}"
958+
972959
if ! ls ${source} 2>&1 > /dev/null ; then
973960
echo "upload_safe_logs: artifacts files not found, nothing will be archived"
974961
return
975962
fi
976963

977-
google_cloud_auth_safe_logs
978-
979-
gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
964+
gcloud storage cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
980965

981-
google_cloud_logout_active_account
966+
echo "GCP logout is not required, the BK plugin will do it for us"
982967
}
983968

984969
clean_safe_logs() {

.buildkite/scripts/trigger_integrations_in_parallel.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,17 @@ for package in ${PACKAGE_LIST}; do
8888
FORCE_CHECK_ALL: "${FORCE_CHECK_ALL}"
8989
SERVERLESS: "false"
9090
UPLOAD_SAFE_LOGS: ${UPLOAD_SAFE_LOGS}
91+
plugins:
92+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
93+
# This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
94+
- elastic/oblt-aws-auth#v0.1.0:
95+
duration: 10800 # seconds
96+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
97+
# This plugin authenticates to Google Cloud using the OIDC token.
98+
- elastic/oblt-google-auth#v1.3.0:
99+
lifetime: 10800 # seconds
100+
project-id: "elastic-observability-ci"
101+
project-number: "911195782929"
91102
artifact_paths:
92103
- build/test-results/*.xml
93104
- build/test-coverage/*.xml

.github/CODEOWNERS

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@
164164
/packages/citrix_adc/data_stream/vpn @elastic/obs-infraobs-integrations
165165
/packages/citrix_waf @elastic/sec-deployment-and-devices
166166
/packages/claroty_ctd @elastic/security-service-integrations
167+
/packages/claroty_xdome @elastic/security-service-integrations
167168
/packages/cloud_defend @elastic/sec-linux-platform
168169
/packages/cloud_security_posture @elastic/cloud-security-posture
169170
/packages/cloud_asset_inventory @elastic/cloud-security-posture
@@ -191,6 +192,9 @@
191192
/packages/elastic_connectors @elastic/search-extract-and-transform
192193
/packages/elastic_package_registry @elastic/ecosystem
193194
/packages/elasticsearch @elastic/stack-monitoring
195+
/packages/endace @elastic/sec-deployment-and-devices @elastic/sec-linux-platform
196+
/packages/endace/data_stream/flow @elastic/sec-linux-platform
197+
/packages/endace/data_stream/log @elastic/sec-deployment-and-devices
194198
/packages/enterprisesearch @elastic/stack-monitoring
195199
/packages/entityanalytics_ad @elastic/security-service-integrations
196200
/packages/entityanalytics_entra_id @elastic/security-service-integrations
@@ -299,6 +303,7 @@
299303
/packages/microsoft_sentinel @elastic/security-service-integrations
300304
/packages/microsoft_sqlserver @elastic/obs-infraobs-integrations
301305
/packages/mimecast @elastic/security-service-integrations
306+
/packages/miniflux @elastic/security-service-integrations
302307
/packages/modsecurity @elastic/sec-deployment-and-devices
303308
/packages/mongodb @elastic/obs-infraobs-integrations
304309
/packages/mongodb_atlas @elastic/obs-infraobs-integrations
@@ -356,6 +361,7 @@
356361
/packages/sailpoint_identity_sc @elastic/security-service-integrations
357362
/packages/salesforce @elastic/obs-infraobs-integrations
358363
/packages/santa @elastic/security-service-integrations
364+
/packages/security_ai_prompts @elastic/security-generative-ai
359365
/packages/security_detection_engine @elastic/protections
360366
/packages/sentinel_one @elastic/security-service-integrations
361367
/packages/sentinel_one_cloud_funnel @elastic/security-service-integrations
@@ -449,6 +455,7 @@
449455
/packages/universal_profiling_symbolizer @elastic/obs-ds-intake-services
450456
/packages/varonis @elastic/security-service-integrations
451457
/packages/vectra_detect @elastic/security-service-integrations
458+
/packages/vectra_rux @elastic/security-service-integrations
452459
/packages/vsphere @elastic/obs-infraobs-integrations
453460
/packages/websocket @elastic/security-service-integrations
454461
/packages/watchguard_firebox @elastic/sec-deployment-and-devices

.github/workflows/bump-elastic-stack-version.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
- uses: actions/checkout@v4
2626

2727
- name: Install Updatecli in the runner
28-
uses: updatecli/updatecli-action@cf942226b953240efac9ff60bf42df2b908c2fa0 #v2.83.0
28+
uses: updatecli/updatecli-action@307ce72e224b82157cc31c78828f168b8e55d47d #v2.84.0
2929

3030
- name: Select diff action
3131
if: ${{ github.event_name == 'pull_request' }}

0 commit comments

Comments
 (0)