Skip to content

Commit fe9303c

Browse files
cafuegocafuego
authored
Merge branch 'main' into route53-log-hostnames
2 parents f7c9392 + d24d91f commit fe9303c

File tree

8,036 files changed

+313356
-522232
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

8,036 files changed

+313356
-522232
lines changed

‎.buildkite/hooks/post-checkout‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
#!/bin/bash
22

3+
# ******************************* WARNING ******************************************
4+
# This post-checkout hook is not the same as in the rest of repositories e.g. beats
5+
# because some steps in this pipeline (in PR context) take a very long time and we
6+
# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every
7+
# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime
8+
# and therefore, some steps (e.g. sonarqube) may end up testing a different commit.
9+
#
10+
# Running builds from branches or tags (out of PR context) maintains the same behavior
11+
# as in the rest of the repositories.
12+
#
13+
# Reference: https://github.com/elastic/integrations/pull/10397
14+
# **********************************************************************************
15+
316
set -euo pipefail
417

518
checkout_merge() {
@@ -13,8 +26,15 @@ checkout_merge() {
1326
fi
1427

1528
git fetch -v origin "${target_branch}"
16-
git checkout FETCH_HEAD
17-
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
29+
if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then
30+
git checkout FETCH_HEAD
31+
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
32+
else
33+
# Use the same commit from target branch as in the other steps.
34+
echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}"
35+
git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}"
36+
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
37+
fi
1838

1939
# create temporal branch to merge the PR with the target branch
2040
git checkout -b ${merge_branch}
@@ -44,6 +64,9 @@ PR_COMMIT="${BUILDKITE_COMMIT}"
4464
PR_ID=${BUILDKITE_PULL_REQUEST}
4565
MERGE_BRANCH="pr_merge_${PR_ID}"
4666

67+
# This meta-data field is populated in the pre-command hook
68+
REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "")
69+
4770
checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
4871

4972
echo "Commit information"

‎.buildkite/hooks/pre-command‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
2929
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3030
export REPO_BUILD_TAG
3131

32-
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
32+
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
3333
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
3434

3535
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
@@ -74,6 +74,17 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
7474
export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
7575
fi
7676

77+
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" && "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
78+
# Get the commit from target branch in the first step (reference-target-branch).
79+
# This step MUST be the first one and not run in parallel with any other step to ensure
80+
# that there is just one value for this variable
81+
if is_pr ; then
82+
git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
83+
commit_main=$(git rev-parse --verify FETCH_HEAD)
84+
buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
85+
fi
86+
fi
87+
7788
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
7889
if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
7990
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
@@ -115,19 +126,13 @@ fi
115126

116127
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
117128
if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
118-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
119-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
120-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
121-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
129+
# Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
130+
# add the AWS credentials as in the integrations pipeline.
122131

123132
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
124133
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
125134
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
126135

127-
# Environment variables required by the service deployer
128-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
129-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
130-
131136
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
132137
export BUILDKITE_API_TOKEN
133138

@@ -142,11 +147,18 @@ fi
142147

143148
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
144149
if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
145-
GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
150+
GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
146151
export GITHUB_USERNAME_SECRET
147-
GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
152+
GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
148153
export GITHUB_EMAIL_SECRET
149154
GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
150155
export GITHUB_TOKEN
151156
fi
152157
fi
158+
159+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" || "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
160+
if [[ "$BUILDKITE_STEP_KEY" == "report-failed-tests" ]]; then
161+
export GITHUB_TOKEN="${VAULT_GITHUB_TOKEN}"
162+
fi
163+
fi
164+

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: integrations-schedule-daily
33

44
env:
5-
SETUP_GVM_VERSION: "v0.5.1"
5+
SETUP_GVM_VERSION: "v0.5.2"
66
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
77

88
# The pipeline is triggered by the scheduler every day
@@ -20,22 +20,19 @@ steps:
2020
build:
2121
env:
2222
SERVERLESS: "false"
23-
SKIP_PUBLISHING: "true"
2423
FORCE_CHECK_ALL: "true"
25-
# STACK_VERSION: 7.17-SNAPSHOT # Using 7.17.19 till https://github.com/elastic/fleet-server/issues/3435 is solved.
26-
STACK_VERSION: 7.17.19
24+
STACK_VERSION: 7.17.23-SNAPSHOT
2725
depends_on:
2826
- step: "check"
2927
allow_failure: false
3028

31-
- label: "Check integrations local stacks - Stack Version v8.14"
29+
- label: "Check integrations local stacks - Stack Version v8.15"
3230
trigger: "integrations"
3331
build:
3432
env:
3533
SERVERLESS: "false"
36-
SKIP_PUBLISHING: "true"
3734
FORCE_CHECK_ALL: "true"
38-
STACK_VERSION: 8.14.0-SNAPSHOT
35+
STACK_VERSION: 8.15.0-SNAPSHOT
3936
PUBLISH_COVERAGE_REPORTS: "true"
4037
depends_on:
4138
- step: "check"

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,14 @@ env:
44
SETUP_GVM_VERSION: "v0.5.2"
55
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
66
DOCKER_COMPOSE_VERSION: "v2.24.1"
7-
DOCKER_VERSION: "false"
7+
DOCKER_VERSION: "false" # not required to set since system tests are not running yet
88
KIND_VERSION: 'v0.20.0'
9-
K8S_VERSION: 'v1.29.0'
9+
K8S_VERSION: 'v1.30.0'
1010
YQ_VERSION: 'v4.35.2'
1111
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
12+
GH_CLI_VERSION: "2.29.0"
13+
# This pipeline is intended to test packages with Elastic Serverless
14+
SERVERLESS: true
1215

1316
# Elastic package settings
1417
# Manage docker output/logs
@@ -19,6 +22,10 @@ env:
1922
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2023
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2124
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
25+
# Enable independent Elastic Agents for all packages
26+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
27+
# Set maximum number of parallel tests to run if package allows it
28+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2229

2330
steps:
2431
- input: "Input values for the variables"
@@ -49,9 +56,8 @@ steps:
4956
- label: "Check integrations in serverless"
5057
key: "test-integrations-serverless-project"
5158
command: ".buildkite/scripts/test_integrations_with_serverless.sh"
52-
timeout_in_minutes: 120
59+
timeout_in_minutes: 240
5360
env:
54-
SERVERLESS: true
5561
FORCE_CHECK_ALL: true
5662
UPLOAD_SAFE_LOGS: 1
5763
agents:
@@ -74,3 +80,16 @@ steps:
7480
artifacts: "build/test-results/*.xml"
7581
agents:
7682
provider: "gcp" # junit plugin requires docker
83+
84+
- label: ":github: Report failed tests"
85+
key: report-failed-tests
86+
command: ".buildkite/scripts/report_issues.sh"
87+
agents:
88+
image: "${LINUX_AGENT_IMAGE}"
89+
cpu: "8"
90+
memory: "4G"
91+
# not fail build if this step fails
92+
soft_fail: true
93+
# run this step when if it is triggered by the daily job
94+
if: >
95+
build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

‎.buildkite/pipeline.yml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ env:
55
DOCKER_COMPOSE_VERSION: "v2.24.1"
66
DOCKER_VERSION: "26.1.2"
77
KIND_VERSION: 'v0.20.0'
8-
K8S_VERSION: 'v1.29.0'
8+
K8S_VERSION: 'v1.30.0'
99
YQ_VERSION: 'v4.35.2'
1010
JQ_VERSION: '1.7'
1111
GH_CLI_VERSION: "2.29.0"
@@ -20,15 +20,28 @@ env:
2020
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2121
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2222
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
23+
# Enable independent Elastic Agents for all packages
24+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
25+
# Set maximum number of parallel tests to run if package allows it
26+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2327

2428
steps:
29+
- label: "Get reference from target branch"
30+
key: "reference-target-branch"
31+
command: "echo 'Get reference from main'"
32+
agents:
33+
image: "${LINUX_AGENT_IMAGE}"
34+
2535
- label: ":white_check_mark: Check go sources"
2636
key: "check"
2737
command: ".buildkite/scripts/check_sources.sh"
2838
agents:
2939
image: "${LINUX_AGENT_IMAGE}"
3040
cpu: "8"
3141
memory: "4G"
42+
depends_on:
43+
- step: "reference-target-branch"
44+
allow_failure: false
3245

3346
- label: "Trigger integrations"
3447
key: "test-integrations"
@@ -53,10 +66,14 @@ steps:
5366
memory: "4G"
5467

5568
- label: ":sonarqube: Continuous Code Inspection"
69+
soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
70+
timeout_in_minutes: 120
5671
env:
5772
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
5873
agents:
5974
image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
75+
cpu: "8"
76+
memory: "4G"
6077
command: ".buildkite/scripts/run_sonar_scanner.sh"
6178
artifact_paths:
6279
- build/test-coverage/coverage_merged.xml
@@ -68,3 +85,16 @@ steps:
6885
failed-download-exit-code: 0 # Not fail the build in case there are no XML files
6986
agents:
7087
provider: "gcp" # junit plugin requires docker
88+
89+
- label: ":github: Report failed tests"
90+
key: report-failed-tests
91+
command: ".buildkite/scripts/report_issues.sh"
92+
agents:
93+
image: "${LINUX_AGENT_IMAGE}"
94+
cpu: "8"
95+
memory: "4G"
96+
# not fail build if this step fails
97+
soft_fail: true
98+
# run this step when if it is triggered by the daily job
99+
if: >
100+
build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

‎.buildkite/scripts/build_packages.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ build_packages() {
8282
}
8383

8484
if [ "${SKIP_PUBLISHING}" == "true" ] ; then
85-
echo "packageStoragePublish: skipping because skip_publishing param is ${SKIP_PUBLISHING}"
85+
echo "packageStoragePublish: skipping because SKIP_PUBLISHING environment variable is ${SKIP_PUBLISHING}"
8686
exit 0
8787
fi
8888

‎.buildkite/scripts/check_sources.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ set -euo pipefail
77
add_bin_path
88
with_mage
99

10-
mage -debug check
10+
mage -v check
1111

1212
check_git_diff

‎.buildkite/scripts/common.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ is_pr_affected() {
692692

693693
echo "[${package}] git-diff: check non-package files"
694694
commit_merge=$(git merge-base "${from}" "${to}")
695-
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|docs/)' ; then
695+
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|README.md|docs/)' ; then
696696
echo "[${package}] PR is affected: found non-package files"
697697
return 0
698698
fi
@@ -819,6 +819,9 @@ test_package_in_serverless() {
819819
if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
820820
return 1
821821
fi
822+
if ! ${ELASTIC_PACKAGE_BIN} test policy ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
823+
return 1
824+
fi
822825
echo ""
823826
return 0
824827
}

‎.buildkite/scripts/merge_xml.sh‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎.buildkite/scripts/report_issues.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
source .buildkite/scripts/common.sh
4+
5+
set -euo pipefail
6+
7+
junit_folder="test-results"
8+
test_results_folder="build/${junit_folder}"
9+
buildkite_pattern="${test_results_folder}/*.xml"
10+
11+
download_test_results() {
12+
mkdir -p "${test_results_folder}"
13+
14+
if ! buildkite-agent artifact download "${buildkite_pattern}" . ; then
15+
message="Could not download XML artifacts. Skip creating issues."
16+
echo "--- :boom: ${message}"
17+
buildkite-agent annotate \
18+
"[Report Failed Tests] ${message}" \
19+
--context "ctx-report-failed-tests-no-files" \
20+
--style "warning"
21+
return 1
22+
fi
23+
return 0
24+
}
25+
26+
if running_on_buildkite ; then
27+
echo "--- Installing tools"
28+
add_bin_path
29+
with_mage
30+
with_github_cli # to list, create and update issues
31+
32+
echo "--- Download Test Results"
33+
if ! download_test_results ; then
34+
exit 0
35+
fi
36+
fi
37+
38+
echo "--- Create GitHub Issues for failed tests"
39+
mage -v ReportFailedTests build/test-results
40+

0 commit comments

Comments
 (0)