Skip to content

Commit 3258ebb

Browse files
AlphayeeeetAlphayeeeet
authored
Merge branch 'main' into remove-event.original6
2 parents ef811e7 + bb6c198 commit 3258ebb

File tree

6,580 files changed

+536216
-290362
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

6,580 files changed

+536216
-290362
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,16 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
3131
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3232
export REPO_BUILD_TAG
3333

34-
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
35-
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
36-
3734
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
3835

39-
EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
4036
EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
4137

4238
# variables required for terraform
4339
export ENVIRONMENT="ci"
4440
export REPO="${REPO_NAME}"
4541

42+
export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
43+
4644
branch_name_label() {
4745
local branch="$1"
4846

@@ -107,40 +105,17 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
107105
fi
108106

109107
if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
110-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
111-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
112-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
113-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
114-
115-
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
116-
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
117-
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
118-
119-
# Environment variables required by the service deployer
120-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
121-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
122-
123108
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
124109
export BUILDKITE_API_TOKEN
125110
fi
126111
fi
127112

128113
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
129114
if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
130-
# Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
131-
# add the AWS credentials as in the integrations pipeline.
132-
133-
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
134-
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
135-
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
136115

137116
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
138117
export BUILDKITE_API_TOKEN
139118

140-
EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
141-
export EC_API_KEY_SECRET
142-
EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
143-
export EC_HOST_SECRET
144119
EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
145120
export EC_REGION_SECRET
146121
fi

‎.buildkite/hooks/pre-exit‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ set -euo pipefail
77
if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
88
# FIXME: update condition depending on the pipeline steps triggered
99
if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
10-
unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
11-
unset ELASTIC_PACKAGE_AWS_SECRET_KEY
12-
unset AWS_ACCESS_KEY_ID
13-
unset AWS_SECRET_ACCESS_KEY
1410

1511
# Ensure that kind cluster is deleted
1612
delete_kind_cluster
@@ -25,27 +21,21 @@ fi
2521

2622
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
2723
if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
28-
unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
29-
unset ELASTIC_PACKAGE_AWS_SECRET_KEY
30-
unset AWS_ACCESS_KEY_ID
31-
unset AWS_SECRET_ACCESS_KEY
3224

3325
# Ensure that kind cluster is deleted
3426
delete_kind_cluster
3527

3628
# Ensure elastic stack is stopped
3729
if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
3830
echo "--- Take down the Elastic stack"
39-
EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v
31+
${ELASTIC_PACKAGE_BIN} stack down -v
4032
fi
4133
fi
4234
fi
4335

4436
unset_secrets
4537
cleanup
4638

47-
google_cloud_logout_active_account
48-
4939
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
5040
cd "${WORKSPACE}"
5141
git config remote.origin.url "https://github.com/elastic/integrations.git"

‎.buildkite/pipeline.backport.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ name: "integrations-backport"
44

55
env:
66
YQ_VERSION: 'v4.35.2'
7+
# Agent images used in pipeline steps
8+
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
79

810
steps:
911

@@ -49,6 +51,8 @@ steps:
4951
- label: "Creating the backport branch"
5052
key: "create-backport-branch"
5153
command: ".buildkite/scripts/backport_branch.sh"
54+
agents:
55+
image: "${LINUX_AGENT_IMAGE}"
5256
depends_on:
5357
- step: "input-variables"
5458
allow_failure: false

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ env:
66
DOCKER_COMPOSE_VERSION: "v2.24.1"
77
DOCKER_VERSION: "false" # not required to set since system tests are not running yet
88
KIND_VERSION: 'v0.27.0'
9-
K8S_VERSION: 'v1.32.0'
9+
K8S_VERSION: 'v1.33.0'
1010
YQ_VERSION: 'v4.35.2'
1111
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
1212
GH_CLI_VERSION: "2.29.0"
@@ -67,6 +67,22 @@ steps:
6767
agents:
6868
provider: "gcp"
6969
image: "${IMAGE_UBUNTU_X86_64}"
70+
plugins:
71+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
72+
# This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
73+
- elastic/oblt-aws-auth#v0.1.0:
74+
duration: 10800 # seconds
75+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
76+
# This plugin authenticates to CI Google Cloud using the OIDC token.
77+
- elastic/oblt-google-auth#v1.3.0:
78+
lifetime: 10800 # seconds
79+
project-id: "elastic-observability-ci"
80+
project-number: "911195782929"
81+
lifetime: 10800 # seconds
82+
- avaly/gcp-secret-manager#v1.2.0:
83+
env:
84+
EC_API_KEY: elastic-cloud-observability-team-qa-api-key
85+
EC_HOST: elastic-cloud-observability-team-qa-endpoint
7086
artifact_paths:
7187
- "build/test-results/*.xml"
7288
- "build/elastic-stack-dump/*/logs/*.log"

‎.buildkite/pipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ env:
44
DOCKER_COMPOSE_VERSION: "v2.24.1"
55
DOCKER_VERSION: "26.1.2"
66
KIND_VERSION: 'v0.27.0'
7-
K8S_VERSION: 'v1.32.0'
7+
K8S_VERSION: 'v1.33.0'
88
YQ_VERSION: 'v4.35.2'
99
JQ_VERSION: '1.7'
1010
GH_CLI_VERSION: "2.29.0"

‎.buildkite/scripts/backport_branch.sh‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,16 @@ createLocalBackportBranch() {
9696

9797
removeOtherPackages() {
9898
local sourceFolder=$1
99+
local currentPackage=""
99100
for dir in "$sourceFolder"/*; do
100101
if [[ -d "$dir" ]] && [[ "$(basename "$dir")" != "$PACKAGE_NAME" ]]; then
101102
echo "Removing directory: $dir"
102103
rm -rf "$dir"
104+
105+
currentPackage=$(basename "${dir}")
106+
echo "Removing ${currentPackage} from .github/CODEOWNERS"
107+
sed -i "/^\/packages\/${currentPackage}\//d" .github/CODEOWNERS
108+
sed -i "/^\/packages\/${currentPackage} /d" .github/CODEOWNERS
103109
fi
104110
done
105111
}
@@ -116,47 +122,90 @@ updateBackportBranchContents() {
116122
local BUILDKITE_FOLDER_PATH=".buildkite"
117123
local JENKINS_FOLDER_PATH=".ci"
118124
local files_cached_num=""
125+
126+
git checkout "$BACKPORT_BRANCH_NAME"
127+
echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
128+
git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
129+
git add $BUILDKITE_FOLDER_PATH
130+
119131
if git ls-tree -d --name-only main:.ci >/dev/null 2>&1; then
120-
git checkout $BACKPORT_BRANCH_NAME
121-
echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
122-
git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
123132
echo "Copying $JENKINS_FOLDER_PATH from $SOURCE_BRANCH..."
124133
git checkout $SOURCE_BRANCH -- $JENKINS_FOLDER_PATH
134+
git add $JENKINS_FOLDER_PATH
125135
else
126-
git checkout $BACKPORT_BRANCH_NAME
127-
echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
128-
git checkout $SOURCE_BRANCH -- $BUILDKITE_FOLDER_PATH
129-
echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
130-
rm -rf "$JENKINS_FOLDER_PATH"
136+
if [ -d "${JENKINS_FOLDER_PATH}" ]; then
137+
echo "Removing $JENKINS_FOLDER_PATH from $BACKPORT_BRANCH_NAME..."
138+
rm -rf "$JENKINS_FOLDER_PATH"
139+
git add "$JENKINS_FOLDER_PATH"
140+
fi
141+
fi
142+
143+
# Update scripts used by mage
144+
local MAGEFILE_SCRIPTS_FOLDER="dev/citools"
145+
local TESTSREPORTER_SCRIPTS_FOLDER="dev/testsreporter"
146+
local COVERAGE_SCRIPTS_FOLDER="dev/coverage"
147+
local CODEOWNERS_SCRIPTS_FOLDER="dev/codeowners"
148+
149+
if git ls-tree -d --name-only main:${MAGEFILE_SCRIPTS_FOLDER} > /dev/null 2>&1 ; then
150+
echo "Copying $MAGEFILE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
151+
git checkout "$SOURCE_BRANCH" -- "${MAGEFILE_SCRIPTS_FOLDER}"
152+
git add ${MAGEFILE_SCRIPTS_FOLDER}
153+
154+
echo "Copying $TESTSREPORTER_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
155+
git checkout "$SOURCE_BRANCH" -- "${TESTSREPORTER_SCRIPTS_FOLDER}"
156+
git add ${TESTSREPORTER_SCRIPTS_FOLDER}
157+
158+
echo "Copying $COVERAGE_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
159+
git checkout "$SOURCE_BRANCH" -- "${COVERAGE_SCRIPTS_FOLDER}"
160+
git add ${COVERAGE_SCRIPTS_FOLDER}
161+
162+
echo "Copying $CODEOWNERS_SCRIPTS_FOLDER from $SOURCE_BRANCH..."
163+
git checkout "$SOURCE_BRANCH" -- "${CODEOWNERS_SCRIPTS_FOLDER}"
164+
git add ${CODEOWNERS_SCRIPTS_FOLDER}
165+
166+
echo "Copying magefile.go from $SOURCE_BRANCH..."
167+
git checkout "$SOURCE_BRANCH" -- "magefile.go"
168+
git add magefile.go
169+
170+
# As this script runs in the context of the main branch (mainly go mod tidy), we need to copy
171+
# the .go-version file from the main branch to the backport branch. This avoids failures
172+
# installing dependencies in the backport Pull Request.
173+
echo "Copying .go-version from $SOURCE_BRANCH..."
174+
git checkout "$SOURCE_BRANCH" -- ".go-version"
175+
git add .go-version
176+
177+
# Run go mod tidy to update just the dependencies related to magefile and dev scripts
178+
go mod tidy
179+
180+
git add go.mod go.sum
131181
fi
132182

133183
if [ "${REMOVE_OTHER_PACKAGES}" == "true" ]; then
134184
echo "Removing all packages from $PACKAGES_FOLDER_PATH folder"
135-
removeOtherPackages "$PACKAGES_FOLDER_PATH"
136-
ls -la $PACKAGES_FOLDER_PATH
185+
removeOtherPackages "${PACKAGES_FOLDER_PATH}"
186+
ls -la "${PACKAGES_FOLDER_PATH}"
187+
188+
git add "${PACKAGES_FOLDER_PATH}/"
189+
git add .github/CODEOWNERS
137190
fi
138191

192+
git status
193+
139194
echo "Setting up git environment..."
140195
update_git_config
141196

142-
echo "Commiting"
143-
git add $BUILDKITE_FOLDER_PATH
144-
if [ -d "${JENKINS_FOLDER_PATH}" ]; then
145-
git add $JENKINS_FOLDER_PATH
146-
fi
147-
git add $PACKAGES_FOLDER_PATH/
148-
git status
149-
150197
files_cached_num=$(git diff --name-only --cached | wc -l)
151198
if [ "${files_cached_num}" -gt 0 ]; then
199+
echo "Committing changes..."
152200
git commit -m "Add $BUILDKITE_FOLDER_PATH and $JENKINS_FOLDER_PATH to backport branch: $BACKPORT_BRANCH_NAME from the $SOURCE_BRANCH branch"
153201
else
154202
echo "Nothing to commit, skip."
155203
fi
156204

157205
if [ "$DRY_RUN" == "true" ];then
158206
echo "DRY_RUN mode, nothing will be pushed."
159-
git diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME
207+
# Show just the relevant files diff (go.mod, go.sum, .buildkite, dev, .go-version, .github/CODEOWNERS and package to be backported)
208+
git --no-pager diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME .buildkite/ dev/ go.sum go.mod .go-version .github/CODEOWNERS "packages/${PACKAGE_NAME}"
160209
else
161210
echo "Pushing..."
162211
git push origin $BACKPORT_BRANCH_NAME

0 commit comments

Comments
 (0)