Skip to content

Commit d5e2c51

Browse files
paul-tavarespaul-tavares
authored
Merge branch 'main' into task/olm-12313-update-3rd-party-edr-action-complete-task-to-support-spaces
2 parents 33c224f + cf2091d commit d5e2c51

2,057 files changed

Lines changed: 42175 additions & 20947 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/package-lock.json‎

Lines changed: 2598 additions & 22 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.buildkite/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
},
1212
"dependencies": {
1313
"@octokit/rest": "^18.10.0",
14+
"adm-zip": "^0.5.16",
1415
"axios": "^1.8.3",
1516
"globby": "^11.1.0",
1617
"js-yaml": "^4.1.0",
@@ -21,11 +22,14 @@
2122
"devDependencies": {
2223
"@types/chai": "^4.3.3",
2324
"@types/js-yaml": "^4.0.9",
25+
"@types/jscodeshift": "^0.12.0",
2426
"@types/minimatch": "^3.0.5",
2527
"@types/minimist": "^1.2.5",
2628
"@types/mocha": "^10.0.1",
2729
"@types/node": "^15.12.2",
2830
"chai": "^4.3.10",
31+
"chai-snapshot-tests": "^0.6.0",
32+
"jscodeshift": "^17.1.2",
2933
"mocha": "^11.0.1",
3034
"nock": "^12.0.2",
3135
"ts-node": "^10.9.2",

‎.buildkite/pipeline-resource-definitions/kibana-chromium-linux-build.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
2+
apiVersion: backstage.io/v1alpha1
3+
kind: Resource
4+
metadata:
5+
name: bk-kibana-chromium-linux-build
6+
description: Create chromium linux builds for the Kibana Reporting feature
7+
links:
8+
- title: Pipeline link
9+
url: https://buildkite.com/elastic/kibana-chromium-linux-build
10+
spec:
11+
type: buildkite-pipeline
12+
owner: group:response-ops
13+
system: buildkite
14+
implementation:
15+
apiVersion: buildkite.elastic.dev/v1
16+
kind: Pipeline
17+
metadata:
18+
name: kibana / Chromium Linux Build
19+
description: Creates headless Chromium Linux builds
20+
spec:
21+
env:
22+
SLACK_NOTIFICATIONS_CHANNEL: "#kibana-alerting"
23+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
24+
repository: elastic/kibana
25+
branch_configuration: main
26+
default_branch: main
27+
pipeline_file: ".buildkite/pipelines/chromium_linux_build/build_chromium.yml"
28+
provider_settings:
29+
trigger_mode: none
30+
teams:
31+
kibana-operations:
32+
access_level: MANAGE_BUILD_AND_READ
33+
appex-sharedux:
34+
access_level: MANAGE_BUILD_AND_READ
35+
response-ops:
36+
access_level: MANAGE_BUILD_AND_READ
37+
everyone:
38+
access_level: READ_ONLY
39+
tags:
40+
- kibana

‎.buildkite/pipeline-resource-definitions/locations.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ spec:
1414
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
1515
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
1616
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-chrome-forward-testing.yml
17+
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-chromium-linux-build.yml
1718
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
1819
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml
1920
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml

‎.buildkite/pipelines/chromium_linux_build/build_chromium.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
env:
2+
GITHUB_PR_TARGET_BRANCH: 'main' # common utils needs this value to be set
3+
4+
steps:
5+
- command: |
6+
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
7+
--message "Linux headless chromium build started at: $BUILDKITE_BUILD_URL" \
8+
--context "chromium-linux-build-job-start" \
9+
--issue-number $GITHUB_ISSUE_NUMBER \
10+
--repository $GITHUB_ISSUE_BASE_REPO \
11+
--repository-owner $GITHUB_ISSUE_BASE_OWNER \
12+
--clear-previous
13+
label: Comment with job URL on issue
14+
agents:
15+
provider: gcp
16+
image: family/kibana-ubuntu-2004
17+
imageProject: elastic-images-prod
18+
machineType: n2-standard-2
19+
timeout_in_minutes: 5
20+
21+
- command: |
22+
.buildkite/scripts/pipelines/chromium_linux_build/chromium_version.sh
23+
label: Infer chromium version for puppeteer version
24+
key: infer_chromium_version
25+
agents:
26+
provider: gcp
27+
image: family/kibana-ubuntu-2004
28+
imageProject: elastic-images-prod
29+
machineType: n2-standard-2
30+
timeout_in_minutes: 5
31+
32+
- command: "PLATFORM_VARIANT={{matrix}} .buildkite/scripts/pipelines/chromium_linux_build/build.sh"
33+
label: "Build {{matrix}} linux headless chromium"
34+
depends_on: "infer_chromium_version"
35+
key: build_chromium
36+
env:
37+
DEBIAN_FRONTEND: noninteractive
38+
agents:
39+
image: family/kibana-ubuntu-2004
40+
imageProject: elastic-images-prod
41+
provider: gcp
42+
machineType: c2d-highcpu-112
43+
diskSizeGb: 275
44+
matrix:
45+
- "x64"
46+
- "arm64"
47+
48+
- command: node .buildkite/scripts/pipelines/chromium_linux_build/issue_feedback/entry.js
49+
label: provided feedback to issue created to update puppeteer
50+
depends_on: "build_chromium"
51+
env:
52+
KIBANA_MACHINE_USERNAME: kibanamachine
53+
KIBANA_MACHINE_EMAIL: 42973632+kibanamachine@users.noreply.github.com
54+
agents:
55+
provider: gcp
56+
image: family/kibana-ubuntu-2004
57+
imageProject: elastic-images-prod
58+
machineType: n2-standard-2
59+
timeout_in_minutes: 30

‎.buildkite/scripts/common/activate_service_account.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ if [[ -z "$EMAIL" ]]; then
7676
"ci-artifacts.kibana.dev")
7777
EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
7878
;;
79+
"kibana-ci-access-chromium-blds")
80+
EMAIL="kibana-ci-access-chromium-blds@$GCLOUD_EMAIL_POSTFIX"
81+
;;
7982
*)
8083
EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
8184
;;

‎.buildkite/scripts/lifecycle/post_command.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
5454
ts-node .buildkite/scripts/lifecycle/annotate_test_failures.ts
5555
fi
5656

57+
if [[ -d 'target/agent_diagnostics' ]]; then
58+
buildkite-agent artifact upload 'target/agent_diagnostics/**/*'
59+
fi
60+
5761
fi
5862

5963
if [[ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]]; then

‎.buildkite/scripts/pipelines/chromium_linux_build/build.sh‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
CHROMIUM_COMMIT_HASH=$(buildkite-agent meta-data get "chromium_commit_hash")
6+
7+
echo "---Preparing to build Chromium of commit hash: $CHROMIUM_COMMIT_HASH"
8+
9+
BUILD_ROOT_DIR="$HOME/chromium"
10+
11+
KIBANA_CHECKOUT_DIR="$(pwd)"
12+
13+
BUILD_SCRIPT="$KIBANA_CHECKOUT_DIR/x-pack/build_chromium"
14+
15+
# Create a dedicated working directory outside of the default buildkite working directory.
16+
mkdir "$BUILD_ROOT_DIR" && cd "$BUILD_ROOT_DIR"
17+
18+
ARTIFACT_STAGING_STORAGE_BUCKET="gs://headless_shell_staging"
19+
ARTIFACT_PROD_STORAGE_BUCKET="gs://headless_shell"
20+
21+
ARTIFACT_QUERY="chromium-${CHROMIUM_COMMIT_HASH:0:7}-.*_$PLATFORM_VARIANT"
22+
23+
## impersonate service account that has access to our storage bucket
24+
"$KIBANA_CHECKOUT_DIR/.buildkite/scripts/common/activate_service_account.sh" "kibana-ci-access-chromium-blds"
25+
26+
# Query to determine if expected build artifact from a prior build exists,
27+
# the build.py script uploads the build artifact to the staging bucket
28+
artifacts=$(gsutil ls "$ARTIFACT_STAGING_STORAGE_BUCKET" | grep "$ARTIFACT_QUERY" || true)
29+
30+
if [[ -z "$artifacts" ]]; then
31+
echo "No files found matching the query: $ARTIFACT_QUERY"
32+
33+
echo "---Chromium build does not exist in the bucket, proceeding with the build"
34+
35+
# Install the required packages for building chromium
36+
sudo apt-get update && \
37+
sudo apt-get install -y curl git build-essential pkg-config gcc gperf python3 && \
38+
sudo rm -rf /var/lib/apt/lists/*
39+
40+
BUILD_SCRIPT_SYMLINK="$BUILD_ROOT_DIR/build_scripts"
41+
42+
# link existing build_chromium directory from kibana
43+
ln -s "$BUILD_SCRIPT" "$BUILD_SCRIPT_SYMLINK"
44+
45+
# Allow our scripts to use depot_tools commands
46+
export PATH=$BUILD_ROOT_DIR/depot_tools:$PATH
47+
48+
# Install the OS packages, configure the environment, download the chromium source (56GB)
49+
python3 "$BUILD_SCRIPT_SYMLINK/init.py"
50+
51+
echo "---Building $PLATFORM_VARIANT Chromium of commit hash: $CHROMIUM_COMMIT_HASH"
52+
53+
# Run the build script with the path to the chromium src directory, the git commit hash
54+
python3 "$BUILD_SCRIPT_SYMLINK/build.py" "$CHROMIUM_COMMIT_HASH" "$PLATFORM_VARIANT"
55+
56+
echo "---Upload build artefact to prod storage bucket"
57+
58+
gsutil cp "$BUILD_ROOT_DIR/chromium/src/out/headless/chromium-*" "$ARTIFACT_PROD_STORAGE_BUCKET"
59+
60+
echo "---Persisting build artefact to buildkite for following steps"
61+
62+
buildkite-agent artifact upload "$BUILD_ROOT_DIR/chromium/src/out/headless/chromium-*"
63+
64+
else
65+
echo "$artifacts" | while read -r file; do
66+
gsutil cp "$file" .
67+
done
68+
69+
shopt -s nullglob
70+
files=("chromium-*")
71+
shopt -u nullglob
72+
73+
if [ ${#files[@]} -gt 0 ]; then
74+
echo "---Chromium build already exists in the bucket, skipping build"
75+
# Upload the existing build artifact to buildkite so it can be used in the next steps,
76+
# and accessible without necessarily having access to the storage bucket itself
77+
buildkite-agent artifact upload "chromium-*"
78+
fi
79+
fi
80+
81+
echo "---Build completed"

‎.buildkite/scripts/pipelines/chromium_linux_build/chromium_version.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common/util.sh
6+
.buildkite/scripts/bootstrap.sh
7+
8+
echo "---Attempting to compute chromium version for provided puppeteer version"
9+
10+
CHROMIUM_VERSION_OUTPUT=$(node scripts/chromium_version $PUPPETEER_VERSION)
11+
12+
echo "$CHROMIUM_VERSION_OUTPUT" | grep -i "chromium commit" | awk '{print $5}' | buildkite-agent meta-data set "chromium_commit_hash"
13+
echo "$CHROMIUM_VERSION_OUTPUT" | grep -i "chrome version" | awk '{print $5}' | buildkite-agent meta-data set "chromium_version"
14+
echo "$CHROMIUM_VERSION_OUTPUT" | grep -i "chromium revision" | awk '{print $5}' | buildkite-agent meta-data set "chromium_revision"

‎.buildkite/scripts/pipelines/chromium_linux_build/issue_feedback/__snapshots__/transform_path_file.test.ts.json‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)