Skip to content

Commit 863b6cc

Browse files
kibanamachinekibanamachine
authored
Merge branch 'master' into add-php-agent-name
2 parents a2ffac4 + 6927d6c commit 863b6cc

1,215 files changed

Lines changed: 31491 additions & 21859 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Kibana / Buildkite
2+
3+
## Directory Structure
4+
5+
- `hooks` - special directory used by Buildkite agents for [hooks](https://buildkite.com/docs/agent/v3/hooks)
6+
- `pipelines` - contains pipeline definitions
7+
- `scripts/common` - scripts that get `source`d by other scripts to set environment variables or import shared functions
8+
- `scripts/lifecycle` - general scripts for tasks that run before or after individual steps or the entire build
9+
- `scripts/steps` - scripts that define something that will run for a step defined in a pipeline
10+
- `scripts/*` - all other scripts are building blocks that make up the tasks in pipelines. They may be run by other scripts, but should not be `source`d

‎.buildkite/agents.json‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
{
2+
"gcp": {
3+
"project": "elastic-kibana-ci",
4+
"zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"],
5+
"serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com",
6+
"imageFamily": "kb-ubuntu",
7+
"subnetwork": "buildkite",
8+
"disableExternalIp": true,
9+
"diskType": "pd-ssd",
10+
"diskSizeGb": 75,
11+
"overprovision": 0,
12+
"minimumAgents": 0,
13+
"maximumAgents": 50,
14+
"gracefulStopAfterMins": 360,
15+
"hardStopAfterMins": 540,
16+
"idleTimeoutMins": 10,
17+
"exitAfterOneJob": false,
18+
19+
"agents": [
20+
{
21+
"queue": "default",
22+
"name": "kb-default",
23+
"minimumAgents": 1,
24+
"maximumAgents": 100,
25+
"idleTimeoutMins": 60,
26+
"machineType": "e2-small"
27+
},
28+
{
29+
"queue": "c2-8",
30+
"name": "kb-c2-8",
31+
"machineType": "c2-standard-8",
32+
"localSsds": 1
33+
},
34+
{
35+
"queue": "c2-4",
36+
"name": "kb-c2-4",
37+
"machineType": "c2-standard-4",
38+
"localSsds": 1
39+
},
40+
{
41+
"queue": "jest",
42+
"name": "kb-jest",
43+
"machineType": "n2-standard-2",
44+
"diskSizeGb": 128
45+
},
46+
{
47+
"queue": "ci-group",
48+
"name": "kb-cigroup",
49+
"machineType": "n2-standard-8",
50+
"diskSizeGb": 256
51+
},
52+
{
53+
"queue": "ci-group-4",
54+
"name": "kb-cigroup-4",
55+
"machineType": "n2-standard-4",
56+
"diskSizeGb": 128
57+
},
58+
{
59+
"queue": "ci-group-4d",
60+
"name": "kb-cigroup-4d",
61+
"machineType": "n2d-standard-4",
62+
"diskSizeGb": 128
63+
},
64+
{
65+
"queue": "ci-group-6",
66+
"name": "kb-cigroup-6",
67+
"machineType": "n2-custom-6-16384",
68+
"diskSizeGb": 128
69+
},
70+
{
71+
"queue": "packer",
72+
"name": "kb-packer",
73+
"serviceAccount": "buildkite-packer-agent@elastic-kibana-ci.iam.gserviceaccount.com",
74+
"maximumAgents": 10,
75+
"machineType": "e2-small"
76+
}
77+
]
78+
}
79+
}

‎.buildkite/hooks/post-command‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
3+
source .buildkite/scripts/lifecycle/post_command.sh

‎.buildkite/hooks/pre-command‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
3+
source .buildkite/scripts/lifecycle/pre_command.sh

‎.buildkite/pipelines/on_merge.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
env:
2+
GITHUB_COMMIT_STATUS_ENABLED: 'true'
3+
GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge'
4+
steps:
5+
- command: .buildkite/scripts/lifecycle/pre_build.sh
6+
label: Pre-Build
7+
8+
- wait
9+
10+
- command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
11+
label: Default Build and Metrics
12+
agents:
13+
queue: c2-8
14+
15+
- command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
16+
label: Build TS Refs and Check Public API Docs
17+
agents:
18+
queue: c2-4
19+
20+
- wait: ~
21+
continue_on_failure: true
22+
23+
- command: .buildkite/scripts/lifecycle/post_build.sh
24+
label: Post-Build

‎.buildkite/scripts/bootstrap.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common/util.sh
6+
7+
echo "--- yarn install and bootstrap"
8+
yarn kbn bootstrap --verbose
9+
10+
###
11+
### upload ts-refs-cache artifacts as quickly as possible so they are available for download
12+
###
13+
if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
14+
echo "--- Upload ts-refs-cache"
15+
cd "$KIBANA_DIR/target/ts_refs_cache"
16+
gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
17+
cd "$KIBANA_DIR"
18+
fi
19+
20+
if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
21+
verify_no_git_changes 'yarn kbn bootstrap'
22+
fi

‎.buildkite/scripts/build_kibana.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
export KBN_NP_PLUGINS_BUILT=true
6+
7+
echo "--- Build Kibana Distribution"
8+
node scripts/build --debug --no-oss
9+
10+
echo "--- Archive Kibana Distribution"
11+
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
12+
installDir="$KIBANA_DIR/install/kibana"
13+
mkdir -p "$installDir"
14+
tar -xzf "$linuxBuild" -C "$installDir" --strip=1
15+
mkdir -p "$KIBANA_BUILD_LOCATION"
16+
cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/"

‎.buildkite/scripts/build_kibana_plugins.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
echo "--- Build Platform Plugins"
6+
node scripts/build_kibana_platform_plugins \
7+
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
8+
--scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
9+
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
10+
--scan-dir "$KIBANA_DIR/examples" \
11+
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
12+
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
13+
--scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
14+
--scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
15+
--scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
16+
--scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
17+
--scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
18+
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
19+
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
20+
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
21+
--scan-dir "$XPACK_DIR/examples" \
22+
--verbose
23+
24+
echo "--- Archive built plugins"
25+
shopt -s globstar
26+
tar -zcf \
27+
target/kibana-default-plugins.tar.gz \
28+
x-pack/plugins/**/target/public \
29+
x-pack/test/**/target/public \
30+
examples/**/target/public \
31+
x-pack/examples/**/target/public \
32+
test/**/target/public

‎.buildkite/scripts/common/env.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/usr/bin/env bash
2+
3+
export CI=true
4+
5+
KIBANA_DIR=$(pwd)
6+
export KIBANA_DIR
7+
export XPACK_DIR="$KIBANA_DIR/x-pack"
8+
9+
export CACHE_DIR="$HOME/.kibana"
10+
PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
11+
export PARENT_DIR
12+
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
13+
14+
KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
15+
export KIBANA_PKG_BRANCH
16+
export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
17+
18+
export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
19+
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
20+
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
21+
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
22+
23+
export NODE_OPTIONS="--max-old-space-size=4096"
24+
25+
export FORCE_COLOR=1
26+
export TEST_BROWSER_HEADLESS=1
27+
28+
export ELASTIC_APM_ENVIRONMENT=ci
29+
export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
30+
31+
CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM
32+
33+
if is_pr; then
34+
export ELASTIC_APM_ACTIVE=false
35+
export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}"
36+
37+
# These can be removed once we're not supporting Jenkins and Buildkite at the same time
38+
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json
39+
export ghprbGhRepository="elastic/kibana"
40+
export ghprbActualCommit="$BUILDKITE_COMMIT"
41+
export BUILD_URL="$BUILDKITE_BUILD_URL"
42+
43+
# set_git_merge_base # TODO for PRs
44+
else
45+
export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}"
46+
export CHECKS_REPORTER_ACTIVE=false
47+
fi
48+
49+
export FLEET_PACKAGE_REGISTRY_PORT=6104
50+
export TEST_CORS_SERVER_PORT=6105
51+
52+
export DETECT_CHROMEDRIVER_VERSION=true
53+
export CHROMEDRIVER_FORCE_DOWNLOAD=true
54+
55+
export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this
56+
57+
export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack"
58+
59+
if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then
60+
export BUILD_TS_REFS_CACHE_ENABLE=false
61+
fi
62+
63+
export BUILD_TS_REFS_DISABLE=true
64+
export DISABLE_BOOTSTRAP_VALIDATION=true
65+
66+
export TEST_KIBANA_HOST=localhost
67+
export TEST_KIBANA_PORT=6101
68+
export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
69+
export TEST_ES_URL="http://elastic:changeme@localhost:6102"
70+
export TEST_ES_TRANSPORT_PORT=6103
71+
export TEST_CORS_SERVER_PORT=6106
72+
export ALERTING_PROXY_PORT=6105

‎.buildkite/scripts/common/setup_bazel.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
4+
export KIBANA_BUILDBUDDY_CI_API_KEY
5+
6+
cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"
7+
8+
###
9+
### append auth token to buildbuddy into "$HOME/.bazelrc";
10+
###
11+
echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc"
12+
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"
13+
14+
###
15+
### remove write permissions on buildbuddy remote cache for prs
16+
###
17+
if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then
18+
{
19+
echo "# Uploads logs & artifacts without writing to cache"
20+
echo "build --noremote_upload_local_results"
21+
} >> "$HOME/.bazelrc"
22+
fi

0 commit comments

Comments
 (0)