Skip to content

Commit db9d5a6

Browse files
alisonelizabethalisonelizabeth
committed
Merge branch '7.x' of https://github.com/elastic/kibana into 7/ua/token-authentication
2 parents 6d6b050 + f05e18d commit db9d5a6

1,895 files changed

Lines changed: 64435 additions & 49144 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/pipelines/es_snapshots/build.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
steps:
2+
- command: .buildkite/scripts/steps/es_snapshots/build.sh
3+
label: Build ES Snapshot
4+
agents:
5+
queue: c2-8

‎.buildkite/pipelines/es_snapshots/promote.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
steps:
2+
- block: 'Promote'
3+
prompt: "Enter the details for the snapshot you'd like to promote"
4+
if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
5+
# Later, this could be a dropdown dynamically filled with recent builds
6+
fields:
7+
- text: 'ES_SNAPSHOT_MANIFEST'
8+
key: 'ES_SNAPSHOT_MANIFEST'
9+
hint: 'URL pointing to the manifest to promote'
10+
required: true
11+
- label: Promote Snapshot
12+
command: .buildkite/scripts/steps/es_snapshots/promote.sh

‎.buildkite/pipelines/es_snapshots/verify.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
env:
2+
IGNORE_SHIP_CI_STATS_ERROR: 'true'
3+
steps:
4+
- block: 'Verify'
5+
prompt: "Enter the details for the snapshot you'd like to verify"
6+
if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
7+
# Later, this could be a dropdown dynamically filled with recent builds
8+
fields:
9+
- text: 'ES_SNAPSHOT_MANIFEST'
10+
key: 'ES_SNAPSHOT_MANIFEST'
11+
hint: 'URL pointing to the manifest to promote'
12+
required: true
13+
14+
- command: .buildkite/scripts/lifecycle/pre_build.sh
15+
label: Pre-Build
16+
17+
- wait
18+
19+
- command: .buildkite/scripts/steps/build_kibana.sh
20+
label: Build Kibana Distribution and Plugins
21+
agents:
22+
queue: c2-8
23+
key: build
24+
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
25+
26+
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
27+
label: 'Default CI Group'
28+
parallelism: 13
29+
agents:
30+
queue: ci-group-6
31+
artifact_paths: target/junit/**/*.xml
32+
depends_on: build
33+
key: default-cigroup
34+
retry:
35+
automatic:
36+
- exit_status: '*'
37+
limit: 1
38+
39+
- command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
40+
label: 'Docker CI Group'
41+
agents:
42+
queue: ci-group-6
43+
artifact_paths: target/junit/**/*.xml
44+
depends_on: build
45+
key: default-cigroup-docker
46+
retry:
47+
automatic:
48+
- exit_status: '*'
49+
limit: 1
50+
51+
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
52+
label: 'OSS CI Group'
53+
parallelism: 12
54+
agents:
55+
queue: ci-group-4d
56+
artifact_paths: target/junit/**/*.xml
57+
depends_on: build
58+
key: oss-cigroup
59+
retry:
60+
automatic:
61+
- exit_status: '*'
62+
limit: 1
63+
64+
- command: .buildkite/scripts/steps/test/jest_integration.sh
65+
label: 'Jest Integration Tests'
66+
agents:
67+
queue: jest
68+
artifact_paths: target/junit/**/*.xml
69+
key: jest-integration
70+
retry:
71+
automatic:
72+
- exit_status: '*'
73+
limit: 1
74+
75+
- command: .buildkite/scripts/steps/test/api_integration.sh
76+
label: 'API Integration Tests'
77+
agents:
78+
queue: jest
79+
artifact_paths: target/junit/**/*.xml
80+
key: api-integration
81+
82+
- command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
83+
label: Trigger promotion
84+
depends_on:
85+
- default-cigroup
86+
- default-cigroup-docker
87+
- oss-cigroup
88+
- jest-integration
89+
- api-integration
90+
91+
- wait: ~
92+
continue_on_failure: true
93+
94+
- plugins:
95+
- junit-annotate#v1.9.0:
96+
artifacts: target/junit/**/*.xml
97+
98+
- wait: ~
99+
continue_on_failure: true
100+
101+
- command: .buildkite/scripts/lifecycle/post_build.sh
102+
label: Post-Build

‎.buildkite/scripts/bootstrap.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -euo pipefail
55
source .buildkite/scripts/common/util.sh
66

77
echo "--- yarn install and bootstrap"
8-
yarn kbn bootstrap --verbose
8+
yarn kbn bootstrap
99

1010
###
1111
### upload ts-refs-cache artifacts as quickly as possible so they are available for download

‎.buildkite/scripts/build_kibana_plugins.sh‎

100644100755
Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ node scripts/build_kibana_platform_plugins \
1818
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
1919
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
2020
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
21-
--scan-dir "$XPACK_DIR/examples" \
22-
--verbose
21+
--scan-dir "$XPACK_DIR/examples"
2322

2423
echo "--- Archive built plugins"
2524
shopt -s globstar

‎.buildkite/scripts/download_build_artifacts.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
77

88
cd "$WORKSPACE"
99

10-
buildkite-agent artifact download kibana-default.tar.gz .
11-
buildkite-agent artifact download kibana-default-plugins.tar.gz .
10+
buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
11+
buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
1212

1313
mkdir -p "$KIBANA_BUILD_LOCATION"
1414
tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

‎.buildkite/scripts/lifecycle/pre_command.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,8 @@ if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
7272
source .buildkite/scripts/common/setup_bazel.sh
7373
fi
7474
fi
75+
76+
PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
77+
if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
78+
source "$PIPELINE_PRE_COMMAND"
79+
fi

‎.buildkite/scripts/steps/build_kibana.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
.buildkite/scripts/bootstrap.sh
6+
.buildkite/scripts/build_kibana.sh
7+
.buildkite/scripts/build_kibana_plugins.sh
8+
.buildkite/scripts/post_build_kibana.sh

‎.buildkite/scripts/steps/es_snapshots/bucket_config.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
module.exports = {
2+
BASE_BUCKET_DAILY: 'kibana-ci-es-snapshots-daily',
3+
BASE_BUCKET_PERMANENT: 'kibana-ci-es-snapshots-permanent',
4+
};

‎.buildkite/scripts/steps/es_snapshots/build.sh‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common/util.sh
6+
7+
echo "--- Cloning Elasticsearch and preparing workspace"
8+
9+
cd ..
10+
destination="$(pwd)/es-build"
11+
rm -rf "$destination"
12+
mkdir -p "$destination"
13+
14+
mkdir -p elasticsearch && cd elasticsearch
15+
16+
export ELASTICSEARCH_BRANCH="${ELASTICSEARCH_BRANCH:-$BUILDKITE_BRANCH}"
17+
18+
if [[ ! -d .git ]]; then
19+
git init
20+
git remote add origin https://github.com/elastic/elasticsearch.git
21+
fi
22+
git fetch origin --depth 1 "$ELASTICSEARCH_BRANCH"
23+
git reset --hard FETCH_HEAD
24+
25+
ELASTICSEARCH_GIT_COMMIT="$(git rev-parse HEAD)"
26+
export ELASTICSEARCH_GIT_COMMIT
27+
28+
ELASTICSEARCH_GIT_COMMIT_SHORT="$(git rev-parse --short HEAD)"
29+
export ELASTICSEARCH_GIT_COMMIT_SHORT
30+
31+
# These turn off automation in the Elasticsearch repo
32+
export BUILD_NUMBER=""
33+
export JENKINS_URL=""
34+
export BUILD_URL=""
35+
export JOB_NAME=""
36+
export NODE_NAME=""
37+
export DOCKER_BUILDKIT=""
38+
39+
# Reads the ES_BUILD_JAVA env var out of .ci/java-versions.properties and exports it
40+
export "$(grep '^ES_BUILD_JAVA' .ci/java-versions.properties | xargs)"
41+
42+
export PATH="$HOME/.java/$ES_BUILD_JAVA/bin:$PATH"
43+
export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
44+
45+
# The Elasticsearch Dockerfile needs to be built with root privileges, but Docker on our servers is running using a non-root user
46+
# So, let's use docker-in-docker to temporarily create a privileged docker daemon to run `docker build` on
47+
# We have to do this, because there's no `docker build --privileged` or similar
48+
49+
echo "--- Setting up Docker-in-Docker for Elasticsearch"
50+
51+
docker rm -f dind || true # If there's an old daemon running that somehow didn't get cleaned up, lets remove it first
52+
CERTS_DIR="$HOME/dind-certs"
53+
rm -rf "$CERTS_DIR"
54+
docker run -d --rm --privileged --name dind --userns host -p 2377:2376 -e DOCKER_TLS_CERTDIR=/certs -v "$CERTS_DIR":/certs docker:dind
55+
56+
trap "docker rm -f dind" EXIT
57+
58+
export DOCKER_TLS_VERIFY=true
59+
export DOCKER_CERT_PATH="$CERTS_DIR/client"
60+
export DOCKER_TLS_CERTDIR="$CERTS_DIR"
61+
export DOCKER_HOST=localhost:2377
62+
63+
echo "--- Build Elasticsearch"
64+
./gradlew -Dbuild.docker=true assemble --parallel
65+
66+
echo "--- Create distribution archives"
67+
find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \;
68+
69+
ls -alh "$destination"
70+
71+
echo "--- Create docker image archives"
72+
docker images "docker.elastic.co/elasticsearch/elasticsearch"
73+
docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
74+
docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
75+
76+
echo "--- Create checksums for snapshot files"
77+
cd "$destination"
78+
find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;
79+
80+
cd "$BUILDKITE_BUILD_CHECKOUT_PATH"
81+
node "$(dirname "${0}")/create_manifest.js" "$destination"
82+
83+
ES_SNAPSHOT_MANIFEST="$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)"
84+
85+
cat << EOF | buildkite-agent annotate --style "info"
86+
- \`ELASTICSEARCH_BRANCH\` - \`$ELASTICSEARCH_BRANCH\`
87+
- \`ELASTICSEARCH_GIT_COMMIT\` - \`$ELASTICSEARCH_GIT_COMMIT\`
88+
- \`ES_SNAPSHOT_MANIFEST\` - \`$ES_SNAPSHOT_MANIFEST\`
89+
- \`ES_SNAPSHOT_VERSION\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_VERSION)\`
90+
- \`ES_SNAPSHOT_ID\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_ID)\`
91+
EOF
92+
93+
cat << EOF | buildkite-agent pipeline upload
94+
steps:
95+
- trigger: 'kibana-elasticsearch-snapshot-verify'
96+
async: true
97+
build:
98+
env:
99+
ES_SNAPSHOT_MANIFEST: '$ES_SNAPSHOT_MANIFEST'
100+
branch: '$BUILDKITE_BRANCH'
101+
EOF

0 commit comments

Comments
 (0)