Skip to content

Commit b45cdce

Browse files
dplumleedplumlee
committed
Merge remote-tracking branch 'upstream/main' into rules-rbac-new
2 parents 5f6fbd5 + 23aa180 commit b45cdce

1,557 files changed

Lines changed: 26273 additions & 12529 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/ftr_platform_stateful_configs.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ enabled:
310310
- x-pack/platform/test/security_api_integration/oidc_implicit_flow.config.ts
311311
- x-pack/platform/test/security_api_integration/oidc.config.ts
312312
- x-pack/platform/test/security_api_integration/oidc.http2.config.ts
313+
- x-pack/platform/test/security_api_integration/oidc_multiple_realms.config.ts
313314
- x-pack/platform/test/security_api_integration/pki.config.ts
314315
- x-pack/platform/test/security_api_integration/saml.config.ts
315316
- x-pack/platform/test/security_api_integration/saml.http2.config.ts

‎.buildkite/package-lock.json‎

Lines changed: 7 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.buildkite/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
"@types/jscodeshift": "^0.12.0",
2525
"@types/minimatch": "^3.0.5",
2626
"@types/minimist": "^1.2.5",
27-
"@types/node": "^22.17.1",
27+
"@types/node": "22.19.0",
2828
"jest": "^30.0.3",
2929
"jscodeshift": "^17.1.2",
3030
"nock": "^12.0.2",

‎.buildkite/pipeline-resource-definitions/kibana-entity-store-performance-from-pr.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
2+
apiVersion: backstage.io/v1alpha1
3+
kind: Resource
4+
metadata:
5+
name: bk-kibana-entity-store-performance-from-pr
6+
description: 'Runs entity store performance tests from a PR'
7+
links:
8+
- url: 'https://buildkite.com/elastic/kibana-entity-store-performance-from-pr'
9+
title: Pipeline link
10+
spec:
11+
type: buildkite-pipeline
12+
system: buildkite
13+
owner: 'group:contextual-security'
14+
implementation:
15+
apiVersion: buildkite.elastic.dev/v1
16+
kind: Pipeline
17+
metadata:
18+
name: kibana / entity store performance from PR
19+
description: 'Runs entity store performance tests from a PR. Requires ci:entity-store-performance label.'
20+
spec:
21+
env:
22+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
23+
24+
allow_rebuilds: false
25+
branch_configuration: main
26+
default_branch: main
27+
repository: elastic/kibana
28+
pipeline_file: .buildkite/pipelines/build_pr_and_test_entity_store_performance.yml
29+
provider_settings:
30+
build_pull_requests: true
31+
prefix_pull_request_fork_branch_names: false
32+
skip_pull_request_builds_for_existing_commits: true
33+
trigger_mode: none
34+
cancel_intermediate_builds: true
35+
teams:
36+
contextual-security:
37+
access_level: MANAGE_BUILD_AND_READ
38+
kibana-operations:
39+
access_level: MANAGE_BUILD_AND_READ
40+
appex-qa:
41+
access_level: MANAGE_BUILD_AND_READ
42+
kibana-tech-leads:
43+
access_level: MANAGE_BUILD_AND_READ
44+
everyone:
45+
access_level: BUILD_AND_READ
46+
tags:
47+
- kibana
48+

‎.buildkite/pipeline-resource-definitions/locations.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ spec:
2121
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
2222
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml
2323
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
24+
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-entity-store-performance-from-pr.yml
2425
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-1.yml
2526
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
2627
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml

‎.buildkite/pipelines/build_pr_and_test_entity_store_performance.yml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
env:
2+
ELASTIC_PR_COMMENTS_ENABLED: 'true'
3+
ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
4+
GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-entity-store-performance-from-pr
5+
6+
steps:
7+
- group: 'Notify if missing labels'
8+
if: "build.env('GITHUB_PR_LABELS') !~ /ci:entity-store-performance/"
9+
10+
steps:
11+
- command: |
12+
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
13+
--message "Entity store performance tests require the \`ci:entity-store-performance\` label. Please add the label and trigger the job through the checkbox again." \
14+
--context "entity-store-performance-job" \
15+
--clear-previous
16+
label: Comment missing labels
17+
agents:
18+
provider: gcp
19+
image: family/kibana-ubuntu-2404
20+
imageProject: elastic-images-prod
21+
machineType: n2-standard-2
22+
timeout_in_minutes: 5
23+
24+
- group: 'Entity Store Performance Tests'
25+
if: "build.env('GITHUB_PR_LABELS') =~ /ci:entity-store-performance/"
26+
27+
steps:
28+
- command: .buildkite/scripts/lifecycle/pre_build.sh
29+
label: Pre-Build
30+
timeout_in_minutes: 10
31+
agents:
32+
provider: gcp
33+
image: family/kibana-ubuntu-2404
34+
imageProject: elastic-images-prod
35+
machineType: n2-standard-2
36+
retry:
37+
automatic:
38+
- exit_status: '*'
39+
limit: 1
40+
41+
- command: |
42+
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
43+
--message "Entity store performance tests started at: $BUILDKITE_BUILD_URL" \
44+
--context "entity-store-performance-job" \
45+
--clear-previous
46+
label: Comment with job URL
47+
agents:
48+
provider: gcp
49+
image: family/kibana-ubuntu-2404
50+
imageProject: elastic-images-prod
51+
machineType: n2-standard-2
52+
timeout_in_minutes: 5
53+
54+
- wait: ~
55+
56+
- command: .buildkite/scripts/steps/build_kibana.sh
57+
label: Build Kibana Distribution
58+
agents:
59+
provider: gcp
60+
image: family/kibana-ubuntu-2404
61+
imageProject: elastic-images-prod
62+
machineType: n2-standard-8
63+
preemptible: true
64+
diskSizeGb: 125
65+
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
66+
timeout_in_minutes: 90
67+
key: build
68+
retry:
69+
automatic:
70+
- exit_status: '-1'
71+
limit: 3
72+
73+
- command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
74+
env:
75+
ES_ZONE_COUNT: 2
76+
ES_HOT_TIER_MEMORY_SIZE: 8192
77+
label: 'Build and Deploy to Cloud'
78+
agents:
79+
provider: gcp
80+
image: family/kibana-ubuntu-2404
81+
imageProject: elastic-images-prod
82+
machineType: n2-standard-2
83+
preemptible: true
84+
timeout_in_minutes: 30
85+
depends_on: build
86+
retry:
87+
automatic:
88+
- exit_status: '-1'
89+
limit: 3
90+
91+
- wait: ~
92+
93+
- command: .buildkite/scripts/steps/entity_store_performance/run_performance_tests_orchestrator.sh
94+
label: 'Run Entity Store Performance Tests'
95+
env:
96+
# SECURITY_DOCS_GEN_BRANCH: branch-name # Optional: checkout a specific branch from security-documents-generator repo (default: main branch)
97+
# SECURITY_DOCS_GEN_PR: 123 # Optional: checkout a specific PR from security-documents-generator repo (alternative to SECURITY_DOCS_GEN_BRANCH)
98+
PERF_INTERVAL: 100
99+
PERF_COUNT: 1
100+
PERF_SAMPLING_INTERVAL: 1
101+
PERF_DATA_FILE: standard
102+
# PERF_NO_TRANSFORMS: true # Optional: set to "true" to disable transforms (default: false)
103+
# Optional: Baseline comparison configuration
104+
PERF_BASELINE_FILE: baselines/baseline-v1_0 # Optional: specific baseline file to use
105+
PERF_DEGRADATION_THRESHOLD: 35 # Optional: degradation threshold % (default: 25)
106+
PERF_WARNING_THRESHOLD: 25 # Optional: warning threshold % (default: 15)
107+
PERF_IMPROVEMENT_THRESHOLD: 30 # Optional: improvement threshold % (default: 15)
108+
agents:
109+
provider: gcp
110+
image: family/kibana-ubuntu-2404
111+
imageProject: elastic-images-prod
112+
machineType: n2-standard-2
113+
timeout_in_minutes: 60
114+
retry:
115+
automatic:
116+
- exit_status: '-1'
117+
limit: 1
118+
119+
- wait: ~
120+
121+
- command: .buildkite/scripts/lifecycle/post_build.sh
122+
label: Post-Build
123+
agents:
124+
provider: gcp
125+
image: family/kibana-ubuntu-2404
126+
imageProject: elastic-images-prod
127+
machineType: n2-standard-2

‎.buildkite/pipelines/pull_request/trigger_entity_store_performance.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
steps:
2+
- command: .buildkite/scripts/steps/entity_store_performance/trigger_performance_tests.sh
3+
label: 'Entity Store Performance Tests'
4+
agents:
5+
provider: gcp
6+
image: family/kibana-ubuntu-2404
7+
imageProject: elastic-images-prod
8+
machineType: n2-standard-2
9+
timeout_in_minutes: 10
10+
soft_fail: true
11+
retry:
12+
automatic:
13+
- exit_status: '-1'
14+
limit: 1
15+

‎.buildkite/pull_requests.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,33 @@
102102
"\\.test\\.(ts|tsx|js|jsx)"
103103
]
104104
},
105+
{
106+
"repoOwner": "elastic",
107+
"repoName": "kibana",
108+
"pipelineSlug": "kibana-entity-store-performance-from-pr",
109+
"skip_ci_labels": [],
110+
"enabled": true,
111+
"allow_org_users": true,
112+
"allowed_repo_permissions": ["admin", "write"],
113+
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
114+
"set_commit_status": true,
115+
"commit_status_context": "kibana-entity-store-performance-from-pr",
116+
"build_on_commit": false,
117+
"build_on_comment": true,
118+
"build_drafts": false,
119+
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:run)\\W+(?:entity-store-performance))$",
120+
"enable_trigger_checkbox": true,
121+
"kibana_versions_check": true,
122+
"kibana_build_reuse": true,
123+
"kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-cloud-from-pr", "kibana-entity-store-performance-from-pr"],
124+
"kibana_build_reuse_regexes": [
125+
"^test/",
126+
"^x-pack/platform/test/",
127+
"^x-pack/solutions/(search|observability|security)/test/",
128+
"/__snapshots__/",
129+
"\\.test\\.(ts|tsx|js|jsx)"
130+
]
131+
},
105132
{
106133
"repoOwner": "elastic",
107134
"repoName": "kibana",

‎.buildkite/scripts/pipelines/pull_request/pipeline.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,12 @@ const FTR_ENABLE_FIPS_AGENT = process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() =
193193
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
194194
}
195195

196+
if (GITHUB_PR_LABELS.includes('ci:entity-store-performance')) {
197+
pipeline.push(
198+
getPipeline('.buildkite/pipelines/pull_request/trigger_entity_store_performance.yml')
199+
);
200+
}
201+
196202
if (
197203
GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
198204
GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||

‎.buildkite/scripts/steps/cloud/build_and_deploy.sh‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,28 @@ case "$KIBANA_MEMORY_SIZE" in
2222
;;
2323
esac
2424

25+
ES_ZONE_COUNT=${ES_ZONE_COUNT:-1}
26+
case "$ES_ZONE_COUNT" in
27+
1|2|3)
28+
echo "--- Elasticsearch zone count: ${ES_ZONE_COUNT}"
29+
;;
30+
*)
31+
echo "Error: ES_ZONE_COUNT must be 1, 2, or 3. Got: $ES_ZONE_COUNT"
32+
exit 1
33+
;;
34+
esac
35+
36+
ES_HOT_TIER_MEMORY_SIZE=${ES_HOT_TIER_MEMORY_SIZE:-2048}
37+
case "$ES_HOT_TIER_MEMORY_SIZE" in
38+
1024|2048|4096|8192|16384|32768)
39+
echo "--- Elasticsearch hot tier memory size: ${ES_HOT_TIER_MEMORY_SIZE}MB"
40+
;;
41+
*)
42+
echo "Error: ES_HOT_TIER_MEMORY_SIZE must be one of: 1024, 2048, 4096, 8192, 16384, 32768. Got: $ES_HOT_TIER_MEMORY_SIZE"
43+
exit 1
44+
;;
45+
esac
46+
2547
echo "--- Download Kibana Distribution"
2648

2749
mkdir -p ./target
@@ -66,7 +88,7 @@ else
6688
--skip-docker-contexts
6789
fi
6890

69-
if is_pr_with_label "ci:cloud-redeploy"; then
91+
if is_pr_with_label "ci:cloud-redeploy" || is_pr_with_label "ci:entity-store-performance"; then
7092
echo "--- Shutdown Previous Deployment"
7193
CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
7294
if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" == "null" ]; then
@@ -87,9 +109,16 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
87109
.resources.kibana[0].plan.kibana.version = "'$VERSION'" |
88110
.resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
89111
.resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'" |
90-
.resources.kibana[0].plan.cluster_topology[0].size.value = '$KIBANA_MEMORY_SIZE'
112+
.resources.kibana[0].plan.cluster_topology[0].size.value = '$KIBANA_MEMORY_SIZE' |
113+
(.resources.elasticsearch[0].plan.cluster_topology[] | select(.zone_count != null) | .zone_count) = '$ES_ZONE_COUNT' |
114+
(.resources.elasticsearch[0].plan.cluster_topology[] | select(.id == "hot_content") | .size.value) = '$ES_HOT_TIER_MEMORY_SIZE'
91115
' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
92116

117+
# Verify that zone_count was set (at least one topology element should have zone_count)
118+
if ! jq -e '.resources.elasticsearch[0].plan.cluster_topology[]? | select(.zone_count != null) | .zone_count' /tmp/deploy.json > /dev/null 2>&1; then
119+
echo "⚠️ Warning: No cluster_topology elements with zone_count found in deployment configuration"
120+
fi
121+
93122
echo "Creating deployment..."
94123
ecctl deployment create --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
95124

@@ -127,9 +156,16 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
127156
else
128157
ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
129158
.resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
130-
(.. | select(.version? != null).version) = "'$VERSION'"
159+
(.. | select(.version? != null).version) = "'$VERSION'" |
160+
(.resources.elasticsearch[0].plan.cluster_topology[]? | select(.zone_count != null) | .zone_count) = '$ES_ZONE_COUNT' |
161+
(.resources.elasticsearch[0].plan.cluster_topology[]? | select(.id == "hot_content") | .size.value) = '$ES_HOT_TIER_MEMORY_SIZE'
131162
' > /tmp/deploy.json
132163

164+
# Verify that zone_count was set (at least one topology element should have zone_count)
165+
if ! jq -e '.resources.elasticsearch[0].plan.cluster_topology[]? | select(.zone_count != null) | .zone_count' /tmp/deploy.json > /dev/null 2>&1; then
166+
echo "⚠️ Warning: No cluster_topology elements with zone_count found in deployment update payload"
167+
fi
168+
133169
echo "Updating deployment..."
134170
ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
135171
fi

0 commit comments

Comments
 (0)