Skip to content

Commit bde7150

Browse files
dej611dej611
authored
Merge branch 'main' into fix/188263
2 parents 35ad286 + b0ef1e6 commit bde7150

2,613 files changed

Lines changed: 37984 additions & 24830 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/ftr_security_serverless_configs.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,20 +81,15 @@ enabled:
8181
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/configs/serverless.config.ts
8282
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/diffable_rule_fields/common_fields/configs/serverless.config.ts
8383
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/diffable_rule_fields/type_specific_fields/configs/serverless.config.ts
84-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless_complete_tier.config.ts
8584
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/serverless_essentials_tier.config.ts
8685
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/serverless.config.ts
8786
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/serverless.config.ts
8887
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/serverless.config.ts
8988
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/serverless.config.ts
9089
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/serverless.config.ts
9190
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_non_customized_prebuilt_rules/feature_enabled/configs/serverless_essentials_tier.config.ts
92-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_non_customized_prebuilt_rules/feature_disabled/configs/serverless_feature_flag_disabled.config.ts
9391
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_enabled/configs/serverless_complete_tier.config.ts
94-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_disabled/configs/serverless_essentials_tier.config.ts
95-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_disabled/configs/serverless_feature_flag_disabled.config.ts
9692
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/export_prebuilt_rules/feature_enabled/configs/serverless_essentials_tier.config.ts
97-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/export_prebuilt_rules/feature_disabled/configs/serverless_feature_flag_disabled.config.ts
9893
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/serverless.config.ts
9994
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/serverless.config.ts
10095
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts

‎.buildkite/ftr_security_stateful_configs.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,20 +63,15 @@ enabled:
6363
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/diffable_rule_fields/common_fields/configs/ess.config.ts
6464
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_enabled/diffable_rule_fields/type_specific_fields/configs/ess.config.ts
6565
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess_basic_license.config.ts
66-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/customization_disabled/configs/ess_trial_license.config.ts
6766
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/ess.config.ts
6867
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/ess.config.ts
6968
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/ess.config.ts
7069
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/ess.config.ts
7170
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/ess.config.ts
7271
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/ess.config.ts
7372
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_non_customized_prebuilt_rules/feature_enabled/configs/ess_basic_license.config.ts
74-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_non_customized_prebuilt_rules/feature_disabled/configs/ess_feature_flag_disabled.config.ts
7573
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_enabled/configs/ess_enterprise_license.config.ts
76-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_disabled/configs/ess_basic_license.config.ts
77-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/import_customized_prebuilt_rules/feature_disabled/configs/ess_feature_flag_disabled.config.ts
7874
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/export_prebuilt_rules/feature_enabled/configs/ess_basic_license.config.ts
79-
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/export_prebuilt_rules/feature_disabled/configs/ess_feature_flag_disabled.config.ts
8075
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/ess.config.ts
8176
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts
8277
- x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts

‎.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,3 +561,57 @@ export async function pickTestGroupRunOrder() {
561561
].flat()
562562
);
563563
}
564+
565+
export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
566+
const bk = new BuildkiteClient();
567+
const envFromlabels: Record<string, string> = collectEnvFromLabels();
568+
569+
if (!Fs.existsSync(scoutConfigsPath)) {
570+
throw new Error(`Scout configs file not found at ${scoutConfigsPath}`);
571+
}
572+
573+
const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8'));
574+
const pluginsWithScoutConfigs: string[] = Object.keys(rawScoutConfigs);
575+
576+
if (pluginsWithScoutConfigs.length === 0) {
577+
// no scout configs found, nothing to need to upload steps
578+
return;
579+
}
580+
581+
const scoutGroups = pluginsWithScoutConfigs.map((plugin) => ({
582+
title: plugin,
583+
key: plugin,
584+
usesParallelWorkers: rawScoutConfigs[plugin].usesParallelWorkers,
585+
group: rawScoutConfigs[plugin].group,
586+
}));
587+
588+
// upload the step definitions to Buildkite
589+
bk.uploadSteps(
590+
[
591+
{
592+
group: 'Scout Configs',
593+
key: 'scout-configs',
594+
depends_on: ['build'],
595+
steps: scoutGroups.map(
596+
({ title, key, group, usesParallelWorkers }): BuildkiteStep => ({
597+
label: `Scout: [ ${group} / ${title} ] plugin`,
598+
command: getRequiredEnv('SCOUT_CONFIGS_SCRIPT'),
599+
timeout_in_minutes: 60,
600+
agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
601+
env: {
602+
SCOUT_CONFIG_GROUP_KEY: key,
603+
SCOUT_CONFIG_GROUP_TYPE: group,
604+
...envFromlabels,
605+
},
606+
retry: {
607+
automatic: [
608+
{ exit_status: '-1', limit: 1 },
609+
{ exit_status: '*', limit: 0 },
610+
],
611+
},
612+
})
613+
),
614+
},
615+
].flat()
616+
);
617+
}

‎.buildkite/pipelines/build_pr_and_deploy_cloud.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ steps:
3434
machineType: n2-standard-2
3535
timeout_in_minutes: 5
3636
37+
- wait: ~
38+
3739
- command: .buildkite/scripts/steps/build_kibana.sh
3840
label: Build Kibana Distribution
3941
agents:
@@ -45,13 +47,12 @@ steps:
4547
diskSizeGb: 125
4648
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
4749
timeout_in_minutes: 90
50+
key: build
4851
retry:
4952
automatic:
5053
- exit_status: '-1'
5154
limit: 3
5255

53-
- wait: ~
54-
5556
- command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
5657
label: 'Build and Deploy to Cloud'
5758
agents:
@@ -61,6 +62,7 @@ steps:
6162
machineType: n2-standard-2
6263
preemptible: true
6364
timeout_in_minutes: 30
65+
depends_on: build
6466
retry:
6567
automatic:
6668
- exit_status: '-1'

‎.buildkite/pipelines/on_merge_unsupported_ftrs.yml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,21 +68,20 @@ steps:
6868
- exit_status: '*'
6969
limit: 1
7070

71-
- command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
72-
label: 'Scout UI Tests'
71+
- command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
72+
label: 'Scout Test Run Builder'
7373
agents:
7474
image: family/kibana-ubuntu-2004
7575
imageProject: elastic-images-prod
7676
provider: gcp
77-
machineType: n2-standard-8
77+
machineType: n2-standard-2
7878
preemptible: true
7979
depends_on: build
80+
timeout_in_minutes: 10
8081
env:
82+
SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
8183
PING_SLACK_TEAM: "@appex-qa-team"
82-
timeout_in_minutes: 60
8384
retry:
8485
automatic:
85-
- exit_status: '-1'
86-
limit: 2
8786
- exit_status: '*'
8887
limit: 1

‎.buildkite/pipelines/pull_request/scout_tests.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
steps:
2+
- command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
3+
label: 'Scout Test Run Builder'
4+
agents:
5+
machineType: n2-standard-2
6+
diskSizeGb: 75
7+
timeout_in_minutes: 10
8+
env:
9+
SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
10+
retry:
11+
automatic:
12+
- exit_status: '*'
13+
limit: 1

‎.buildkite/pipelines/pull_request/scout_ui_tests.yml‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.buildkite/scripts/pipelines/pull_request/pipeline.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -458,13 +458,18 @@ const getPipeline = (filename: string, removeSteps = true) => {
458458

459459
if (
460460
(await doAnyChangesMatch([
461-
/^x-pack\/platform\/plugins\/private\/discover_enhanced\/ui_tests/,
462-
/^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
463461
/^src\/platform\/packages\/shared\/kbn-scout/,
462+
/^src\/platform\/packages\/private\/kbn-scout-info/,
463+
/^src\/platform\/packages\/private\/kbn-scout-reporting/,
464+
/^x-pack\/platform\/plugins\/shared\/maps/,
465+
/^x-pack\/platform\/plugins\/private\/discover_enhanced/,
466+
/^x-pack\/solutions\/observability\/packages\/kbn-scout-oblt/,
467+
/^x-pack\/solutions\/observability\/plugins\/apm/,
468+
/^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
464469
])) ||
465470
GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
466471
) {
467-
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
472+
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
468473
}
469474

470475
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));

‎.buildkite/scripts/steps/functional/scout_ui_tests.sh‎

Lines changed: 0 additions & 42 deletions
This file was deleted.

‎.buildkite/scripts/steps/test/scout_configs.sh‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/steps/functional/common.sh
6+
7+
BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
8+
SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
9+
SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
10+
11+
if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
12+
echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
13+
exit 1
14+
fi
15+
16+
if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
17+
echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
18+
exit 1
19+
fi
20+
21+
EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
22+
test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
23+
24+
export JOB="$SCOUT_CONFIG_GROUP_KEY"
25+
26+
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
27+
28+
configs=""
29+
group=$SCOUT_CONFIG_GROUP_TYPE
30+
31+
# The first retry should only run the configs that failed in the previous attempt
32+
# Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
33+
if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
34+
configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
35+
if [[ "$configs" ]]; then
36+
echo "--- Retrying only failed configs"
37+
echo "$configs"
38+
fi
39+
fi
40+
41+
if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
42+
echo "--- downloading scout test configuration"
43+
download_artifact scout_playwright_configs.json .
44+
configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
45+
fi
46+
47+
if [ "$configs" == "" ]; then
48+
echo "unable to determine configs to run"
49+
exit 1
50+
fi
51+
52+
# Define run modes based on group
53+
declare -A RUN_MODES
54+
RUN_MODES["platform"]="--stateful --serverless=es --serverless=oblt --serverless=security"
55+
RUN_MODES["observability"]="--stateful --serverless=oblt"
56+
RUN_MODES["search"]="--stateful --serverless=es"
57+
RUN_MODES["security"]="--stateful --serverless=security"
58+
59+
# Determine valid run modes for the group
60+
RUN_MODE_LIST=${RUN_MODES[$group]}
61+
62+
if [[ -z "$RUN_MODE_LIST" ]]; then
63+
echo "Unknown group: $group"
64+
exit 1
65+
fi
66+
67+
results=()
68+
failedConfigs=()
69+
configWithoutTests=()
70+
passedConfigs=()
71+
72+
FINAL_EXIT_CODE=0
73+
74+
# Run tests for each config
75+
while read -r config_path; do
76+
if [[ -z "$config_path" ]]; then
77+
continue
78+
fi
79+
80+
for mode in $RUN_MODE_LIST; do
81+
echo "--- Running tests: $config_path ($mode)"
82+
83+
# prevent non-zero exit code from breaking the loop
84+
set +e;
85+
node scripts/scout run-tests "$mode" --config "$config_path" --kibana-install-dir "$KIBANA_BUILD_LOCATION"
86+
EXIT_CODE=$?
87+
set -e;
88+
89+
if [[ $EXIT_CODE -eq 2 ]]; then
90+
configWithoutTests+=("$config_path ($mode)")
91+
elif [[ $EXIT_CODE -ne 0 ]]; then
92+
failedConfigs+=("$config_path ($mode) ❌")
93+
FINAL_EXIT_CODE=10 # Ensure we exit with failure if any test fails with (exit code 10 to match FTR)
94+
else
95+
results+=("$config_path ($mode) ✅")
96+
fi
97+
done
98+
done <<< "$configs"
99+
100+
echo "--- Scout Test Run Complete: Summary"
101+
echo "✅ Passed: ${#results[@]}"
102+
echo "⚠️ Configs without tests: ${#configWithoutTests[@]}"
103+
echo "❌ Failed: ${#failedConfigs[@]}"
104+
105+
if [[ ${#results[@]} -gt 0 ]]; then
106+
echo "✅ Successful tests:"
107+
printf '%s\n' "${results[@]}"
108+
fi
109+
110+
if [[ ${#configWithoutTests[@]} -gt 0 ]]; then
111+
{
112+
echo "Scout Playwright configs without tests:"
113+
echo ""
114+
for config in "${configWithoutTests[@]}"; do
115+
echo "- $config"
116+
done
117+
} | buildkite-agent annotate --style "warning" --context "no-tests"
118+
fi
119+
120+
if [[ ${#failedConfigs[@]} -gt 0 ]]; then
121+
echo "❌ Failed tests:"
122+
printf '%s\n' "${failedConfigs[@]}"
123+
buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
124+
fi
125+
126+
exit $FINAL_EXIT_CODE # Exit with 10 only if there were config failures

0 commit comments

Comments
 (0)