Skip to content

Commit b291bc3

Browse files
cee-chencee-chen
authored
Merge branch 'main' into ml-log-rate-analysis-fix-table
2 parents df5fc14 + 7fa867c commit b291bc3

File tree

1,414 files changed

+9842
-9660
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,414 files changed

+9842
-9660
lines changed

‎.buildkite/ftr_configs.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,7 @@ enabled:
322322
- x-pack/test/functional/apps/security/config.ts
323323
- x-pack/test/functional/apps/snapshot_restore/config.ts
324324
- x-pack/test/functional/apps/spaces/config.ts
325+
- x-pack/test/functional/apps/spaces/in_solution_navigation/config.ts
325326
- x-pack/test/functional/apps/status_page/config.ts
326327
- x-pack/test/functional/apps/transform/creation/index_pattern/config.ts
327328
- x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts

‎.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
2+
apiVersion: backstage.io/v1alpha1
3+
kind: Resource
4+
metadata:
5+
name: bk-kibana-fips-daily
6+
description: Run Kibana FIPS smoke tests
7+
links:
8+
- title: Pipeline link
9+
url: https://buildkite.com/elastic/kibana-fips
10+
spec:
11+
type: buildkite-pipeline
12+
owner: group:kibana-operations
13+
system: buildkite
14+
implementation:
15+
apiVersion: buildkite.elastic.dev/v1
16+
kind: Pipeline
17+
metadata:
18+
name: kibana / fips
19+
description: Run Kibana FIPS smoke tests
20+
spec:
21+
env:
22+
SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
23+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
24+
repository: elastic/kibana
25+
branch_configuration: main
26+
default_branch: main
27+
pipeline_file: ".buildkite/pipelines/fips.yml"
28+
provider_settings:
29+
trigger_mode: none
30+
schedules:
31+
daily:
32+
branch: main
33+
cronline: 0 9 * * * America/New_York
34+
teams:
35+
kibana-operations:
36+
access_level: MANAGE_BUILD_AND_READ
37+
everyone:
38+
access_level: READ_ONLY

‎.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ spec:
2828
pipeline_file: .buildkite/pipelines/fleet/packages_daily.yml
2929
skip_intermediate_builds: false
3030
provider_settings:
31+
trigger_mode: none
3132
publish_commit_status: false
3233
prefix_pull_request_fork_branch_names: false
3334
skip_pull_request_builds_for_existing_commits: false

‎.buildkite/pipeline-resource-definitions/locations.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ spec:
1212
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
1313
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
1414
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
15+
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml
1516
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
1617
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml
1718
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml

‎.buildkite/pipelines/fips.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
env:
2+
DISABLE_CI_STATS_SHIPPING: "true"
3+
steps:
4+
- command: .buildkite/scripts/steps/build_kibana.sh
5+
label: Build Kibana Distribution and Plugins
6+
agents:
7+
image: family/kibana-ubuntu-2004
8+
imageProject: elastic-images-qa
9+
provider: gcp
10+
machineType: n2-standard-16
11+
preemptible: true
12+
key: build
13+
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
14+
timeout_in_minutes: 60
15+
retry:
16+
automatic:
17+
- exit_status: "-1"
18+
limit: 3
19+
20+
- wait
21+
22+
- command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh
23+
label: "Smoke testing for FIPS"
24+
agents:
25+
image: family/kibana-ubuntu-2004
26+
imageProject: elastic-images-qa
27+
provider: gcp
28+
enableNestedVirtualization: true
29+
localSsds: 1
30+
localSsdInterface: nvme
31+
machineType: n2-standard-4
32+
timeout_in_minutes: 600

‎.buildkite/pipelines/security_solution/api_integration.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ steps:
4747
- exit_status: '*'
4848
limit: 2
4949

50-
- label: Running exception_operators_keyword:qa:serverless
51-
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_keyword:qa:serverless
52-
key: exception_operators_keyword:qa:serverless
50+
- label: Running exception_operators_keyword:essentials:qa:serverless
51+
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_keyword:essentials:qa:serverless
52+
key: exception_operators_keyword:essentials:qa:serverless
5353
agents:
5454
image: family/kibana-ubuntu-2004
5555
imageProject: elastic-images-qa
@@ -62,9 +62,9 @@ steps:
6262
- exit_status: '*'
6363
limit: 2
6464

65-
- label: Running exception_operators_ips:qa:serverless
66-
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_ips:qa:serverless
67-
key: exception_operators_ips:qa:serverless
65+
- label: Running exception_operators_ips:essentials:qa:serverless
66+
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_ips:essentials:qa:serverless
67+
key: exception_operators_ips:essentials:qa:serverless
6868
agents:
6969
image: family/kibana-ubuntu-2004
7070
imageProject: elastic-images-qa
@@ -92,9 +92,9 @@ steps:
9292
- exit_status: '1'
9393
limit: 2
9494

95-
- label: Running exception_operators_text:qa:serverless
96-
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_text:qa:serverless
97-
key: exception_operators_text:qa:serverless
95+
- label: Running exception_operators_text:essentials:qa:serverless
96+
command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_text:essentials:qa:serverless
97+
key: exception_operators_text:essentials:qa:serverless
9898
agents:
9999
image: family/kibana-ubuntu-2004
100100
imageProject: elastic-images-qa

‎.buildkite/scripts/steps/artifacts/docker_image.sh‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,16 +99,9 @@ gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMM
9999
gcloud auth revoke "$GCS_SA_CDN_EMAIL"
100100

101101
echo "--- Validate CDN assets"
102-
(
103-
shopt -s globstar
104-
cd $CDN_ASSETS_FOLDER
105-
for CDN_ASSET in **/*; do
106-
if [[ -f "$CDN_ASSET" ]]; then
107-
echo -n "Testing $CDN_ASSET..."
108-
curl --retry 10 --retry-max-time 600 --connect-timeout 120 --max-time 120 -I --write-out '%{http_code}\n' --fail --silent --output /dev/null "$GCS_SA_CDN_URL/$CDN_ASSET"
109-
fi
110-
done
111-
)
102+
ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts" \
103+
"$GCS_SA_CDN_URL" \
104+
"$CDN_ASSETS_FOLDER"
112105

113106
echo "--- Upload archives"
114107
buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-x86_64.tar.gz"

‎.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import * as glob from 'glob';
10+
import axios from 'axios';
11+
12+
const CDN_URL_PREFIX = process.argv[2];
13+
const CDN_ASSETS_FOLDER = process.argv[3];
14+
15+
async function main() {
16+
const allAssets = glob.sync('**/*', { cwd: CDN_ASSETS_FOLDER, nodir: true });
17+
const totalAssetCount = allAssets.length;
18+
let assetsProcessed = 0;
19+
let assetsFound = 0;
20+
21+
const batchSize = 50;
22+
23+
console.log(`Starting CDN asset validation for ${totalAssetCount} assets...`);
24+
25+
while (assetsProcessed < totalAssetCount) {
26+
const batch = allAssets.slice(assetsProcessed, assetsProcessed + batchSize);
27+
const results = await Promise.all(batch.map((url) => headAssetUrlWithRetry(url)));
28+
assetsProcessed += results.length;
29+
results.forEach((result) => {
30+
if (result.status === 200) {
31+
console.log(`Testing ${result.assetPath}...${result.status}`);
32+
assetsFound++;
33+
} else {
34+
console.error(`Testing ${result.assetPath}...${result.status} (${result.testUrl})`);
35+
}
36+
});
37+
}
38+
39+
return {
40+
totalAssetCount,
41+
assetsProcessed,
42+
assetsFound,
43+
assetsNotFound: totalAssetCount - assetsFound,
44+
};
45+
}
46+
47+
async function headAssetUrl(assetPath: string) {
48+
const testUrl = `${CDN_URL_PREFIX}/${assetPath}`;
49+
const response = await axios.head(testUrl, {
50+
timeout: 1000,
51+
});
52+
return {
53+
status: response.status,
54+
testUrl,
55+
assetPath,
56+
};
57+
}
58+
59+
async function headAssetUrlWithRetry(
60+
assetPath: string,
61+
retries = 5
62+
): Promise<{
63+
status: number;
64+
testUrl: string;
65+
assetPath: string;
66+
}> {
67+
const result = await headAssetUrl(assetPath);
68+
if (result.status === 200) {
69+
return result;
70+
} else if (retries > 0) {
71+
console.log(`Retrying ${assetPath}...(retries left: ${retries})`);
72+
await new Promise((resolve) => setTimeout(resolve, 1000));
73+
return headAssetUrlWithRetry(assetPath, retries - 1);
74+
} else {
75+
return {
76+
status: result.status || 0,
77+
testUrl: result.testUrl,
78+
assetPath,
79+
};
80+
}
81+
}
82+
83+
main()
84+
.then(({ totalAssetCount, assetsNotFound }) => {
85+
if (assetsNotFound) {
86+
console.error(`Couldn't find ${assetsNotFound} assets on CDN.`);
87+
process.exit(1);
88+
} else {
89+
console.log(`All ${totalAssetCount} found on CDN.`);
90+
process.exit(0);
91+
}
92+
})
93+
.catch((error) => {
94+
console.error(error.message);
95+
process.exit(1);
96+
});

‎.buildkite/scripts/steps/fips/build.sh‎

100644100755
File mode changed.

‎.buildkite/scripts/steps/fips/smoke_test.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
3+
if [ -z "$KIBANA_BUILD_LOCATION" ]; then
4+
export KIBANA_BUILD_LOCATION="/usr/share/kibana"
5+
fi
6+
7+
# a FTR failure will result in the script returning an exit code of 10
8+
exitCode=0
9+
10+
configs=(
11+
"x-pack/test/reporting_functional/reporting_and_security.config.ts"
12+
"x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts"
13+
"x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts"
14+
"x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts"
15+
"x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts"
16+
"x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts"
17+
"x-pack/test/functional/apps/saved_objects_management/config.ts"
18+
"x-pack/test/functional/apps/user_profiles/config.ts"
19+
"x-pack/test/functional/apps/security/config.ts"
20+
)
21+
22+
cd /home/vagrant/kibana
23+
24+
for config in "${configs[@]}"; do
25+
set +e
26+
node /home/vagrant/kibana/scripts/functional_tests \
27+
--bail \
28+
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
29+
--config="$config"
30+
lastCode=$?
31+
set -e
32+
33+
if [ $lastCode -ne 0 ]; then
34+
exitCode=10
35+
echo "FTR exited with code $lastCode"
36+
echo "^^^ +++"
37+
38+
if [[ "$failedConfigs" ]]; then
39+
failedConfigs="${failedConfigs}"$'\n'"$config"
40+
else
41+
failedConfigs="$config"
42+
fi
43+
fi
44+
done
45+
46+
if [[ "$failedConfigs" ]]; then
47+
echo "$failedConfigs" >/home/vagrant/ftr_failed_configs
48+
fi
49+
50+
echo "--- FIPS smoke test complete"
51+
52+
exit $exitCode

0 commit comments

Comments
 (0)