Skip to content

Commit ec92e42

Browse files
maxcoldmaxcold
committed
Merge branch 'main' into csp-aws-sec-hub-full-posture
2 parents 83bb07d + 820f2ca commit ec92e42

File tree

1,969 files changed

+164603
-34312
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,969 files changed

+164603
-34312
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ git config --global core.pager 'cat'
1111
export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
1212
export SERVERLESS=${SERVERLESS:-"false"}
1313
export STACK_VERSION=${STACK_VERSION:-""}
14+
export ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
15+
export STACK_LOGSDB_ENABLED=${STACK_LOGSDB_ENABLED:-"false"}
1416
export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
1517
export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
1618

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ steps:
2525
depends_on:
2626
- step: "check"
2727
allow_failure: false
28+
if: |
29+
build.env('TEST_PACKAGES_7_BRANCH') == "true"
2830
2931
- label: "Check integrations local stacks - Stack Version v8.19"
3032
trigger: "integrations"
@@ -37,6 +39,8 @@ steps:
3739
depends_on:
3840
- step: "check"
3941
allow_failure: false
42+
if: |
43+
build.env('TEST_PACKAGES_8_BRANCH') == "true"
4044
4145
- label: "Check integrations local stacks - Stack Version v8.19 - LogsDB"
4246
trigger: "integrations"
@@ -50,6 +54,37 @@ steps:
5054
depends_on:
5155
- step: "check"
5256
allow_failure: false
57+
if: |
58+
build.env('TEST_PACKAGES_8_BRANCH') == "true"
59+
60+
- label: "Check integrations local stacks and basic subscription and LogsDB"
61+
trigger: "integrations"
62+
build:
63+
env:
64+
SERVERLESS: "false"
65+
FORCE_CHECK_ALL: "true"
66+
PUBLISH_COVERAGE_REPORTS: "false"
67+
ELASTIC_SUBSCRIPTION: "basic"
68+
STACK_LOGSDB_ENABLED: "true"
69+
depends_on:
70+
- step: "check"
71+
allow_failure: false
72+
if: |
73+
build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true"
74+
75+
- label: "Check integrations local stacks and basic subscription"
76+
trigger: "integrations"
77+
build:
78+
env:
79+
SERVERLESS: "false"
80+
FORCE_CHECK_ALL: "true"
81+
PUBLISH_COVERAGE_REPORTS: "false"
82+
ELASTIC_SUBSCRIPTION: "basic"
83+
depends_on:
84+
- step: "check"
85+
allow_failure: false
86+
if: |
87+
build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true"
5388
5489
- label: "Check integrations local stacks - Stack Version v9.1"
5590
trigger: "integrations"
@@ -62,6 +97,8 @@ steps:
6297
depends_on:
6398
- step: "check"
6499
allow_failure: false
100+
if: |
101+
build.env('TEST_PACKAGES_9_BRANCH') == "true"
65102
66103
- label: "Check integrations in serverless - project: Observability"
67104
key: "trigger-integrations-serverless-obs"
@@ -72,6 +109,8 @@ steps:
72109
depends_on:
73110
- step: "check"
74111
allow_failure: false
112+
if: |
113+
build.env('TEST_PACKAGES_SERVERLESS') == "true"
75114
76115
- label: "Check integrations in serverless - project: Security"
77116
key: "trigger-integrations-serverless-security"
@@ -82,10 +121,14 @@ steps:
82121
depends_on:
83122
- step: "check"
84123
allow_failure: false
124+
if: |
125+
build.env('TEST_PACKAGES_SERVERLESS') == "true"
85126
86127
- label: ":package: Publish missing packages"
87128
key: "trigger-integrations-publish"
88129
trigger: "integrations-publish"
89130
depends_on:
90131
- step: "check"
91132
allow_failure: false
133+
if: |
134+
build.env('REPUBLISH_PACKAGES') == "true"

‎.buildkite/scripts/common.sh‎

Lines changed: 96 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/
1818
COVERAGE_FORMAT="generic"
1919
COVERAGE_OPTIONS="--test-coverage --coverage-format=${COVERAGE_FORMAT}"
2020

21+
FATAL_ERROR="Fatal Error"
22+
2123
running_on_buildkite() {
2224
if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
2325
return 0
@@ -318,20 +320,12 @@ delete_kind_cluster() {
318320
}
319321

320322
kibana_version_manifest() {
321-
local kibana_version
322-
kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
323-
if [ "${kibana_version}" != "null" ]; then
324-
echo "${kibana_version}"
325-
return
326-
fi
327-
328-
kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
329-
if [ "${kibana_version}" != "null" ]; then
330-
echo "${kibana_version}"
331-
return
323+
local kibana_version=""
324+
if ! kibana_version=$(mage -d "${WORKSPACE}" -w . kibanaConstraintPackage) ; then
325+
return 1
332326
fi
333-
334-
echo "null"
327+
echo "${kibana_version}"
328+
return 0
335329
}
336330

337331
capabilities_manifest() {
@@ -457,35 +451,25 @@ is_supported_stack() {
457451
return 0
458452
fi
459453

460-
local kibana_version
461-
kibana_version=$(kibana_version_manifest)
462-
if [ "${kibana_version}" == "null" ]; then
463-
return 0
464-
fi
465-
if [[ ( ! "${kibana_version}" =~ \^7\. ) && "${STACK_VERSION}" =~ ^7\. ]]; then
466-
return 1
467-
fi
468-
if [[ ( ! "${kibana_version}" =~ \^8\. ) && "${STACK_VERSION}" =~ ^8\. ]]; then
469-
return 1
470-
fi
471-
472-
# TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too.
473-
# This workaround should be removed once packages have been updated their constraints for 9.0 stack.
474-
if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then
454+
local supported
455+
if ! supported=$(mage -d "${WORKSPACE}" -w . isSupportedStack "${STACK_VERSION}"); then
475456
return 1
476457
fi
477-
458+
echo "${supported}"
478459
return 0
479460
}
480461

481462
oldest_supported_version() {
482463
local kibana_version
483-
kibana_version=$(kibana_version_manifest)
464+
if ! kibana_version=$(kibana_version_manifest); then
465+
return 1
466+
fi
484467
if [ "$kibana_version" != "null" ]; then
485468
python3 "${SCRIPTS_BUILDKITE_PATH}/find_oldest_supported_version.py" --manifest-path manifest.yml
486-
return
469+
return 0
487470
fi
488471
echo "null"
472+
return 0
489473
}
490474

491475
create_elastic_package_profile() {
@@ -497,21 +481,48 @@ create_elastic_package_profile() {
497481
prepare_stack() {
498482
echo "--- Prepare stack"
499483

484+
local requiredSubscription="${ELASTIC_SUBSCRIPTION:-""}"
485+
local requiredLogsDB="${STACK_LOGSDB_ENABLED:-"false"}"
486+
500487
local args="-v"
488+
local version_set=""
501489
if [ -n "${STACK_VERSION}" ]; then
502490
args="${args} --version ${STACK_VERSION}"
491+
version_set="${STACK_VERSION}"
503492
else
504493
local version
505-
version=$(oldest_supported_version)
494+
if ! version=$(oldest_supported_version); then
495+
return 1
496+
fi
497+
if [[ "${requiredLogsDB}" == "true" ]]; then
498+
# If LogsDB index mode is enabled, the required Elastic stack should be at least 8.17.0
499+
# In 8.17.0 LogsDB index mode was made GA.
500+
local less_than=""
501+
if ! less_than=$(mage -d "${WORKSPACE}" -w . isVersionLessThanLogsDBGA "${version}") ; then
502+
echo "${FATAL_ERROR}"
503+
return 1
504+
fi
505+
if [[ "${less_than}" == "true" ]]; then
506+
version="8.17.0"
507+
fi
508+
fi
506509
if [[ "${version}" != "null" ]]; then
507510
args="${args} --version ${version}"
511+
version_set="${version}"
508512
fi
509513
fi
514+
echoerr "- Stack Version: \"${version_set}\""
510515

511-
if [ "${STACK_LOGSDB_ENABLED:-false}" == "true" ]; then
516+
if [ "${requiredLogsDB:-false}" == "true" ]; then
517+
echoerr "- Enable LogsDB"
512518
args="${args} -U stack.logsdb_enabled=true"
513519
fi
514520

521+
if [ "${requiredSubscription}" != "" ]; then
522+
echoerr "- Set Subscription ${ELASTIC_SUBSCRIPTION}"
523+
args="${args} -U stack.elastic_subscription=${requiredSubscription}"
524+
fi
525+
515526
if [[ "${STACK_VERSION}" =~ ^7\.17 ]]; then
516527
# Required starting with STACK_VERSION 7.17.21
517528
export ELASTIC_AGENT_IMAGE_REF_OVERRIDE="docker.elastic.co/beats/elastic-agent-complete:${STACK_VERSION}-amd64"
@@ -668,16 +679,58 @@ get_to_changeset() {
668679
echo "${to}"
669680
}
670681

682+
is_subscription_compatible() {
683+
local reason=""
684+
685+
if ! reason=$(mage -d "${WORKSPACE}" -w . isSubscriptionCompatible) ; then
686+
return 1
687+
fi
688+
echo "${reason}"
689+
return 0
690+
}
691+
692+
is_logsdb_compatible() {
693+
if [[ "${STACK_VERSION:-""}" != "" ]]; then
694+
# Assumption that if this variable is set, it is supported
695+
echo "true"
696+
return 0
697+
fi
698+
699+
if ! reason=$(mage -d "${WORKSPACE}" -w . isLogsDBSupportedInPackage); then
700+
return 1
701+
fi
702+
echo "${reason}"
703+
return 0
704+
}
705+
671706
is_pr_affected() {
672707
local package="${1}"
673708
local from="${2}"
674709
local to="${3}"
675710

676-
if ! is_supported_stack ; then
711+
local stack_supported=""
712+
if ! stack_supported=$(is_supported_stack) ; then
713+
echo "${FATAL_ERROR}"
714+
return 1
715+
fi
716+
if [[ "${stack_supported}" == "false" ]]; then
677717
echo "[${package}] PR is not affected: unsupported stack (${STACK_VERSION})"
678718
return 1
679719
fi
680720

721+
if [[ "${STACK_LOGSDB_ENABLED:-"false"}" == "true" ]]; then
722+
# Packages require to support 8.17.0 or higher as part of their Kibana constraints (manifest)
723+
local logsdb_compatible=""
724+
if ! logsdb_compatible=$(is_logsdb_compatible); then
725+
echo "${FATAL_ERROR}"
726+
return 1
727+
fi
728+
if [[ "${logsdb_compatible}" == "false" ]]; then
729+
echo "[${package}] PR is not affected: not supported LogsDB (${STACK_VERSION})"
730+
return 1
731+
fi
732+
fi
733+
681734
if is_serverless; then
682735
if is_package_excluded_in_config "${package}" "${WORKSPACE}/kibana.serverless.config.yml"; then
683736
echo "[${package}] PR is not affected: package ${package} excluded in Kibana config for ${SERVERLESS_PROJECT}"
@@ -698,6 +751,15 @@ is_pr_affected() {
698751
return 1
699752
fi
700753
fi
754+
local compatible=""
755+
if ! compatible=$(is_subscription_compatible); then
756+
echo "${FATAL_ERROR}"
757+
return 1
758+
fi
759+
if [[ "${compatible}" == "false" ]]; then
760+
echo "[${package}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}"
761+
return 1
762+
fi
701763

702764
if [[ "${FORCE_CHECK_ALL}" == "true" ]];then
703765
echo "[${package}] PR is affected: \"force_check_all\" parameter enabled"

‎.buildkite/scripts/test_integrations_with_serverless.sh‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ if [ ! -d packages ]; then
4343
exit 1
4444
fi
4545

46+
echo "--- Install requirements"
4647
add_bin_path
4748

4849
with_yq
@@ -60,6 +61,7 @@ sleep 120
6061
echo "Done."
6162

6263
# setting range of changesets to check differences
64+
echo "--- Get from and to changesets"
6365
from="$(get_from_changeset)"
6466
if [[ "${from}" == "" ]]; then
6567
echo "Missing \"from\" changset".
@@ -76,15 +78,28 @@ any_package_failing=0
7678

7779
pushd packages > /dev/null
7880
for package in $(list_all_directories); do
81+
echo "--- [$package] check if it is required to be tested"
7982
pushd "${package}" > /dev/null
83+
skip_package=false
84+
failure=false
8085
if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then
81-
echo "${reason}"
86+
skip_package=true
87+
if [[ "${reason}" == "${FATAL_ERROR}" ]]; then
88+
failure=true
89+
fi
90+
fi
91+
popd > /dev/null
92+
if [[ "${failure}" == "true" ]]; then
93+
echo "Unexpected failure checking ${package}"
94+
exit 1
95+
fi
96+
97+
echo "${reason}"
98+
99+
if [[ "${skip_package}" == "true" ]]; then
82100
echo "- ${reason}" >> "${SKIPPED_PACKAGES_FILE_PATH}"
83-
popd > /dev/null
84101
continue
85102
fi
86-
echo "${reason}"
87-
popd > /dev/null
88103

89104
if ! process_package "${package}" "${FAILED_PACKAGES_FILE_PATH}" ; then
90105
any_package_failing=1
@@ -94,10 +109,12 @@ popd > /dev/null
94109

95110
if running_on_buildkite ; then
96111
if [ -f "${SKIPPED_PACKAGES_FILE_PATH}" ]; then
112+
echo "--- Create Skip Buildkite annotation"
97113
create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" "${SKIPPED_PACKAGES_FILE_PATH}" "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
98114
fi
99115

100116
if [ -f "${FAILED_PACKAGES_FILE_PATH}" ]; then
117+
echo "--- Create Failed Buildkite annotation"
101118
create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" "${FAILED_PACKAGES_FILE_PATH}" "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
102119
fi
103120
fi

0 commit comments

Comments
 (0)