Skip to content

Commit cbb8ed0

Browse files
anil-elasticanil-elastic
authored
Merge branch 'main' into integration--10695
2 parents d1d8fe9 + 4d6643d commit cbb8ed0

File tree

7,576 files changed

+441394
-304624
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

7,576 files changed

+441394
-304624
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
3333
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
3434

3535
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
36-
GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
3736

3837
EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
3938
EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
@@ -101,7 +100,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
101100
if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
102101
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
103102
export BUILDKITE_API_TOKEN
104-
GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
103+
GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
105104
export GITHUB_TOKEN
106105
fi
107106

@@ -147,12 +146,10 @@ fi
147146

148147
if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
149148
if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
150-
GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
151-
export GITHUB_USERNAME_SECRET
152-
GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
153-
export GITHUB_EMAIL_SECRET
154-
GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
155-
export GITHUB_TOKEN
149+
GITHUB_USERNAME="elastic-vault-github-plugin-prod"
150+
GITHUB_EMAIL="elasticmachine@elastic.co"
151+
GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
152+
export GITHUB_TOKEN GITHUB_EMAIL GITHUB_USERNAME
156153
fi
157154
fi
158155

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ steps:
2121
env:
2222
SERVERLESS: "false"
2323
FORCE_CHECK_ALL: "true"
24-
STACK_VERSION: 7.17.24-SNAPSHOT
24+
STACK_VERSION: 7.17.25-SNAPSHOT
2525
depends_on:
2626
- step: "check"
2727
allow_failure: false
@@ -38,6 +38,31 @@ steps:
3838
- step: "check"
3939
allow_failure: false
4040

41+
- label: "Check integrations local stacks - Stack Version v8.16 - LogsDB"
42+
trigger: "integrations"
43+
build:
44+
env:
45+
SERVERLESS: "false"
46+
FORCE_CHECK_ALL: "true"
47+
STACK_VERSION: 8.16.0-SNAPSHOT
48+
STACK_LOGSDB_ENABLED: "true"
49+
PUBLISH_COVERAGE_REPORTS: "false"
50+
depends_on:
51+
- step: "check"
52+
allow_failure: false
53+
54+
- label: "Check integrations local stacks - Stack Version v9.0"
55+
trigger: "integrations"
56+
build:
57+
env:
58+
SERVERLESS: "false"
59+
FORCE_CHECK_ALL: "true"
60+
STACK_VERSION: 9.0.0-SNAPSHOT
61+
PUBLISH_COVERAGE_REPORTS: "false"
62+
depends_on:
63+
- step: "check"
64+
allow_failure: false
65+
4166
- label: "Check integrations in serverless - project: Observability"
4267
key: "trigger-integrations-serverless-obs"
4368
trigger: "integrations-serverless"

‎.buildkite/pipeline.schedule-weekly.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,16 @@ steps:
2727
depends_on:
2828
- step: "check"
2929
allow_failure: false
30+
31+
- label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.0"
32+
trigger: "integrations"
33+
build:
34+
env:
35+
SERVERLESS: "false"
36+
FORCE_CHECK_ALL: "true"
37+
STACK_VERSION: 9.0.0-SNAPSHOT
38+
PUBLISH_COVERAGE_REPORTS: "false"
39+
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
40+
depends_on:
41+
- step: "check"
42+
allow_failure: false

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ env:
2626
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
2727
# Set maximum number of parallel tests to run if package allows it
2828
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
29+
# Enable/Disable the usage of wolfi images for Elastic Agent
30+
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
31+
# Disable checking for newer versions
32+
ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
2933

3034
steps:
3135
- input: "Input values for the variables"

‎.buildkite/pipeline.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@ env:
2626
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
2727
# Set maximum number of parallel tests to run if package allows it
2828
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
29-
# Disable the usage of wolfi images for Elastic Agent
30-
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-true}"
29+
# Enable/Disable the usage of wolfi images for Elastic Agent
30+
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
31+
# Disable checking for newer versions
32+
ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
3133

3234
steps:
3335
- label: "Get reference from target branch"
@@ -102,5 +104,8 @@ steps:
102104
# not fail build if this step fails
103105
soft_fail: true
104106
# run this step when if it is triggered by the daily job
105-
if: >
106-
build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"
107+
# skipped for now notification for tests related to STACK_VERSION: 9
108+
if: |
109+
build.source == "trigger_job" &&
110+
build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" &&
111+
build.env('STACK_VERSION') !~ /^9/

‎.buildkite/scripts/backport_branch.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,9 @@ removeOtherPackages() {
106106

107107
update_git_config() {
108108
pushd $WORKSPACE > /dev/null
109-
git config --global user.name "${GITHUB_USERNAME_SECRET}"
110-
git config --global user.email "${GITHUB_EMAIL_SECRET}"
109+
git config --global user.name "${GITHUB_USERNAME}"
110+
git config --global user.email "${GITHUB_EMAIL}"
111111

112-
git config remote.origin.url "https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/elastic/integrations.git"
113112
popd > /dev/null
114113
}
115114

‎.buildkite/scripts/build_packages.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ report_build_failure() {
4343

4444
# if running in Buildkite , add an annotation
4545
if [ -n "${BUILDKITE_BRANCH+x}" ]; then
46-
buildkite-agent annotate "Build package ${package} failed, not published." --ctx "ctx-build-${package}" --style "warning"
46+
buildkite-agent annotate "Build package ${package} failed, not published." --context "ctx-build-${package}" --style "warning"
4747
fi
4848
}
4949

‎.buildkite/scripts/common.sh‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,6 @@ with_mage() {
126126

127127
local install_packages=(
128128
"github.com/magefile/mage"
129-
"github.com/elastic/go-licenser"
130-
"golang.org/x/tools/cmd/goimports"
131129
"github.com/jstemmer/go-junit-report"
132130
"gotest.tools/gotestsum"
133131
)
@@ -464,12 +462,19 @@ is_supported_stack() {
464462
if [ "${kibana_version}" == "null" ]; then
465463
return 0
466464
fi
467-
if [[ ! "${kibana_version}" =~ \^7\. && "${STACK_VERSION}" =~ ^7\. ]]; then
465+
if [[ ( ! "${kibana_version}" =~ \^7\. ) && "${STACK_VERSION}" =~ ^7\. ]]; then
468466
return 1
469467
fi
470-
if [[ ! "${kibana_version}" =~ \^8\. && "${STACK_VERSION}" =~ ^8\. ]]; then
468+
if [[ ( ! "${kibana_version}" =~ \^8\. ) && "${STACK_VERSION}" =~ ^8\. ]]; then
471469
return 1
472470
fi
471+
472+
# TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too.
473+
# This workaround should be removed once packages have been updated their constraints for 9.0 stack.
474+
if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then
475+
return 1
476+
fi
477+
473478
return 0
474479
}
475480

@@ -503,6 +508,10 @@ prepare_stack() {
503508
fi
504509
fi
505510

511+
if [ "${STACK_LOGSDB_ENABLED:-false}" == "true" ]; then
512+
args="${args} -U stack.logsdb_enabled=true"
513+
fi
514+
506515
echo "Boot up the Elastic stack"
507516
if ! ${ELASTIC_PACKAGE_BIN} stack up -d ${args} ; then
508517
return 1

‎.buildkite/scripts/find_oldest_supported_version.py‎

Lines changed: 21 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ def fetch_version():
2626

2727

2828
def find_oldest_supported_version(kibana_version_condition: str) -> str:
29-
# The logic of this function is copied from https://github.com/elastic/apm-pipeline-library/blob/main/vars/findOldestSupportedVersion.groovy
30-
3129
if "||" in kibana_version_condition and kibana_version_condition.index("||") >= 0:
3230
return handle_or(kibana_version_condition)
3331

@@ -55,33 +53,6 @@ def find_oldest_supported_version(kibana_version_condition: str) -> str:
5553
if not next_patch_exists and (snapshot_version in available_versions):
5654
return snapshot_version
5755

58-
# Use the version as is if it exists.
59-
if version in available_versions:
60-
return version
61-
62-
# Old minors may not be available in artifacts-api, if it is older
63-
# than the others in the same major, return the version as is.
64-
older = True
65-
for available_version in available_versions:
66-
available_parts = available_version.split(".")
67-
if len(available_parts) < 2:
68-
continue
69-
70-
available_major = available_parts[0]
71-
available_minor = available_parts[1]
72-
if major == available_major and minor > available_minor:
73-
older = False
74-
break
75-
if older:
76-
return version
77-
78-
# If no version has been found so far, try with the snapshot of the next version
79-
# in the current major.
80-
major_snapshot = f"{major}.x-SNAPSHOT"
81-
if major_snapshot in available_aliases:
82-
return major_snapshot
83-
84-
# Otherwise, return it, whatever this is.
8556
return version
8657

8758

@@ -170,19 +141,22 @@ class TestFindOldestSupportVersion(unittest.TestCase):
170141
"8.8.0",
171142
"8.8.1",
172143
"8.8.2",
173-
"8.9.0",
174144
"8.9.1-SNAPSHOT",
175145
"8.9.1",
176146
"8.9.2-SNAPSHOT",
177147
"8.9.2",
148+
"8.9.4",
178149
"8.10.0-SNAPSHOT",
179150
"8.10.0",
180151
"8.10.1-SNAPSHOT",
152+
"8.10.1",
181153
"8.11.0-SNAPSHOT"
182154
],
183155
"aliases": [
156+
"7.x-SNAPSHOT",
184157
"7.17-SNAPSHOT",
185158
"7.17",
159+
"8.x-SNAPSHOT",
186160
"8.7",
187161
"8.8",
188162
"8.9-SNAPSHOT",
@@ -225,6 +199,23 @@ def test_available_version_contains_kibana_version(self):
225199
def test_too_old_to_be_in_api(self):
226200
self.assertEqual(find_oldest_supported_version("7.16.0"), "7.16.0")
227201
self.assertEqual(find_oldest_supported_version("8.6.0"), "8.6.0")
202+
self.assertEqual(find_oldest_supported_version("7.6.0"), "7.6.0")
203+
204+
def test_newer_major_or_minor_versions_not_shown_in_api(self):
205+
# next minor from 7.x
206+
self.assertEqual(find_oldest_supported_version("7.19.0"), "7.19.0")
207+
# next minor from 8.x
208+
self.assertEqual(find_oldest_supported_version("8.12.0"), "8.12.0")
209+
# next patch from 8.8.x
210+
self.assertEqual(find_oldest_supported_version("8.8.3"), "8.8.3")
211+
# next major 9.0
212+
self.assertEqual(find_oldest_supported_version("9.0.0"), "9.0.0")
213+
214+
def test_missing_older_versions_in_api_response(self):
215+
# exists 8.9.1 and 8.8.x, but not 8.9.0
216+
self.assertEqual(find_oldest_supported_version("8.9.0"), "8.9.0")
217+
# exists 8.9.2 and 8.9.4, but not 8.9.3
218+
self.assertEqual(find_oldest_supported_version("8.9.3"), "8.9.3")
228219

229220
def test_or(self):
230221
self.assertEqual(find_oldest_supported_version("8.6.0||8.7.0"), "8.6.0")

‎.github/CODEOWNERS‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
/packages/aws/data_stream/route53_resolver_logs @elastic/security-service-integrations
7070
/packages/aws/data_stream/vpcflow @elastic/security-service-integrations
7171
/packages/aws/data_stream/waf @elastic/security-service-integrations
72-
/packages/aws/kibana @elastic/obs-ds-hosted-services @elastic/kibana-visualizations @elastic/obs-infraobs-integrations
72+
/packages/aws/kibana @elastic/obs-ds-hosted-services @elastic/obs-infraobs-integrations
7373
/packages/aws/manifest.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
7474
/packages/aws_bedrock @elastic/security-service-integrations
7575
/packages/aws_logs @elastic/obs-ds-hosted-services
@@ -140,8 +140,15 @@
140140
/packages/cisco_secure_email_gateway @elastic/sec-deployment-and-devices
141141
/packages/cisco_secure_endpoint @elastic/security-service-integrations
142142
/packages/cisco_umbrella @elastic/security-service-integrations
143-
/packages/citrix_adc @elastic/obs-infraobs-integrations
143+
/packages/citrix_adc @elastic/obs-infraobs-integrations @elastic/sec-deployment-and-devices
144+
/packages/citrix_adc/data_stream/interface @elastic/obs-infraobs-integrations
145+
/packages/citrix_adc/data_stream/lbvserver @elastic/obs-infraobs-integrations
146+
/packages/citrix_adc/data_stream/log @elastic/sec-deployment-and-devices
147+
/packages/citrix_adc/data_stream/service @elastic/obs-infraobs-integrations
148+
/packages/citrix_adc/data_stream/system @elastic/obs-infraobs-integrations
149+
/packages/citrix_adc/data_stream/vpn @elastic/obs-infraobs-integrations
144150
/packages/citrix_waf @elastic/sec-deployment-and-devices
151+
/packages/claroty_ctd @elastic/security-service-integrations
145152
/packages/cloud_defend @elastic/sec-linux-platform
146153
/packages/cloud_security_posture @elastic/cloud-security-posture
147154
/packages/cloud_asset_inventory @elastic/cloud-security-posture
@@ -164,6 +171,7 @@
164171
/packages/digital_guardian @elastic/security-service-integrations
165172
/packages/docker @elastic/obs-cloudnative-monitoring
166173
/packages/elastic_agent @elastic/elastic-agent
174+
/packages/elastic_connectors @elastic/search-extract-and-transform
167175
/packages/elastic_package_registry @elastic/ecosystem
168176
/packages/elasticsearch @elastic/stack-monitoring
169177
/packages/enterprisesearch @elastic/stack-monitoring
@@ -177,6 +185,7 @@
177185
/packages/falco @elastic/security-service-integrations
178186
/packages/fim @elastic/sec-linux-platform
179187
/packages/fireeye @elastic/security-service-integrations
188+
/packages/first_epss @elastic/security-service-integrations
180189
/packages/fleet_server @elastic/fleet
181190
/packages/forcepoint_web @elastic/security-service-integrations
182191
/packages/forgerock @elastic/security-service-integrations
@@ -207,6 +216,7 @@
207216
/packages/gcp/data_stream/vpcflow @elastic/security-service-integrations
208217
/packages/gcp_metrics @elastic/obs-ds-hosted-services
209218
/packages/gcp_pubsub @elastic/security-service-integrations
219+
/packages/gcp_vertexai @elastic/obs-infraobs-integrations
210220
/packages/gigamon @elastic/security-service-integrations
211221
/packages/github @elastic/security-service-integrations
212222
/packages/gitlab @elastic/security-service-integrations
@@ -232,6 +242,8 @@
232242
/packages/iptables @elastic/sec-deployment-and-devices
233243
/packages/istio @elastic/obs-cloudnative-monitoring
234244
/packages/jamf_compliance_reporter @elastic/security-service-integrations
245+
/packages/jamf_pro @elastic/security-service-integrations
246+
/packages/jamf_protect @elastic/security-service-integrations
235247
/packages/jolokia_input @elastic/obs-infraobs-integrations
236248
/packages/journald @elastic/elastic-agent-data-plane
237249
/packages/jumpcloud @elastic/security-service-integrations
@@ -243,7 +255,8 @@
243255
/packages/keycloak @elastic/security-service-integrations
244256
/packages/kibana @elastic/stack-monitoring
245257
/packages/kubernetes @elastic/obs-cloudnative-monitoring
246-
/packages/kubernetes/kibana @elastic/obs-cloudnative-monitoring @elastic/kibana-visualizations
258+
/packages/kubernetes/kibana @elastic/obs-cloudnative-monitoring
259+
/packages/kubernetes_otel @elastic/obs-cloudnative-monitoring
247260
/packages/lastpass @elastic/security-service-integrations
248261
/packages/linux @elastic/elastic-agent-data-plane
249262
/packages/lmd @elastic/ml-ui @elastic/sec-applied-ml
@@ -301,6 +314,7 @@
301314
/packages/prometheus_input @elastic/obs-infraobs-integrations
302315
/packages/proofpoint_on_demand @elastic/security-service-integrations
303316
/packages/proofpoint_tap @elastic/security-service-integrations
317+
/packages/proxysg @elastic/sec-deployment-and-devices
304318
/packages/pulse_connect_secure @elastic/security-service-integrations
305319
/packages/qualys_vmdr @elastic/security-service-integrations
306320
/packages/qnap_nas @elastic/sec-deployment-and-devices
@@ -334,6 +348,7 @@
334348
/packages/symantec_endpoint_security @elastic/security-service-integrations
335349
/packages/synthetics @elastic/obs-ux-infra_services-team
336350
/packages/synthetics_dashboards @elastic/obs-ux-infra_services-team
351+
/packages/sysdig @elastic/security-service-integrations
337352
/packages/sysmon_linux @elastic/sec-linux-platform
338353
/packages/system @elastic/obs-infraobs-integrations
339354
/packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
@@ -376,6 +391,7 @@
376391
/packages/ti_otx @elastic/security-service-integrations
377392
/packages/ti_rapid7_threat_command @elastic/security-service-integrations
378393
/packages/ti_recordedfuture @elastic/security-service-integrations
394+
/packages/ti_custom @elastic/security-service-integrations
379395
/packages/ti_threatconnect @elastic/security-service-integrations
380396
/packages/ti_threatq @elastic/security-service-integrations
381397
/packages/ti_util @elastic/security-service-integrations
@@ -386,6 +402,7 @@
386402
/packages/trellix_epo_cloud @elastic/security-service-integrations
387403
/packages/trend_micro_vision_one @elastic/security-service-integrations
388404
/packages/trendmicro @elastic/security-service-integrations
405+
/packages/tychon @elastic/security-service-integrations
389406
/packages/udp @elastic/sec-deployment-and-devices
390407
/packages/universal_profiling_agent @elastic/profiling
391408
/packages/universal_profiling_collector @elastic/obs-ds-intake-services
@@ -417,4 +434,5 @@
417434
/packages/zoom @elastic/security-service-integrations
418435
/packages/zscaler_zia @elastic/security-service-integrations
419436
/packages/zscaler_zpa @elastic/security-service-integrations
420-
/packages/jamf_protect @elastic/security-service-integrations
437+
/packages/cisco_meraki_metrics @elastic/obs-infraobs-integrations
438+
/packages/panw_metrics @elastic/obs-infraobs-integrations

0 commit comments

Comments
 (0)