Skip to content

Commit bcfcc57

Browse files
harnish-crest-dataharnish-crest-data
committed
Merge branch 'main' of https://github.com/milan-elastic/integrations into azure-add-dataset-filters-dashboard
2 parents 88aa002 + 3be250b commit bcfcc57

File tree

1,265 files changed

+179521
-6730
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,265 files changed

+179521
-6730
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
2929
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3030
export REPO_BUILD_TAG
3131

32-
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
32+
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
3333
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
3434

3535
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
@@ -126,19 +126,13 @@ fi
126126

127127
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
128128
if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
129-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
130-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
131-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
132-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
129+
# Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
130+
# add the AWS credentials as in the integrations pipeline.
133131

134132
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
135133
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
136134
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
137135

138-
# Environment variables required by the service deployer
139-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
140-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
141-
142136
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
143137
export BUILDKITE_API_TOKEN
144138

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ steps:
3232
env:
3333
SERVERLESS: "false"
3434
FORCE_CHECK_ALL: "true"
35-
STACK_VERSION: 8.15.0-SNAPSHOT
35+
STACK_VERSION: 8.16.0-SNAPSHOT
3636
PUBLISH_COVERAGE_REPORTS: "true"
3737
depends_on:
3838
- step: "check"

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ env:
2222
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2323
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2424
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
25+
# Enable independent Elastic Agents for all packages
26+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
27+
# Set maximum number of parallel tests to run if package allows it
28+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2529

2630
steps:
2731
- input: "Input values for the variables"

‎.buildkite/pipeline.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ env:
2020
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2121
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2222
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
23+
# Enable independent Elastic Agents for all packages
24+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
25+
# Set maximum number of parallel tests to run if package allows it
26+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2327

2428
steps:
2529
- label: "Get reference from target branch"

‎.buildkite/pull-requests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"pipelineSlug": "integrations",
66
"allow_org_users": true,
77
"allowed_repo_permissions": ["admin", "write"],
8-
"allowed_list": ["dependabot[bot]", "mergify[bot]"],
8+
"allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]"],
99
"set_commit_status": true,
1010
"build_on_commit": true,
1111
"build_on_comment": true,

‎.buildkite/scripts/backport_branch.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default ${DRY_RUN:-"true"})"
1717
BASE_COMMIT="$(buildkite-agent meta-data get BASE_COMMIT --default ${BASE_COMMIT:-""})"
1818
PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default ${PACKAGE_NAME:-""})"
1919
PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default ${PACKAGE_VERSION:-""})"
20-
REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get PREMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
20+
REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get REMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
2121

2222
if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then
2323
buildkite-agent annotate "The variables **PACKAGE_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning"

‎.buildkite/scripts/common.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ is_pr_affected() {
692692

693693
echo "[${package}] git-diff: check non-package files"
694694
commit_merge=$(git merge-base "${from}" "${to}")
695-
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|docs/)' ; then
695+
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
696696
echo "[${package}] PR is affected: found non-package files"
697697
return 0
698698
fi
@@ -819,6 +819,9 @@ test_package_in_serverless() {
819819
if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
820820
return 1
821821
fi
822+
if ! ${ELASTIC_PACKAGE_BIN} test policy ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
823+
return 1
824+
fi
822825
echo ""
823826
return 0
824827
}

‎.buildkite/scripts/find_oldest_supported_version.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,27 @@
22
import argparse
33
import requests
44
import sys
5-
import yaml
65
import unittest
6+
import yaml
77

8-
VERSION_URL = "https://artifacts-api.elastic.co/v1/versions?x-elastic-no-kpi=true"
8+
from requests.adapters import HTTPAdapter, Retry
9+
10+
ARTIFACTS_URL = "https://artifacts-api.elastic.co"
11+
VERSION_URL = ARTIFACTS_URL + "/v1/versions?x-elastic-no-kpi=true"
912

1013

1114
def fetch_version():
12-
return requests.get(VERSION_URL).json()
15+
# Retry forever on connection or 500 errors, assume the artifacts API
16+
# will come back. If it doesn't come back we cannot continue executing
17+
# jobs in any case.
18+
retries = Retry(
19+
total=None,
20+
backoff_factor=0.5,
21+
status_forcelist=[500, 502, 503, 504],
22+
)
23+
session = requests.Session()
24+
session.mount(ARTIFACTS_URL, HTTPAdapter(max_retries=retries))
25+
return session.get(VERSION_URL).json()
1326

1427

1528
def find_oldest_supported_version(kibana_version_condition: str) -> str:

‎.buildkite/scripts/trigger_integrations_in_parallel.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ for package in ${PACKAGE_LIST}; do
5050
- label: "Check integrations ${package}"
5151
key: "test-integrations-${package}"
5252
command: ".buildkite/scripts/test_one_package.sh ${package} ${from} ${to}"
53+
timeout_in_minutes: 240
5354
agents:
5455
provider: gcp
5556
image: ${IMAGE_UBUNTU_X86_64}

‎.github/CODEOWNERS‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
/packages/aws/changelog.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
2828
/packages/aws/data_stream/apigateway_logs @elastic/obs-infraobs-integrations
2929
/packages/aws/data_stream/apigateway_metrics @elastic/obs-infraobs-integrations
30+
/packages/aws/data_stream/awshealth @elastic/obs-infraobs-integrations
3031
/packages/aws/data_stream/billing @elastic/obs-infraobs-integrations
3132
/packages/aws/data_stream/cloudfront_logs @elastic/obs-infraobs-integrations
3233
/packages/aws/data_stream/cloudtrail @elastic/obs-infraobs-integrations
@@ -50,14 +51,22 @@
5051
/packages/aws/data_stream/s3_daily_storage @elastic/obs-ds-hosted-services
5152
/packages/aws/data_stream/s3_request @elastic/obs-ds-hosted-services
5253
/packages/aws/data_stream/s3_storage_lens @elastic/obs-infraobs-integrations
53-
/packages/aws/data_stream/s3access @elastic/obs-ds-hosted-services @elastic/security-service-integrations
54+
/packages/aws/data_stream/s3access @elastic/obs-ds-hosted-services
5455
/packages/aws/data_stream/securityhub_findings @elastic/security-service-integrations
5556
/packages/aws/data_stream/securityhub_insights @elastic/security-service-integrations
5657
/packages/aws/data_stream/sns @elastic/obs-infraobs-integrations
5758
/packages/aws/data_stream/sqs @elastic/obs-infraobs-integrations
5859
/packages/aws/data_stream/transitgateway @elastic/obs-infraobs-integrations
5960
/packages/aws/data_stream/usage @elastic/obs-infraobs-integrations
6061
/packages/aws/data_stream/vpn @elastic/obs-infraobs-integrations
62+
/packages/aws/data_stream/firewall_logs @elastic/security-service-integrations
63+
/packages/aws/data_stream/firewall_metrics @elastic/security-service-integrations
64+
/packages/aws/data_stream/inspector @elastic/security-service-integrations
65+
/packages/aws/data_stream/kafka_metrics @elastic/obs-infraobs-integrations
66+
/packages/aws/data_stream/route53_public_logs @elastic/security-service-integrations
67+
/packages/aws/data_stream/route53_resolver_logs @elastic/security-service-integrations
68+
/packages/aws/data_stream/vpcflow @elastic/security-service-integrations
69+
/packages/aws/data_stream/waf @elastic/security-service-integrations
6170
/packages/aws/kibana @elastic/obs-ds-hosted-services @elastic/kibana-visualizations @elastic/obs-infraobs-integrations
6271
/packages/aws/manifest.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
6372
/packages/aws_bedrock @elastic/security-service-integrations
@@ -75,6 +84,7 @@
7584
/packages/azure/data_stream/springcloudlogs @elastic/obs-infraobs-integrations
7685
/packages/azure/data_stream/application_gateway @elastic/security-service-integrations
7786
/packages/azure/data_stream/firewall_logs @elastic/security-service-integrations
87+
/packages/azure/data_stream/graphactivitylogs @elastic/security-service-integrations
7888
/packages/azure_app_service @elastic/obs-infraobs-integrations
7989
/packages/azure_app_service/data_stream/app_service_logs @elastic/obs-infraobs-integrations
8090
/packages/azure_application_insights @elastic/obs-infraobs-integrations
@@ -274,11 +284,15 @@
274284
/packages/platform_observability @elastic/stack-monitoring
275285
/packages/postgresql @elastic/obs-infraobs-integrations
276286
/packages/pps @elastic/security-service-integrations
287+
/packages/prisma_access @elastic/security-service-integrations
277288
/packages/prisma_cloud @elastic/security-service-integrations
278289
/packages/problemchild @elastic/ml-ui @elastic/sec-applied-ml
279290
/packages/prometheus @elastic/obs-infraobs-integrations
280291
/packages/prometheus/data_stream/remote_write @elastic/obs-cloudnative-monitoring
292+
/packages/prometheus/data_stream/collector @elastic/obs-infraobs-integrations
293+
/packages/prometheus/data_stream/query @elastic/obs-infraobs-integrations
281294
/packages/prometheus_input @elastic/obs-infraobs-integrations
295+
/packages/proofpoint_on_demand @elastic/security-service-integrations
282296
/packages/proofpoint_tap @elastic/security-service-integrations
283297
/packages/pulse_connect_secure @elastic/security-service-integrations
284298
/packages/qualys_vmdr @elastic/security-service-integrations
@@ -314,8 +328,23 @@
314328
/packages/sysmon_linux @elastic/sec-linux-platform
315329
/packages/system @elastic/obs-infraobs-integrations
316330
/packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
317-
/packages/system/data_stream/auth @elastic/sec-windows-platform
318-
/packages/system/data_stream/security @elastic/sec-linux-platform @elastic/sec-windows-platform
331+
/packages/system/data_stream/auth @elastic/sec-linux-platform
332+
/packages/system/data_stream/security @elastic/sec-windows-platform
333+
/packages/system/data_stream/application @elastic/sec-windows-platform
334+
/packages/system/data_stream/core @elastic/obs-infraobs-integrations
335+
/packages/system/data_stream/cpu @elastic/obs-infraobs-integrations
336+
/packages/system/data_stream/diskio @elastic/obs-infraobs-integrations
337+
/packages/system/data_stream/filesystem @elastic/obs-infraobs-integrations
338+
/packages/system/data_stream/fsstat @elastic/obs-infraobs-integrations
339+
/packages/system/data_stream/load @elastic/obs-infraobs-integrations
340+
/packages/system/data_stream/memory @elastic/obs-infraobs-integrations
341+
/packages/system/data_stream/network @elastic/obs-infraobs-integrations
342+
/packages/system/data_stream/process @elastic/obs-infraobs-integrations
343+
/packages/system/data_stream/process_summary @elastic/obs-infraobs-integrations
344+
/packages/system/data_stream/socket_summary @elastic/obs-infraobs-integrations
345+
/packages/system/data_stream/syslog @elastic/obs-infraobs-integrations
346+
/packages/system/data_stream/system @elastic/obs-infraobs-integrations
347+
/packages/system/data_stream/uptime @elastic/obs-infraobs-integrations
319348
/packages/system/manifest.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
320349
/packages/system_audit @elastic/sec-linux-platform
321350
/packages/tanium @elastic/security-service-integrations
@@ -354,6 +383,8 @@
354383
/packages/universal_profiling_symbolizer @elastic/obs-ds-intake-services
355384
/packages/vectra_detect @elastic/security-service-integrations
356385
/packages/vsphere @elastic/obs-infraobs-integrations
386+
/packages/websocket @elastic/security-service-integrations
387+
/packages/watchguard_firebox @elastic/sec-deployment-and-devices
357388
/packages/websphere_application_server @elastic/obs-infraobs-integrations
358389
/packages/windows @elastic/elastic-agent-data-plane @elastic/sec-windows-platform
359390
/packages/windows/data_stream/applocker_exe_and_dll @elastic/sec-windows-platform
@@ -366,6 +397,7 @@
366397
/packages/windows/data_stream/powershell_operational @elastic/sec-windows-platform
367398
/packages/windows/data_stream/service @elastic/elastic-agent-data-plane
368399
/packages/windows/data_stream/sysmon_operational @elastic/sec-windows-platform
400+
/packages/windows/data_stream/windows_defender @elastic/sec-windows-platform
369401
/packages/windows_etw @elastic/sec-windows-platform
370402
/packages/winlog @elastic/sec-windows-platform
371403
/packages/wiz @elastic/security-service-integrations

0 commit comments

Comments
 (0)