Skip to content

Commit ff6f176

Browse files
haetamoudihaetamoudi
committed
Merge branch 'main' into handle-multiline-processor
2 parents 9d2902f + 25dfab5 commit ff6f176

118 files changed

Lines changed: 4699 additions & 1022 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/hooks/pre-command‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,13 @@ export SERVERLESS=${SERVERLESS:-"false"}
1010
WORKSPACE=$(pwd)
1111
export WORKSPACE
1212

13-
1413
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
1514
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
1615

1716
EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
1817
EC_DATA_PATH=secret/ci/elastic-elastic-package/ec_data
1918

20-
# variables required for terraform
19+
# variables required for Terraform
2120
export ENVIRONMENT="ci"
2221
REPO=$(repo_name "${BUILDKITE_REPO}")
2322
export REPO
@@ -44,8 +43,6 @@ export BRANCH_NAME_LOWER_CASE
4443
# This variable contains the build number https://buildkite.com/elastic/elastic-package/<number>
4544
export BUILD_ID="${BUILDKITE_BUILD_NUMBER}"
4645
# get current timestamp in milliseconds
47-
# From Jenkins
48-
# CREATED_DATE = "${new Date().getTime()}"
4946
CREATED_DATE=$(date +%s%3N)
5047
export CREATED_DATE
5148

@@ -80,30 +77,6 @@ if is_step_required_to_upload_safe_logs; then
8077
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
8178
fi
8279

83-
is_step_testing_aws () {
84-
if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
85-
return 1
86-
fi
87-
if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-true" ]]; then
88-
return 0
89-
fi
90-
if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-true" ]]; then
91-
return 0
92-
fi
93-
return 1
94-
}
95-
96-
if is_step_testing_aws; then
97-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
98-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
99-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
100-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
101-
102-
# Environment variables required by the service deployer
103-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
104-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
105-
fi
106-
10780
if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "release" ]]; then
10881
GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
10982
export GITHUB_TOKEN

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ env:
66
KIND_VERSION: 'v0.20.0'
77
K8S_VERSION: 'v1.29.0'
88
GH_CLI_VERSION: "2.29.0"
9+
YQ_VERSION: 'v4.44.3'
10+
911
# Agent images used in pipeline steps
1012
UBUNTU_X86_64_AGENT_IMAGE: "family/core-ubuntu-2204"
1113

@@ -14,6 +16,9 @@ env:
1416
ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
1517
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
1618
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
19+
# Enable/Disable the usage of wolfi images for Elastic Agent
20+
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
21+
1722

1823
steps:
1924
- input: "Input values for the variables"

‎.buildkite/pipeline.trigger.integration.tests.sh‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,10 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
8383
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
8484
echo " artifact_paths:"
8585
echo " - build/test-results/*.xml"
86+
echo " - build/test-results/*.xml.expected-errors.txt" # these files are uploaded in case it is needed to review the xUnit files in case of CI reports success the step
8687
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
8788
done
88-
89-
popd > /dev/null
89+
popd > /dev/null
9090

9191
pushd test/packages/parallel > /dev/null
9292
for independent_agent in false true; do
@@ -127,28 +127,38 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
127127
echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
128128
done
129129
done
130-
131130
popd > /dev/null
132131

133-
echo " - label: \":go: Integration test: build-zip\""
132+
# TODO: Missing docker & docker-compose in MACOS ARM agent image, skip installation of packages in the meantime.
133+
# If docker and docker-compose are available for this platform/architecture, it could be added a step to test the stack commands (or even replace this one).
134+
echo " - label: \":macos: :go: Integration test: build-zip\""
134135
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
135136
echo " agents:"
137+
echo " provider: \"orka\""
138+
echo " imagePrefix: \"${MACOS_ARM_AGENT_IMAGE}\""
139+
echo " artifact_paths:"
140+
echo " - build/elastic-stack-dump/build-zip/logs/*.log"
141+
echo " - build/packages/*.sig"
142+
143+
echo " - label: \":go: Integration test: build-install-zip\""
144+
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-build-install-zip"
145+
echo " agents:"
136146
echo " provider: \"gcp\""
137147
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
138148
echo " artifact_paths:"
139149
echo " - build/elastic-stack-dump/build-zip/logs/*.log"
140150
echo " - build/packages/*.sig"
141151

142-
echo " - label: \":go: Integration test: install-zip\""
143-
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip"
152+
echo " - label: \":go: Integration test: build-install-zip-file\""
153+
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-build-install-zip-file"
144154
echo " agents:"
145155
echo " provider: \"gcp\""
146156
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
147157
echo " artifact_paths:"
148158
echo " - build/elastic-stack-dump/install-zip/logs/*.log"
149159

150-
echo " - label: \":go: Integration test: install-zip-shellinit\""
151-
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip-shellinit"
160+
echo " - label: \":go: Integration test: build-install-zip-file-shellinit\""
161+
echo " command: ./.buildkite/scripts/integration_tests.sh -t test-build-install-zip-file-shellinit"
152162
echo " agents:"
153163
echo " provider: \"gcp\""
154164
echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""

‎.buildkite/pipeline.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ env:
1818
ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
1919
# Set maximum number of parallel tests to run if package allows it
2020
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: 3
21+
# Enable/Disable the usage of wolfi images for Elastic Agent
22+
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
23+
2124

2225
steps:
2326
- label: ":go: Run check-static"

‎.buildkite/scripts/integration_tests.sh‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ PARALLEL_TARGET="test-check-packages-parallel"
3131
FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
3232
KIND_TARGET="test-check-packages-with-kind"
3333
SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
34+
TEST_BUILD_ZIP_TARGET="test-build-zip"
35+
3436
GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
3537

3638
REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
@@ -80,18 +82,18 @@ if [[ "$SERVERLESS" == "false" ]]; then
8082
echo "--- install go"
8183
with_go
8284

83-
echo "--- install docker"
84-
with_docker
85+
if [[ "${TARGET}" != "${TEST_BUILD_ZIP_TARGET}" ]]; then
86+
# Not supported in Macos ARM
87+
echo "--- install docker"
88+
with_docker
8589

86-
echo "--- install docker-compose plugin"
87-
with_docker_compose_plugin
90+
echo "--- install docker-compose plugin"
91+
with_docker_compose_plugin
92+
fi
8893
fi
8994

90-
if [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
91-
# just required for now while testing false positive targets
92-
echo "--- install yq"
93-
with_yq
94-
fi
95+
echo "--- install yq"
96+
with_yq
9597

9698
if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
9799
echo "--- install kubectl & kind"

‎.buildkite/scripts/test_packages_with_serverless.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ with_go
3737

3838
echo "--- Install docker"
3939
with_docker
40+
4041
echo "--- Install docker-compose"
4142
with_docker_compose_plugin
4243

‎.github/workflows/bump-elastic-stack-version.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
echo "UPDATECLI_ACTION=apply" >> $GITHUB_ENV
3636
3737
- name: Install Updatecli in the runner
38-
uses: updatecli/updatecli-action@d0950ebbe80f4f80c3392b288d6a218fae872f69 #v2.66.0
38+
uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f #v2.67.0
3939

4040
- name: Update default stack version
4141
# --experimental needed for commitusingapi option.

‎Makefile‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ test-stack-command-86:
7373
./scripts/test-stack-command.sh 8.6.2
7474

7575
test-stack-command-8x:
76-
./scripts/test-stack-command.sh 8.16.0-54ba7abd-SNAPSHOT
76+
./scripts/test-stack-command.sh 8.16.0-a744fd5c-SNAPSHOT
7777

7878
test-stack-command-with-apm-server:
7979
APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh
@@ -109,11 +109,14 @@ test-check-packages-with-custom-agent:
109109
test-build-zip:
110110
./scripts/test-build-zip.sh
111111

112-
test-install-zip:
113-
./scripts/test-install-zip.sh
112+
test-build-install-zip:
113+
./scripts/test-build-install-zip.sh
114114

115-
test-install-zip-shellinit:
116-
./scripts/test-install-zip.sh -s
115+
test-build-install-zip-file:
116+
./scripts/test-build-install-zip-file.sh
117+
118+
test-build-install-zip-file-shellinit:
119+
./scripts/test-build-install-zip-file.sh -s
117120

118121
test-system-test-flags:
119122
./scripts/test-system-test-flags.sh
@@ -124,7 +127,7 @@ test-profiles-command:
124127
test-check-update-version:
125128
./scripts/test-check-update-version.sh
126129

127-
test: test-go test-stack-command test-check-packages test-profiles-command test-build-zip test-check-update-version
130+
test: test-go test-stack-command test-check-packages test-profiles-command test-build-install-zip test-build-zip test-build-install-zip-file test-build-install-zip-file-shellinit test-check-update-version test-profiles-command test-system-test-flags
128131

129132
check-git-clean:
130133
git update-index --really-refresh

‎README.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -694,11 +694,10 @@ There are available some environment variables that could be used to change some
694694
- `ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS`: If set to `true`, the results from pipeline tests are not compared to avoid errors from GeoIP.
695695
- `ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI`: If set to `true`, the Elastic Agent image used for running agents will be using the Ubuntu docker images
696696
(e.g. `docker.elastic.co/elastic-agent/elastic-agent-complete`). If set to `false`, the Elastic Agent image used for the running agents will be based on the wolfi
697-
<<<<<<< HEAD
698-
images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `true`.
699-
=======
700697
images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `false`.
701-
>>>>>>> upstream/main
698+
- `ELASTIC_PACKAGE_TEST_DUMP_SCENARIO_DOCS. If the variable is set, elastic-package will dump to a file the documents generated
699+
by system tests before they are verified. This is useful to know exactly what fields are being verified when investigating
700+
issues on this step. Documents are dumped to a file in the system temporary directory. It is disabled by default.
702701

703702
- To configure the Elastic stack to be used by `elastic-package`:
704703
- `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)

‎docs/howto/system_testing.md‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -612,22 +612,28 @@ Fleet (along with the Agent Policies) and a new Elastic Agent will be created fo
612612
file.
613613

614614
These Elastic Agents can be customized adding the required settings for the tests in the test configuration file.
615-
For example, the `oracle/memory` data stream's [`test-memory-config.yml`](https://github.com/elastic/elastic-package/blob/6338a33c255f8753107f61673245ef352fbac0b6/test/packages/parallel/oracle/data_stream/memory/_dev/test/system/test-memory-config.yml) is shown below:
615+
For example, the `oracle/memory` data stream's [`test-memory-config.yml`](https://github.com/elastic/elastic-package/blob/19b2d35c0d7aea7357ccfc572398f39812ff08bc/test/packages/parallel/oracle/data_stream/memory/_dev/test/system/test-memory-config.yml) is shown below:
616616
```yaml
617617
vars:
618618
hosts:
619619
- "oracle://sys:Oradoc_db1@{{ Hostname }}:{{ Port }}/ORCLCDB.localdomain?sysdba=1"
620620
agent:
621621
runtime: docker
622622
provisioning_script:
623-
language: "bash"
623+
language: "sh"
624624
contents: |
625-
apt-get update && apt-get -y install libaio1 wget unzip
625+
set -eu
626+
if grep wolfi /etc/os-release > /dev/null ; then
627+
apk update && apk add libaio wget unzip
628+
else
629+
apt-get update && apt-get -y install libaio1 wget unzip
630+
fi
626631
mkdir -p /opt/oracle
627632
cd /opt/oracle
628-
wget https://download.oracle.com/otn_software/linux/instantclient/214000/instantclient-basic-linux.x64-21.4.0.0.0dbru.zip && unzip -o instantclient-basic-linux.x64-21.4.0.0.0dbru.zip
629-
wget https://download.oracle.com/otn_software/linux/instantclient/217000/instantclient-sqlplus-linux.x64-21.7.0.0.0dbru.zip && unzip -o instantclient-sqlplus-linux.x64-21.7.0.0.0dbru.zip
630-
echo /opt/oracle/instantclient_21_4 > /etc/ld.so.conf.d/oracle-instantclient.conf && ldconfig
633+
wget https://download.oracle.com/otn_software/linux/instantclient/214000/instantclient-basic-linux.x64-21.4.0.0.0dbru.zip && unzip -o instantclient-basic-linux.x64-21.4.0.0.0dbru.zip || exit 1
634+
wget https://download.oracle.com/otn_software/linux/instantclient/217000/instantclient-sqlplus-linux.x64-21.7.0.0.0dbru.zip && unzip -o instantclient-sqlplus-linux.x64-21.7.0.0.0dbru.zip || exit 1
635+
mkdir -p /etc/ld.so.conf.d/
636+
echo /opt/oracle/instantclient_21_4 > /etc/ld.so.conf.d/oracle-instantclient.conf && ldconfig || exit 1
631637
cp /opt/oracle/instantclient_21_7/glogin.sql /opt/oracle/instantclient_21_7/libsqlplus.so /opt/oracle/instantclient_21_7/libsqlplusic.so /opt/oracle/instantclient_21_7/sqlplus /opt/oracle/instantclient_21_4/
632638
pre_start_script:
633639
language: "sh"
@@ -637,6 +643,9 @@ agent:
637643
cd /opt/oracle/instantclient_21_4
638644
```
639645

646+
**IMPORTANT**: The provisioning script must exit with a code different from zero in case any of the commands defined fails.
647+
That will ensure that the docker build step run by `elastic-package` fails too.
648+
640649
Another example setting capabilities to the agent ([`auditid_manager` test package](https://github.com/elastic/elastic-package/blob/6338a33c255f8753107f61673245ef352fbac0b6/test/packages/parallel/auditd_manager/data_stream/auditd/_dev/test/system/test-default-config.yml)):
641650
```yaml
642651
data_stream:

0 commit comments

Comments
 (0)