Skip to content

Commit a808309

Browse files
donoghucyaauie
donoghuc
authored and
yaauie
committed
Add a step to exhaustive tests for observabilitySRE accetpance testing (#17623)
* Add a step to exhaustive tests for observabilitySRE accetpance testing This commit shows the proposed pattern for adding acceptance testing for the observability SRE image. This will run when exhaustive tests run. A new gradle task will hook in to rspec similar to how it is done for the smoke tests. The main difference is that instead of building a container, the latest is pulled from the container registry and run on a fips configured host VM. * WIP: Idea for how to handle multipe container configs for acceptance tests This commit shows the rough structure for how I am planning on handling docker compose networks for acceptance tests. The main idea is to use interpolation in the docker compose file to point to different configuration files for filebeat/logstash/elasticsearch. This is mainly due to the nature of these tests showing behavior when the system is and is not configured properly for FIPS. The breakdown in responsibility is: 1. Gradle handles cert generation (similar to smoke test, this avoids checking in PKI) 2. Rspec handles stopping/starting docker compose and managing environment vars for intperolation in docker compose manifests (different from smoke tests where a single static docker compose is started in gradle) 3. Rspec handles deciding when containers are ready and querying state about data flowing through the system 4. Gradle cleans up certs THis is just a rough sketch, there are still bugs to be worked out but before i get too far in to it I want to get the idea out there. * Add tests describing behavior of LS -> ES with non-fips config This commit adds a test to show that data will not flow from LS to ES when weak non fips config is used. * Use latest ES image This will be handled separately in a separate PR, but taking this commit for now on this branch. * Remove custom entrypoint from new container The latest ES images do not require this workaround. * Take up code review suggestions 1. Remove rogue character from test file causing interpreter failure 2. Split out helpers for docker compose orchestration 3. Only send a single message instead of infinite through to ES * Add full prefix name for new image * Test filebeat -> LS -> ES using fips config As described in elastic/ingest-dev#5471 this commit adds a test for filebeat sending data through logstash to elasticsearch using fips config. * Test LS wont accept input from non fips configured filebeat This test ensures logstash will not accept data from filebeat when using weak tls configuration. See elastic/ingest-dev#5472 * Fix a funny typo. Crytpo is actually kind of a funny. * Ensure we are using the purpose build ES image in testing Similar to #17627 * Ensure JAVA_HOME is set etc Use the same buildkite agent script for setting up a vm based runner as other pipes
1 parent 0236ba4 commit a808309

16 files changed

Lines changed: 483 additions & 1 deletion

File tree

‎.buildkite/scripts/exhaustive-tests/generate-steps.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,26 @@ def acceptance_docker_steps()-> list[typing.Any]:
168168

169169
return steps
170170

171+
def fips_test_runner_step() -> dict[str, typing.Any]:
172+
step = {
173+
"label": "Observability SRE Acceptance Tests",
174+
"key": "observabilitySRE-acceptance-tests",
175+
"agents": {
176+
"provider": "aws",
177+
"instanceType": "m6i.xlarge",
178+
"diskSizeGb": 60,
179+
"instanceMaxAge": 1440,
180+
"imagePrefix": "platform-ingest-logstash-ubuntu-2204-fips"
181+
},
182+
"retry": {"automatic": [{"limit": 1}]},
183+
"command": LiteralScalarString("""#!/usr/bin/env bash
184+
set -euo pipefail
185+
source .buildkite/scripts/common/vm-agent.sh
186+
./gradlew observabilitySREacceptanceTests --stacktrace
187+
"""),
188+
}
189+
return step
190+
171191
if __name__ == "__main__":
172192
LINUX_OS_ENV_VAR_OVERRIDE = os.getenv("LINUX_OS")
173193
WINDOWS_OS_ENV_VAR_OVERRIDE = os.getenv("WINDOWS_OS")
@@ -215,5 +235,12 @@ def acceptance_docker_steps()-> list[typing.Any]:
215235
"steps": acceptance_docker_steps(),
216236
})
217237

238+
structure["steps"].append({
239+
"group": "Observability SRE Acceptance Tests",
240+
"key": "acceptance-observability-sre",
241+
"depends_on": ["testing-phase"],
242+
"steps": [fips_test_runner_step()],
243+
})
244+
218245
print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
219246
YAML().dump(structure, sys.stdout)

‎x-pack/build.gradle‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ tasks.register("buildFipsValidationGem") {
7575
rake(rootProject.projectDir, rootProject.buildDir, 'plugin:build-fips-validation-plugin')
7676
}
7777
}
78+
7879
tasks.register("observabilitySREsmokeTests", Test) {
7980
description = "Run ObservabilitySRE smoke tests using docker-compose and RSpec"
8081
// Need to have set up the ruby environment for rspec even through we are running in container
@@ -114,3 +115,25 @@ tasks.register("observabilitySREsmokeTests", Test) {
114115
delete fileTree("distributions/internal/observabilitySRE/qa/smoke/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
115116
}
116117
}
118+
119+
tasks.register("observabilitySREacceptanceTests", Test) {
120+
description = "Run ObservabilitySRE acceptance tests"
121+
// Need to have set up the ruby environment for rspec even through we are running in container
122+
dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
123+
124+
inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
125+
doFirst {
126+
// Generate the certificates first
127+
exec {
128+
workingDir file("distributions/internal/observabilitySRE/qa/acceptance/docker/certs")
129+
commandLine 'bash', './generate.sh'
130+
ignoreExitValue = false
131+
}
132+
}
133+
systemProperty 'logstash.root.dir', projectDir.parent
134+
include '**/org/logstash/xpack/test/RSpecObservabilitySREAcceptanceTests.class'
135+
doLast {
136+
// Clean up the generated certificates
137+
delete fileTree("distributions/internal/observabilitySRE/qa/acceptance/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
138+
}
139+
}

‎x-pack/distributions/internal/observabilitySRE/plugin/logstash-integration-fips_validation/lib/logstash/fips_validation.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def before_bootstrap_checks(runner)
4242
# ensure Bouncycastle is configured and ready
4343
begin
4444
if Java::org.bouncycastle.crypto.CryptoServicesRegistrar.isInApprovedOnlyMode
45-
accumulator.success "Bouncycastle Crytpo is in `approved-only` mode"
45+
accumulator.success "Bouncycastle Crypto is in `approved-only` mode"
4646
else
4747
accumulator.failure "Bouncycastle Crypto is not in 'approved-only' mode"
4848
end

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*.crt
2+
*.csr
3+
*.key

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/certs/generate.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
echo "Generating CA certificate"
4+
openssl req -x509 -newkey rsa:3072 -days 365 -nodes -keyout ca.key -out ca.crt -subj "/CN=Elastic-CA" -sha256
5+
6+
echo "Generating Elasticsearch certificate"
7+
openssl req -newkey rsa:3072 -nodes -keyout elasticsearch.key -out elasticsearch.csr -subj "/CN=elasticsearch" -sha256
8+
openssl x509 -req -in elasticsearch.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out elasticsearch.crt -days 365 -sha256
9+
10+
echo "Generating Logstash certificate"
11+
openssl req -newkey rsa:3072 -nodes -keyout logstash.key -out logstash.csr -subj "/CN=logstash" -sha256
12+
openssl x509 -req -in logstash.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out logstash.crt -days 365 -sha256
13+
14+
echo "Generating Filebeat certificate"
15+
openssl req -newkey rsa:3072 -nodes -keyout filebeat.key -out filebeat.csr -subj "/CN=filebeat" -sha256
16+
openssl x509 -req -in filebeat.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out filebeat.crt -days 365 -sha256
17+
18+
chmod 644 *.crt *.key

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/docker-compose.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
services:
2+
logstash:
3+
image: docker.elastic.co/logstash/logstash-observability-sre:${OBSERVABILITY_SRE_IMAGE_VERSION:-8.19.0-SNAPSHOT}
4+
container_name: fips_test_logstash
5+
ports:
6+
- "5044:5044"
7+
volumes:
8+
- ./logstash/config/${LOGSTASH_CONFIG:-logstash-fips.yml}:/usr/share/logstash/config/logstash.yml
9+
- ./logstash/pipeline/${LOGSTASH_PIPELINE:-logstash-to-elasticsearch.conf}:/usr/share/logstash/pipeline/logstash.conf
10+
- ./certs:/usr/share/logstash/config/certs
11+
networks:
12+
- elastic
13+
depends_on:
14+
- elasticsearch
15+
16+
elasticsearch:
17+
image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
18+
container_name: fips_test_elasticsearch
19+
ports:
20+
- "9200:9200"
21+
volumes:
22+
- ./elasticsearch/config/${ELASTICSEARCH_CONFIG:-elasticsearch-fips.yml}:/usr/share/elasticsearch/config/elasticsearch.yml
23+
- ./certs:/usr/share/elasticsearch/config/certs
24+
environment:
25+
- discovery.type=single-node
26+
- ES_JAVA_OPTS=-Xms1g -Xmx1g
27+
- ELASTIC_PASSWORD=changeme
28+
networks:
29+
- elastic
30+
31+
filebeat:
32+
# The filebeat shipped with the elasticsearch-fips container is built for FIPS support
33+
# There is no stand alone distribution. This uses the shipped version for testing.
34+
image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${FILEBEAT_IMAGE_VERSION:-8.19.0-SNAPSHOT}
35+
container_name: fips_test_filebeat
36+
working_dir: /usr/share/filebeat
37+
entrypoint: ["/bin/bash", "-c"]
38+
# Start Filebeat with /tmp for data (always writable)
39+
command:
40+
- |
41+
exec /opt/filebeat/filebeat -e \
42+
--strict.perms=false \
43+
-c /usr/share/filebeat/filebeat.yml \
44+
--path.data /tmp/filebeat_data
45+
volumes:
46+
- ./filebeat/config/${FILEBEAT_CONFIG:-filebeat-fips.yml}:/usr/share/filebeat/filebeat.yml:ro
47+
- ./filebeat/data:/data/logs:ro
48+
- ./certs:/usr/share/filebeat/certs:ro
49+
networks:
50+
- elastic
51+
depends_on:
52+
- logstash
53+
54+
networks:
55+
elastic:
56+
driver: bridge

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/elasticsearch/config/elasticsearch-fips.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Elasticsearch settings
2+
discovery.type: single-node
3+
http.port: 9200
4+
network.host: 0.0.0.0
5+
# Security settings
6+
xpack.security.enabled: true
7+
xpack.security.transport.ssl.enabled: true
8+
xpack.security.transport.ssl.verification_mode: certificate
9+
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
10+
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
11+
xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
12+
xpack.security.http.ssl.enabled: true
13+
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
14+
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
15+
xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/config/filebeat-fips.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
filebeat.inputs:
2+
- type: log
3+
enabled: true
4+
paths:
5+
- /data/logs/sample_logs.txt
6+
7+
output.logstash:
8+
hosts: ["logstash:5044"]
9+
ssl.enabled: true
10+
ssl.certificate: "/usr/share/filebeat/certs/filebeat.crt"
11+
ssl.key: "/usr/share/filebeat/certs/filebeat.key"
12+
ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca.crt"]
13+
ssl.verification_mode: "certificate"
14+
15+
# Add debugging
16+
logging.level: debug
17+
logging.to_stderr: true
18+
19+
# Keep registry in the anonymous volume to avoid host pollution
20+
path.data: /tmp/filebeat_data

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/data/sample_logs.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
TEST-LOG: FIPS filebeat test message

‎x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/logstash/config/logstash-fips.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
api.http.host: "0.0.0.0"
2+
xpack.monitoring.enabled: false
3+
4+
pipeline.ordered: false
5+
pipeline.workers: 2
6+
pipeline.buffer.type: heap

0 commit comments

Comments
 (0)