Skip to content

Commit ffaac9a

Browse files
mashhursmashhurs
committed
Health Report integration tests bootstrapper and initial slow start scenario implementation.
1 parent 2b8c473 commit ffaac9a

12 files changed

Lines changed: 402 additions & 0 deletions

File tree

‎.buildkite/scripts/health-report-tests/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
## Description
2+
This package for integration tests of the Health Report API.
3+
Export `LS_VERSION` (major and minor version such as 8.x) to run on a specific branch. By default, it uses the main branch.
4+
5+
## How to run the Health Report Integration test?
6+
### Prerequisites
7+
Make sure you have python installed. Install the integration test dependencies with the following command:
8+
```shell
9+
python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
10+
```
11+
12+
### Run the integration tests
13+
```shell
14+
python3 .buildkite/scripts/health-report-tests/main.py
15+
```

‎.buildkite/scripts/health-report-tests/__init__.py‎

Whitespace-only changes.

‎.buildkite/scripts/health-report-tests/bootstrap.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
"""
2+
Health Report Integration test bootstrapper with Python script
3+
- A script to resolve Logstash version if not provided
4+
- Download LS docker image and spin up
5+
- When tests finished, teardown the Logstash
6+
"""
7+
import os
8+
import subprocess
9+
import util
10+
import yaml
11+
12+
13+
class Bootstrap:
14+
ELASTIC_STACK_VERSIONS_URL = "https://artifacts-api.elastic.co/v1/versions"
15+
16+
def __init__(self) -> None:
17+
f"""
18+
A constructor of the {Bootstrap}.
19+
Returns:
20+
Resolves Logstash branch considering provided LS_VERSION
21+
Checks out git branch
22+
"""
23+
logstash_version = os.environ.get("LS_VERSION")
24+
if logstash_version is None:
25+
# version is not specified, use the main branch, no need to git checkout
26+
print(f"LS_VERSION is not specified, using main branch.")
27+
else:
28+
# LS_VERSION accepts major latest as a major.x or specific version as X.Y
29+
if logstash_version.find(".x") == -1:
30+
print(f"Using specified branch: {logstash_version}")
31+
util.git_check_out_branch(logstash_version)
32+
else:
33+
major_version = logstash_version.split(".")[0]
34+
if major_version and major_version.isnumeric():
35+
resolved_version = self.__resolve_latest_stack_version_for(major_version)
36+
minor_version = resolved_version.split(".")[1]
37+
branch = major_version + "." + minor_version
38+
print(f"Using resolved branch: {branch}")
39+
util.git_check_out_branch(branch)
40+
else:
41+
raise ValueError(f"Invalid value set to LS_VERSION. Please set it properly (ex: 8.x or 9.0) and "
42+
f"rerun again")
43+
44+
def __resolve_latest_stack_version_for(self, major_version: str) -> None:
45+
resolved_version = ""
46+
response = util.call_url_with_retry(self.ELASTIC_STACK_VERSIONS_URL)
47+
release_versions = response.json()["versions"]
48+
for release_version in reversed(release_versions):
49+
if release_version.find("SNAPSHOT") > 0:
50+
continue
51+
if release_version.split(".")[0] == major_version:
52+
print(f"Resolved latest version for {major_version} is {release_version}.")
53+
resolved_version = release_version
54+
break
55+
56+
if resolved_version == "":
57+
raise ValueError(f"Cannot resolve latest version for {major_version} major")
58+
return resolved_version
59+
60+
def install_plugin(self, plugin_path: str) -> None:
61+
util.run_or_raise_error(
62+
["bin/logstash-plugin", "install", plugin_path],
63+
f"Failed to install {plugin_path}")
64+
65+
def build_logstash(self):
66+
print(f"Building Logstash.")
67+
util.run_or_raise_error(
68+
["./gradlew", "clean", "bootstrap", "assemble", "installDefaultGems"],
69+
"Failed to build Logstash")
70+
print(f"Logstash has successfully built.")
71+
72+
def apply_config(self, config: dict) -> None:
73+
with open(os.getcwd() + "/config/pipelines.yml", 'w') as pipelines_file:
74+
yaml.dump(config, pipelines_file)
75+
76+
def run_logstash(self) -> subprocess.Popen:
77+
process = subprocess.Popen(["bin/logstash"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
78+
if process.poll() is not None:
79+
print(f"Logstash failed to run, check the the config and logs, then rerun.")
80+
return None
81+
82+
# Read stdout and stderr in real-time
83+
logs = []
84+
for stdout_line in iter(process.stdout.readline, ""):
85+
# print("STDOUT:", stdout_line.strip())
86+
logs.append(stdout_line.strip())
87+
if "Starting pipeline" in stdout_line:
88+
break
89+
if "Logstash shut down" in stdout_line:
90+
print(f"Logstash couldn't spin up.")
91+
print(logs)
92+
return None
93+
94+
print(f"Logstash is running with PID: {process.pid}.")
95+
return process
96+
97+
def stop_logstash(self, process: subprocess.Popen) -> None:
98+
process.terminate()
99+
print(f"Stopping Logstash...")

‎.buildkite/scripts/health-report-tests/config_validator.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import yaml
2+
from typing import Any, List, Dict, Union
3+
4+
5+
class ConfigValidator:
6+
7+
REQUIRED_KEYS: Dict[str, List[str]] = {
8+
"config": ["pipeline.id", "config.string"],
9+
"expectation": ["status", "symptom", "diagnosis", "impacts", "details"],
10+
"diagnosis": ["cause"],
11+
"impacts": ["description", "impact_areas"],
12+
"details": ["run_state"],
13+
}
14+
15+
def __init__(self):
16+
self.yaml_content = None
17+
18+
def __validate_keys(self, actual_keys: List[str], expected_keys: List[str], section: str) -> bool:
19+
"""Validate the keys at the current level."""
20+
missing_keys = set(expected_keys) - set(actual_keys)
21+
if len(missing_keys) == len(expected_keys):
22+
print(f"Missing keys in {section}: {missing_keys}")
23+
return False
24+
return True
25+
26+
def __validate_config(self, config_list: List[Dict[str, Any]]) -> bool:
27+
"""Validate the 'config' section."""
28+
for config_item in config_list:
29+
if not self.__validate_keys(list(config_item.keys()), self.REQUIRED_KEYS["config"], "config"):
30+
return False
31+
return True
32+
33+
def __validate_expectation(self, expectation_list: List[Dict[str, Any]]) -> bool:
34+
"""Validate the 'expectation' section."""
35+
for expectation_item in expectation_list:
36+
if not self.__validate_keys(list(expectation_item.keys()), self.REQUIRED_KEYS["expectation"], "expectation"):
37+
return False
38+
if "diagnosis" in expectation_item:
39+
for diagnosis in expectation_item["diagnosis"]:
40+
if not self.__validate_keys(list(diagnosis.keys()), self.REQUIRED_KEYS["diagnosis"], "diagnosis"):
41+
return False
42+
if "impacts" in expectation_item:
43+
for impact in expectation_item["impacts"]:
44+
if not self.__validate_keys(list(impact.keys()), self.REQUIRED_KEYS["impacts"], "impacts"):
45+
return False
46+
if "details" in expectation_item:
47+
for detail in expectation_item["details"]:
48+
if not self.__validate_keys(list(detail.keys()), self.REQUIRED_KEYS["details"], "details"):
49+
return False
50+
return True
51+
52+
def load(self, file_path: str) -> None:
53+
"""Load the YAML file content into self.yaml_content."""
54+
self.yaml_content: Union[List[Dict[str, Any]], None] = None
55+
try:
56+
with open(file_path, 'r') as file:
57+
self.yaml_content = yaml.safe_load(file)
58+
except yaml.YAMLError as exc:
59+
print(f"Error in YAML file: {exc}")
60+
self.yaml_content = None
61+
62+
def is_valid(self) -> bool:
63+
"""Validate the entire YAML structure."""
64+
if self.yaml_content is None:
65+
print(f"YAML content is empty.")
66+
return False
67+
68+
if not isinstance(self.yaml_content, list):
69+
print(f"YAML structure is not as expected, it should start with a list.")
70+
return False
71+
72+
for item in self.yaml_content:
73+
if "config" in item and not self.__validate_config(item["config"]):
74+
return False
75+
76+
if "expectation" in item and not self.__validate_expectation(item["expectation"]):
77+
return False
78+
79+
print(f"YAML file validation successful!")
80+
return True

‎.buildkite/scripts/health-report-tests/logstash_health_report.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""
2+
A class to provide information about Logstash node stats.
3+
"""
4+
5+
import util
6+
7+
8+
class LogstashHealthReport:
9+
LOGSTASH_HEALTH_REPORT_URL = "http://localhost:9600/_health_report"
10+
11+
def __init__(self):
12+
pass
13+
14+
def get(self):
15+
response = util.call_url_with_retry(self.LOGSTASH_HEALTH_REPORT_URL)
16+
return response.json()

‎.buildkite/scripts/health-report-tests/logstash_stats.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""
2+
A class to provide information about Logstash node stats.
3+
"""
4+
5+
import util
6+
7+
8+
class LogstashStats:
9+
LOGSTASH_STATS_URL = "http://localhost:9600/_node/stats"
10+
11+
def __init__(self):
12+
pass
13+
14+
def get(self):
15+
response = util.call_url_with_retry(self.LOGSTASH_STATS_URL)
16+
return response.json()

‎.buildkite/scripts/health-report-tests/main.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"""
2+
Main entry point of the LS health report API integration test suites
3+
"""
4+
import glob
5+
import os
6+
from bootstrap import Bootstrap
7+
from scenario_executor import ScenarioExecutor
8+
from config_validator import ConfigValidator
9+
import yaml
10+
import util
11+
12+
13+
class BootstrapContextManager:
14+
15+
def __init__(self):
16+
pass
17+
18+
def __enter__(self):
19+
print(f"Starting Logstash Health Report Integration test.")
20+
self.bootstrap = Bootstrap()
21+
# self.bootstrap.build_logstash()
22+
23+
plugin_path = os.getcwd() + "/qa/support/logstash-integration-failure_injector/logstash-integration" \
24+
"-failure_injector-*.gem"
25+
matching_files = glob.glob(plugin_path)
26+
if len(matching_files) == 0:
27+
raise ValueError(f"Could not find logstash-integration-failure_injector plugin.")
28+
29+
# self.bootstrap.install_plugin(matching_files[0])
30+
print(f"logstash-integration-failure_injector successfully installed.")
31+
return self.bootstrap
32+
33+
def apply_config(self, bootstrap: Bootstrap, config: str):
34+
bootstrap.apply_config(config)
35+
36+
def __exit__(self, exc_type, exc_value, traceback):
37+
if exc_type is not None:
38+
traceback.print_exception(exc_type, exc_value, traceback)
39+
40+
41+
def main():
42+
with BootstrapContextManager() as bootstrap:
43+
scenario_executor = ScenarioExecutor()
44+
config_validator = ConfigValidator()
45+
46+
working_dir = os.getcwd()
47+
scenario_files_path = working_dir + "/.buildkite/scripts/health-report-tests/tests/*.yaml"
48+
scenario_files = glob.glob(scenario_files_path)
49+
50+
for scenario_file in scenario_files:
51+
print(f"Validating {scenario_file} scenario file.")
52+
config_validator.load(scenario_file)
53+
if not config_validator.is_valid():
54+
print(f"{scenario_file} scenario file is not valid.")
55+
return
56+
57+
for scenario_file in scenario_files:
58+
with open(scenario_file, 'r') as file:
59+
# scenario_content: Union[List[Dict[str, Any]], None] = None
60+
scenario_content = yaml.safe_load(file)
61+
scenario_name = util.get_element_of_array(scenario_content, 'name')
62+
config = util.get_element_of_array(scenario_content, 'config')
63+
if config is not None:
64+
bootstrap.apply_config(config)
65+
expectation = util.get_element_of_array(scenario_content, 'expectation')
66+
process = bootstrap.run_logstash()
67+
if process is not None:
68+
scenario_executor.on(scenario_name, expectation)
69+
process.terminate()
70+
break
71+
72+
73+
if __name__ == "__main__":
74+
main()

‎.buildkite/scripts/health-report-tests/main.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env bash
2+
set -eo pipefail
3+
4+
# Install prerequisites and run integration tests
5+
python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
6+
python3 .buildkite/scripts/health-report-tests/main.py

‎.buildkite/scripts/health-report-tests/requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
requests==2.32.3
2+
deepdiff==8.0.1
3+
pyyaml==6.0.2

‎.buildkite/scripts/health-report-tests/scenario_executor.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"""
2+
A class to execute the given scenario for Logstash Health Report integration test
3+
"""
4+
from deepdiff import DeepDiff
5+
from logstash_health_report import LogstashHealthReport
6+
7+
8+
class ScenarioExecutor:
9+
logstash_health_report_api = LogstashHealthReport()
10+
11+
def __init__(self):
12+
pass
13+
14+
def __meets_expectation(self, scenario_content: list) -> None:
15+
logstash_health = self.logstash_health_report_api.get()
16+
print(f"Logstash health report: {logstash_health}")
17+
18+
differences = []
19+
for index, item in enumerate(scenario_content):
20+
if "expectation" in item:
21+
key = f"Item {index + 1}"
22+
stat_value = logstash_health.get(key, {}).get("expectation")
23+
24+
if stat_value:
25+
diff = DeepDiff(item["expectation"], stat_value, ignore_order=True).to_dict()
26+
if diff:
27+
differences.append({key: diff})
28+
else:
29+
print(f"Stats do not contain an 'expectation' entry for {key}")
30+
31+
if differences:
32+
print("Differences found in 'expectation' section between YAML content and stats:")
33+
for diff in differences:
34+
print(diff)
35+
return False
36+
else:
37+
print("YAML 'expectation' section matches the stats.")
38+
return True
39+
40+
def on(self, scenario_name: str, scenario_content: list) -> None:
41+
print(f"Testing the scenario: {scenario_content}")
42+
if self.__meets_expectation(scenario_content) is False:
43+
raise Exception(f"{scenario_name} failed.")

0 commit comments

Comments
 (0)