Skip to content

Commit b044504

Browse files
yaauieyaauie
authored
Merge branch 'feature/health-report-api' into yaauie-8x-to-feature-branch
2 parents c2c62fd + a13d251 commit b044504

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2955
-19
lines changed

‎.buildkite/scripts/health-report-tests/README.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
## Description
2+
This package for integration tests of the Health Report API.
3+
Export `LS_BRANCH` to run on a specific branch. By default, it uses the main branch.
4+
5+
## How to run the Health Report Integration test?
6+
### Prerequisites
7+
Make sure you have python installed. Install the integration test dependencies with the following command:
8+
```shell
9+
python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
10+
```
11+
12+
### Run the integration tests
13+
```shell
14+
python3 .buildkite/scripts/health-report-tests/main.py
15+
```
16+
17+
### Troubleshooting
18+
- If you get `WARNING: pip is configured with locations that require TLS/SSL,...` warning message, make sure you have python >=3.12.4 installed.

‎.buildkite/scripts/health-report-tests/__init__.py‎

Whitespace-only changes.

‎.buildkite/scripts/health-report-tests/bootstrap.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
"""
2+
Health Report Integration test bootstrapper with Python script
3+
- A script to resolve Logstash version if not provided
4+
- Download LS docker image and spin up
5+
- When tests finished, teardown the Logstash
6+
"""
7+
import os
8+
import subprocess
9+
import util
10+
import yaml
11+
12+
13+
class Bootstrap:
14+
ELASTIC_STACK_VERSIONS_URL = "https://artifacts-api.elastic.co/v1/versions"
15+
16+
def __init__(self) -> None:
17+
f"""
18+
A constructor of the {Bootstrap}.
19+
Returns:
20+
Resolves Logstash branch considering provided LS_BRANCH
21+
Checks out git branch
22+
"""
23+
logstash_branch = os.environ.get("LS_BRANCH")
24+
if logstash_branch is None:
25+
# version is not specified, use the main branch, no need to git checkout
26+
print(f"LS_BRANCH is not specified, using main branch.")
27+
else:
28+
# LS_BRANCH accepts major latest as a major.x or specific branch as X.Y
29+
if logstash_branch.find(".x") == -1:
30+
print(f"Using specified branch: {logstash_branch}")
31+
util.git_check_out_branch(logstash_branch)
32+
else:
33+
major_version = logstash_branch.split(".")[0]
34+
if major_version and major_version.isnumeric():
35+
resolved_version = self.__resolve_latest_stack_version_for(major_version)
36+
minor_version = resolved_version.split(".")[1]
37+
branch = major_version + "." + minor_version
38+
print(f"Using resolved branch: {branch}")
39+
util.git_check_out_branch(branch)
40+
else:
41+
raise ValueError(f"Invalid value set to LS_BRANCH. Please set it properly (ex: 8.x or 9.0) and "
42+
f"rerun again")
43+
44+
def __resolve_latest_stack_version_for(self, major_version: str) -> str:
45+
resolved_version = ""
46+
response = util.call_url_with_retry(self.ELASTIC_STACK_VERSIONS_URL)
47+
release_versions = response.json()["versions"]
48+
for release_version in reversed(release_versions):
49+
if release_version.find("SNAPSHOT") > 0:
50+
continue
51+
if release_version.split(".")[0] == major_version:
52+
print(f"Resolved latest version for {major_version} is {release_version}.")
53+
resolved_version = release_version
54+
break
55+
56+
if resolved_version == "":
57+
raise ValueError(f"Cannot resolve latest version for {major_version} major")
58+
return resolved_version
59+
60+
def install_plugin(self, plugin_path: str) -> None:
61+
util.run_or_raise_error(
62+
["bin/logstash-plugin", "install", plugin_path],
63+
f"Failed to install {plugin_path}")
64+
65+
def build_logstash(self):
66+
print(f"Building Logstash.")
67+
util.run_or_raise_error(
68+
["./gradlew", "clean", "bootstrap", "assemble", "installDefaultGems"],
69+
"Failed to build Logstash")
70+
print(f"Logstash has successfully built.")
71+
72+
def apply_config(self, config: dict) -> None:
73+
with open(os.getcwd() + "/.buildkite/scripts/health-report-tests/config/pipelines.yml", 'w') as pipelines_file:
74+
yaml.dump(config, pipelines_file)
75+
76+
def run_logstash(self, full_start_required: bool) -> subprocess.Popen:
77+
# --config.reload.automatic is to make instance active
78+
# it is helpful when testing crash pipeline cases
79+
config_path = os.getcwd() + "/.buildkite/scripts/health-report-tests/config"
80+
process = subprocess.Popen(["bin/logstash", "--config.reload.automatic", "--path.settings", config_path,
81+
"-w 1"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=False)
82+
if process.poll() is not None:
83+
print(f"Logstash failed to run, check the the config and logs, then rerun.")
84+
return None
85+
86+
# Read stdout and stderr in real-time
87+
logs = []
88+
for stdout_line in iter(process.stdout.readline, ""):
89+
logs.append(stdout_line.strip())
90+
# we don't wait for Logstash fully start as we also test slow pipeline start scenarios
91+
if full_start_required is False and "Starting pipeline" in stdout_line:
92+
break
93+
if full_start_required is True and "Pipeline started" in stdout_line:
94+
break
95+
if "Logstash shut down" in stdout_line or "Logstash stopped" in stdout_line:
96+
print(f"Logstash couldn't spin up.")
97+
print(logs)
98+
return None
99+
100+
print(f"Logstash is running with PID: {process.pid}.")
101+
return process

‎.buildkite/scripts/health-report-tests/config/pipelines.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Intentionally left blank

‎.buildkite/scripts/health-report-tests/config_validator.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import yaml
2+
from typing import Any, List, Dict
3+
4+
5+
class ConfigValidator:
6+
REQUIRED_KEYS = {
7+
"root": ["name", "config", "conditions", "expectation"],
8+
"config": ["pipeline.id", "config.string"],
9+
"conditions": ["full_start_required"],
10+
"expectation": ["status", "symptom", "indicators"],
11+
"indicators": ["pipelines"],
12+
"pipelines": ["status", "symptom", "indicators"],
13+
"DYNAMIC": ["status", "symptom", "diagnosis", "impacts", "details"],
14+
"details": ["status"],
15+
"status": ["state"]
16+
}
17+
18+
def __init__(self):
19+
self.yaml_content = None
20+
21+
def __has_valid_keys(self, data: any, key_path: str, repeated: bool) -> bool:
22+
if isinstance(data, str) or isinstance(data, bool): # we reached values
23+
return True
24+
25+
# we have two indicators section and for the next repeated ones, we go deeper
26+
first_key = next(iter(data))
27+
data = data[first_key] if repeated and key_path == "indicators" else data
28+
29+
if isinstance(data, dict):
30+
# pipeline-id is a DYNAMIC
31+
required = self.REQUIRED_KEYS.get("DYNAMIC" if repeated and key_path == "indicators" else key_path, [])
32+
repeated = not repeated if key_path == "indicators" else repeated
33+
for key in required:
34+
if key not in data:
35+
print(f"Missing key '{key}' in '{key_path}'")
36+
return False
37+
else:
38+
dic_keys_result = self.__has_valid_keys(data[key], key, repeated)
39+
if dic_keys_result is False:
40+
return False
41+
elif isinstance(data, list):
42+
for item in data:
43+
list_keys_result = self.__has_valid_keys(item, key_path, repeated)
44+
if list_keys_result is False:
45+
return False
46+
return True
47+
48+
def load(self, file_path: str) -> None:
49+
"""Load the YAML file content into self.yaml_content."""
50+
self.yaml_content: [Dict[str, Any]] = None
51+
try:
52+
with open(file_path, 'r') as file:
53+
self.yaml_content = yaml.safe_load(file)
54+
except yaml.YAMLError as exc:
55+
print(f"Error in YAML file: {exc}")
56+
self.yaml_content = None
57+
58+
def is_valid(self) -> bool:
59+
"""Validate the entire YAML structure."""
60+
if self.yaml_content is None:
61+
print(f"YAML content is empty.")
62+
return False
63+
64+
if not isinstance(self.yaml_content, dict):
65+
print(f"YAML structure is not as expected, it should start with a Dict.")
66+
return False
67+
68+
result = self.__has_valid_keys(self.yaml_content, "root", False)
69+
return True if result is True else False

‎.buildkite/scripts/health-report-tests/logstash_health_report.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""
2+
A class to provide information about Logstash node stats.
3+
"""
4+
5+
import util
6+
7+
8+
class LogstashHealthReport:
9+
LOGSTASH_HEALTH_REPORT_URL = "http://localhost:9600/_health_report"
10+
11+
def __init__(self):
12+
pass
13+
14+
def get(self):
15+
response = util.call_url_with_retry(self.LOGSTASH_HEALTH_REPORT_URL)
16+
return response.json()

‎.buildkite/scripts/health-report-tests/main.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
"""
2+
Main entry point of the LS health report API integration test suites
3+
"""
4+
import glob
5+
import os
6+
import time
7+
import traceback
8+
import yaml
9+
from bootstrap import Bootstrap
10+
from scenario_executor import ScenarioExecutor
11+
from config_validator import ConfigValidator
12+
13+
14+
class BootstrapContextManager:
15+
16+
def __init__(self):
17+
pass
18+
19+
def __enter__(self):
20+
print(f"Starting Logstash Health Report Integration test.")
21+
self.bootstrap = Bootstrap()
22+
self.bootstrap.build_logstash()
23+
24+
plugin_path = os.getcwd() + "/qa/support/logstash-integration-failure_injector/logstash-integration" \
25+
"-failure_injector-*.gem"
26+
matching_files = glob.glob(plugin_path)
27+
if len(matching_files) == 0:
28+
raise ValueError(f"Could not find logstash-integration-failure_injector plugin.")
29+
30+
self.bootstrap.install_plugin(matching_files[0])
31+
print(f"logstash-integration-failure_injector successfully installed.")
32+
return self.bootstrap
33+
34+
def __exit__(self, exc_type, exc_value, exc_traceback):
35+
if exc_type is not None:
36+
print(traceback.format_exception(exc_type, exc_value, exc_traceback))
37+
38+
39+
def main():
40+
with BootstrapContextManager() as bootstrap:
41+
scenario_executor = ScenarioExecutor()
42+
config_validator = ConfigValidator()
43+
44+
working_dir = os.getcwd()
45+
scenario_files_path = working_dir + "/.buildkite/scripts/health-report-tests/tests/*.yaml"
46+
scenario_files = glob.glob(scenario_files_path)
47+
48+
for scenario_file in scenario_files:
49+
print(f"Validating {scenario_file} scenario file.")
50+
config_validator.load(scenario_file)
51+
if config_validator.is_valid() is False:
52+
print(f"{scenario_file} scenario file is not valid.")
53+
return
54+
else:
55+
print(f"Validation succeeded.")
56+
57+
has_failed_scenario = False
58+
for scenario_file in scenario_files:
59+
with open(scenario_file, 'r') as file:
60+
# scenario_content: Dict[str, Any] = None
61+
scenario_content = yaml.safe_load(file)
62+
print(f"Testing `{scenario_content.get('name')}` scenario.")
63+
scenario_name = scenario_content['name']
64+
65+
is_full_start_required = next(sub.get('full_start_required') for sub in
66+
scenario_content.get('conditions') if 'full_start_required' in sub)
67+
config = scenario_content['config']
68+
if config is not None:
69+
bootstrap.apply_config(config)
70+
expectations = scenario_content.get("expectation")
71+
process = bootstrap.run_logstash(is_full_start_required)
72+
if process is not None:
73+
try:
74+
scenario_executor.on(scenario_name, expectations)
75+
except Exception as e:
76+
print(e)
77+
has_failed_scenario = True
78+
process.terminate()
79+
time.sleep(5) # leave some window to terminate the process
80+
81+
if has_failed_scenario:
82+
# intentionally fail due to visibility
83+
raise Exception("Some of scenarios failed, check the log for details.")
84+
85+
86+
if __name__ == "__main__":
87+
main()

‎.buildkite/scripts/health-report-tests/main.sh‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
#!/usr/bin/env bash
22
set -eo pipefail
33

4-
# TODO:
5-
# if branch is specified with X.Y, pull branches from ACTIVE_BRANCHES_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/branches.json", parse and use
6-
# build Logstash from specificed (ex: 8.x -> translates to 8.latest, 8.16) branch, defaults to main
7-
# install requirements of the python package and run main.py
8-
9-
4+
python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
5+
python3 .buildkite/scripts/health-report-tests/main.py

‎.buildkite/scripts/health-report-tests/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
requests==2.32.3
2+
pyyaml==6.0.2

‎.buildkite/scripts/health-report-tests/scenario_executor.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""
2+
A class to execute the given scenario for Logstash Health Report integration test
3+
"""
4+
import time
5+
from logstash_health_report import LogstashHealthReport
6+
7+
8+
class ScenarioExecutor:
9+
logstash_health_report_api = LogstashHealthReport()
10+
11+
def __init__(self):
12+
pass
13+
14+
def __has_intersection(self, expects, results):
15+
# we expect expects to be existing in results
16+
for expect in expects:
17+
for result in results:
18+
if result.get('help_url') and "health-report-pipeline-status.html#" not in result.get('help_url'):
19+
return False
20+
if not all(key in result and result[key] == value for key, value in expect.items()):
21+
return False
22+
return True
23+
24+
def __get_difference(self, differences: list, expectations: dict, reports: dict) -> dict:
25+
for key in expectations.keys():
26+
27+
if type(expectations.get(key)) != type(reports.get(key)):
28+
differences.append(f"Scenario expectation and Health API report structure differs for {key}.")
29+
return differences
30+
31+
if isinstance(expectations.get(key), str):
32+
if expectations.get(key) != reports.get(key):
33+
differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
34+
continue
35+
elif isinstance(expectations.get(key), dict):
36+
self.__get_difference(differences, expectations.get(key), reports.get(key))
37+
elif isinstance(expectations.get(key), list):
38+
if not self.__has_intersection(expectations.get(key), reports.get(key)):
39+
differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
40+
return differences
41+
42+
def __is_expected(self, expectations: dict) -> None:
43+
reports = self.logstash_health_report_api.get()
44+
differences = self.__get_difference([], expectations, reports)
45+
if differences:
46+
print("Differences found in 'expectation' section between YAML content and stats:")
47+
for diff in differences:
48+
print(f"Difference: {diff}")
49+
return False
50+
else:
51+
return True
52+
53+
def on(self, scenario_name: str, expectations: dict) -> None:
54+
# retriable check the expectations
55+
attempts = 5
56+
while self.__is_expected(expectations) is False:
57+
attempts = attempts - 1
58+
if attempts == 0:
59+
break
60+
time.sleep(1)
61+
62+
if attempts == 0:
63+
raise Exception(f"{scenario_name} failed.")
64+
else:
65+
print(f"Scenario `{scenario_name}` expectaion meets the health report stats.")

0 commit comments

Comments
 (0)