Skip to content

Commit 4d29bf1

Browse files
jmchiltonjmchilton
committed
Rework Galaxy testing to use structured data instead of XUnit data.
This switches planemo in a subtle way from to depending more on the sturctured data json than the XUnit - this will be benefical for non-Galaxy tool testing when XUnit will be a product of the reporting stuff and not an input to it. This refactoring/reworking should allow most of planemo/galaxy/test/actions to be used with the newer planemo-driven testing stuff. Adds tests and doc strings.
1 parent e38c436 commit 4d29bf1

File tree

13 files changed

+561
-417
lines changed

13 files changed

+561
-417
lines changed

‎Makefile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,17 @@ quick-test: ## run quickest tests with the default Python
7070
tox: ## run tests with tox in the specified ENV, defaults to py27
7171
$(IN_VENV) tox -e $(ENV) -- $(ARGS)
7272

73-
coverage: ## check code coverage quickly with the default Python
73+
_coverage-report: ## build coverage report with the default Python
7474
coverage run --source $(SOURCE_DIR) setup.py $(TEST_DIR)
7575
coverage report -m
7676
coverage html
77+
78+
_open-coverage: ## open coverage report using open
7779
open htmlcov/index.html || xdg-open htmlcov/index.html
7880

79-
ready-docs:
81+
coverage: _coverage-report open-coverage ## check code coverage quickly with the default Python
82+
83+
ready-docs: ## rebuild docs folder ahead of running docs or lint-docs
8084
rm -f docs/$(SOURCE_DIR).rst
8185
rm -f docs/planemo_ext.rst
8286
rm -f docs/modules.rst

‎planemo/galaxy/test/__init__.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
1+
"""Entry point and interface for ``planemo.galaxy.test`` package."""
12
from .actions import run_in_config
23
from .structures import StructuredData
34
from .actions import handle_reports
5+
from .actions import handle_reports_and_summary
46

57

6-
__all__ = ["run_in_config", "StructuredData", "handle_reports"]
8+
__all__ = [
9+
"handle_reports",
10+
"handle_reports_and_summary",
11+
"run_in_config",
12+
"StructuredData",
13+
]

‎planemo/galaxy/test/actions.py‎

Lines changed: 83 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@
55

66
from . import structures as test_structures
77
from planemo.io import error, info, warn, shell_join
8+
from planemo.exit_codes import (
9+
EXIT_CODE_OK,
10+
EXIT_CODE_GENERIC_FAILURE,
11+
EXIT_CODE_NO_SUCH_TARGET,
12+
)
813
from planemo.galaxy.run import (
914
run_galaxy_command,
1015
setup_venv,
@@ -91,31 +96,52 @@ def run_in_config(ctx, config, run=run_galaxy_command, **kwds):
9196
shell('cp -r "%s"/* "%s"' % update_cp_args)
9297

9398
_check_test_outputs(xunit_report_file_tracker, structured_report_file_tracker)
94-
9599
test_results = test_structures.GalaxyTestResults(
96100
structured_report_file,
97101
xunit_report_file,
98102
html_report_file,
99103
return_code,
100104
)
101105

102-
test_data = test_results.structured_data
103-
handle_reports(ctx, test_data, kwds)
104-
_handle_summary(
105-
test_results,
106-
**kwds
106+
structured_data = test_results.structured_data
107+
return handle_reports_and_summary(
108+
ctx,
109+
structured_data,
110+
exit_code=test_results.exit_code,
111+
kwds=kwds
107112
)
108113

109-
return return_code
114+
115+
def handle_reports_and_summary(ctx, structured_data, exit_code=None, kwds={}):
116+
"""Produce reports and print summary, return 0 if tests passed.
117+
118+
If ``exit_code`` is set - use underlying test source for return
119+
code and test success determination, otherwise infer from supplied
120+
test data.
121+
"""
122+
handle_reports(ctx, structured_data, kwds)
123+
summary_exit_code = _handle_summary(
124+
structured_data,
125+
**kwds
126+
)
127+
return exit_code if exit_code is not None else summary_exit_code
110128

111129

112-
def handle_reports(ctx, test_data, kwds):
130+
def handle_reports(ctx, structured_data, kwds):
113131
"""Write reports based on user specified kwds."""
114132
exceptions = []
133+
structured_report_file = kwds.get("test_output_json", None)
134+
if structured_report_file and not os.path.exists(structured_report_file):
135+
try:
136+
with open(structured_report_file, "w") as f:
137+
json.dump(structured_report_file, f)
138+
except Exception as e:
139+
exceptions.append(e)
140+
115141
for report_type in ["html", "markdown", "text"]:
116142
try:
117143
_handle_test_output_file(
118-
ctx, report_type, test_data, kwds
144+
ctx, report_type, structured_data, kwds
119145
)
120146
except Exception as e:
121147
exceptions.append(e)
@@ -159,48 +185,48 @@ def _handle_test_output_file(ctx, report_type, test_data, kwds):
159185

160186

161187
def _handle_summary(
162-
test_results,
188+
structured_data,
163189
**kwds
164190
):
191+
summary_dict = _get_dict_value("summary", structured_data)
192+
num_tests = _get_dict_value("num_tests", summary_dict)
193+
num_failures = _get_dict_value("num_failures", summary_dict)
194+
num_errors = _get_dict_value("num_errors", summary_dict)
195+
num_problems = num_failures + num_errors
196+
197+
summary_exit_code = EXIT_CODE_OK
198+
if num_problems > 0:
199+
summary_exit_code = EXIT_CODE_GENERIC_FAILURE
200+
elif num_tests == 0:
201+
summary_exit_code = EXIT_CODE_NO_SUCH_TARGET
202+
165203
summary_style = kwds.get("summary")
166-
if summary_style == "none":
167-
return
204+
if summary_style != "none":
205+
if num_tests == 0:
206+
warn(NO_TESTS_MESSAGE)
207+
elif num_problems == 0:
208+
info(ALL_TESTS_PASSED_MESSAGE % num_tests)
209+
elif num_problems:
210+
html_report_file = kwds.get("test_output")
211+
message_args = (num_problems, num_tests, html_report_file)
212+
message = PROBLEM_COUNT_MESSAGE % message_args
213+
warn(message)
168214

169-
if test_results.has_details:
170215
_summarize_tests_full(
171-
test_results,
216+
structured_data,
172217
**kwds
173218
)
174-
else:
175-
if test_results.exit_code:
176-
warn(GENERIC_PROBLEMS_MESSAGE % test_results.output_html_path)
177-
else:
178-
info(GENERIC_TESTS_PASSED_MESSAGE)
219+
220+
return summary_exit_code
179221

180222

181223
def _summarize_tests_full(
182-
test_results,
224+
structured_data,
183225
**kwds
184226
):
185-
num_tests = test_results.num_tests
186-
num_problems = test_results.num_problems
187-
188-
if num_tests == 0:
189-
warn(NO_TESTS_MESSAGE)
190-
return
191-
192-
if num_problems == 0:
193-
info(ALL_TESTS_PASSED_MESSAGE % num_tests)
194-
195-
if num_problems:
196-
html_report_file = test_results.output_html_path
197-
message_args = (num_problems, num_tests, html_report_file)
198-
message = PROBLEM_COUNT_MESSAGE % message_args
199-
warn(message)
200-
201-
for testcase_el in test_results.xunit_testcase_elements:
202-
structured_data_tests = test_results.structured_data_tests
203-
_summarize_test_case(structured_data_tests, testcase_el, **kwds)
227+
tests = _get_dict_value("tests", structured_data)
228+
for test_case_data in tests:
229+
_summarize_test_case(test_case_data, **kwds)
204230

205231

206232
def passed(xunit_testcase_el):
@@ -211,10 +237,16 @@ def passed(xunit_testcase_el):
211237
return did_pass
212238

213239

214-
def _summarize_test_case(structured_data, testcase_el, **kwds):
240+
def _summarize_test_case(structured_data, **kwds):
215241
summary_style = kwds.get("summary")
216-
test_id = test_structures.case_id(testcase_el)
217-
if not passed(testcase_el):
242+
test_id = test_structures.case_id(
243+
raw_id=_get_dict_value("id", structured_data)
244+
)
245+
status = _get_dict_value(
246+
"status",
247+
_get_dict_value("data", structured_data)
248+
)
249+
if status != "success":
218250
state = click.style("failed", bold=True, fg='red')
219251
else:
220252
state = click.style("passed", bold=True, fg='green')
@@ -223,14 +255,7 @@ def _summarize_test_case(structured_data, testcase_el, **kwds):
223255
_print_command_line(structured_data, test_id)
224256

225257

226-
def _print_command_line(structured_data, test_id):
227-
try:
228-
test = [d for d in structured_data if d["id"] == test_id.id][0]["data"]
229-
except (KeyError, IndexError):
230-
# Failed to find structured data for this test - likely targetting
231-
# and older Galaxy version.
232-
return
233-
258+
def _print_command_line(test, test_id):
234259
execution_problem = test.get("execution_problem", None)
235260
if execution_problem:
236261
click.echo("| command: *could not execute job, no command generated* ")
@@ -245,6 +270,13 @@ def _print_command_line(structured_data, test_id):
245270
click.echo("| command: %s" % command)
246271

247272

273+
def _get_dict_value(key, data):
274+
try:
275+
return data[key]
276+
except (KeyError, TypeError):
277+
raise KeyError("No key [%s] in [%s]" % (key, data))
278+
279+
248280
def _check_test_outputs(
249281
xunit_report_file_tracker,
250282
structured_report_file_tracker
@@ -321,4 +353,5 @@ def changed(self):
321353
__all__ = [
322354
"run_in_config",
323355
"handle_reports",
356+
"handle_reports_and_summary",
324357
]

‎planemo/galaxy/test/structures.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -172,11 +172,9 @@ def __init__(
172172
self.structured_data_tests = sd.structured_data_tests
173173
self.structured_data_by_id = sd.structured_data_by_id
174174

175-
if output_xml_path:
176-
self.xunit_tree = parse_xunit_report(output_xml_path)
177-
sd.merge_xunit(self._xunit_root)
178-
else:
179-
self.xunit_tree = ET.fromstring("<testsuite />")
175+
self.xunit_tree = parse_xunit_report(output_xml_path)
176+
sd.merge_xunit(self._xunit_root)
177+
180178
self.sd.set_exit_code(exit_code)
181179
self.sd.read_summary()
182180
self.sd.update()
@@ -223,13 +221,15 @@ def find_cases(xunit_root):
223221
return xunit_root.findall("testcase")
224222

225223

226-
def case_id(testcase_el):
227-
name_raw = testcase_el.attrib["name"]
228-
if "TestForTool_" in name_raw:
229-
raw_id = name_raw
230-
else:
231-
class_name = testcase_el.attrib["classname"]
232-
raw_id = "{0}.{1}".format(class_name, name_raw)
224+
def case_id(testcase_el=None, raw_id=None):
225+
if raw_id is None:
226+
assert testcase_el is not None
227+
name_raw = testcase_el.attrib["name"]
228+
if "TestForTool_" in name_raw:
229+
raw_id = name_raw
230+
else:
231+
class_name = testcase_el.attrib["classname"]
232+
raw_id = "{0}.{1}".format(class_name, name_raw)
233233

234234
name = None
235235
num = None

‎planemo/io.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ def _echo(message, err=False):
7272

7373

7474
def shell_join(*args):
75+
"""Join potentially empty commands together with ;."""
7576
return "; ".join([c for c in args if c])
7677

7778

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ def get_var(var_name):
4646
'planemo.reports',
4747
'planemo.shed',
4848
'planemo.shed2tap',
49+
'planemo.test',
4950
'planemo.xml',
5051
]
5152
ENTRY_POINTS = '''

0 commit comments

Comments
 (0)