Skip to content

Commit 99ee51a

Browse files
committed
Add test_report command for rebuilding reports from structured JSON.
Refactor Galaxy testing abstractions toward better exception handling (print more, fail harder, try more - if one report type fails try the others but still throw an exception at the end) and toward greater reuse without being a full test context (i.e. try to be able to do more from just json).
1 parent 7572e99 commit 99ee51a

File tree

9 files changed

+346
-48
lines changed

9 files changed

+346
-48
lines changed

planemo/commands/cmd_share_test.py

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import click
44

55
from planemo.cli import pass_context
6+
from planemo import options
67
from planemo.io import info
78
from planemo import github_util
89

@@ -11,20 +12,9 @@
1112
"?test_data_url=%s"
1213
)
1314

14-
target_path = click.Path(
15-
file_okay=True,
16-
dir_okay=False,
17-
resolve_path=True,
18-
)
19-
2015

2116
@click.command("share_test")
22-
@click.argument(
23-
'path',
24-
metavar="FILE_PATH",
25-
type=target_path,
26-
default="tool_test_output.json",
27-
)
17+
@options.tool_test_json()
2818
@pass_context
2919
def cli(ctx, path, **kwds):
3020
"""Publish JSON test results to Github Gist and produce sharable URL.

planemo/commands/cmd_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
)
2525
@options.galaxy_target_options()
2626
@options.galaxy_config_options()
27-
@options.test_options()
27+
@options.test_report_options()
2828
@pass_context
2929
def cli(ctx, paths, **kwds):
3030
"""Run the tests in the specified tool tests in a Galaxy instance.
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import os
2+
3+
import click
4+
5+
from planemo.cli import pass_context
6+
from planemo import io
7+
from planemo import options
8+
from planemo.galaxy_test import StructuredData, handle_test_reports
9+
10+
11+
@click.command('test_reports')
12+
@options.tool_test_json()
13+
@options.test_report_options()
14+
@pass_context
15+
def cli(ctx, path, **kwds):
16+
"""Generate various tool test reports (HTML, text, markdown) from
17+
structure output from tests (tool_test_output.json).
18+
"""
19+
if not os.path.exists(path):
20+
io.error("Failed to tool test json file at %s" % path)
21+
return 1
22+
23+
test_data = StructuredData(path)
24+
handle_test_reports(ctx, test_data, **kwds)

planemo/galaxy_test/__init__.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
from .actions import run_in_config
2+
from .structures import StructuredData
3+
from .actions import handle_test_reports
24

3-
__all__ = ["run_in_config"]
5+
6+
__all__ = ["run_in_config", "StructuredData", "handle_test_reports"]

planemo/galaxy_test/actions.py

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -83,27 +83,8 @@ def run_in_config(ctx, config, **kwds):
8383
return_code,
8484
)
8585

86-
try:
87-
test_data = test_results.structured_data
88-
89-
if 'test_output' in kwds:
90-
output_path = kwds['test_output']
91-
if output_path is not None:
92-
with open(output_path, 'w') as handle:
93-
handle.write(build_report.build_report(test_data))
94-
95-
for kw_name in ('markdown', 'text'):
96-
if 'test_output_%s' % kw_name in kwds:
97-
output_path = kwds['test_output_%s' % kw_name]
98-
if output_path is None:
99-
continue
100-
101-
with open(output_path, 'w') as handle:
102-
handle.write(build_report.build_report(test_data, report_type=kw_name))
103-
104-
except Exception:
105-
ctx.vlog("Problem producing test output.", exception=True)
106-
86+
test_data = test_results.structured_data
87+
handle_test_reports(ctx, test_data, **kwds)
10788
__handle_summary(
10889
test_results,
10990
**kwds
@@ -112,6 +93,54 @@ def run_in_config(ctx, config, **kwds):
11293
return return_code
11394

11495

96+
def handle_test_reports(ctx, test_data, **kwds):
97+
exceptions = []
98+
for report_type in ["html", "markdown", "text"]:
99+
try:
100+
_handle_test_output_file(
101+
ctx, report_type, test_data, **kwds
102+
)
103+
except Exception as e:
104+
exceptions.append(e)
105+
continue
106+
107+
if len(exceptions) > 0:
108+
raise exceptions[0]
109+
110+
111+
def _handle_test_output_file(ctx, report_type, test_data, **kwds):
112+
kwd_name = "test_output"
113+
if report_type != "html":
114+
kwd_name = "test_output_%s" % report_type
115+
116+
path = kwds.get(kwd_name, None)
117+
if path is None:
118+
message = "No file specified for %s, skipping test output." % kwd_name
119+
ctx.vlog(message)
120+
return
121+
122+
try:
123+
contents = build_report.build_report(
124+
report_type, report_type=report_type
125+
)
126+
except Exception:
127+
message = "Problem producing report file %s for %s" % (
128+
path, kwd_name
129+
)
130+
ctx.vlog(message, exception=True)
131+
raise
132+
133+
try:
134+
with open(path, 'w') as handle:
135+
handle.write(contents)
136+
except Exception:
137+
message = "Problem writing output file %s for %s" % (
138+
kwd_name, path
139+
)
140+
ctx.vlog(message, exception=True)
141+
raise
142+
143+
115144
def __handle_summary(
116145
test_results,
117146
**kwds

planemo/galaxy_test/structures.py

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,15 @@ def __init__(self, json_path):
7676
self.structured_data_by_id = structured_data_by_id
7777
self.has_details = "summary" in structured_data
7878
if self.has_details:
79-
self._read_summary()
79+
self.read_summary()
8080

8181
def update(self):
8282
with open(self.json_path, "w") as out_f:
8383
json.dump(self.structured_data, out_f)
8484

85+
def set_exit_code(self, exit_code):
86+
self.structured_data["exit_code"] = exit_code
87+
8588
def merge_xunit(self, xunit_root):
8689
self.has_details = True
8790
xunit_attrib = xunit_root.attrib
@@ -97,8 +100,6 @@ def merge_xunit(self, xunit_root):
97100
)
98101

99102
self.structured_data["summary"] = summary
100-
self.num_tests = num_tests
101-
self.num_problems = num_skips + num_errors + num_failures
102103

103104
for testcase_el in xunit_t_elements_from_root(xunit_root):
104105
test = case_id(testcase_el)
@@ -118,9 +119,17 @@ def merge_xunit(self, xunit_root):
118119
status = "success"
119120
test_data["status"] = status
120121

121-
def _read_summary(self):
122-
# TODO: read relevant data out of summary object.
123-
pass
122+
def read_summary(self):
123+
summary = self.structured_data["summary"]
124+
num_tests = summary["num_tests"]
125+
num_failures = summary["num_failures"]
126+
num_skips = summary["num_skips"]
127+
num_errors = summary["num_errors"]
128+
129+
self.num_tests = num_tests
130+
self.num_problems = num_skips + num_errors + num_failures
131+
132+
self.exit_code = summary["exit_code"]
124133

125134
@property
126135
def failed_ids(self):
@@ -159,8 +168,14 @@ def __init__(
159168
sd.merge_xunit(self._xunit_root)
160169
else:
161170
self.xunit_tree = ET.fromstring("<testsuite />")
171+
self.sd.set_exit_code(exit_code)
172+
self.sd.read_summary()
162173
self.sd.update()
163174

175+
@property
176+
def exit_code(self):
177+
return self.sd.exit_code
178+
164179
@property
165180
def has_details(self):
166181
return self.sd.has_details

planemo/options.py

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -616,14 +616,22 @@ def recursive_option(help="Recursively perform command for subdirectories."):
616616
)
617617

618618

619-
def test_options():
619+
def tool_test_json():
620+
target_path = click.Path(
621+
file_okay=True,
622+
dir_okay=False,
623+
resolve_path=True,
624+
)
625+
return click.argument(
626+
'path',
627+
metavar="FILE_PATH",
628+
type=target_path,
629+
default="tool_test_output.json",
630+
)
631+
632+
633+
def test_report_options():
620634
return _compose(
621-
click.option(
622-
"--update_test_data",
623-
is_flag=True,
624-
help="Update test-data directory with job outputs (normally"
625-
" written to directory --job_output_files if specified.)"
626-
),
627635
click.option(
628636
"--test_output",
629637
type=click.Path(file_okay=True, resolve_path=True),
@@ -648,6 +656,18 @@ def test_options():
648656
"computers)"),
649657
default=None,
650658
),
659+
)
660+
661+
662+
def test_options():
663+
return _compose(
664+
click.option(
665+
"--update_test_data",
666+
is_flag=True,
667+
help="Update test-data directory with job outputs (normally"
668+
" written to directory --job_output_files if specified.)"
669+
),
670+
test_report_options(),
651671
click.option(
652672
"--test_output_xunit",
653673
type=click.Path(file_okay=True, resolve_path=True),

0 commit comments

Comments
 (0)