|
| 1 | +""" |
| 2 | +This script takes a pytest CSV file produced by pytest --csv foo.csv |
| 3 | +and summarizes it into a more minimal CSV that is good for uploading |
| 4 | +to Google Sheets. We have been using this with dynamic shapes to |
| 5 | +understand how many tests fail when we turn on dynamic shapes. If |
| 6 | +you have a test suite with a lot of skips or xfails, if force the |
| 7 | +tests to run anyway, this can help you understand what the actual |
| 8 | +errors things are failing with are. |
| 9 | +
|
| 10 | +The resulting csv is written to stdout. An easy way to get the csv |
| 11 | +onto your local file system is to send it to GitHub Gist: |
| 12 | +
|
| 13 | + $ python scripts/analysis/format_test_csv.py foo.csv | gh gist create - |
| 14 | +
|
| 15 | +See also scripts/analysis/run_test_csv.sh |
| 16 | +""" |
| 17 | + |
| 18 | +import csv |
| 19 | +import subprocess |
| 20 | +import sys |
| 21 | +import argparse |
| 22 | + |
| 23 | +parser = argparse.ArgumentParser(description=__doc__, |
| 24 | + formatter_class=argparse.RawDescriptionHelpFormatter) |
| 25 | +parser.add_argument("--log-url", type=str, default="", help="URL of raw logs") |
| 26 | +parser.add_argument("file", help="pytest CSV file to format") |
| 27 | +args = parser.parse_args() |
| 28 | + |
| 29 | +out = csv.writer(sys.stdout, dialect="excel") |
| 30 | +hash = subprocess.check_output( |
| 31 | + "git rev-parse HEAD".split(" "), encoding="utf-8" |
| 32 | +).rstrip() |
| 33 | + |
| 34 | +out.writerow([hash, args.log_url, ""]) |
| 35 | + |
| 36 | +with open(args.file, "r") as f: |
| 37 | + reader = csv.DictReader(f) |
| 38 | + for row in reader: |
| 39 | + if row["status"] not in {"failed", "error"}: |
| 40 | + continue |
| 41 | + msg = row["message"].split("\n")[0] |
| 42 | + msg.replace( |
| 43 | + " - erroring out! It's likely that this is caused by data-dependent control flow or similar.", |
| 44 | + "", |
| 45 | + ) |
| 46 | + msg.replace("\t", " ") |
| 47 | + # Feel free to edit this; the idea is to remove prefixes that are |
| 48 | + # just gooping up the resulting spreadsheet outpu |
| 49 | + name = row["name"].replace("test_make_fx_symbolic_exhaustive_", "") |
| 50 | + out.writerow([name, msg, ""]) |
0 commit comments