Skip to content

[Feature] Add message of test failures to output #4

@siegfriedpammer

Description

@siegfriedpammer

For all failed tests the message should be added to the output. The junit XML format contains the following information for failures:

    <testcase classname="ICSharpCode.Decompiler.Tests.PrettyTestRunner" name="LiftedOperators(UseRoslynLatest)" time="0.7345250">
      <failure type="failure" message="  ...&#xD;&#xA; 821  604     &#x9;&#x9;}&#xD;&#xA; 822  605     &#x9;&#x9;public static long ReturnChainWithComputation(int? a, short? b, long? c, byte d)&#xD;&#xA; 823  606     &#x9;&#x9;{&#xD;&#xA; 824          &#x9;&#x9;&#x9;//TODO: unnecessary casts&#xD;&#xA; 825          &#x9;&#x9;&#x9;//return (a + 1) ?? (b + 2) ?? (c + 3) ?? (d + 4);&#xD;&#xA; 826  607     &#x9;&#x9;&#x9;return 0L;&#xD;&#xA; 827  608     &#x9;&#x9;}&#xD;&#xA; 828  609     &#x9;}&#xD;&#xA; 829          &#x9;// dummy structure for testing custom operators&#xD;&#xA; 830  610     &#x9;[StructLayout(LayoutKind.Sequential, Size = 1)]&#xD;&#xA; 831      (-) &#x9;public struct TS&#xD;&#xA;      611 (+) &#x9;public record struct TS&#xD;&#xA; 832  612     &#x9;{&#xD;&#xA; 833          &#x9;&#x9;// unary&#xD;&#xA; 834  613     &#x9;&#x9;public static TS operator +(TS a)&#xD;&#xA; 835  614     &#x9;&#x9;{&#xD;&#xA; 836  615     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 837  616     &#x9;&#x9;}&#xD;&#xA; 838  617     &#x9;&#x9;public static TS operator -(TS a)&#xD;&#xA; 839  618     &#x9;&#x9;{&#xD;&#xA; 840  619     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 841  620     &#x9;&#x9;}&#xD;&#xA;  ...&#xD;&#xA; 894  672     &#x9;&#x9;}&#xD;&#xA; 895  673     &#x9;&#x9;public static TS operator &lt;&lt;(TS a, int b)&#xD;&#xA; 896  674     &#x9;&#x9;{&#xD;&#xA; 897  675     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 898  676     &#x9;&#x9;}&#xD;&#xA; 899  677     &#x9;&#x9;public static TS operator &gt;&gt;(TS a, int b)&#xD;&#xA; 900  678     &#x9;&#x9;{&#xD;&#xA; 901  679     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 902  680     &#x9;&#x9;}&#xD;&#xA; 903          &#x9;&#x9;// comparisons&#xD;&#xA; 904       -  &#x9;&#x9;public static bool operator ==(TS a, TS b)&#xD;&#xA; 905       -  &#x9;&#x9;{&#xD;&#xA; 906       -  &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 907       -  &#x9;&#x9;}&#xD;&#xA; 908       -  &#x9;&#x9;public static bool operator !=(TS a, TS b)&#xD;&#xA; 909       -  &#x9;&#x9;{&#xD;&#xA; 910       -  &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 911       -  &#x9;&#x9;}&#xD;&#xA; 912  681     &#x9;&#x9;public static bool operator &lt;(TS a, TS b)&#xD;&#xA; 913  682     &#x9;&#x9;{&#xD;&#xA; 914  683     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 915  684     &#x9;&#x9;}&#xD;&#xA; 916  685     &#x9;&#x9;public static bool operator &lt;=(TS a, TS b)&#xD;&#xA; 917  686     &#x9;&#x9;{&#xD;&#xA; 918  687     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 919  688     &#x9;&#x9;}&#xD;&#xA; 920  689     &#x9;&#x9;public static bool operator &gt;(TS a, TS b)&#xD;&#xA; 921  690     &#x9;&#x9;{&#xD;&#xA; 922  691     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 923  692     &#x9;&#x9;}&#xD;&#xA; 924  693     &#x9;&#x9;public static bool operator &gt;=(TS a, TS b)&#xD;&#xA; 925       -  &#x9;&#x9;{&#xD;&#xA; 926       -  &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 927       -  &#x9;&#x9;}&#xD;&#xA; 928       -  &#x9;&#x9;public override bool Equals(object obj)&#xD;&#xA; 929  694     &#x9;&#x9;{&#xD;&#xA; 930  695     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 931  696     &#x9;&#x9;}&#xD;&#xA; 932  697     &#x9;&#x9;public override int GetHashCode()&#xD;&#xA; 933  698     &#x9;&#x9;{&#xD;&#xA; 934  699     &#x9;&#x9;&#x9;throw null;&#xD;&#xA; 935  700     &#x9;&#x9;}&#xD;&#xA; 936  701     &#x9;}&#xD;&#xA; 937  702     }&#xD;&#xA;">at ICSharpCode.Decompiler.Tests.Helpers.CodeAssert.AreEqual(String input1, String input2, String[] definedSymbols) in D:\a\ILSpy\ILSpy\ICSharpCode.Decompiler.Tests\Helpers\CodeAssert.cs:line 27
   at ICSharpCode.Decompiler.Tests.Helpers.CodeAssert.FilesAreEqual(String fileName1, String fileName2, String[] definedSymbols) in D:\a\ILSpy\ILSpy\ICSharpCode.Decompiler.Tests\Helpers\CodeAssert.cs:line 19
   at ICSharpCode.Decompiler.Tests.PrettyTestRunner.Run(String testName, AssemblerOptions asmOptions, CompilerOptions cscOptions, DecompilerSettings decompilerSettings) in D:\a\ILSpy\ILSpy\ICSharpCode.Decompiler.Tests\PrettyTestRunner.cs:line 747
   at ICSharpCode.Decompiler.Tests.PrettyTestRunner.RunForLibrary(String testName, AssemblerOptions asmOptions, CompilerOptions cscOptions, DecompilerSettings decompilerSettings) in D:\a\ILSpy\ILSpy\ICSharpCode.Decompiler.Tests\PrettyTestRunner.cs:line 719
   at ICSharpCode.Decompiler.Tests.PrettyTestRunner.LiftedOperators(CompilerOptions cscOptions) in D:\a\ILSpy\ILSpy\ICSharpCode.Decompiler.Tests\PrettyTestRunner.cs:line 320
   at NUnit.Framework.Internal.TaskAwaitAdapter.GenericAdapter`1.BlockUntilCompleted()
   at NUnit.Framework.Internal.MessagePumpStrategy.NoMessagePumpStrategy.WaitForCompletion(AwaitAdapter awaiter)
   at NUnit.Framework.Internal.AsyncToSyncAdapter.Await(Func`1 invoke)
   at NUnit.Framework.Internal.Commands.TestMethodCommand.RunTestMethod(TestExecutionContext context)
   at NUnit.Framework.Internal.Commands.TestMethodCommand.Execute(TestExecutionContext context)
   at NUnit.Framework.Internal.Execution.SimpleWorkItem.&lt;&gt;c__DisplayClass4_0.&lt;PerformWork&gt;b__0()
   at NUnit.Framework.Internal.ContextUtils.&lt;&gt;c__DisplayClass1_0`1.&lt;DoIsolated&gt;b__0(Object _)</failure>
    </testcase>

Currently only the stack trace is added to the summary output, which is not very useful as it's almost always the same in our case. Adding the message and (even better) stdout/stderr would be a nice addition.
As for stdout/stderr, I don't think it's possible using the junit format as there is only one global system-out and system-err node.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions