Skip to content

Commit 6c308b5

Browse files
jakubch1jakubch1
committed
acceptance tests changes
1 parent 1b56b78 commit 6c308b5

5 files changed

Lines changed: 211 additions & 25 deletions

File tree

‎src/Microsoft.TestPlatform.CrossPlatEngine/MultiTestRunFinalization/MultiTestRunFinalizationManager.cs‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@ private async Task<Collection<AttachmentSet>> InternalFinalizeMultiTestRunAsync(
7272

7373
if (completedTask == task)
7474
{
75-
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(false, false, null), await task, eventHandler);
75+
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(false, false, null), await task, stopwatch, eventHandler);
7676
}
7777
else
7878
{
7979
eventHandler?.HandleLogMessage(ObjectModel.Logging.TestMessageLevel.Informational, "Finalization was cancelled.");
80-
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(true, false, null), attachments, eventHandler);
80+
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(true, false, null), attachments, stopwatch, eventHandler);
8181
}
8282
}
8383
}
@@ -87,19 +87,14 @@ private async Task<Collection<AttachmentSet>> InternalFinalizeMultiTestRunAsync(
8787
{
8888
EqtTrace.Warning("MultiTestRunFinalizationManager: operation was cancelled.");
8989
}
90-
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(true, false, null), attachments, eventHandler);
90+
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(true, false, null), attachments, stopwatch, eventHandler);
9191
}
9292
catch (Exception e)
9393
{
9494
EqtTrace.Error("MultiTestRunFinalizationManager: Exception in FinalizeMultiTestRunAsync: " + e);
9595

9696
eventHandler?.HandleLogMessage(ObjectModel.Logging.TestMessageLevel.Error, e.Message);
97-
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(false, true, e), attachments, eventHandler);
98-
}
99-
finally
100-
{
101-
stopwatch.Stop();
102-
requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecForFinalization, stopwatch.Elapsed.TotalSeconds);
97+
return FinalizeOperation(requestData, new MultiTestRunFinalizationCompleteEventArgs(false, true, e), attachments, stopwatch, eventHandler);
10398
}
10499
}
105100

@@ -111,6 +106,7 @@ private Collection<AttachmentSet> ProcessAttachments(Collection<AttachmentSet> a
111106
{
112107
IDataCollectorAttachments dataCollectorAttachmentsHandler = dataCollectorAttachmentsHandlers[i];
113108
string attachmentsHandlerName = dataCollectorAttachmentsHandler.GetType().FullName;
109+
int attachmentsHandlerIndex = i + 1;
114110

115111
Uri attachementUri = dataCollectorAttachmentsHandler.GetExtensionUri();
116112
if (attachementUri != null)
@@ -125,8 +121,10 @@ private Collection<AttachmentSet> ProcessAttachments(Collection<AttachmentSet> a
125121

126122
IProgress<int> progressReporter = new Progress<int>((int progress) =>
127123
eventsHandler?.HandleMultiTestRunFinalizationProgress(
128-
new MultiTestRunFinalizationProgressEventArgs(i + 1, attachmentsHandlerName, progress, dataCollectorAttachmentsHandlers.Length)));
124+
new MultiTestRunFinalizationProgressEventArgs(attachmentsHandlerIndex, attachmentsHandlerName, progress, dataCollectorAttachmentsHandlers.Length)));
125+
129126
ICollection<AttachmentSet> processedAttachments = dataCollectorAttachmentsHandler.HandleDataCollectionAttachmentSets(new Collection<AttachmentSet>(attachmentsToBeProcessed), progressReporter, cancellationToken);
127+
130128
foreach (var attachment in processedAttachments)
131129
{
132130
attachments.Add(attachment);
@@ -138,12 +136,15 @@ private Collection<AttachmentSet> ProcessAttachments(Collection<AttachmentSet> a
138136
return attachments;
139137
}
140138

141-
private Collection<AttachmentSet> FinalizeOperation(IRequestData requestData, MultiTestRunFinalizationCompleteEventArgs completeArgs, Collection<AttachmentSet> attachments, IMultiTestRunFinalizationEventsHandler eventHandler)
139+
private Collection<AttachmentSet> FinalizeOperation(IRequestData requestData, MultiTestRunFinalizationCompleteEventArgs completeArgs, Collection<AttachmentSet> attachments, Stopwatch stopwatch, IMultiTestRunFinalizationEventsHandler eventHandler)
142140
{
143141
testPlatformEventSource.MultiTestRunFinalizationStop(attachments.Count);
144142
requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterFinalization, attachments.Count);
145143
requestData.MetricsCollection.Add(TelemetryDataConstants.FinalizationState, completeArgs.IsAborted ? FinalizationFailed : completeArgs.IsCanceled ? FinalizationCanceled : FinalizationCompleted);
146144

145+
stopwatch.Stop();
146+
requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecForFinalization, stopwatch.Elapsed.TotalSeconds);
147+
147148
completeArgs.Metrics = requestData.MetricsCollection.Metrics;
148149
eventHandler?.HandleMultiTestRunFinalizationComplete(completeArgs, attachments);
149150

‎src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,13 @@ private string MergeCodeCoverageFiles(IList<string> files, IProgress<int> progre
7979
for (int i = 1; i < files.Count; i++)
8080
{
8181
cancellationToken.ThrowIfCancellationRequested();
82-
methodInfo.Invoke(null, new object[] { files[i], outputfileName, fileName, true });
82+
progressReporter?.Report(100 * i / files.Count);
8383

8484
cancellationToken.ThrowIfCancellationRequested();
85-
File.Copy(fileName, outputfileName, true);
85+
methodInfo.Invoke(null, new object[] { files[i], outputfileName, fileName, true });
8686

87-
progressReporter?.Report(100 * i / files.Count);
87+
cancellationToken.ThrowIfCancellationRequested();
88+
File.Copy(fileName, outputfileName, true);
8889
}
8990

9091
File.Delete(fileName);

‎test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs‎

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33

44
namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests
55
{
6+
using Castle.Core.Internal;
67
using Microsoft.TestPlatform.TestUtilities;
78
using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces;
9+
using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
810
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
911
using Microsoft.VisualStudio.TestTools.UnitTesting;
1012
using System;
@@ -109,6 +111,66 @@ public async Task TestRunWithCodeCoverageAndFinalization(RunnerInfo runnerInfo)
109111
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 1 : 2, this.multiTestRunFinalizationEventHandler.Attachments.Count);
110112

111113
AssertCoverageResults(this.multiTestRunFinalizationEventHandler.Attachments);
114+
115+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsAborted);
116+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsCanceled);
117+
Assert.IsNull(multiTestRunFinalizationEventHandler.CompleteArgs.Error);
118+
119+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 2 : 0, multiTestRunFinalizationEventHandler.ProgressArgs.Count);
120+
for (int i = 0; i < multiTestRunFinalizationEventHandler.ProgressArgs.Count; i++)
121+
{
122+
VisualStudio.TestPlatform.ObjectModel.Client.MultiTestRunFinalizationProgressEventArgs progressArgs = multiTestRunFinalizationEventHandler.ProgressArgs[i];
123+
Assert.AreEqual(1, progressArgs.CurrentHandlerIndex);
124+
Assert.AreEqual("Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler", progressArgs.CurrentHandlerName);
125+
Assert.AreEqual(1, progressArgs.HandlersCount);
126+
Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentHandlerProgress);
127+
}
128+
129+
Assert.AreEqual("Completed", multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.FinalizationState]);
130+
Assert.AreEqual(2L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsSentForFinalization]);
131+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 1L : 2L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsAfterFinalization]);
132+
Assert.IsTrue(multiTestRunFinalizationEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForFinalization));
133+
}
134+
135+
[TestMethod]
136+
[NetFullTargetFrameworkDataSource]
137+
[NetCoreTargetFrameworkDataSource]
138+
public async Task TestRunWithCodeCoverageAndFinalizationNoMetrics(RunnerInfo runnerInfo)
139+
{
140+
// arrange
141+
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
142+
this.Setup();
143+
144+
this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler);
145+
this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Skip(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler);
146+
147+
Assert.AreEqual(6, this.runEventHandler.TestResults.Count);
148+
Assert.AreEqual(2, this.runEventHandler.Attachments.Count);
149+
150+
// act
151+
await this.vstestConsoleWrapper.FinalizeMultiTestRunAsync(runEventHandler.Attachments, false, multiTestRunFinalizationEventHandler, CancellationToken.None);
152+
153+
// Assert
154+
multiTestRunFinalizationEventHandler.EnsureSuccess();
155+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 1 : 2, this.multiTestRunFinalizationEventHandler.Attachments.Count);
156+
157+
AssertCoverageResults(this.multiTestRunFinalizationEventHandler.Attachments);
158+
159+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsAborted);
160+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsCanceled);
161+
Assert.IsNull(multiTestRunFinalizationEventHandler.CompleteArgs.Error);
162+
163+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 2 : 0, multiTestRunFinalizationEventHandler.ProgressArgs.Count);
164+
for (int i = 0; i < multiTestRunFinalizationEventHandler.ProgressArgs.Count; i++)
165+
{
166+
VisualStudio.TestPlatform.ObjectModel.Client.MultiTestRunFinalizationProgressEventArgs progressArgs = multiTestRunFinalizationEventHandler.ProgressArgs[i];
167+
Assert.AreEqual(1, progressArgs.CurrentHandlerIndex);
168+
Assert.AreEqual("Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler", progressArgs.CurrentHandlerName);
169+
Assert.AreEqual(1, progressArgs.HandlersCount);
170+
Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentHandlerProgress);
171+
}
172+
173+
Assert.IsTrue(multiTestRunFinalizationEventHandler.CompleteArgs.Metrics.IsNullOrEmpty());
112174
}
113175

114176
[TestMethod]
@@ -135,6 +197,25 @@ public async Task TestRunWithCodeCoverageAndFinalizationModuleDuplicated(RunnerI
135197
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 1 : 3, this.multiTestRunFinalizationEventHandler.Attachments.Count);
136198

137199
AssertCoverageResults(this.multiTestRunFinalizationEventHandler.Attachments);
200+
201+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsAborted);
202+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsCanceled);
203+
Assert.IsNull(multiTestRunFinalizationEventHandler.CompleteArgs.Error);
204+
205+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 3 : 0, multiTestRunFinalizationEventHandler.ProgressArgs.Count);
206+
for (int i = 0; i < multiTestRunFinalizationEventHandler.ProgressArgs.Count; i++)
207+
{
208+
VisualStudio.TestPlatform.ObjectModel.Client.MultiTestRunFinalizationProgressEventArgs progressArgs = multiTestRunFinalizationEventHandler.ProgressArgs[i];
209+
Assert.AreEqual(1, progressArgs.CurrentHandlerIndex);
210+
Assert.AreEqual("Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler", progressArgs.CurrentHandlerName);
211+
Assert.AreEqual(1, progressArgs.HandlersCount);
212+
Assert.AreEqual(i == 0 ? 33 : i == 1 ? 66 : 100, progressArgs.CurrentHandlerProgress);
213+
}
214+
215+
Assert.AreEqual("Completed", multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.FinalizationState]);
216+
Assert.AreEqual(3L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsSentForFinalization]);
217+
Assert.AreEqual(testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 1L : 3L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsAfterFinalization]);
218+
Assert.IsTrue(multiTestRunFinalizationEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForFinalization));
138219
}
139220

140221
[TestMethod]
@@ -146,6 +227,8 @@ public async Task TestRunWithCodeCoverageAndFinalizationCancelled(RunnerInfo run
146227
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
147228
this.Setup();
148229

230+
if (!testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework)) return;
231+
149232
this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler);
150233
Assert.AreEqual(3, this.runEventHandler.TestResults.Count);
151234
Assert.AreEqual(1, this.runEventHandler.Attachments.Count);
@@ -155,7 +238,20 @@ public async Task TestRunWithCodeCoverageAndFinalizationCancelled(RunnerInfo run
155238
CancellationTokenSource cts = new CancellationTokenSource();
156239

157240
Task finalization = this.vstestConsoleWrapper.FinalizeMultiTestRunAsync(attachments, true, multiTestRunFinalizationEventHandler, cts.Token);
158-
await Task.Delay(TimeSpan.FromSeconds(1));
241+
242+
while (true)
243+
{
244+
try
245+
{
246+
if (multiTestRunFinalizationEventHandler.ProgressArgs.Count >= 3)
247+
break;
248+
}
249+
catch
250+
{
251+
// ignore
252+
}
253+
await Task.Delay(100);
254+
}
159255

160256
// act
161257
cts.Cancel();
@@ -164,7 +260,34 @@ public async Task TestRunWithCodeCoverageAndFinalizationCancelled(RunnerInfo run
164260
await finalization;
165261
multiTestRunFinalizationEventHandler.EnsureSuccess();
166262

167-
Assert.AreEqual(1000, this.multiTestRunFinalizationEventHandler.Attachments.Count);
263+
Assert.AreEqual(1000, this.multiTestRunFinalizationEventHandler.Attachments.Count);
264+
265+
Assert.IsFalse(multiTestRunFinalizationEventHandler.CompleteArgs.IsAborted);
266+
Assert.IsTrue(multiTestRunFinalizationEventHandler.CompleteArgs.IsCanceled);
267+
Assert.IsNull(multiTestRunFinalizationEventHandler.CompleteArgs.Error);
268+
269+
Assert.IsTrue((testEnvironment.RunnerFramework.Equals(IntegrationTestBase.DesktopRunnerFramework) ? 3 : 0) <= multiTestRunFinalizationEventHandler.ProgressArgs.Count);
270+
for (int i = 0; i < multiTestRunFinalizationEventHandler.ProgressArgs.Count; i++)
271+
{
272+
VisualStudio.TestPlatform.ObjectModel.Client.MultiTestRunFinalizationProgressEventArgs progressArgs = multiTestRunFinalizationEventHandler.ProgressArgs[i];
273+
Assert.AreEqual(1, progressArgs.CurrentHandlerIndex);
274+
Assert.AreEqual("Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler", progressArgs.CurrentHandlerName);
275+
Assert.AreEqual(1, progressArgs.HandlersCount);
276+
277+
if (i == 5)
278+
{
279+
Assert.AreEqual(0, progressArgs.CurrentHandlerProgress);
280+
}
281+
else if (i == 15)
282+
{
283+
Assert.AreEqual(1, progressArgs.CurrentHandlerProgress);
284+
}
285+
}
286+
287+
Assert.AreEqual("Canceled", multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.FinalizationState]);
288+
Assert.AreEqual(1000L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsSentForFinalization]);
289+
Assert.AreEqual(1000L, multiTestRunFinalizationEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.NumberOfAttachmentsAfterFinalization]);
290+
Assert.IsTrue(multiTestRunFinalizationEventHandler.CompleteArgs.Metrics.ContainsKey(TelemetryDataConstants.TimeTakenInSecForFinalization));
168291
}
169292

170293
[TestMethod]

‎test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/MultiTestRunFinalizationEventHandler.cs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests
1313
/// <inheritdoc />
1414
public class MultiTestRunFinalizationEventHandler : IMultiTestRunFinalizationEventsHandler
1515
{
16-
/// <summary>
17-
/// Gets the attachments.
18-
/// </summary>
1916
public List<AttachmentSet> Attachments { get; private set; }
2017

21-
/// <summary>
22-
/// Gets the metrics.
23-
/// </summary>
24-
public IDictionary<string, object> Metrics { get; private set; }
18+
public MultiTestRunFinalizationCompleteEventArgs CompleteArgs { get; private set; }
19+
20+
public List<MultiTestRunFinalizationProgressEventArgs> ProgressArgs { get; private set; }
2521

2622
/// <summary>
2723
/// Gets the log message.
@@ -39,6 +35,7 @@ public MultiTestRunFinalizationEventHandler()
3935
{
4036
this.Errors = new List<string>();
4137
this.Attachments = new List<AttachmentSet>();
38+
this.ProgressArgs = new List<MultiTestRunFinalizationProgressEventArgs>();
4239
}
4340

4441
public void EnsureSuccess()
@@ -53,7 +50,8 @@ public void HandleLogMessage(TestMessageLevel level, string message)
5350
{
5451
this.LogMessage = message;
5552
this.TestMessageLevel = level;
56-
if (level == TestMessageLevel.Error) {
53+
if (level == TestMessageLevel.Error)
54+
{
5755
this.Errors.Add(message);
5856
}
5957
}
@@ -94,6 +92,8 @@ public void HandleMultiTestRunFinalizationComplete(MultiTestRunFinalizationCompl
9492
{
9593
Errors.Add(finalizationCompleteEventArgs.Error.Message);
9694
}
95+
96+
CompleteArgs = finalizationCompleteEventArgs;
9797
}
9898

9999
public void HandleFinalisedAttachments(IEnumerable<AttachmentSet> attachments)
@@ -103,7 +103,7 @@ public void HandleFinalisedAttachments(IEnumerable<AttachmentSet> attachments)
103103

104104
public void HandleMultiTestRunFinalizationProgress(MultiTestRunFinalizationProgressEventArgs finalizationProgressEventArgs)
105105
{
106-
throw new NotImplementedException();
106+
ProgressArgs.Add(finalizationProgressEventArgs);
107107
}
108108
}
109109
}

0 commit comments

Comments
 (0)