Skip to content

Commit a399f46

Browse files
feerrenrutfeerrenrut
authored
Merge 75f3fd9 into 2392d13 
2 parents 2392d13 + 75f3fd9 commit a399f46

File tree

12 files changed

+486
-148
lines changed

12 files changed

+486
-148
lines changed

‎appveyor/scripts/tests/beforeTests.ps1‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,14 @@ New-Item -ItemType directory -Path testOutput
22
New-Item -ItemType directory -Path testOutput\unit
33
New-Item -ItemType directory -Path testOutput\system
44
New-Item -ItemType directory -Path testOutput\lint
5+
# The first system test to run that requires chrome occasionally fails.
6+
# This has been replicated on developer machines after chrome updates,
7+
# however it is difficult to reproduce, and difficult to detect/recover-from
8+
# in an automated way.
9+
# It may be possible to handle this better within NVDA, however it seems to actually affect users
10+
# infrequently, and they seem to be able to recover by waiting a few seconds and de/refocus chrome.
11+
# In the mean time, the system tests failing delays development.
12+
#
13+
# Theory: Chrome is busy with post install tasks, so start chrome in the background ahead of the tests.
14+
cmd /c start /min chrome.exe
515
Set-AppveyorBuildVariable "testFailExitCode" 0

‎tests/system/libraries/ChromeLib.py‎

Lines changed: 105 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"""
99

1010
# imported methods start with underscore (_) so they don't get imported into robot files as keywords
11+
import datetime as _datetime
1112
from os.path import join as _pJoin
1213
import tempfile as _tempfile
1314
from typing import Optional as _Optional
@@ -18,7 +19,6 @@
1819
from SystemTestSpy.windows import (
1920
CloseWindow,
2021
GetWindowWithTitle,
21-
GetForegroundWindowTitle,
2222
Window,
2323
)
2424
import re
@@ -29,22 +29,26 @@
2929
from robot.libraries.Process import Process as _ProcessLib
3030
from AssertsLib import AssertsLib as _AssertsLib
3131
import NvdaLib as _NvdaLib
32+
import WindowsLib as _WindowsLib
3233

3334
builtIn: BuiltIn = BuiltIn()
3435
opSys: _OpSysLib = _getLib('OperatingSystem')
3536
process: _ProcessLib = _getLib('Process')
3637
assertsLib: _AssertsLib = _getLib('AssertsLib')
38+
windowsLib: _WindowsLib = _getLib('WindowsLib')
3739

3840

3941
# In Robot libraries, class name must match the name of the module. Use caps for both.
4042
class ChromeLib:
4143
_testFileStagingPath = _tempfile.mkdtemp()
4244

43-
def __init__(self):
44-
self.chromeWindow: _Optional[Window] = None
45-
"""Chrome Hwnd used to control Chrome via Windows functions."""
46-
self.processRFHandleForStart: _Optional[int] = None
47-
"""RF process handle, will wait for the chrome process to exit."""
45+
# Use class variables for state that should be tied to the RF library instance.
46+
# These variables will be available in the teardown
47+
_chromeWindow: _Optional[Window] = None
48+
"""Chrome Hwnd used to control Chrome via Windows functions."""
49+
_processRFHandleForStart: _Optional[int] = None
50+
"""RF process handle, will wait for the chrome process to exit."""
51+
4852

4953
@staticmethod
5054
def _getTestCasePath(filename):
@@ -54,19 +58,31 @@ def close_chrome_tab(self):
5458
spy = _NvdaLib.getSpyLib()
5559
builtIn.log(
5660
# True is expected due to /wait argument.
61+
# Note: if chrome was already open when this test started, the start process will no longer be open
62+
# Assumption:
63+
# An additionally started chrome process merely communicates the intent to open a URI and then exits.
64+
# Start is tracking only this process.
5765
"Is Start process still running (True expected): "
58-
f"{process.is_process_running(self.processRFHandleForStart)}"
66+
f"{process.is_process_running(ChromeLib._processRFHandleForStart)}"
5967
)
68+
69+
if not windowsLib.isWindowInForeground(ChromeLib._chromeWindow):
70+
builtIn.log(
71+
"Unable to close tab, window not in foreground: "
72+
f"({ChromeLib._chromeWindow.title} - {ChromeLib._chromeWindow.hwndVal})"
73+
)
74+
return
75+
6076
spy.emulateKeyPress('control+w')
6177
process.wait_for_process(
62-
self.processRFHandleForStart,
63-
timeout="1 minute",
78+
ChromeLib._processRFHandleForStart,
79+
timeout="10 seconds",
6480
on_timeout="continue"
6581
)
6682
builtIn.log(
6783
# False is expected, chrome should have allowed "Start" to exit.
6884
"Is Start process still running (False expected): "
69-
f"{process.is_process_running(self.processRFHandleForStart)}"
85+
f"{process.is_process_running(ChromeLib._processRFHandleForStart)}"
7086
)
7187

7288
def exit_chrome(self):
@@ -79,41 +95,58 @@ def exit_chrome(self):
7995
if _window is not None:
8096
res = CloseWindow(_window)
8197
if not res:
82-
builtIn.log(f"Unable to task kill chrome hwnd: {self.chromeWindow.hwndVal}", level="ERROR")
98+
builtIn.log(f"Unable to task kill chrome hwnd: {ChromeLib._chromeWindow.hwndVal}", level="ERROR")
8399
else:
84-
self.chromeWindow = _window = None
100+
ChromeLib._chromeWindow = _window = None
85101
else:
86102
builtIn.log("No chrome handle, unable to task kill", level="WARN")
87103

88104
def start_chrome(self, filePath: str, testCase: str) -> Window:
89105
builtIn.log(f"starting chrome: {filePath}")
90-
self.processRFHandleForStart = process.start_process(
106+
ChromeLib._processRFHandleForStart = process.start_process(
91107
"start" # windows utility to start a process
92108
# https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/start
93-
" /wait" # Starts an application and waits for it to end.
109+
#
110+
" /wait"
111+
# /wait: Starts an application and waits for it to end.
112+
# If chrome was already open, the start.exe process won't be able to track the full lifetime of the
113+
# intent to view the URI.
114+
# Assumption.
115+
# An additionally started chrome process merely communicates the intent to open a URI and then exits.
116+
# Start is tracking only this process.
117+
#
94118
" chrome" # Start Chrome
95119
" --force-renderer-accessibility"
96120
" --suppress-message-center-popups"
97121
" --disable-notifications"
98122
" --no-experiments"
99123
" --no-default-browser-check"
124+
" --disable-session-crashed-bubble"
125+
# --disable-session-crashed-bubble: If chrome crashes, don't cause subsequent tests to fail.
126+
# However, the config can be checked to determine if a crash occurred.
127+
# See https://superuser.com/a/1343331
128+
# Use C:\Users\username\AppData\Local\Google\Chrome\User Data\Default\Preference
129+
# Values:
130+
# "exit_type": "none",
131+
# "exited_cleanly":true,
132+
#
100133
f' "{filePath}"',
101134
shell=True,
102135
alias='chromeStartAlias',
103136
)
104-
process.process_should_be_running(self.processRFHandleForStart)
137+
process.process_should_be_running(ChromeLib._processRFHandleForStart)
105138
titlePattern = self.getUniqueTestCaseTitleRegex(testCase)
106-
success, self.chromeWindow = _blockUntilConditionMet(
139+
success, ChromeLib._chromeWindow = _blockUntilConditionMet(
107140
getValue=lambda: GetWindowWithTitle(titlePattern, lambda message: builtIn.log(message, "DEBUG")),
108-
giveUpAfterSeconds=3,
141+
giveUpAfterSeconds=10, # Chrome has been taking ~3 seconds to open a new tab on appveyor.
109142
shouldStopEvaluator=lambda _window: _window is not None,
110143
intervalBetweenSeconds=0.5,
111144
errorMessage="Unable to get chrome window"
112145
)
113146

114-
if not success or self.chromeWindow is None:
147+
if not success or ChromeLib._chromeWindow is None:
115148
builtIn.fatal_error("Unable to get chrome window")
116-
return self.chromeWindow
149+
return ChromeLib._chromeWindow
117150

118151
_testCaseTitle = "NVDA Browser Test Case"
119152
_beforeMarker = "Before Test Case Marker"
@@ -159,88 +192,89 @@ def _waitForStartMarker(self) -> bool:
159192
@return: False on failure
160193
"""
161194
spy = _NvdaLib.getSpyLib()
162-
spy.emulateKeyPress('alt+d') # focus the address bar, chrome shortcut
163195
spy.wait_for_speech_to_finish()
164-
addressSpeechIndex = spy.get_last_speech_index()
196+
expectedAddressBarSpeech = "Address and search bar"
197+
moveToAddressBarSpeech = _NvdaLib.getSpeechAfterKey('nvda+tab') # report current focus.
198+
if expectedAddressBarSpeech not in moveToAddressBarSpeech:
199+
moveToAddressBarSpeech = _NvdaLib.getSpeechAfterKey('alt+d') # focus the address bar, chrome shortcut
200+
if expectedAddressBarSpeech not in moveToAddressBarSpeech:
201+
builtIn.log(
202+
f"Didn't read '{expectedAddressBarSpeech}' after alt+d, instead got: {moveToAddressBarSpeech}"
203+
)
204+
return False
165205

166-
spy.emulateKeyPress('control+F6') # focus web content, chrome shortcut.
167-
spy.wait_for_speech_to_finish()
168-
afterControlF6Speech = spy.get_speech_at_index_until_now(addressSpeechIndex)
169-
if f"document\n{ChromeLib._beforeMarker}" not in afterControlF6Speech:
170-
builtIn.log(afterControlF6Speech, level="DEBUG")
206+
afterControlF6Speech = _NvdaLib.getSpeechAfterKey('control+F6') # focus web content, chrome shortcut.
207+
documentDescriptor = f"document\n{ChromeLib._beforeMarker}"
208+
if documentDescriptor not in afterControlF6Speech:
209+
builtIn.log(
210+
f"Didn't get '{documentDescriptor}' after moving to document, instead got: {afterControlF6Speech}"
211+
)
171212
return False
172213

173-
spy.emulateKeyPress('control+home') # ensure we start at the top of the document
174-
controlHomeSpeechIndex = spy.get_last_speech_index()
175-
spy.emulateKeyPress('numpad8') # report current line
214+
# ensure we start at the top of the document
215+
_NvdaLib.getSpeechAfterKey('control+home')
176216
spy.wait_for_speech_to_finish()
177-
afterNumPad8Speech = spy.get_speech_at_index_until_now(controlHomeSpeechIndex)
217+
218+
afterNumPad8Speech = _NvdaLib.getSpeechAfterKey('numpad8') # report current line
178219
if ChromeLib._beforeMarker not in afterNumPad8Speech:
179-
builtIn.log(afterNumPad8Speech, level="DEBUG")
220+
builtIn.log(
221+
f"Didn't get {ChromeLib._beforeMarker} after reporting the current line"
222+
f", instead got: {afterNumPad8Speech}"
223+
)
180224
return False
181225
return True
182226

183-
def toggleFocusChrome(self) -> None:
184-
"""Remove focus, then refocus chrome
185-
Attempt to work around NVDA missing focus / foreground events when chrome first opens.
186-
Forcing chrome to send another foreground event by focusing the desktop, then using alt+tab to return
187-
chrome to the foreground.
188-
@remarks If another application raises to the foreground after chrome, this approach won't resolve that
189-
situation.
190-
We don't have evidence that another application taking focus is a cause of failure yet.
191-
"""
192-
spy = _NvdaLib.getSpyLib()
193-
spy.emulateKeyPress('windows+d')
194-
_blockUntilConditionMet(
195-
giveUpAfterSeconds=5,
196-
getValue=GetForegroundWindowTitle,
197-
shouldStopEvaluator=lambda _title: self.chromeWindow.title != _title,
198-
errorMessage="Chrome didn't lose focus"
227+
def canChromeTitleBeReported(self, chromeTitleSpeechPattern: re.Pattern) -> bool:
228+
speech = _NvdaLib.getSpeechAfterKey('NVDA+t')
229+
return bool(
230+
chromeTitleSpeechPattern.search(speech)
199231
)
200-
spy.emulateKeyPress('alt+tab')
201-
_blockUntilConditionMet(
202-
giveUpAfterSeconds=5,
203-
getValue=GetForegroundWindowTitle,
204-
shouldStopEvaluator=lambda _title: self.chromeWindow.title == _title,
205-
errorMessage="Chrome didn't gain focus"
206-
)
207-
spy.wait_for_speech_to_finish()
208232

209-
def ensureChromeTitleCanBeReported(self, applicationTitle: str) -> int:
210-
spy = _NvdaLib.getSpyLib()
211-
afterFocusToggleIndex = spy.get_last_speech_index()
212-
spy.emulateKeyPress('NVDA+t')
213-
appTitleIndex = spy.wait_for_specific_speech(applicationTitle, afterIndex=afterFocusToggleIndex)
214-
return appTitleIndex
215-
216-
def prepareChrome(self, testCase: str, _doToggleFocus: bool = False) -> None:
233+
def prepareChrome(self, testCase: str, _alwaysDoToggleFocus: bool = False) -> None:
217234
"""
218235
Starts Chrome opening a file containing the HTML sample
219236
@param testCase - The HTML sample to test.
220-
@param _doToggleFocus - When True, Chrome will be intentionally de-focused and re-focused
237+
@param _alwaysDoToggleFocus - When True, Chrome will be intentionally de-focused and re-focused
221238
"""
239+
testCase = testCase + (
240+
"\n<!-- " # new line, start a HTML comment
241+
"Sample generation time, to ensure that the test case title is reproducibly unique purely from"
242+
" this test case string: \n"
243+
f"{ _datetime.datetime.now().isoformat()} "
244+
f" -->" # end HTML comment
245+
)
222246
spy = _NvdaLib.getSpyLib()
223247
_chromeLib: "ChromeLib" = _getLib('ChromeLib') # using the lib gives automatic 'keyword' logging.
224248
path = self._writeTestFile(testCase)
225249

226250
spy.wait_for_speech_to_finish()
227-
lastSpeechIndex = spy.get_last_speech_index()
228251
_chromeLib.start_chrome(path, testCase)
252+
windowsLib.logForegroundWindowTitle()
253+
229254
applicationTitle = ChromeLib.getUniqueTestCaseTitle(testCase)
255+
# application title will be something like "NVDA Browser Test Case (499078752)"
256+
# the parentheses could be escaped, instead we can just replace them with "match any char".
257+
patternSafeTitleString = applicationTitle.replace('(', '.').replace(')', '.')
258+
chromeTitleSpeechPattern = re.compile(patternSafeTitleString)
230259

231-
_chromeLib.ensureChromeTitleCanBeReported(applicationTitle)
232-
spy.wait_for_speech_to_finish()
260+
if (
261+
_alwaysDoToggleFocus # may work around focus/foreground event missed issues for tests.
262+
or not _chromeLib.canChromeTitleBeReported(chromeTitleSpeechPattern)
263+
):
264+
windowsLib.taskSwitchToItemMatching(targetWindowNamePattern=chromeTitleSpeechPattern)
265+
windowsLib.logForegroundWindowTitle()
233266

234-
if _doToggleFocus: # may work around focus/foreground event missed issues for tests.
235-
_chromeLib.toggleFocusChrome()
236-
spy.wait_for_speech_to_finish()
267+
if not _chromeLib.canChromeTitleBeReported(chromeTitleSpeechPattern):
268+
raise AssertionError("NVDA unable to report chrome title")
269+
270+
spy.wait_for_speech_to_finish()
237271

238272
if not self._waitForStartMarker():
239273
builtIn.fail(
240274
"Unable to locate 'before sample' marker."
241275
" See NVDA log for full speech."
242276
)
243-
# Move to the loading status line, and wait fore it to become complete
277+
# Move to the loading status line, and wait for it to become complete
244278
# the page has fully loaded.
245279
spy.emulateKeyPress('downArrow')
246280
for x in range(10):

0 commit comments

Comments
 (0)