Merge 56cec12 into 1ef4dfb

sthibaul · web-flow · commit ddc85900404a · 2020-08-21T09:49:18.000+02:00
diff --git a/devDocs/developerGuide.t2t b/devDocs/developerGuide.t2t
@@ -72,6 +72,7 @@ Subsequent lines contain a textual identifier used to identify the symbol, a tab
 For example:
 ```
 . sentence ending	(?<=[^\s.])\.(?=[\"')\s]|$)
+dates .	\b(\d\d)\.(\d\d)\.(\d{2}|\d{4})\b
 ```
 
 Again, the English symbols are inherited by all other locales, so you need not include any complex symbols already defined for English.
@@ -98,6 +99,8 @@ Certain characters cannot be typed into the file, so the following special seque
  - \f: form feed
  - \#: # character (needed because # at the start of a line denotes a comment)
 - replacement: The text which should be spoken for the symbol.
+If the symbol is a complex symbol, \1, \2, etc. can be used to refer to the groups matches, which will be inlined in the replacement, allowing for simpler rules.
+This also means that to get a \ character in the replacement, one has to type \\.
 - level: The symbol level at which the symbol should be spoken.
 The symbol level is configured by the user and specifies the amount of symbols that should be spoken.
 This field should contain one of the levels "none", "some", "most", "all" or "char", or "-" to use the default.
@@ -133,6 +136,13 @@ It means that the ". sentence ending" complex symbol should be spoken as "point"
 Level and preserve are not specified, so they will be taken from English.
 A display name is provided so that French users will know what the symbol represents.
 
+```
+dates .	\1 point \2 point \3	all	norep	# point de date
+```
+This line appears in the French symbols.dic file.
+It means that the first, second, and third groups of the match will be included, separated by spaces.
+The effect is thus to remove the dots from the date.
+
 Please see the file locale\en\symbols.dic for the English definitions which are inherited for all locales.
 This is also a good full example.
 
diff --git a/source/characterProcessing.py b/source/characterProcessing.py
@@ -519,6 +519,35 @@ def __init__(self, locale):
 			log.error("Invalid complex symbol regular expression in locale %s: %s" % (locale, e))
 			raise LookupError
 
+	def _replaceGroups(self, m: re.Match, string: str) -> str:
+		"""Replace matching group references (\\1, \\2, ...) with the corresponding matched groups.
+		Also replace \\\\ with \\ and reject other escapes, for escaping coherency.
+		@param m: The currently-matched group
+		@param string: The match replacement string which may contain group references
+		"""
+		result = ''
+
+		in_escape = False
+		for char in string:
+			if not in_escape:
+				if char == '\\':
+					in_escape = True
+				else:
+					result += char
+			else:
+				if char == '\\':
+					result += '\\'
+				elif char >= '0' and char <= '9':
+					result += m.group(m.lastindex + ord(char) - ord('0'))
+				else:
+					log.error("Invalid reference \\%string" % char)
+					raise LookupError
+				in_escape = False
+		if in_escape:
+			log.error("Unterminated backslash")
+			raise LookupError
+		return result
+
 	def _regexpRepl(self, m):
 		group = m.lastgroup
 
@@ -540,16 +569,19 @@ def _regexpRepl(self, m):
 			if group == "simple":
 				# Simple symbol.
 				symbol = self.computedSymbols[text]
+				replacement = symbol.replacement
 			else:
 				# Complex symbol.
 				index = int(group[1:])
 				symbol = self._computedComplexSymbolsList[index]
+				replacement = self._replaceGroups(m, symbol.replacement)
+
 			if symbol.preserve == SYMPRES_ALWAYS or (symbol.preserve == SYMPRES_NOREP and self._level < symbol.level):
 				suffix = text
 			else:
 				suffix = " "
-			if self._level >= symbol.level and symbol.replacement:
-				return u" {repl}{suffix}".format(repl=symbol.replacement, suffix=suffix)
+			if self._level >= symbol.level and replacement:
+				return u" {repl}{suffix}".format(repl=replacement, suffix=suffix)
 			else:
 				return suffix
 
diff --git a/tests/unit/test_characterProcessing.py b/tests/unit/test_characterProcessing.py
@@ -0,0 +1,122 @@
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2020 NV Access Limited
+
+"""Unit tests for the characterProcessing module.
+"""
+
+import unittest
+import re
+from characterProcessing import SYMLVL_ALL
+from characterProcessing import SpeechSymbolProcessor
+from characterProcessing import processSpeechSymbols as process
+
+
+class TestComplex(unittest.TestCase):
+	"""Test the complex symbols rules.
+	"""
+
+	def _replace_cb(self, replacement, name=None):
+		"""Return a regexp callback which replaces matches of the given
+		group name (or all groups if no name is given) with the
+		replacement string, with support for replacement of group
+		references.
+		"""
+		def replace(m):
+			if name is None or m.lastgroup == name:
+				return SpeechSymbolProcessor._replaceGroups(self, m, replacement)
+			return m.group()
+		return replace
+
+	def _replace(self, string, pattern, replacement, name=None):
+		"""Perform a pattern-based replacement on a string, for the
+		given named group (or all groups if no name is given), with
+		support for replacement of group references.
+		"""
+		regexp = re.compile(pattern, re.UNICODE)
+		return regexp.sub(self._replace_cb(replacement, name), string)
+
+	def test_group_replacement(self):
+		"""Test that plain text gets properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement="a"
+		)
+		self.assertEqual(replaced, "a")
+
+	def test_backslash_replacement(self):
+		"""Test that backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\"
+		)
+		self.assertEqual(replaced, "\\")
+
+	def test_double_backslash_replacement(self):
+		"""Test that double backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\\\"
+		)
+		self.assertEqual(replaced, r"\\")
+
+	def test_unknown_escape(self):
+		"""Test that a non-supported escaped character (i.e. not \\1,
+		\\2, ... \\9 and \\\\) in the replacement raises an error
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\a"
+			)
+
+	def test_missing_group(self):
+		"""Test that a reference in the replacement to an non-existing
+		group raises an error
+		"""
+		with self.assertRaises(IndexError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\2"
+			)
+
+	def test_unterminated_escape(self):
+		"""Test that an escape at the end of replacement raises an
+		error, since there is nothing to be escaped there
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement="\\"
+			)
+
+	def test_group_replacements(self):
+		"""Test that group references get properly replaced
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1"
+		)
+		self.assertEqual(replaced, "BAT>bar")
+
+	def test_multiple_group_replacement(self):
+		"""Test that group indexing is correct with multiple groups
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(baz)|(?P<foo>([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1",
+			name="foo"
+		)
+		self.assertEqual(replaced, "BAT>bar")
