bpo-30540: regrtest: add --matchfile option (#1909)

vstinner · web-flow · commit ef8320cf6f09 · 2017-06-09T10:18:48.000+02:00
* Add a new option taking a filename to get a list of test names to
  filter tests.
* support.match_tests becomes a list.
* Modify run_unittest() to accept to match the whole test identifier,
  not just a part of a test identifier.

For example, the following command only runs test_default_timeout()
of the BarrierTests class of test_threading:

$ ./python -m test -v test_threading -m test.test_threading.BarrierTests.test_default_timeout

Remove also some empty lines from test_regrtest.py to make flake8
tool happy.
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
@@ -117,6 +117,13 @@
 To enable all resources except one, use '-uall,-<resource>'.  For
 example, to run all the tests except for the gui tests, give the
 option '-uall,-gui'.
+
+--matchfile filters tests using a text file, one pattern per line.
+Pattern examples:
+
+- test method: test_stat_attributes
+- test class: FileTests
+- test identifier: test_os.FileTests.test_stat_attributes
 """
 
 
@@ -189,8 +196,12 @@ def _create_parser():
                        help='single step through a set of tests.' +
                             more_details)
     group.add_argument('-m', '--match', metavar='PAT',
-                       dest='match_tests',
+                       dest='match_tests', action='append',
                        help='match test cases and methods with glob pattern PAT')
+    group.add_argument('--matchfile', metavar='FILENAME',
+                       dest='match_filename',
+                       help='similar to --match but get patterns from a '
+                            'text file, one pattern per line')
     group.add_argument('-G', '--failfast', action='store_true',
                        help='fail as soon as a test fails (only with -v or -W)')
     group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -350,5 +361,12 @@ def _parse_args(args, **kwargs):
         print("WARNING: Disable --verbose3 because it's incompatible with "
               "--huntrleaks: see http://bugs.python.org/issue27103",
               file=sys.stderr)
+    if ns.match_filename:
+        if ns.match_tests is None:
+            ns.match_tests = []
+        filename = os.path.join(support.SAVEDCWD, ns.match_filename)
+        with open(filename) as fp:
+            for line in fp:
+                ns.match_tests.append(line.strip())
 
     return ns
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -1922,9 +1922,15 @@ def run_unittest(*classes):
     def case_pred(test):
         if match_tests is None:
             return True
-        for name in test.id().split("."):
-            if fnmatch.fnmatchcase(name, match_tests):
+        test_id = test.id()
+
+        for match_test in match_tests:
+            if fnmatch.fnmatchcase(test_id, match_test):
                 return True
+
+            for name in test_id.split("."):
+                if fnmatch.fnmatchcase(name, match_test):
+                    return True
         return False
     _filter_suite(suite, case_pred)
     _run_suite(suite)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
@@ -159,9 +159,24 @@ def test_match(self):
         for opt in '-m', '--match':
             with self.subTest(opt=opt):
                 ns = libregrtest._parse_args([opt, 'pattern'])
-                self.assertEqual(ns.match_tests, 'pattern')
+                self.assertEqual(ns.match_tests, ['pattern'])
                 self.checkError([opt], 'expected one argument')
 
+        ns = libregrtest._parse_args(['-m', 'pattern1',
+                                      '-m', 'pattern2'])
+        self.assertEqual(ns.match_tests, ['pattern1', 'pattern2'])
+
+        self.addCleanup(support.unlink, support.TESTFN)
+        with open(support.TESTFN, "w") as fp:
+            print('matchfile1', file=fp)
+            print('matchfile2', file=fp)
+
+        filename = os.path.abspath(support.TESTFN)
+        ns = libregrtest._parse_args(['-m', 'match',
+                                      '--matchfile', filename])
+        self.assertEqual(ns.match_tests,
+                         ['match', 'matchfile1', 'matchfile2'])
+
     def test_failfast(self):
         for opt in '-G', '--failfast':
             with self.subTest(opt=opt):
@@ -275,7 +290,6 @@ def test_forever(self):
                 ns = libregrtest._parse_args([opt])
                 self.assertTrue(ns.forever)
 
-
     def test_unrecognized_argument(self):
         self.checkError(['--xxx'], 'usage:')
 
@@ -457,7 +471,6 @@ def run_command(self, args, input=None, exitcode=0, **kw):
             self.fail(msg)
         return proc
 
-
     def run_python(self, args, **kw):
         args = [sys.executable, '-X', 'faulthandler', '-I', *args]
         proc = self.run_command(args, **kw)
@@ -823,6 +836,52 @@ def test_crashed(self):
         self.check_executed_tests(output, tests, failed=crash_test,
                                   randomize=True)
 
+    def parse_methods(self, output):
+        regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
+        return [match.group(1) for match in regex.finditer(output)]
+
+    def test_matchfile(self):
+        # Any code which causes a crash
+        code = textwrap.dedent("""
+            import unittest
+
+            class Tests(unittest.TestCase):
+                def test_method1(self):
+                    pass
+                def test_method2(self):
+                    pass
+                def test_method3(self):
+                    pass
+                def test_method4(self):
+                    pass
+        """)
+        all_methods = ['test_method1', 'test_method2',
+                       'test_method3', 'test_method4']
+        testname = self.create_test(code=code)
+
+        # by default, all methods should be run
+        output = self.run_tests("-v", testname)
+        methods = self.parse_methods(output)
+        self.assertEqual(methods, all_methods)
+
+        # only run a subset
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+
+        subset = [
+            # only match the method name
+            'test_method1',
+            # match the full identifier
+            '%s.Tests.test_method3' % testname]
+        with open(filename, "w") as fp:
+            for name in subset:
+                print(name, file=fp)
+
+        output = self.run_tests("-v", "--matchfile", filename, testname)
+        methods = self.parse_methods(output)
+        subset = ['test_method1', 'test_method3']
+        self.assertEqual(methods, subset)
+
 
 if __name__ == '__main__':
     unittest.main()
