bpo-32539: Fix OSError for os.listdir() for extended-length paths on Windows (#5169)

asottile · serhiy-storchaka · commit 27f32e938ff5 · 2018-01-15T23:39:04.000+02:00
See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath Paths that begin with `\\?\` are "extended-length paths".
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
@@ -900,6 +900,56 @@ def test_CTRL_BREAK_EVENT(self):
         self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT")
 
 
+@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32ListdirTests(unittest.TestCase):
+    """Test listdir on Windows."""
+
+    def setUp(self):
+        self.created_paths = []
+        for i in range(2):
+            dir_name = 'SUB%d' % i
+            dir_path = os.path.join(support.TESTFN, dir_name)
+            file_name = 'FILE%d' % i
+            file_path = os.path.join(support.TESTFN, file_name)
+            os.makedirs(dir_path)
+            with open(file_path, 'w') as f:
+                f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
+            self.created_paths.extend([dir_name, file_name])
+        self.created_paths.sort()
+
+    def tearDown(self):
+        shutil.rmtree(support.TESTFN)
+
+    def test_listdir_no_extended_path(self):
+        """Test when the path is not an "extended" path."""
+        # unicode
+        fs_encoding = sys.getfilesystemencoding()
+        self.assertEqual(
+                sorted(os.listdir(support.TESTFN.decode(fs_encoding))),
+                [path.decode(fs_encoding) for path in self.created_paths])
+
+        # bytes
+        self.assertEqual(
+                sorted(os.listdir(os.fsencode(support.TESTFN))),
+                self.created_paths)
+
+    def test_listdir_extended_path(self):
+        """Test when the path starts with '\\\\?\\'."""
+        # See: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+        # unicode
+        fs_encoding = sys.getfilesystemencoding()
+        path = u'\\\\?\\' + os.path.abspath(support.TESTFN.decode(fs_encoding))
+        self.assertEqual(
+                sorted(os.listdir(path)),
+                [path.decode(fs_encoding) for path in self.created_paths])
+
+        # bytes
+        path = b'\\\\?\\' + os.path.abspath(support.TESTFN)
+        self.assertEqual(
+                sorted(os.listdir(path)),
+                self.created_paths)
+
+
 class SpawnTests(unittest.TestCase):
     def _test_invalid_env(self, spawn):
         args = [sys.executable, '-c', 'pass']
diff --git a/Misc/NEWS.d/next/Library/2018-01-12-09-20-22.bpo-32539.D7AbdE.rst b/Misc/NEWS.d/next/Library/2018-01-12-09-20-22.bpo-32539.D7AbdE.rst
@@ -0,0 +1,2 @@
+Fix ``OSError`` for ``os.listdir`` with deep paths (starting with ``\\?\``) on
+windows.  Patch by Anthony Sottile.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
@@ -2385,7 +2385,7 @@ posix_listdir(PyObject *self, PyObject *args)
     if (len > 0) {
         char ch = namebuf[len-1];
         if (ch != SEP && ch != ALTSEP && ch != ':')
-            namebuf[len++] = '/';
+            namebuf[len++] = SEP;
         strcpy(namebuf + len, "*.*");
     }
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Fix ``OSError`` for ``os.listdir`` with deep paths (starting with ``\\?\``) on
	`2`	`+windows. Patch by Anthony Sottile.`
Original file line number	Diff line number	Diff line change
`@@ -2385,7 +2385,7 @@ posix_listdir(PyObject self, PyObject args)`
`2385`	`2385`	`if (len > 0) {`
`2386`	`2386`	`char ch = namebuf[len-1];`
`2387`	`2387`	`if (ch != SEP && ch != ALTSEP && ch != ':')`
`2388`		`- namebuf[len++] = '/';`
	`2388`	`+ namebuf[len++] = SEP;`
`2389`	`2389`	`strcpy(namebuf + len, ".");`
`2390`	`2390`	`}`
`2391`	`2391`