bpo-33684: json.tool: Use utf-8 for infile and outfile. (GH-17460)

methane · web-flow · commit e0f148e66354 · 2019-12-04T19:26:22.000+09:00
(cherry picked from commit 808769f) Co-authored-by: Inada Naoki <songofacandy@gmail.com>
diff --git a/Lib/json/tool.py b/Lib/json/tool.py
@@ -20,9 +20,9 @@ def main():
     description = ('A simple command line interface for json module '
                    'to validate and pretty-print JSON objects.')
     parser = argparse.ArgumentParser(prog=prog, description=description)
-    parser.add_argument('infile', nargs='?', type=argparse.FileType(),
+    parser.add_argument('infile', nargs='?', type=argparse.FileType(encoding="utf-8"),
                         help='a JSON file to be validated or pretty-printed')
-    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
+    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w', encoding="utf-8"),
                         help='write the output of infile to outfile')
     parser.add_argument('--sort-keys', action='store_true', default=False,
                         help='sort the output of dictionaries alphabetically by key')
diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py
@@ -67,11 +67,11 @@ def test_stdin_stdout(self):
         self.assertEqual(out.splitlines(), self.expect.encode().splitlines())
         self.assertEqual(err, b'')
 
-    def _create_infile(self):
+    def _create_infile(self, data=None):
         infile = support.TESTFN
-        with open(infile, "w") as fp:
+        with open(infile, "w", encoding="utf-8") as fp:
             self.addCleanup(os.remove, infile)
-            fp.write(self.data)
+            fp.write(data or self.data)
         return infile
 
     def test_infile_stdout(self):
@@ -81,6 +81,21 @@ def test_infile_stdout(self):
         self.assertEqual(out.splitlines(), self.expect.encode().splitlines())
         self.assertEqual(err, b'')
 
+    def test_non_ascii_infile(self):
+        data = '{"msg": "\u3053\u3093\u306b\u3061\u306f"}'
+        expect = textwrap.dedent('''\
+        {
+            "msg": "\\u3053\\u3093\\u306b\\u3061\\u306f"
+        }
+        ''').encode()
+
+        infile = self._create_infile(data)
+        rc, out, err = assert_python_ok('-m', 'json.tool', infile)
+
+        self.assertEqual(rc, 0)
+        self.assertEqual(out.splitlines(), expect.splitlines())
+        self.assertEqual(err, b'')
+
     def test_infile_outfile(self):
         infile = self._create_infile()
         outfile = support.TESTFN + '.out'
diff --git a/Misc/NEWS.d/next/Library/2019-12-04-15-28-40.bpo-33684.QeSmQP.rst b/Misc/NEWS.d/next/Library/2019-12-04-15-28-40.bpo-33684.QeSmQP.rst
@@ -0,0 +1,2 @@
+Fix ``json.tool`` failed to read a JSON file with non-ASCII characters when
+locale encoding is not UTF-8.

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Fix ``json.tool`` failed to read a JSON file with non-ASCII characters when
	`2`	`+locale encoding is not UTF-8.`