changeset:   89352:0f0e9b7d4f1d
branch:      2.7
parent:      89347:a9464e900705
user:        Terry Jan Reedy <tjreedy@udel.edu>
date:        Sun Feb 23 23:32:59 2014 -0500
files:       Lib/test/test_tokenize.py Lib/tokenize.py
description:
Issue #9974: When untokenizing, use row info to insert backslash+newline.
Original patches by A. Kuchling and G. Rees (#12691).


diff -r a9464e900705 -r 0f0e9b7d4f1d Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py	Mon Feb 24 00:41:14 2014 +0100
+++ b/Lib/test/test_tokenize.py	Sun Feb 23 23:32:59 2014 -0500
@@ -4,7 +4,7 @@
     >>> import glob, random, sys
 
 The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARKER is omitted for
 brevity.
 
     >>> dump_tokens("1 + 1")
@@ -618,6 +618,7 @@
 class UntokenizeTest(TestCase):
 
     def test_bad_input_order(self):
+        # raise if previous row
         u = Untokenizer()
         u.prev_row = 2
         u.prev_col = 2
@@ -625,8 +626,21 @@
             u.add_whitespace((1,3))
         self.assertEqual(cm.exception.args[0],
                 'start (1,3) precedes previous end (2,2)')
+        # raise if previous column in row
         self.assertRaises(ValueError, u.add_whitespace, (2,1))
 
+    def test_backslash_continuation(self):
+        # The problem is that <whitespace>\<newline> leaves no token
+        u = Untokenizer()
+        u.prev_row = 1
+        u.prev_col =  1
+        u.tokens = []
+        u.add_whitespace((2, 0))
+        self.assertEqual(u.tokens, ['\\\n'])
+        u.prev_row = 2
+        u.add_whitespace((4, 4))
+        self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', '    '])
+
     def test_iter_compat(self):
         u = Untokenizer()
         token = (NAME, 'Hello')
diff -r a9464e900705 -r 0f0e9b7d4f1d Lib/tokenize.py
--- a/Lib/tokenize.py	Mon Feb 24 00:41:14 2014 +0100
+++ b/Lib/tokenize.py	Sun Feb 23 23:32:59 2014 -0500
@@ -188,6 +188,10 @@
         if row < self.prev_row or row == self.prev_row and col < self.prev_col:
             raise ValueError("start ({},{}) precedes previous end ({},{})"
                              .format(row, col, self.prev_row, self.prev_col))
+        row_offset = row - self.prev_row
+        if row_offset:
+            self.tokens.append("\\\n" * row_offset)
+            self.prev_col = 0
         col_offset = col - self.prev_col
         if col_offset:
             self.tokens.append(" " * col_offset)
@@ -199,6 +203,8 @@
                 self.compat(t, it)
                 break
             tok_type, token, start, end, line = t
+            if tok_type == ENDMARKER:
+                break
             self.add_whitespace(start)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end