Emit T_BAD_CHARACTER for unexpected characters

nikic · nikic · commit 9ad094e37196 · 2019-07-15T12:51:01.000+02:00
Avoid having holes in the token stream which are annoying and
inefficient to reconstruct on the consumer side.
diff --git a/UPGRADING b/UPGRADING
@@ -132,6 +132,10 @@ PHP 7.4 UPGRADE NOTES
     PASSWORD_BCRYPT, PASSWORD_ARGON2I, and PASSWORD_ARGON2ID will continue to
     function correctly.
 
+- Tokenizer:
+  . token_get_all() will now emit a T_BAD_CHARACTER token for unexpected
+    characters instead of leaving behind holes in the token stream.
+
 ========================================
 2. New Features
 ========================================
diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y
@@ -221,6 +221,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token T_COALESCE        "?? (T_COALESCE)"
 %token T_POW             "** (T_POW)"
 %token T_POW_EQUAL       "**= (T_POW_EQUAL)"
+%token T_BAD_CHARACTER   "invalid character (T_BAD_CHARACTER)"
 
 /* Token used to force a parse error from the lexer */
 %token T_ERROR
diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l
@@ -2847,8 +2847,12 @@ nowdoc_scan_done:
 		RETURN_TOKEN(END);
 	}
 
-	zend_error(E_COMPILE_WARNING,"Unexpected character in input:  '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);
-	goto restart;
+	zend_error(E_COMPILE_WARNING, "Unexpected character in input:  '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);
+	if (PARSER_MODE()) {
+		goto restart;
+	} else {
+		RETURN_TOKEN(T_BAD_CHARACTER);
+	}
 }
 
 */
diff --git a/ext/tokenizer/tests/bad_character.phpt b/ext/tokenizer/tests/bad_character.phpt
@@ -0,0 +1,47 @@
+--TEST--
+token_get_all() produces T_BAD_CHARACTER for unexpected characters
+--FILE--
+<?php
+
+$codes = [
+    "<?php \0 foo",
+    "<?php \1 bar",
+    "<?php \1\2 bar \3",
+];
+
+foreach ($codes as $code) {
+    foreach (token_get_all($code) as $token) {
+        if (is_array($token)) {
+            echo token_name($token[0]), " ", strlen($token[1]), "\n";
+        } else {
+            echo $token, "\n";
+        }
+    }
+}
+
+?>
+--EXPECTF--
+Warning: Unexpected character in input:  ' in %s on line %d
+T_OPEN_TAG 6
+T_BAD_CHARACTER 1
+T_WHITESPACE 1
+T_STRING 3
+
+Warning: Unexpected character in input:  '%s' (ASCII=1) state=0 in %s on line %d
+T_OPEN_TAG 6
+T_BAD_CHARACTER 1
+T_WHITESPACE 1
+T_STRING 3
+
+Warning: Unexpected character in input:  '%s' (ASCII=1) state=0 in %s on line %d
+
+Warning: Unexpected character in input:  '%s' (ASCII=2) state=0 in %s on line %d
+
+Warning: Unexpected character in input:  '%s' (ASCII=3) state=0 in %s on line %d
+T_OPEN_TAG 6
+T_BAD_CHARACTER 1
+T_BAD_CHARACTER 1
+T_WHITESPACE 1
+T_STRING 3
+T_WHITESPACE 1
+T_BAD_CHARACTER 1
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
@@ -162,6 +162,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
 }
 
@@ -304,6 +305,7 @@ char *get_token_type_name(int token_type)
 		case T_NS_C: return "T_NS_C";
 		case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
 		case T_ELLIPSIS: return "T_ELLIPSIS";
+		case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
 
 	}
 	return "UNKNOWN";

Original file line number	Diff line number	Diff line change
`@@ -2847,8 +2847,12 @@ nowdoc_scan_done:`
`2847`	`2847`	`RETURN_TOKEN(END);`
`2848`	`2848`	`}`
`2849`	`2849`
`2850`		`- zend_error(E_COMPILE_WARNING,"Unexpected character in input: '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);`
`2851`		`- goto restart;`
	`2850`	`+ zend_error(E_COMPILE_WARNING, "Unexpected character in input: '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);`
	`2851`	`+ if (PARSER_MODE()) {`
	`2852`	`+ goto restart;`
	`2853`	`+ } else {`
	`2854`	`+ RETURN_TOKEN(T_BAD_CHARACTER);`
	`2855`	`+ }`
`2852`	`2856`	`}`
`2853`	`2857`
`2854`	`2858`	`*/`
Original file line number	Diff line number	Diff line change
`@@ -162,6 +162,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {`
`162`	`162`	`REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS \| CONST_PERSISTENT);`
`163`	`163`	`REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS \| CONST_PERSISTENT);`
`164`	`164`	`REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS \| CONST_PERSISTENT);`
	`165`	`+ REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS \| CONST_PERSISTENT);`
`165`	`166`	`REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS \| CONST_PERSISTENT);`
`166`	`167`	`}`
`167`	`168`
`@@ -304,6 +305,7 @@ char *get_token_type_name(int token_type)`
`304`	`305`	`case T_NS_C: return "T_NS_C";`
`305`	`306`	`case T_NS_SEPARATOR: return "T_NS_SEPARATOR";`
`306`	`307`	`case T_ELLIPSIS: return "T_ELLIPSIS";`
	`308`	`+ case T_BAD_CHARACTER: return "T_BAD_CHARACTER";`
`307`	`309`
`308`	`310`	`}`
`309`	`311`	`return "UNKNOWN";`