PHP 7.4: backfill for tokenizing numeric sequences with an underscore separator #2546
Description
PHP 7.4 will contain the ability to use an underscore in numeric literals to visually separate groups of digits. This RFC has already been implemented in the PHP 7.4 branch and can be tested with the alpha builds available.
Now, from a PHPCS/Tokenizer point of view, there are two things we need to consider:
- While the PHP 7.4 tokenizer will correctly tokenize these sequences as
T_LNUMBERorT_DNUMBER, the tokenizer in any PHP version below PHP 7.4 will not.
IMO, this is something which can and should be back-filled in PHPCS. - The token
contentvalue for any such token - including the backfilled ones - would contain the_(s). This is problematic when a sniff would examine thecontentof aT_LNUMBERorT_DNUMBER, for instance to check that the parameter value passed to a certain function call is higher or lower than a set value.
If that sniff would run on PHP 7.4, all would be fine, but on any older PHP version the sniff would break as soon as it would try to do a numeric comparison with thecontentof the token.
To prevent such breakage of existing (userland) sniffs, I would like to suggest for the tokenizer - after backfilling the token parts to one token -, to save the token value as received from the tokenizer toorig_contentand to strip out the underscores for thecontentvalue of the token.
Does that make sense ?
Code example based on the RFC:
<?php
// OK: ordinary numeric sequences.
$a = 1000000000;
$b = 107925284.88;
$discount = 13500;
// OK: Constant name starting with an underscore is perfectly valid.
const _100 = 100;
// PHP 7.4 numeric sequences with a numeric literal separator.
$threshold = 1_000_000_000; // a billion!
$testValue = 107_925_284.88; // scale is hundreds of millions
$discount = 135_00; // $135, stored as cents
$a = 6.674_083e-11; // float
$a = 299_792_458; // decimal
$a = 0xCAFE_F00D; // hexadecimal
$a = 0b0101_1111; // binary
$a = 0137_041; // octal
// More test cases.
$a = 1_2.3_4e1_23;
// Invalid use of underscores in numeric sequences.
// Each underscore in a numeric literal must be directly between two digits.
// These below all produce "Parse error: syntax error" in PHP 7.4.
$a = 100_; // trailing
$a = 1__1; // next to underscore
$a = 1_.0; // next to decimal point
$a = 1._0; // next to decimal point
$a = 0x_123; // next to x
$a = 0b_101; // next to b
$a = 1_e2; // next to e
$a = 1e_2; // next to eThis code sample is tokenized like this in PHP 7.3.6:
0 :: L001 :: C1 :: T_OPEN_TAG :: (5) :: <?php
1 :: L002 :: C1 :: T_WHITESPACE :: (0) ::
2 :: L003 :: C1 :: T_COMMENT :: (34) :: // OK: ordinary numeric sequences.
3 :: L004 :: C1 :: T_VARIABLE :: (2) :: $a
4 :: L004 :: C3 :: T_WHITESPACE :: (1) ::
5 :: L004 :: C4 :: T_EQUAL :: (1) :: =
6 :: L004 :: C5 :: T_WHITESPACE :: (1) ::
7 :: L004 :: C6 :: T_LNUMBER :: (10) :: 1000000000
8 :: L004 :: C16 :: T_SEMICOLON :: (1) :: ;
9 :: L004 :: C17 :: T_WHITESPACE :: (0) ::
10 :: L005 :: C1 :: T_VARIABLE :: (2) :: $b
11 :: L005 :: C3 :: T_WHITESPACE :: (1) ::
12 :: L005 :: C4 :: T_EQUAL :: (1) :: =
13 :: L005 :: C5 :: T_WHITESPACE :: (1) ::
14 :: L005 :: C6 :: T_DNUMBER :: (12) :: 107925284.88
15 :: L005 :: C18 :: T_SEMICOLON :: (1) :: ;
16 :: L005 :: C19 :: T_WHITESPACE :: (0) ::
17 :: L006 :: C1 :: T_VARIABLE :: (9) :: $discount
18 :: L006 :: C10 :: T_WHITESPACE :: (1) ::
19 :: L006 :: C11 :: T_EQUAL :: (1) :: =
20 :: L006 :: C12 :: T_WHITESPACE :: (1) ::
21 :: L006 :: C13 :: T_LNUMBER :: (5) :: 13500
22 :: L006 :: C18 :: T_SEMICOLON :: (1) :: ;
23 :: L006 :: C19 :: T_WHITESPACE :: (0) ::
24 :: L007 :: C1 :: T_WHITESPACE :: (0) ::
25 :: L008 :: C1 :: T_COMMENT :: (68) :: // OK: Constant name starting with an underscore is perfectly valid.
26 :: L009 :: C1 :: T_CONST :: (5) :: const
27 :: L009 :: C6 :: T_WHITESPACE :: (1) ::
28 :: L009 :: C7 :: T_STRING :: (4) :: _100
29 :: L009 :: C11 :: T_WHITESPACE :: (1) ::
30 :: L009 :: C12 :: T_EQUAL :: (1) :: =
31 :: L009 :: C13 :: T_WHITESPACE :: (1) ::
32 :: L009 :: C14 :: T_LNUMBER :: (3) :: 100
33 :: L009 :: C17 :: T_SEMICOLON :: (1) :: ;
34 :: L009 :: C18 :: T_WHITESPACE :: (0) ::
35 :: L010 :: C1 :: T_WHITESPACE :: (0) ::
36 :: L011 :: C1 :: T_COMMENT :: (62) :: // PHP 7.4 numeric sequences with a numeric literal separator.
37 :: L012 :: C1 :: T_VARIABLE :: (10) :: $threshold
38 :: L012 :: C11 :: T_WHITESPACE :: (1) ::
39 :: L012 :: C12 :: T_EQUAL :: (1) :: =
40 :: L012 :: C13 :: T_WHITESPACE :: (1) ::
41 :: L012 :: C14 :: T_LNUMBER :: (1) :: 1
42 :: L012 :: C15 :: T_STRING :: (12) :: _000_000_000
43 :: L012 :: C27 :: T_SEMICOLON :: (1) :: ;
44 :: L012 :: C28 :: T_WHITESPACE :: (2) ::
45 :: L012 :: C30 :: T_COMMENT :: (13) :: // a billion!
46 :: L013 :: C1 :: T_VARIABLE :: (10) :: $testValue
47 :: L013 :: C11 :: T_WHITESPACE :: (1) ::
48 :: L013 :: C12 :: T_EQUAL :: (1) :: =
49 :: L013 :: C13 :: T_WHITESPACE :: (1) ::
50 :: L013 :: C14 :: T_LNUMBER :: (3) :: 107
51 :: L013 :: C17 :: T_STRING :: (8) :: _925_284
52 :: L013 :: C25 :: T_DNUMBER :: (3) :: .88
53 :: L013 :: C28 :: T_SEMICOLON :: (1) :: ;
54 :: L013 :: C29 :: T_WHITESPACE :: (1) ::
55 :: L013 :: C30 :: T_COMMENT :: (32) :: // scale is hundreds of millions
56 :: L014 :: C1 :: T_VARIABLE :: (9) :: $discount
57 :: L014 :: C10 :: T_WHITESPACE :: (2) ::
58 :: L014 :: C12 :: T_EQUAL :: (1) :: =
59 :: L014 :: C13 :: T_WHITESPACE :: (1) ::
60 :: L014 :: C14 :: T_LNUMBER :: (3) :: 135
61 :: L014 :: C17 :: T_STRING :: (3) :: _00
62 :: L014 :: C20 :: T_SEMICOLON :: (1) :: ;
63 :: L014 :: C21 :: T_WHITESPACE :: (9) ::
64 :: L014 :: C30 :: T_COMMENT :: (24) :: // $135, stored as cents
65 :: L015 :: C1 :: T_WHITESPACE :: (0) ::
66 :: L016 :: C1 :: T_VARIABLE :: (2) :: $a
67 :: L016 :: C3 :: T_WHITESPACE :: (1) ::
68 :: L016 :: C4 :: T_EQUAL :: (1) :: =
69 :: L016 :: C5 :: T_WHITESPACE :: (1) ::
70 :: L016 :: C6 :: T_DNUMBER :: (5) :: 6.674
71 :: L016 :: C11 :: T_STRING :: (5) :: _083e
72 :: L016 :: C16 :: T_MINUS :: (1) :: -
73 :: L016 :: C17 :: T_LNUMBER :: (2) :: 11
74 :: L016 :: C19 :: T_SEMICOLON :: (1) :: ;
75 :: L016 :: C20 :: T_WHITESPACE :: (1) ::
76 :: L016 :: C21 :: T_COMMENT :: (8) :: // float
77 :: L017 :: C1 :: T_VARIABLE :: (2) :: $a
78 :: L017 :: C3 :: T_WHITESPACE :: (1) ::
79 :: L017 :: C4 :: T_EQUAL :: (1) :: =
80 :: L017 :: C5 :: T_WHITESPACE :: (1) ::
81 :: L017 :: C6 :: T_LNUMBER :: (3) :: 299
82 :: L017 :: C9 :: T_STRING :: (8) :: _792_458
83 :: L017 :: C17 :: T_SEMICOLON :: (1) :: ;
84 :: L017 :: C18 :: T_WHITESPACE :: (3) ::
85 :: L017 :: C21 :: T_COMMENT :: (10) :: // decimal
86 :: L018 :: C1 :: T_VARIABLE :: (2) :: $a
87 :: L018 :: C3 :: T_WHITESPACE :: (1) ::
88 :: L018 :: C4 :: T_EQUAL :: (1) :: =
89 :: L018 :: C5 :: T_WHITESPACE :: (1) ::
90 :: L018 :: C6 :: T_LNUMBER :: (6) :: 0xCAFE
91 :: L018 :: C12 :: T_STRING :: (5) :: _F00D
92 :: L018 :: C17 :: T_SEMICOLON :: (1) :: ;
93 :: L018 :: C18 :: T_WHITESPACE :: (3) ::
94 :: L018 :: C21 :: T_COMMENT :: (14) :: // hexadecimal
95 :: L019 :: C1 :: T_VARIABLE :: (2) :: $a
96 :: L019 :: C3 :: T_WHITESPACE :: (1) ::
97 :: L019 :: C4 :: T_EQUAL :: (1) :: =
98 :: L019 :: C5 :: T_WHITESPACE :: (1) ::
99 :: L019 :: C6 :: T_LNUMBER :: (6) :: 0b0101
100 :: L019 :: C12 :: T_STRING :: (5) :: _1111
101 :: L019 :: C17 :: T_SEMICOLON :: (1) :: ;
102 :: L019 :: C18 :: T_WHITESPACE :: (3) ::
103 :: L019 :: C21 :: T_COMMENT :: (9) :: // binary
104 :: L020 :: C1 :: T_VARIABLE :: (2) :: $a
105 :: L020 :: C3 :: T_WHITESPACE :: (1) ::
106 :: L020 :: C4 :: T_EQUAL :: (1) :: =
107 :: L020 :: C5 :: T_WHITESPACE :: (1) ::
108 :: L020 :: C6 :: T_LNUMBER :: (4) :: 0137
109 :: L020 :: C10 :: T_STRING :: (4) :: _041
110 :: L020 :: C14 :: T_SEMICOLON :: (1) :: ;
111 :: L020 :: C15 :: T_WHITESPACE :: (6) ::
112 :: L020 :: C21 :: T_COMMENT :: (8) :: // octal
113 :: L021 :: C1 :: T_WHITESPACE :: (0) ::
114 :: L022 :: C1 :: T_COMMENT :: (19) :: // More test cases.
115 :: L023 :: C1 :: T_VARIABLE :: (2) :: $a
116 :: L023 :: C3 :: T_WHITESPACE :: (1) ::
117 :: L023 :: C4 :: T_EQUAL :: (1) :: =
118 :: L023 :: C5 :: T_WHITESPACE :: (1) ::
119 :: L023 :: C6 :: T_LNUMBER :: (1) :: 1
120 :: L023 :: C7 :: T_STRING :: (2) :: _2
121 :: L023 :: C9 :: T_DNUMBER :: (2) :: .3
122 :: L023 :: C11 :: T_STRING :: (7) :: _4e1_23
123 :: L023 :: C18 :: T_SEMICOLON :: (1) :: ;
124 :: L023 :: C19 :: T_WHITESPACE :: (0) ::
125 :: L024 :: C1 :: T_WHITESPACE :: (0) ::
126 :: L025 :: C1 :: T_COMMENT :: (51) :: // Invalid use of underscores in numeric sequences.
127 :: L026 :: C1 :: T_COMMENT :: (76) :: // Each underscore in a numeric literal must be directly between two digits.
128 :: L027 :: C1 :: T_COMMENT :: (66) :: // These below all produce "Parse error: syntax error" in PHP 7.4.
129 :: L028 :: C1 :: T_VARIABLE :: (2) :: $a
130 :: L028 :: C3 :: T_WHITESPACE :: (1) ::
131 :: L028 :: C4 :: T_EQUAL :: (1) :: =
132 :: L028 :: C5 :: T_WHITESPACE :: (1) ::
133 :: L028 :: C6 :: T_LNUMBER :: (3) :: 100
134 :: L028 :: C9 :: T_STRING :: (1) :: _
135 :: L028 :: C10 :: T_SEMICOLON :: (1) :: ;
136 :: L028 :: C11 :: T_WHITESPACE :: (3) ::
137 :: L028 :: C14 :: T_COMMENT :: (11) :: // trailing
138 :: L029 :: C1 :: T_VARIABLE :: (2) :: $a
139 :: L029 :: C3 :: T_WHITESPACE :: (1) ::
140 :: L029 :: C4 :: T_EQUAL :: (1) :: =
141 :: L029 :: C5 :: T_WHITESPACE :: (1) ::
142 :: L029 :: C6 :: T_LNUMBER :: (1) :: 1
143 :: L029 :: C7 :: T_STRING :: (3) :: __1
144 :: L029 :: C10 :: T_SEMICOLON :: (1) :: ;
145 :: L029 :: C11 :: T_WHITESPACE :: (3) ::
146 :: L029 :: C14 :: T_COMMENT :: (21) :: // next to underscore
147 :: L030 :: C1 :: T_VARIABLE :: (2) :: $a
148 :: L030 :: C3 :: T_WHITESPACE :: (1) ::
149 :: L030 :: C4 :: T_EQUAL :: (1) :: =
150 :: L030 :: C5 :: T_WHITESPACE :: (1) ::
151 :: L030 :: C6 :: T_LNUMBER :: (1) :: 1
152 :: L030 :: C7 :: T_STRING :: (1) :: _
153 :: L030 :: C8 :: T_DNUMBER :: (2) :: .0
154 :: L030 :: C10 :: T_SEMICOLON :: (1) :: ;
155 :: L030 :: C11 :: T_WHITESPACE :: (3) ::
156 :: L030 :: C14 :: T_COMMENT :: (24) :: // next to decimal point
157 :: L031 :: C1 :: T_VARIABLE :: (2) :: $a
158 :: L031 :: C3 :: T_WHITESPACE :: (1) ::
159 :: L031 :: C4 :: T_EQUAL :: (1) :: =
160 :: L031 :: C5 :: T_WHITESPACE :: (1) ::
161 :: L031 :: C6 :: T_DNUMBER :: (2) :: 1.
162 :: L031 :: C8 :: T_STRING :: (2) :: _0
163 :: L031 :: C10 :: T_SEMICOLON :: (1) :: ;
164 :: L031 :: C11 :: T_WHITESPACE :: (3) ::
165 :: L031 :: C14 :: T_COMMENT :: (24) :: // next to decimal point
166 :: L032 :: C1 :: T_VARIABLE :: (2) :: $a
167 :: L032 :: C3 :: T_WHITESPACE :: (1) ::
168 :: L032 :: C4 :: T_EQUAL :: (1) :: =
169 :: L032 :: C5 :: T_WHITESPACE :: (1) ::
170 :: L032 :: C6 :: T_LNUMBER :: (1) :: 0
171 :: L032 :: C7 :: T_STRING :: (5) :: x_123
172 :: L032 :: C12 :: T_SEMICOLON :: (1) :: ;
173 :: L032 :: C13 :: T_WHITESPACE :: (1) ::
174 :: L032 :: C14 :: T_COMMENT :: (12) :: // next to x
175 :: L033 :: C1 :: T_VARIABLE :: (2) :: $a
176 :: L033 :: C3 :: T_WHITESPACE :: (1) ::
177 :: L033 :: C4 :: T_EQUAL :: (1) :: =
178 :: L033 :: C5 :: T_WHITESPACE :: (1) ::
179 :: L033 :: C6 :: T_LNUMBER :: (1) :: 0
180 :: L033 :: C7 :: T_STRING :: (5) :: b_101
181 :: L033 :: C12 :: T_SEMICOLON :: (1) :: ;
182 :: L033 :: C13 :: T_WHITESPACE :: (1) ::
183 :: L033 :: C14 :: T_COMMENT :: (12) :: // next to b
184 :: L034 :: C1 :: T_VARIABLE :: (2) :: $a
185 :: L034 :: C3 :: T_WHITESPACE :: (1) ::
186 :: L034 :: C4 :: T_EQUAL :: (1) :: =
187 :: L034 :: C5 :: T_WHITESPACE :: (1) ::
188 :: L034 :: C6 :: T_LNUMBER :: (1) :: 1
189 :: L034 :: C7 :: T_STRING :: (3) :: _e2
190 :: L034 :: C10 :: T_SEMICOLON :: (1) :: ;
191 :: L034 :: C11 :: T_WHITESPACE :: (3) ::
192 :: L034 :: C14 :: T_COMMENT :: (12) :: // next to e
193 :: L035 :: C1 :: T_VARIABLE :: (2) :: $a
194 :: L035 :: C3 :: T_WHITESPACE :: (1) ::
195 :: L035 :: C4 :: T_EQUAL :: (1) :: =
196 :: L035 :: C5 :: T_WHITESPACE :: (1) ::
197 :: L035 :: C6 :: T_LNUMBER :: (1) :: 1
198 :: L035 :: C7 :: T_STRING :: (3) :: e_2
199 :: L035 :: C10 :: T_SEMICOLON :: (1) :: ;
200 :: L035 :: C11 :: T_WHITESPACE :: (3) ::
201 :: L035 :: C14 :: T_COMMENT :: (12) :: // next to e
This code sample is tokenized like this in PHP 7.4-alpha2:
0 :: L001 :: C1 :: T_OPEN_TAG :: (5) :: <?php
1 :: L002 :: C1 :: T_WHITESPACE :: (0) ::
2 :: L003 :: C1 :: T_COMMENT :: (34) :: // OK: ordinary numeric sequences.
3 :: L004 :: C1 :: T_VARIABLE :: (2) :: $a
4 :: L004 :: C3 :: T_WHITESPACE :: (1) ::
5 :: L004 :: C4 :: T_EQUAL :: (1) :: =
6 :: L004 :: C5 :: T_WHITESPACE :: (1) ::
7 :: L004 :: C6 :: T_LNUMBER :: (10) :: 1000000000
8 :: L004 :: C16 :: T_SEMICOLON :: (1) :: ;
9 :: L004 :: C17 :: T_WHITESPACE :: (0) ::
10 :: L005 :: C1 :: T_VARIABLE :: (2) :: $b
11 :: L005 :: C3 :: T_WHITESPACE :: (1) ::
12 :: L005 :: C4 :: T_EQUAL :: (1) :: =
13 :: L005 :: C5 :: T_WHITESPACE :: (1) ::
14 :: L005 :: C6 :: T_DNUMBER :: (12) :: 107925284.88
15 :: L005 :: C18 :: T_SEMICOLON :: (1) :: ;
16 :: L005 :: C19 :: T_WHITESPACE :: (0) ::
17 :: L006 :: C1 :: T_VARIABLE :: (9) :: $discount
18 :: L006 :: C10 :: T_WHITESPACE :: (1) ::
19 :: L006 :: C11 :: T_EQUAL :: (1) :: =
20 :: L006 :: C12 :: T_WHITESPACE :: (1) ::
21 :: L006 :: C13 :: T_LNUMBER :: (5) :: 13500
22 :: L006 :: C18 :: T_SEMICOLON :: (1) :: ;
23 :: L006 :: C19 :: T_WHITESPACE :: (0) ::
24 :: L007 :: C1 :: T_WHITESPACE :: (0) ::
25 :: L008 :: C1 :: T_COMMENT :: (68) :: // OK: Constant name starting with an underscore is perfectly valid.
26 :: L009 :: C1 :: T_CONST :: (5) :: const
27 :: L009 :: C6 :: T_WHITESPACE :: (1) ::
28 :: L009 :: C7 :: T_STRING :: (4) :: _100
29 :: L009 :: C11 :: T_WHITESPACE :: (1) ::
30 :: L009 :: C12 :: T_EQUAL :: (1) :: =
31 :: L009 :: C13 :: T_WHITESPACE :: (1) ::
32 :: L009 :: C14 :: T_LNUMBER :: (3) :: 100
33 :: L009 :: C17 :: T_SEMICOLON :: (1) :: ;
34 :: L009 :: C18 :: T_WHITESPACE :: (0) ::
35 :: L010 :: C1 :: T_WHITESPACE :: (0) ::
36 :: L011 :: C1 :: T_COMMENT :: (62) :: // PHP 7.4 numeric sequences with a numeric literal separator.
37 :: L012 :: C1 :: T_VARIABLE :: (10) :: $threshold
38 :: L012 :: C11 :: T_WHITESPACE :: (1) ::
39 :: L012 :: C12 :: T_EQUAL :: (1) :: =
40 :: L012 :: C13 :: T_WHITESPACE :: (1) ::
41 :: L012 :: C14 :: T_LNUMBER :: (13) :: 1_000_000_000
42 :: L012 :: C27 :: T_SEMICOLON :: (1) :: ;
43 :: L012 :: C28 :: T_WHITESPACE :: (2) ::
44 :: L012 :: C30 :: T_COMMENT :: (13) :: // a billion!
45 :: L013 :: C1 :: T_VARIABLE :: (10) :: $testValue
46 :: L013 :: C11 :: T_WHITESPACE :: (1) ::
47 :: L013 :: C12 :: T_EQUAL :: (1) :: =
48 :: L013 :: C13 :: T_WHITESPACE :: (1) ::
49 :: L013 :: C14 :: T_DNUMBER :: (14) :: 107_925_284.88
50 :: L013 :: C28 :: T_SEMICOLON :: (1) :: ;
51 :: L013 :: C29 :: T_WHITESPACE :: (1) ::
52 :: L013 :: C30 :: T_COMMENT :: (32) :: // scale is hundreds of millions
53 :: L014 :: C1 :: T_VARIABLE :: (9) :: $discount
54 :: L014 :: C10 :: T_WHITESPACE :: (2) ::
55 :: L014 :: C12 :: T_EQUAL :: (1) :: =
56 :: L014 :: C13 :: T_WHITESPACE :: (1) ::
57 :: L014 :: C14 :: T_LNUMBER :: (6) :: 135_00
58 :: L014 :: C20 :: T_SEMICOLON :: (1) :: ;
59 :: L014 :: C21 :: T_WHITESPACE :: (9) ::
60 :: L014 :: C30 :: T_COMMENT :: (24) :: // $135, stored as cents
61 :: L015 :: C1 :: T_WHITESPACE :: (0) ::
62 :: L016 :: C1 :: T_VARIABLE :: (2) :: $a
63 :: L016 :: C3 :: T_WHITESPACE :: (1) ::
64 :: L016 :: C4 :: T_EQUAL :: (1) :: =
65 :: L016 :: C5 :: T_WHITESPACE :: (1) ::
66 :: L016 :: C6 :: T_DNUMBER :: (13) :: 6.674_083e-11
67 :: L016 :: C19 :: T_SEMICOLON :: (1) :: ;
68 :: L016 :: C20 :: T_WHITESPACE :: (1) ::
69 :: L016 :: C21 :: T_COMMENT :: (8) :: // float
70 :: L017 :: C1 :: T_VARIABLE :: (2) :: $a
71 :: L017 :: C3 :: T_WHITESPACE :: (1) ::
72 :: L017 :: C4 :: T_EQUAL :: (1) :: =
73 :: L017 :: C5 :: T_WHITESPACE :: (1) ::
74 :: L017 :: C6 :: T_LNUMBER :: (11) :: 299_792_458
75 :: L017 :: C17 :: T_SEMICOLON :: (1) :: ;
76 :: L017 :: C18 :: T_WHITESPACE :: (3) ::
77 :: L017 :: C21 :: T_COMMENT :: (10) :: // decimal
78 :: L018 :: C1 :: T_VARIABLE :: (2) :: $a
79 :: L018 :: C3 :: T_WHITESPACE :: (1) ::
80 :: L018 :: C4 :: T_EQUAL :: (1) :: =
81 :: L018 :: C5 :: T_WHITESPACE :: (1) ::
82 :: L018 :: C6 :: T_DNUMBER :: (11) :: 0xCAFE_F00D
83 :: L018 :: C17 :: T_SEMICOLON :: (1) :: ;
84 :: L018 :: C18 :: T_WHITESPACE :: (3) ::
85 :: L018 :: C21 :: T_COMMENT :: (14) :: // hexadecimal
86 :: L019 :: C1 :: T_VARIABLE :: (2) :: $a
87 :: L019 :: C3 :: T_WHITESPACE :: (1) ::
88 :: L019 :: C4 :: T_EQUAL :: (1) :: =
89 :: L019 :: C5 :: T_WHITESPACE :: (1) ::
90 :: L019 :: C6 :: T_LNUMBER :: (11) :: 0b0101_1111
91 :: L019 :: C17 :: T_SEMICOLON :: (1) :: ;
92 :: L019 :: C18 :: T_WHITESPACE :: (3) ::
93 :: L019 :: C21 :: T_COMMENT :: (9) :: // binary
94 :: L020 :: C1 :: T_VARIABLE :: (2) :: $a
95 :: L020 :: C3 :: T_WHITESPACE :: (1) ::
96 :: L020 :: C4 :: T_EQUAL :: (1) :: =
97 :: L020 :: C5 :: T_WHITESPACE :: (1) ::
98 :: L020 :: C6 :: T_LNUMBER :: (8) :: 0137_041
99 :: L020 :: C14 :: T_SEMICOLON :: (1) :: ;
100 :: L020 :: C15 :: T_WHITESPACE :: (6) ::
101 :: L020 :: C21 :: T_COMMENT :: (8) :: // octal
102 :: L021 :: C1 :: T_WHITESPACE :: (0) ::
103 :: L022 :: C1 :: T_COMMENT :: (19) :: // More test cases.
104 :: L023 :: C1 :: T_VARIABLE :: (2) :: $a
105 :: L023 :: C3 :: T_WHITESPACE :: (1) ::
106 :: L023 :: C4 :: T_EQUAL :: (1) :: =
107 :: L023 :: C5 :: T_WHITESPACE :: (1) ::
108 :: L023 :: C6 :: T_DNUMBER :: (12) :: 1_2.3_4e1_23
109 :: L023 :: C18 :: T_SEMICOLON :: (1) :: ;
110 :: L023 :: C19 :: T_WHITESPACE :: (0) ::
111 :: L024 :: C1 :: T_WHITESPACE :: (0) ::
112 :: L025 :: C1 :: T_COMMENT :: (51) :: // Invalid use of underscores in numeric sequences.
113 :: L026 :: C1 :: T_COMMENT :: (76) :: // Each underscore in a numeric literal must be directly between two digits.
114 :: L027 :: C1 :: T_COMMENT :: (66) :: // These below all produce "Parse error: syntax error" in PHP 7.4.
115 :: L028 :: C1 :: T_VARIABLE :: (2) :: $a
116 :: L028 :: C3 :: T_WHITESPACE :: (1) ::
117 :: L028 :: C4 :: T_EQUAL :: (1) :: =
118 :: L028 :: C5 :: T_WHITESPACE :: (1) ::
119 :: L028 :: C6 :: T_LNUMBER :: (3) :: 100
120 :: L028 :: C9 :: T_STRING :: (1) :: _
121 :: L028 :: C10 :: T_SEMICOLON :: (1) :: ;
122 :: L028 :: C11 :: T_WHITESPACE :: (3) ::
123 :: L028 :: C14 :: T_COMMENT :: (11) :: // trailing
124 :: L029 :: C1 :: T_VARIABLE :: (2) :: $a
125 :: L029 :: C3 :: T_WHITESPACE :: (1) ::
126 :: L029 :: C4 :: T_EQUAL :: (1) :: =
127 :: L029 :: C5 :: T_WHITESPACE :: (1) ::
128 :: L029 :: C6 :: T_LNUMBER :: (1) :: 1
129 :: L029 :: C7 :: T_STRING :: (3) :: __1
130 :: L029 :: C10 :: T_SEMICOLON :: (1) :: ;
131 :: L029 :: C11 :: T_WHITESPACE :: (3) ::
132 :: L029 :: C14 :: T_COMMENT :: (21) :: // next to underscore
133 :: L030 :: C1 :: T_VARIABLE :: (2) :: $a
134 :: L030 :: C3 :: T_WHITESPACE :: (1) ::
135 :: L030 :: C4 :: T_EQUAL :: (1) :: =
136 :: L030 :: C5 :: T_WHITESPACE :: (1) ::
137 :: L030 :: C6 :: T_LNUMBER :: (1) :: 1
138 :: L030 :: C7 :: T_STRING :: (1) :: _
139 :: L030 :: C8 :: T_DNUMBER :: (2) :: .0
140 :: L030 :: C10 :: T_SEMICOLON :: (1) :: ;
141 :: L030 :: C11 :: T_WHITESPACE :: (3) ::
142 :: L030 :: C14 :: T_COMMENT :: (24) :: // next to decimal point
143 :: L031 :: C1 :: T_VARIABLE :: (2) :: $a
144 :: L031 :: C3 :: T_WHITESPACE :: (1) ::
145 :: L031 :: C4 :: T_EQUAL :: (1) :: =
146 :: L031 :: C5 :: T_WHITESPACE :: (1) ::
147 :: L031 :: C6 :: T_DNUMBER :: (2) :: 1.
148 :: L031 :: C8 :: T_STRING :: (2) :: _0
149 :: L031 :: C10 :: T_SEMICOLON :: (1) :: ;
150 :: L031 :: C11 :: T_WHITESPACE :: (3) ::
151 :: L031 :: C14 :: T_COMMENT :: (24) :: // next to decimal point
152 :: L032 :: C1 :: T_VARIABLE :: (2) :: $a
153 :: L032 :: C3 :: T_WHITESPACE :: (1) ::
154 :: L032 :: C4 :: T_EQUAL :: (1) :: =
155 :: L032 :: C5 :: T_WHITESPACE :: (1) ::
156 :: L032 :: C6 :: T_LNUMBER :: (1) :: 0
157 :: L032 :: C7 :: T_STRING :: (5) :: x_123
158 :: L032 :: C12 :: T_SEMICOLON :: (1) :: ;
159 :: L032 :: C13 :: T_WHITESPACE :: (1) ::
160 :: L032 :: C14 :: T_COMMENT :: (12) :: // next to x
161 :: L033 :: C1 :: T_VARIABLE :: (2) :: $a
162 :: L033 :: C3 :: T_WHITESPACE :: (1) ::
163 :: L033 :: C4 :: T_EQUAL :: (1) :: =
164 :: L033 :: C5 :: T_WHITESPACE :: (1) ::
165 :: L033 :: C6 :: T_LNUMBER :: (1) :: 0
166 :: L033 :: C7 :: T_STRING :: (5) :: b_101
167 :: L033 :: C12 :: T_SEMICOLON :: (1) :: ;
168 :: L033 :: C13 :: T_WHITESPACE :: (1) ::
169 :: L033 :: C14 :: T_COMMENT :: (12) :: // next to b
170 :: L034 :: C1 :: T_VARIABLE :: (2) :: $a
171 :: L034 :: C3 :: T_WHITESPACE :: (1) ::
172 :: L034 :: C4 :: T_EQUAL :: (1) :: =
173 :: L034 :: C5 :: T_WHITESPACE :: (1) ::
174 :: L034 :: C6 :: T_LNUMBER :: (1) :: 1
175 :: L034 :: C7 :: T_STRING :: (3) :: _e2
176 :: L034 :: C10 :: T_SEMICOLON :: (1) :: ;
177 :: L034 :: C11 :: T_WHITESPACE :: (3) ::
178 :: L034 :: C14 :: T_COMMENT :: (12) :: // next to e
179 :: L035 :: C1 :: T_VARIABLE :: (2) :: $a
180 :: L035 :: C3 :: T_WHITESPACE :: (1) ::
181 :: L035 :: C4 :: T_EQUAL :: (1) :: =
182 :: L035 :: C5 :: T_WHITESPACE :: (1) ::
183 :: L035 :: C6 :: T_LNUMBER :: (1) :: 1
184 :: L035 :: C7 :: T_STRING :: (3) :: e_2
185 :: L035 :: C10 :: T_SEMICOLON :: (1) :: ;
186 :: L035 :: C11 :: T_WHITESPACE :: (3) ::
187 :: L035 :: C14 :: T_COMMENT :: (12) :: // next to e