Skip to content

Commit 119a702

Browse files
gsherwoodgsherwood
committed
PHP 7.4 numeric separators are now tokenized in the same way when using older PHP versions (ref #2546)
1 parent eb378ac commit 119a702

File tree

4 files changed

+193
-0
lines changed

4 files changed

+193
-0
lines changed

‎package.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
3232
-- The token after the statement (normally a semicolon) becomes the scope closer
3333
-- The token is also associated with the opening and closing parenthesis of the statement
3434
-- Any functions named "fn" will cause have a T_FN token for the function name, but have no scope information
35+
- PHP 7.4 numeric separators are now tokenized in the same way when using older PHP versions
36+
-- Previously, a number like 1_000 would tokenize as T_LNUMBER (1), T_STRING (_000)
37+
-- Now, the number tokenizes as T_LNUMBER (1_000)
38+
-- Sniff developers should consider how numbers with underscores impact their custom sniffs
3539
- The PHPCS file cache now takes file permissions into account
3640
-- The cache is now invalidated for a file when its permissions are changed
3741
- File::getMethodParameters() now supports arrow functions
@@ -125,6 +129,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
125129
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
126130
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
127131
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
132+
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
133+
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
128134
</dir>
129135
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
130136
<file baseinstalldir="" name="AllTests.php" role="test" />
@@ -1960,6 +1966,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
19601966
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
19611967
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
19621968
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
1969+
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
1970+
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
19631971
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
19641972
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
19651973
</filelist>
@@ -2001,6 +2009,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20012009
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
20022010
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
20032011
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
2012+
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
2013+
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
20042014
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20052015
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20062016
<ignore name="bin/phpcs.bat" />

‎src/Tokenizers/PHP.php

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,66 @@ protected function tokenize($string)
968968
continue;
969969
}
970970

971+
/*
972+
Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
973+
tokens split the token with a T_STRING. So look for
974+
and change these tokens in earlier versions.
975+
*/
976+
977+
if ($tokenIsArray === true
978+
&& ($token[0] === T_LNUMBER
979+
|| $token[0] === T_DNUMBER)
980+
&& isset($tokens[($stackPtr + 1)]) === true
981+
&& is_array($tokens[($stackPtr + 1)]) === true
982+
&& $tokens[($stackPtr + 1)][0] === T_STRING
983+
&& $tokens[($stackPtr + 1)][1][0] === '_'
984+
) {
985+
$newContent = $token[1];
986+
$newType = $token[0];
987+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
988+
if (is_array($tokens[$i]) === false) {
989+
break;
990+
}
991+
992+
if ($tokens[$i][0] === T_LNUMBER
993+
|| $tokens[$i][0] === T_DNUMBER
994+
|| ($tokens[$i][0] === T_STRING
995+
&& $tokens[$i][1][0] === '_')
996+
) {
997+
$newContent .= $tokens[$i][1];
998+
999+
// Any T_DNUMBER token needs to make the
1000+
// new number a T_DNUMBER as well.
1001+
if ($tokens[$i][0] === T_DNUMBER) {
1002+
$newType = T_DNUMBER;
1003+
}
1004+
1005+
// Support floats.
1006+
if ($tokens[$i][0] === T_STRING
1007+
&& substr(strtolower($tokens[$i][1]), -1) === 'e'
1008+
&& $tokens[($i + 1)] === '-'
1009+
) {
1010+
$newContent .= '-';
1011+
$i++;
1012+
}
1013+
1014+
continue;
1015+
}
1016+
1017+
break;
1018+
}//end for
1019+
1020+
$newToken = [];
1021+
$newToken['code'] = $newType;
1022+
$newToken['type'] = Util\Tokens::tokenName($token[0]);
1023+
$newToken['content'] = $newContent;
1024+
$finalTokens[$newStackPtr] = $newToken;
1025+
1026+
$newStackPtr++;
1027+
$stackPtr = ($i - 1);
1028+
continue;
1029+
}//end if
1030+
9711031
/*
9721032
Convert ? to T_NULLABLE OR T_INLINE_THEN
9731033
*/

‎tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?php
2+
3+
/* testSimpleLNumber */
4+
$foo = 1_000_000_000;
5+
6+
/* testSimpleDNumber */
7+
‪$foo = 107_925_284.88;
8+
9+
/* testFloat */
10+
$foo = 6.674_083e-11;
11+
12+
/* testHex */
13+
$foo = 0xCAFE_F00D;
14+
15+
/* testHexMultiple */
16+
$foo = 0x42_72_6F_77_6E;
17+
18+
/* testBinary */
19+
$foo = 0b0101_1111;
20+
21+
/* testOctal */
22+
$foo = 0137_041;

‎tests/Core/Tokenizer/BackfillNumericSeparatorTest.php

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<?php
2+
/**
3+
* Tests the backfilling of numeric seperators to PHP < 7.4.
4+
*
5+
* @author Greg Sherwood <gsherwood@squiz.net>
6+
* @copyright 2019 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
11+
12+
use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
13+
14+
class BackfillNumericSeparatorTest extends AbstractMethodUnitTest
15+
{
16+
17+
18+
/**
19+
* Test that numbers using numeric seperators are tokenized correctly.
20+
*
21+
* @param array $testData The data required for the specific test case.
22+
*
23+
* @dataProvider dataTestBackfill
24+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
25+
*
26+
* @return void
27+
*/
28+
public function testBackfill($testData)
29+
{
30+
$tokens = self::$phpcsFile->getTokens();
31+
$number = $this->getTargetToken($testData['marker'], $testData['type']);
32+
$this->assertSame($tokens[$number]['content'], $testData['value']);
33+
34+
}//end testBackfill()
35+
36+
37+
/**
38+
* Data provider.
39+
*
40+
* @see testBackfill()
41+
*
42+
* @return array
43+
*/
44+
public function dataTestBackfill()
45+
{
46+
return [
47+
[
48+
[
49+
'marker' => '/* testSimpleLNumber */',
50+
'type' => T_LNUMBER,
51+
'value' => '1_000_000_000',
52+
],
53+
],
54+
[
55+
[
56+
'marker' => '/* testSimpleDNumber */',
57+
'type' => T_DNUMBER,
58+
'value' => '107_925_284.88',
59+
],
60+
],
61+
[
62+
[
63+
'marker' => '/* testFloat */',
64+
'type' => T_DNUMBER,
65+
'value' => '6.674_083e-11',
66+
],
67+
],
68+
[
69+
[
70+
'marker' => '/* testHex */',
71+
'type' => T_LNUMBER,
72+
'value' => '0xCAFE_F00D',
73+
],
74+
],
75+
[
76+
[
77+
'marker' => '/* testHexMultiple */',
78+
'type' => T_LNUMBER,
79+
'value' => '0x42_72_6F_77_6E',
80+
],
81+
],
82+
[
83+
[
84+
'marker' => '/* testBinary */',
85+
'type' => T_LNUMBER,
86+
'value' => '0b0101_1111',
87+
],
88+
],
89+
[
90+
[
91+
'marker' => '/* testOctal */',
92+
'type' => T_LNUMBER,
93+
'value' => '0137_041',
94+
],
95+
],
96+
];
97+
98+
}//end dataTestBackfill()
99+
100+
101+
}//end class

0 commit comments

Comments
 (0)