Skip to content

Commit 51afb54

Browse files
gsherwoodgsherwood
committed
T_FN tokens now have scope openers and closers (ref #2523)
This changed the way the T_FN token was backfilled as it now needs to set before processAdditional kicks in so it also works in PHP 7.4. Scope information is added in processAdditional because the way the opener and closer is calulated is too specific to put into the generic scope mapping code.
1 parent 033b431 commit 51afb54

File tree

4 files changed

+354
-36
lines changed

4 files changed

+354
-36
lines changed

‎package.xml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ http://pear.php.net/dtd/package-2.0.xsd">
2828
<notes>
2929
- The PHP 7.4 T_FN token has been made available for older versions
3030
-- T_FN represents the fn string used for arrow functions
31-
-- The token is associated with the opening and closing parenthesis of the statement
31+
-- The double arrow becomes the scope opener
32+
-- The token after the statement (normally a semicolon) becomes the scope closer
33+
-- The token is also associated with the opening and closing parenthesis of the statement
34+
-- Any functions named "fn" will cause have a T_FN token for the function name, but have no scope information
3235
- File::getMethodParameters() now supports arrow functions
3336
- File::getMethodProperties() now supports arrow functions
3437
- Generic.CodeAnalysis.EmptyPhpStatement now reports unnecessary semicolons after control structure closing braces

‎src/Tokenizers/PHP.php

Lines changed: 98 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,6 +1217,28 @@ function return types. We want to keep the parenthesis map clean,
12171217
continue;
12181218
}
12191219

1220+
/*
1221+
Backfill the T_FN token for PHP versions < 7.4.
1222+
*/
1223+
1224+
if ($tokenIsArray === true
1225+
&& $token[0] === T_STRING
1226+
&& strtolower($token[1]) === 'fn'
1227+
) {
1228+
$finalTokens[$newStackPtr] = [
1229+
'content' => $token[1],
1230+
'code' => T_FN,
1231+
'type' => 'T_FN',
1232+
];
1233+
1234+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1235+
echo "\t\t* token $stackPtr changed from T_STRING to T_FN".PHP_EOL;
1236+
}
1237+
1238+
$newStackPtr++;
1239+
continue;
1240+
}
1241+
12201242
/*
12211243
PHP doesn't assign a token to goto labels, so we have to.
12221244
These are just string tokens with a single colon after them. Double
@@ -1606,9 +1628,7 @@ protected function processAdditional()
16061628
}//end if
16071629

16081630
continue;
1609-
} else if ($this->tokens[$i]['code'] === T_STRING
1610-
&& strtolower($this->tokens[$i]['content']) === 'fn'
1611-
) {
1631+
} else if ($this->tokens[$i]['code'] === T_FN) {
16121632
// Possible arrow function.
16131633
for ($x = ($i + 1); $i < $numTokens; $x++) {
16141634
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
@@ -1618,22 +1638,84 @@ protected function processAdditional()
16181638
}
16191639

16201640
if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
1621-
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1622-
$line = $this->tokens[$i]['line'];
1623-
echo "\t* token $i on line $line changed from T_STRING to T_FN".PHP_EOL;
1641+
$ignore = Util\Tokens::$emptyTokens;
1642+
$ignore += [
1643+
T_STRING => T_STRING,
1644+
T_COLON => T_COLON,
1645+
];
1646+
1647+
$closer = $this->tokens[$x]['parenthesis_closer'];
1648+
for ($arrow = ($closer + 1); $arrow < $numTokens; $arrow++) {
1649+
if (isset($ignore[$this->tokens[$arrow]['code']]) === false) {
1650+
break;
1651+
}
16241652
}
16251653

1626-
$this->tokens[$i]['code'] = T_FN;
1627-
$this->tokens[$i]['type'] = 'T_FN';
1628-
$this->tokens[$i]['parenthesis_owner'] = $i;
1629-
$this->tokens[$i]['parenthesis_opener'] = $x;
1630-
$this->tokens[$i]['parenthesis_closer'] = $this->tokens[$x]['parenthesis_closer'];
1654+
if ($this->tokens[$arrow]['code'] === T_DOUBLE_ARROW) {
1655+
$endTokens = [
1656+
T_COLON => true,
1657+
T_COMMA => true,
1658+
T_DOUBLE_ARROW => true,
1659+
T_SEMICOLON => true,
1660+
T_CLOSE_PARENTHESIS => true,
1661+
T_CLOSE_SQUARE_BRACKET => true,
1662+
T_CLOSE_CURLY_BRACKET => true,
1663+
T_CLOSE_SHORT_ARRAY => true,
1664+
T_OPEN_TAG => true,
1665+
T_CLOSE_TAG => true,
1666+
];
16311667

1632-
$opener = $this->tokens[$i]['parenthesis_opener'];
1633-
$closer = $this->tokens[$i]['parenthesis_closer'];
1634-
$this->tokens[$opener]['parenthesis_owner'] = $i;
1635-
$this->tokens[$closer]['parenthesis_owner'] = $i;
1636-
}
1668+
for ($scopeCloser = ($arrow + 1); $scopeCloser < $numTokens; $scopeCloser++) {
1669+
if (isset($endTokens[$this->tokens[$scopeCloser]['code']]) === true) {
1670+
break;
1671+
}
1672+
1673+
if (isset($this->tokens[$scopeCloser]['scope_closer']) === true) {
1674+
// We minus 1 here in case the closer can be shared with us.
1675+
$scopeCloser = ($this->tokens[$scopeCloser]['scope_closer'] - 1);
1676+
continue;
1677+
}
1678+
1679+
if (isset($this->tokens[$scopeCloser]['parenthesis_closer']) === true) {
1680+
$scopeCloser = $this->tokens[$scopeCloser]['parenthesis_closer'];
1681+
continue;
1682+
}
1683+
1684+
if (isset($this->tokens[$scopeCloser]['bracket_closer']) === true) {
1685+
$scopeCloser = $this->tokens[$scopeCloser]['bracket_closer'];
1686+
continue;
1687+
}
1688+
}//end for
1689+
1690+
if ($scopeCloser !== $numTokens) {
1691+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1692+
$line = $this->tokens[$i]['line'];
1693+
echo "\t* token $i on line $line changed from T_STRING to T_FN".PHP_EOL;
1694+
}
1695+
1696+
$this->tokens[$i]['code'] = T_FN;
1697+
$this->tokens[$i]['type'] = 'T_FN';
1698+
$this->tokens[$i]['scope_condition'] = $i;
1699+
$this->tokens[$i]['scope_opener'] = $arrow;
1700+
$this->tokens[$i]['scope_closer'] = $scopeCloser;
1701+
$this->tokens[$i]['parenthesis_owner'] = $i;
1702+
$this->tokens[$i]['parenthesis_opener'] = $x;
1703+
$this->tokens[$i]['parenthesis_closer'] = $this->tokens[$x]['parenthesis_closer'];
1704+
1705+
$this->tokens[$arrow]['scope_condition'] = $i;
1706+
$this->tokens[$arrow]['scope_opener'] = $arrow;
1707+
$this->tokens[$arrow]['scope_closer'] = $scopeCloser;
1708+
$this->tokens[$scopeCloser]['scope_condition'] = $i;
1709+
$this->tokens[$scopeCloser]['scope_opener'] = $arrow;
1710+
$this->tokens[$scopeCloser]['scope_closer'] = $scopeCloser;
1711+
1712+
$opener = $this->tokens[$i]['parenthesis_opener'];
1713+
$closer = $this->tokens[$i]['parenthesis_closer'];
1714+
$this->tokens[$opener]['parenthesis_owner'] = $i;
1715+
$this->tokens[$closer]['parenthesis_owner'] = $i;
1716+
}//end if
1717+
}//end if
1718+
}//end if
16371719
} else if ($this->tokens[$i]['code'] === T_OPEN_SQUARE_BRACKET) {
16381720
if (isset($this->tokens[$i]['bracket_closer']) === false) {
16391721
continue;

‎tests/Core/Tokenizer/BackfillFnTokenTest.inc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,24 @@ $fn1 = fn /* comment here */ ($x) => $x + $y;
1414

1515
/* testFunctionName */
1616
function fn() {}
17+
18+
/* testNested */
19+
$fn = fn($x) => fn($y) => $x * $y + $z;
20+
21+
/* testFunctionCall */
22+
$extended = fn($c) => $callable($factory($c), $c);
23+
24+
/* testClosure */
25+
$extended = fn($c) => $callable(function() {
26+
for ($x = 1; $x < 10; $x++) {
27+
echo $x;
28+
}
29+
30+
echo 'done';
31+
}, $c);
32+
33+
$result = array_map(
34+
/* testReturnType */
35+
static fn(int $number) : int => $number + 1,
36+
$numbers
37+
);

0 commit comments

Comments
 (0)