Skip to content

Commit b768302

Browse files
jrfnljrfnl
committed
Tokenizer: apply tab replacement to "yield from"
The `yield from` keyword(s) can be separated by spaces, tabs, new lines and since PHP 8.3, even comments. The PHPCS `Tokenizer` previously did not execute tab replacement on this token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. Previously, this affected all `T_YIELD_FROM` tokens. After the change to support PHP 8.3 "yield ... comment... from" syntax, this now only affects single line `yield from` expressions where the keywords are separated by a tab or a mix of tabs and spaces. All the same, consistency is key, so this commit adds the `T_YIELD_FROM` token to the array of tokens for which to do tab replacement, which should make them more consistent with the rest of PHPCS. Includes unit tests safeguarding this change.
1 parent 62cda40 commit b768302

File tree

4 files changed

+121
-1
lines changed

4 files changed

+121
-1
lines changed

‎src/Tokenizers/Tokenizer.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ private function createPositionMap()
200200
T_END_HEREDOC => true,
201201
T_END_NOWDOC => true,
202202
T_INLINE_HTML => true,
203+
T_YIELD_FROM => true,
203204
];
204205

205206
$this->numTokens = count($this->tokens);

‎tests/Core/Tokenizers/PHP/YieldTest.php

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,11 @@ public function testYieldFromKeywordSingleToken($testMarker, $expectedContent)
100100
$this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)');
101101
$this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)');
102102

103-
$this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation');
103+
if (isset($tokenArray['orig_content']) === true) {
104+
$this->assertSame($expectedContent, $tokenArray['orig_content'], 'Token (orig) content does not match expectation');
105+
} else {
106+
$this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation');
107+
}
104108

105109
}//end testYieldFromKeywordSingleToken()
106110

‎tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?php
2+
3+
function myGenerator() {
4+
/* testYieldFromHasSingleSpace */
5+
yield from gen2();
6+
7+
/* testYieldFromHasMultiSpace */
8+
yield from gen2();
9+
10+
/* testYieldFromHasTabs */
11+
yield from gen2();
12+
13+
/* testYieldFromMixedTabsSpaces */
14+
Yield From gen2();
15+
}

‎tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of "yield from" tokens.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
13+
14+
/**
15+
* Yield from token test.
16+
*
17+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
18+
*/
19+
final class CreatePositionMapYieldFromTest extends AbstractTokenizerTestCase
20+
{
21+
22+
23+
/**
24+
* Verify that spaces/tabs in "yield from" tokens get the tab replacement treatment.
25+
*
26+
* @param string $testMarker The comment prefacing the target token.
27+
* @param array<string, int|string|null> $expected Expectations for the token array.
28+
* @param string $content Optional. The test token content to search for.
29+
* Defaults to null.
30+
*
31+
* @dataProvider dataYieldFromTabReplacement
32+
*
33+
* @return void
34+
*/
35+
public function testYieldFromTabReplacement($testMarker, $expected, $content=null)
36+
{
37+
$tokens = $this->phpcsFile->getTokens();
38+
$target = $this->getTargetToken($testMarker, [T_YIELD_FROM], $content);
39+
40+
foreach ($expected as $key => $value) {
41+
if ($key === 'orig_content' && $value === null) {
42+
$this->assertArrayNotHasKey($key, $tokens[$target], "Unexpected 'orig_content' key found in the token array.");
43+
continue;
44+
}
45+
46+
$this->assertArrayHasKey($key, $tokens[$target], "Key $key not found in the token array.");
47+
$this->assertSame($value, $tokens[$target][$key], "Value for key $key does not match expectation.");
48+
}
49+
50+
}//end testYieldFromTabReplacement()
51+
52+
53+
/**
54+
* Data provider.
55+
*
56+
* @see testYieldFromTabReplacement()
57+
*
58+
* @return array<string, array<string, string|array<string, int|string|null>>>
59+
*/
60+
public static function dataYieldFromTabReplacement()
61+
{
62+
return [
63+
'Yield from, single line, single space' => [
64+
'testMarker' => '/* testYieldFromHasSingleSpace */',
65+
'expected' => [
66+
'length' => 10,
67+
'content' => 'yield from',
68+
'orig_content' => null,
69+
],
70+
],
71+
'Yield from, single line, multiple spaces' => [
72+
'testMarker' => '/* testYieldFromHasMultiSpace */',
73+
'expected' => [
74+
'length' => 14,
75+
'content' => 'yield from',
76+
'orig_content' => null,
77+
],
78+
],
79+
'Yield from, single line, has tabs' => [
80+
'testMarker' => '/* testYieldFromHasTabs */',
81+
'expected' => [
82+
'length' => 16,
83+
'content' => 'yield from',
84+
'orig_content' => 'yield from',
85+
],
86+
],
87+
'Yield from, single line, mix of tabs and spaces' => [
88+
'testMarker' => '/* testYieldFromMixedTabsSpaces */',
89+
'expected' => [
90+
'length' => 20,
91+
'content' => 'Yield From',
92+
'orig_content' => 'Yield From',
93+
],
94+
],
95+
];
96+
97+
}//end dataYieldFromTabReplacement()
98+
99+
100+
}//end class

0 commit comments

Comments
 (0)