Skip to content

Commit aafc72f

Browse files
jrfnljrfnl
authored
Merge pull request #645 from PHPCSStandards/feature/tokenizer-php-yield-from-add-tests+minor-bug-fix
Tokenizer/PHP: add tests for tokenization of yield and yield from + minor bug fix
2 parents de3ca90 + df8bfe9 commit aafc72f

File tree

3 files changed

+314
-46
lines changed

3 files changed

+314
-46
lines changed

‎src/Tokenizers/PHP.php

Lines changed: 21 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,27 @@ protected function tokenize($string)
605605
echo PHP_EOL;
606606
}
607607

608+
/*
609+
Before PHP 5.5, the yield keyword was tokenized as
610+
T_STRING. So look for and change this token in
611+
earlier versions.
612+
*/
613+
614+
if (PHP_VERSION_ID < 50500
615+
&& $tokenIsArray === true
616+
&& $token[0] === T_STRING
617+
&& strtolower($token[1]) === 'yield'
618+
&& isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false
619+
) {
620+
// Could still be a context sensitive keyword or "yield from" and potentially multi-line,
621+
// so adjust the token stack in place.
622+
$token[0] = T_YIELD;
623+
624+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
625+
echo "\t\t* token $stackPtr changed from T_STRING to T_YIELD".PHP_EOL;
626+
}
627+
}
628+
608629
/*
609630
Tokenize context sensitive keyword as string when it should be string.
610631
*/
@@ -1499,7 +1520,6 @@ protected function tokenize($string)
14991520
*/
15001521

15011522
if (PHP_VERSION_ID < 70000
1502-
&& PHP_VERSION_ID >= 50500
15031523
&& $tokenIsArray === true
15041524
&& $token[0] === T_YIELD
15051525
&& isset($tokens[($stackPtr + 1)]) === true
@@ -1524,51 +1544,6 @@ protected function tokenize($string)
15241544
$tokens[($stackPtr + 2)] = null;
15251545
}
15261546

1527-
/*
1528-
Before PHP 5.5, the yield keyword was tokenized as
1529-
T_STRING. So look for and change this token in
1530-
earlier versions.
1531-
Checks also if it is just "yield" or "yield from".
1532-
*/
1533-
1534-
if (PHP_VERSION_ID < 50500
1535-
&& $tokenIsArray === true
1536-
&& $token[0] === T_STRING
1537-
&& strtolower($token[1]) === 'yield'
1538-
&& isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false
1539-
) {
1540-
if (isset($tokens[($stackPtr + 1)]) === true
1541-
&& isset($tokens[($stackPtr + 2)]) === true
1542-
&& $tokens[($stackPtr + 1)][0] === T_WHITESPACE
1543-
&& $tokens[($stackPtr + 2)][0] === T_STRING
1544-
&& strtolower($tokens[($stackPtr + 2)][1]) === 'from'
1545-
) {
1546-
// Could be multi-line, so just just the token stack.
1547-
$token[0] = T_YIELD_FROM;
1548-
$token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1];
1549-
1550-
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1551-
for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) {
1552-
$type = Tokens::tokenName($tokens[$i][0]);
1553-
$content = Common::prepareForOutput($tokens[$i][1]);
1554-
echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content".PHP_EOL;
1555-
}
1556-
}
1557-
1558-
$tokens[($stackPtr + 1)] = null;
1559-
$tokens[($stackPtr + 2)] = null;
1560-
} else {
1561-
$newToken = [];
1562-
$newToken['code'] = T_YIELD;
1563-
$newToken['type'] = 'T_YIELD';
1564-
$newToken['content'] = $token[1];
1565-
$finalTokens[$newStackPtr] = $newToken;
1566-
1567-
$newStackPtr++;
1568-
continue;
1569-
}//end if
1570-
}//end if
1571-
15721547
/*
15731548
Before PHP 5.6, the ... operator was tokenized as three
15741549
T_STRING_CONCAT tokens in a row. So look for and combine

‎tests/Core/Tokenizers/PHP/YieldTest.inc

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<?php
2+
3+
function generator()
4+
{
5+
/* testYield */
6+
yield 1;
7+
8+
/* testYieldFollowedByComment */
9+
YIELD/*comment*/ 2;
10+
11+
/* testYieldFrom */
12+
yield from gen2();
13+
14+
/* testYieldFromWithExtraSpacesBetween */
15+
Yield From gen2();
16+
17+
/* testYieldFromWithTabBetween */
18+
yield from gen2();
19+
20+
/* testYieldFromSplitByNewLines */
21+
yield
22+
23+
FROM
24+
gen2();
25+
}
26+
27+
/* testYieldUsedAsClassName */
28+
class Yield {
29+
/* testYieldUsedAsClassConstantName */
30+
const Type YIELD = 'foo';
31+
32+
/* testYieldUsedAsMethodName */
33+
public function yield() {
34+
/* testYieldUsedAsPropertyName1 */
35+
echo $obj->yield;
36+
37+
/* testYieldUsedAsPropertyName2 */
38+
echo $obj?->yield();
39+
40+
/* testYieldUsedForClassConstantAccess1 */
41+
echo MyClass::YIELD;
42+
/* testFromUsedForClassConstantAccess1 */
43+
echo MyClass::FROM;
44+
}
45+
46+
/* testYieldUsedAsMethodNameReturnByRef */
47+
public function &yield() {}
48+
}
49+
50+
function myGen() {
51+
/* testYieldLiveCoding */
52+
yield

‎tests/Core/Tokenizers/PHP/YieldTest.php

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of the `yield` and `yield from` keywords.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2021 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
/**
16+
* Tests the tokenization of the `yield` and `yield from` keywords.
17+
*
18+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
19+
*/
20+
final class YieldTest extends AbstractTokenizerTestCase
21+
{
22+
23+
24+
/**
25+
* Test that the yield keyword is tokenized as such.
26+
*
27+
* @param string $testMarker The comment which prefaces the target token in the test file.
28+
*
29+
* @dataProvider dataYieldKeyword
30+
*
31+
* @return void
32+
*/
33+
public function testYieldKeyword($testMarker)
34+
{
35+
$tokens = $this->phpcsFile->getTokens();
36+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
37+
$tokenArray = $tokens[$target];
38+
39+
$this->assertSame(T_YIELD, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (code)');
40+
41+
// This assertion would fail on PHP 5.4 with PHPUnit 4 as PHPUnit polyfills the `T_YIELD` token too, but
42+
// with a different value, which causes the token 'type' to be set to `UNKNOWN`.
43+
// This issue _only_ occurs when running the tests, not when running PHPCS outside of a test situation.
44+
// The PHPUnit polyfilled token is declared in the PHP_CodeCoverage_Report_HTML_Renderer_File class
45+
// in vendor/phpunit/php-code-coverage/src/CodeCoverage/Report/HTML/Renderer/File.php.
46+
if (PHP_VERSION_ID >= 50500) {
47+
$this->assertSame('T_YIELD', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (type)');
48+
}
49+
50+
}//end testYieldKeyword()
51+
52+
53+
/**
54+
* Data provider.
55+
*
56+
* @see testYieldKeyword()
57+
*
58+
* @return array<string, array<string>>
59+
*/
60+
public static function dataYieldKeyword()
61+
{
62+
return [
63+
'yield' => ['/* testYield */'],
64+
'yield followed by comment' => ['/* testYieldFollowedByComment */'],
65+
'yield at end of file, live coding' => ['/* testYieldLiveCoding */'],
66+
];
67+
68+
}//end dataYieldKeyword()
69+
70+
71+
/**
72+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
73+
* and only has whitespace between the words.
74+
*
75+
* @param string $testMarker The comment which prefaces the target token in the test file.
76+
* @param string $content Optional. The test token content to search for.
77+
* Defaults to null.
78+
*
79+
* @dataProvider dataYieldFromKeywordSingleToken
80+
*
81+
* @return void
82+
*/
83+
public function testYieldFromKeywordSingleToken($testMarker, $content=null)
84+
{
85+
$tokens = $this->phpcsFile->getTokens();
86+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING], $content);
87+
$tokenArray = $tokens[$target];
88+
89+
$this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)');
90+
$this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)');
91+
92+
}//end testYieldFromKeywordSingleToken()
93+
94+
95+
/**
96+
* Data provider.
97+
*
98+
* @see testYieldFromKeywordSingleToken()
99+
*
100+
* @return array<string, array<string>>
101+
*/
102+
public static function dataYieldFromKeywordSingleToken()
103+
{
104+
return [
105+
'yield from' => [
106+
'testMarker' => '/* testYieldFrom */',
107+
],
108+
'yield from with extra space between' => [
109+
'testMarker' => '/* testYieldFromWithExtraSpacesBetween */',
110+
],
111+
'yield from with tab between' => [
112+
'testMarker' => '/* testYieldFromWithTabBetween */',
113+
],
114+
];
115+
116+
}//end dataYieldFromKeywordSingleToken()
117+
118+
119+
/**
120+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
121+
* and only has whitespace between the words.
122+
*
123+
* @param string $testMarker The comment which prefaces the target token in the test file.
124+
* @param array<array<string, string>> $expectedTokens The tokenization expected.
125+
*
126+
* @dataProvider dataYieldFromKeywordMultiToken
127+
*
128+
* @return void
129+
*/
130+
public function testYieldFromKeywordMultiToken($testMarker, $expectedTokens)
131+
{
132+
$tokens = $this->phpcsFile->getTokens();
133+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
134+
135+
foreach ($expectedTokens as $nr => $tokenInfo) {
136+
$this->assertSame(
137+
constant($tokenInfo['type']),
138+
$tokens[$target]['code'],
139+
'Token tokenized as '.Tokens::tokenName($tokens[$target]['code']).', not '.$tokenInfo['type'].' (code)'
140+
);
141+
$this->assertSame(
142+
$tokenInfo['type'],
143+
$tokens[$target]['type'],
144+
'Token tokenized as '.$tokens[$target]['type'].', not '.$tokenInfo['type'].' (type)'
145+
);
146+
$this->assertSame(
147+
$tokenInfo['content'],
148+
$tokens[$target]['content'],
149+
'Content of token '.($nr + 1).' ('.$tokens[$target]['type'].') does not match expectations'
150+
);
151+
152+
++$target;
153+
}
154+
155+
}//end testYieldFromKeywordMultiToken()
156+
157+
158+
/**
159+
* Data provider.
160+
*
161+
* @see testYieldFromKeywordMultiToken()
162+
*
163+
* @return array<string, array<string, string|array<array<string, string>>>>
164+
*/
165+
public static function dataYieldFromKeywordMultiToken()
166+
{
167+
return [
168+
'yield from with new line' => [
169+
'testMarker' => '/* testYieldFromSplitByNewLines */',
170+
'expectedTokens' => [
171+
[
172+
'type' => 'T_YIELD_FROM',
173+
'content' => 'yield
174+
',
175+
],
176+
[
177+
'type' => 'T_YIELD_FROM',
178+
'content' => '
179+
',
180+
],
181+
[
182+
'type' => 'T_YIELD_FROM',
183+
'content' => ' FROM',
184+
],
185+
[
186+
'type' => 'T_WHITESPACE',
187+
'content' => '
188+
',
189+
],
190+
],
191+
],
192+
];
193+
194+
}//end dataYieldFromKeywordMultiToken()
195+
196+
197+
/**
198+
* Test that 'yield' or 'from' when not used as the reserved keyword are tokenized as `T_STRING`.
199+
*
200+
* @param string $testMarker The comment which prefaces the target token in the test file.
201+
*
202+
* @dataProvider dataYieldNonKeyword
203+
*
204+
* @return void
205+
*/
206+
public function testYieldNonKeyword($testMarker)
207+
{
208+
$tokens = $this->phpcsFile->getTokens();
209+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
210+
$tokenArray = $tokens[$target];
211+
212+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
213+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
214+
215+
}//end testYieldNonKeyword()
216+
217+
218+
/**
219+
* Data provider.
220+
*
221+
* @see testYieldNonKeyword()
222+
*
223+
* @return array<string, array<string>>
224+
*/
225+
public static function dataYieldNonKeyword()
226+
{
227+
return [
228+
'yield used as class name' => ['/* testYieldUsedAsClassName */'],
229+
'yield used as class constant name' => ['/* testYieldUsedAsClassConstantName */'],
230+
'yield used as method name' => ['/* testYieldUsedAsMethodName */'],
231+
'yield used as property access 1' => ['/* testYieldUsedAsPropertyName1 */'],
232+
'yield used as property access 2' => ['/* testYieldUsedAsPropertyName2 */'],
233+
'yield used as class constant access' => ['/* testYieldUsedForClassConstantAccess1 */'],
234+
'from used as class constant access' => ['/* testFromUsedForClassConstantAccess1 */'],
235+
'yield used as method name with ref' => ['/* testYieldUsedAsMethodNameReturnByRef */'],
236+
];
237+
238+
}//end dataYieldNonKeyword()
239+
240+
241+
}//end class

0 commit comments

Comments
 (0)