Skip to content

Commit bd7137c

Browse files
jrfnljrfnl
committed
Tokenizer/PHP: add tests for tokenization of yield and yield from
.. to document and safeguard the existing behaviour. Includes skipping one particular assertion on PHP 5.4. This assertion would fail on PHP 5.4 with PHPUnit 4 as PHPUnit polyfills the `T_YIELD` token too, but with a different value, which causes the token 'type' to be set to `UNKNOWN`. This issue _only_ occurs when running the tests, not when running PHPCS outside of a test situation, so this is not a real problem when running PHPCS on PHP 5.4. For reference: the PHPUnit polyfilled token is declared in the PHP_CodeCoverage_Report_HTML_Renderer_File class in vendor/phpunit/php-code-coverage/src/CodeCoverage/Report/HTML/Renderer/File.php
1 parent de3ca90 commit bd7137c

File tree

2 files changed

+289
-0
lines changed

2 files changed

+289
-0
lines changed

‎tests/Core/Tokenizers/PHP/YieldTest.inc

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?php
2+
3+
function generator()
4+
{
5+
/* testYield */
6+
yield 1;
7+
8+
/* testYieldFollowedByComment */
9+
YIELD/*comment*/ 2;
10+
11+
/* testYieldFrom */
12+
yield from gen2();
13+
14+
/* testYieldFromWithExtraSpacesBetween */
15+
Yield From gen2();
16+
17+
/* testYieldFromWithTabBetween */
18+
yield from gen2();
19+
20+
/* testYieldFromSplitByNewLines */
21+
yield
22+
23+
FROM
24+
gen2();
25+
}
26+
27+
/* testYieldUsedAsClassName */
28+
class Yield {
29+
/* testYieldUsedAsClassConstantName */
30+
const Type YIELD = 'foo';
31+
32+
/* testYieldUsedAsMethodName */
33+
public function yield() {
34+
/* testYieldUsedAsPropertyName1 */
35+
echo $obj->yield;
36+
37+
/* testYieldUsedAsPropertyName2 */
38+
echo $obj?->yield();
39+
40+
/* testYieldUsedForClassConstantAccess1 */
41+
echo MyClass::YIELD;
42+
/* testFromUsedForClassConstantAccess1 */
43+
echo MyClass::FROM;
44+
}
45+
}
46+
47+
function myGen() {
48+
/* testYieldLiveCoding */
49+
yield

‎tests/Core/Tokenizers/PHP/YieldTest.php

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of the `yield` and `yield from` keywords.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2021 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
/**
16+
* Tests the tokenization of the `yield` and `yield from` keywords.
17+
*
18+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
19+
*/
20+
final class YieldTest extends AbstractTokenizerTestCase
21+
{
22+
23+
24+
/**
25+
* Test that the yield keyword is tokenized as such.
26+
*
27+
* @param string $testMarker The comment which prefaces the target token in the test file.
28+
*
29+
* @dataProvider dataYieldKeyword
30+
*
31+
* @return void
32+
*/
33+
public function testYieldKeyword($testMarker)
34+
{
35+
$tokens = $this->phpcsFile->getTokens();
36+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
37+
$tokenArray = $tokens[$target];
38+
39+
$this->assertSame(T_YIELD, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (code)');
40+
41+
// This assertion would fail on PHP 5.4 with PHPUnit 4 as PHPUnit polyfills the `T_YIELD` token too, but
42+
// with a different value, which causes the token 'type' to be set to `UNKNOWN`.
43+
// This issue _only_ occurs when running the tests, not when running PHPCS outside of a test situation.
44+
// The PHPUnit polyfilled token is declared in the PHP_CodeCoverage_Report_HTML_Renderer_File class
45+
// in vendor/phpunit/php-code-coverage/src/CodeCoverage/Report/HTML/Renderer/File.php.
46+
if (PHP_VERSION_ID >= 50500) {
47+
$this->assertSame('T_YIELD', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD (type)');
48+
}
49+
50+
}//end testYieldKeyword()
51+
52+
53+
/**
54+
* Data provider.
55+
*
56+
* @see testYieldKeyword()
57+
*
58+
* @return array<string, array<string>>
59+
*/
60+
public static function dataYieldKeyword()
61+
{
62+
return [
63+
'yield' => ['/* testYield */'],
64+
'yield followed by comment' => ['/* testYieldFollowedByComment */'],
65+
'yield at end of file, live coding' => ['/* testYieldLiveCoding */'],
66+
];
67+
68+
}//end dataYieldKeyword()
69+
70+
71+
/**
72+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
73+
* and only has whitespace between the words.
74+
*
75+
* @param string $testMarker The comment which prefaces the target token in the test file.
76+
* @param string $content Optional. The test token content to search for.
77+
* Defaults to null.
78+
*
79+
* @dataProvider dataYieldFromKeywordSingleToken
80+
*
81+
* @return void
82+
*/
83+
public function testYieldFromKeywordSingleToken($testMarker, $content=null)
84+
{
85+
$tokens = $this->phpcsFile->getTokens();
86+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING], $content);
87+
$tokenArray = $tokens[$target];
88+
89+
$this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)');
90+
$this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)');
91+
92+
}//end testYieldFromKeywordSingleToken()
93+
94+
95+
/**
96+
* Data provider.
97+
*
98+
* @see testYieldFromKeywordSingleToken()
99+
*
100+
* @return array<string, array<string>>
101+
*/
102+
public static function dataYieldFromKeywordSingleToken()
103+
{
104+
return [
105+
'yield from' => [
106+
'testMarker' => '/* testYieldFrom */',
107+
],
108+
'yield from with extra space between' => [
109+
'testMarker' => '/* testYieldFromWithExtraSpacesBetween */',
110+
],
111+
'yield from with tab between' => [
112+
'testMarker' => '/* testYieldFromWithTabBetween */',
113+
],
114+
];
115+
116+
}//end dataYieldFromKeywordSingleToken()
117+
118+
119+
/**
120+
* Test that the yield from keyword is tokenized as a single token when it in on a single line
121+
* and only has whitespace between the words.
122+
*
123+
* @param string $testMarker The comment which prefaces the target token in the test file.
124+
* @param array<array<string, string>> $expectedTokens The tokenization expected.
125+
*
126+
* @dataProvider dataYieldFromKeywordMultiToken
127+
*
128+
* @return void
129+
*/
130+
public function testYieldFromKeywordMultiToken($testMarker, $expectedTokens)
131+
{
132+
$tokens = $this->phpcsFile->getTokens();
133+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
134+
135+
foreach ($expectedTokens as $nr => $tokenInfo) {
136+
$this->assertSame(
137+
constant($tokenInfo['type']),
138+
$tokens[$target]['code'],
139+
'Token tokenized as '.Tokens::tokenName($tokens[$target]['code']).', not '.$tokenInfo['type'].' (code)'
140+
);
141+
$this->assertSame(
142+
$tokenInfo['type'],
143+
$tokens[$target]['type'],
144+
'Token tokenized as '.$tokens[$target]['type'].', not '.$tokenInfo['type'].' (type)'
145+
);
146+
$this->assertSame(
147+
$tokenInfo['content'],
148+
$tokens[$target]['content'],
149+
'Content of token '.($nr + 1).' ('.$tokens[$target]['type'].') does not match expectations'
150+
);
151+
152+
++$target;
153+
}
154+
155+
}//end testYieldFromKeywordMultiToken()
156+
157+
158+
/**
159+
* Data provider.
160+
*
161+
* @see testYieldFromKeywordMultiToken()
162+
*
163+
* @return array<string, array<string, string|array<array<string, string>>>>
164+
*/
165+
public static function dataYieldFromKeywordMultiToken()
166+
{
167+
return [
168+
'yield from with new line' => [
169+
'testMarker' => '/* testYieldFromSplitByNewLines */',
170+
'expectedTokens' => [
171+
[
172+
'type' => 'T_YIELD_FROM',
173+
'content' => 'yield
174+
',
175+
],
176+
[
177+
'type' => 'T_YIELD_FROM',
178+
'content' => '
179+
',
180+
],
181+
[
182+
'type' => 'T_YIELD_FROM',
183+
'content' => ' FROM',
184+
],
185+
[
186+
'type' => 'T_WHITESPACE',
187+
'content' => '
188+
',
189+
],
190+
],
191+
],
192+
];
193+
194+
}//end dataYieldFromKeywordMultiToken()
195+
196+
197+
/**
198+
* Test that 'yield' or 'from' when not used as the reserved keyword are tokenized as `T_STRING`.
199+
*
200+
* @param string $testMarker The comment which prefaces the target token in the test file.
201+
*
202+
* @dataProvider dataYieldNonKeyword
203+
*
204+
* @return void
205+
*/
206+
public function testYieldNonKeyword($testMarker)
207+
{
208+
$tokens = $this->phpcsFile->getTokens();
209+
$target = $this->getTargetToken($testMarker, [T_YIELD, T_YIELD_FROM, T_STRING]);
210+
$tokenArray = $tokens[$target];
211+
212+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
213+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
214+
215+
}//end testYieldNonKeyword()
216+
217+
218+
/**
219+
* Data provider.
220+
*
221+
* @see testYieldNonKeyword()
222+
*
223+
* @return array<string, array<string>>
224+
*/
225+
public static function dataYieldNonKeyword()
226+
{
227+
return [
228+
'yield used as class name' => ['/* testYieldUsedAsClassName */'],
229+
'yield used as class constant name' => ['/* testYieldUsedAsClassConstantName */'],
230+
'yield used as method name' => ['/* testYieldUsedAsMethodName */'],
231+
'yield used as property access 1' => ['/* testYieldUsedAsPropertyName1 */'],
232+
'yield used as property access 2' => ['/* testYieldUsedAsPropertyName2 */'],
233+
'yield used as class constant access' => ['/* testYieldUsedForClassConstantAccess1 */'],
234+
'from used as class constant access' => ['/* testFromUsedForClassConstantAccess1 */'],
235+
];
236+
237+
}//end dataYieldNonKeyword()
238+
239+
240+
}//end class

0 commit comments

Comments
 (0)