Skip to content

Commit de3ca90

Browse files
jrfnljrfnl
authored
Merge pull request #644 from PHPCSStandards/feature/tokenizer-replacetabsintoken-add-tests
Tokenizer::replaceTabsInToken(): add tests
2 parents 3a47cd6 + be0d959 commit de3ca90

9 files changed

+996
-1
lines changed

‎src/Tokenizers/Tokenizer.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth
620620
if ($content !== '') {
621621
$newContent .= $content;
622622
if ($checkEncoding === true) {
623-
// Not using the default encoding, so take a bit more care.
623+
// Not using ASCII encoding, so take a bit more care.
624624
$oldLevel = error_reporting();
625625
error_reporting(0);
626626
$contentLength = iconv_strlen($content, $this->config->encoding);

‎tests/Core/Tokenizers/Tokenizer/CreatePositionMapTabWidth0Test.php

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?php
2+
/**
3+
* Tests the tab replacement logic.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
11+
12+
/**
13+
* Tab replacement test using tab width 0, means no tab replacement will take place.
14+
*
15+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
16+
*/
17+
final class CreatePositionMapTabWidth0Test extends ReplaceTabsInTokenTestCase
18+
{
19+
20+
/**
21+
* The tab width setting to use when tokenizing the file.
22+
*
23+
* @var integer
24+
*/
25+
protected $tabWidth = 0;
26+
27+
28+
/**
29+
* Data provider helper.
30+
*
31+
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
32+
*
33+
* @return array<string, array<string, int|string|null>>
34+
*/
35+
public static function getTabReplacementExpected()
36+
{
37+
return [
38+
'Tab indentation' => [
39+
'length' => 2,
40+
'content' => ' ',
41+
'orig_content' => null,
42+
],
43+
'Mixed tab/space indentation' => [
44+
'length' => 3,
45+
'content' => ' ',
46+
'orig_content' => null,
47+
],
48+
'Inline: single tab in text string' => [
49+
'length' => 15,
50+
'content' => "'tab separated'",
51+
'orig_content' => null,
52+
],
53+
'Inline: single tab between each word in text string' => [
54+
'length' => 24,
55+
'content' => '"tab $between each word"',
56+
'orig_content' => null,
57+
],
58+
'Inline: multiple tabs in heredoc' => [
59+
'length' => 15,
60+
'content' => 'tab separated
61+
',
62+
'orig_content' => null,
63+
],
64+
'Inline: multiple tabs between each word in nowdoc' => [
65+
'length' => 27,
66+
'content' => 'tab between each word
67+
',
68+
'orig_content' => null,
69+
],
70+
'Inline: mixed spaces/tabs in text string' => [
71+
'length' => 20,
72+
'content' => "'tab separated'",
73+
'orig_content' => null,
74+
],
75+
'Inline: mixed spaces/tabs between each word in text string' => [
76+
'length' => 31,
77+
'content' => '"tab $between each word"',
78+
'orig_content' => null,
79+
],
80+
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
81+
'length' => 50,
82+
'content' => '// -123 With tabwidth 4, the tab size should be 1.
83+
',
84+
'orig_content' => null,
85+
],
86+
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
87+
'length' => 52,
88+
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
89+
'orig_content' => null,
90+
],
91+
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
92+
'length' => 45,
93+
'content' => '-1 With tabwidth 4, the tab size should be 3.',
94+
'orig_content' => null,
95+
],
96+
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
97+
'length' => 47,
98+
'content' => '// - With tabwidth 4, the tab size should be 4.
99+
',
100+
'orig_content' => null,
101+
],
102+
];
103+
104+
}//end getTabReplacementExpected()
105+
106+
107+
}//end class

‎tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenMiscTest.php

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<?php
2+
/**
3+
* Tests the tab replacement logic.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
11+
12+
use PHP_CodeSniffer\Files\DummyFile;
13+
use PHP_CodeSniffer\Ruleset;
14+
use PHP_CodeSniffer\Tests\ConfigDouble;
15+
use PHPUnit\Framework\TestCase;
16+
17+
/**
18+
* Miscellaneous tests for tab replacement.
19+
*
20+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
21+
*/
22+
final class ReplaceTabsInTokenMiscTest extends TestCase
23+
{
24+
25+
26+
/**
27+
* Test that when no tab width is set or passed, the tab width will be set to 1.
28+
*
29+
* @return void
30+
*/
31+
public function testTabWidthNotSet()
32+
{
33+
$config = new ConfigDouble();
34+
$ruleset = new Ruleset($config);
35+
36+
$content = <<<EOD
37+
<?php
38+
echo 'foo';
39+
EOD;
40+
$phpcsFile = new DummyFile($content, $ruleset, $config);
41+
$phpcsFile->parse();
42+
43+
$tokens = $phpcsFile->getTokens();
44+
$target = $phpcsFile->findNext(T_WHITESPACE, 0);
45+
46+
// Verify initial state.
47+
$this->assertTrue(is_int($target), 'Target token was not found');
48+
$this->assertSame(' ', $tokens[$target]['content'], 'Content after initial parsing does not contain tabs');
49+
$this->assertSame(2, $tokens[$target]['length'], 'Length after initial parsing is not as expected');
50+
$this->assertArrayNotHasKey('orig_content', $tokens[$target], "Key 'orig_content' found in the initial token array.");
51+
52+
$phpcsFile->tokenizer->replaceTabsInToken($tokens[$target]);
53+
54+
// Verify tab replacement.
55+
$this->assertSame(' ', $tokens[$target]['content'], 'Content after tab replacement is not as expected');
56+
$this->assertSame(2, $tokens[$target]['length'], 'Length after tab replacement is not as expected');
57+
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");
58+
59+
}//end testTabWidthNotSet()
60+
61+
62+
/**
63+
* Test that the length calculation handles text in non-ascii encodings correctly.
64+
*
65+
* @requires extension iconv
66+
*
67+
* @return void
68+
*/
69+
public function testLengthSettingRespectsEncoding()
70+
{
71+
$config = new ConfigDouble();
72+
$config->tabWidth = 4;
73+
$ruleset = new Ruleset($config);
74+
75+
$content = <<<EOD
76+
<?php
77+
echo 'пасха пасха';
78+
EOD;
79+
$phpcsFile = new DummyFile($content, $ruleset, $config);
80+
$phpcsFile->parse();
81+
82+
$tokens = $phpcsFile->getTokens();
83+
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);
84+
85+
$this->assertTrue(is_int($target), 'Target token was not found');
86+
$this->assertSame("'пасха пасха'", $tokens[$target]['content'], 'Content is not as expected');
87+
$this->assertSame(17, $tokens[$target]['length'], 'Length is not as expected');
88+
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");
89+
$this->assertSame("'пасха пасха'", $tokens[$target]['orig_content'], 'Orig_content is not as expected');
90+
91+
}//end testLengthSettingRespectsEncoding()
92+
93+
94+
/**
95+
* Test that the length calculation falls back to byte length if iconv detects an illegal character.
96+
*
97+
* @requires extension iconv
98+
*
99+
* @return void
100+
*/
101+
public function testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()
102+
{
103+
$config = new ConfigDouble();
104+
$config->tabWidth = 4;
105+
$ruleset = new Ruleset($config);
106+
107+
$content = <<<EOD
108+
<?php
109+
echo "aa\xC3\xC3 \xC3\xB8aa";
110+
EOD;
111+
$phpcsFile = new DummyFile($content, $ruleset, $config);
112+
$phpcsFile->parse();
113+
114+
$tokens = $phpcsFile->getTokens();
115+
$target = $phpcsFile->findNext(T_CONSTANT_ENCAPSED_STRING, 0);
116+
117+
$this->assertTrue(is_int($target), 'Target token was not found');
118+
$this->assertSame(11, $tokens[$target]['length'], 'Length is not as expected');
119+
$this->assertArrayHasKey('orig_content', $tokens[$target], "Key 'orig_content' not found in the token array.");
120+
121+
}//end testLengthSettingFallsBackToBytesWhenTextContainsIllegalChars()
122+
123+
124+
}//end class

‎tests/Core/Tokenizers/Tokenizer/ReplaceTabsInTokenTabWidth1Test.php

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
<?php
2+
/**
3+
* Tests the tab replacement logic.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
11+
12+
/**
13+
* Tab replacement test using tab width 1.
14+
*
15+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::replaceTabsInToken
16+
*/
17+
final class ReplaceTabsInTokenTabWidth1Test extends ReplaceTabsInTokenTestCase
18+
{
19+
20+
/**
21+
* The tab width setting to use when tokenizing the file.
22+
*
23+
* @var integer
24+
*/
25+
protected $tabWidth = 1;
26+
27+
28+
/**
29+
* Data provider helper.
30+
*
31+
* @see ReplaceTabsInTokenTestCase::dataTabReplacement()
32+
*
33+
* @return array<string, array<string, int|string>>
34+
*/
35+
public static function getTabReplacementExpected()
36+
{
37+
return [
38+
'Tab indentation' => [
39+
'length' => 2,
40+
'content' => ' ',
41+
'orig_content' => ' ',
42+
],
43+
'Mixed tab/space indentation' => [
44+
'length' => 3,
45+
'content' => ' ',
46+
'orig_content' => ' ',
47+
],
48+
'Inline: single tab in text string' => [
49+
'length' => 15,
50+
'content' => "'tab separated'",
51+
'orig_content' => "'tab separated'",
52+
],
53+
'Inline: single tab between each word in text string' => [
54+
'length' => 24,
55+
'content' => '"tab $between each word"',
56+
'orig_content' => '"tab $between each word"',
57+
],
58+
'Inline: multiple tabs in heredoc' => [
59+
'length' => 15,
60+
'content' => 'tab separated
61+
',
62+
'orig_content' => 'tab separated
63+
',
64+
],
65+
'Inline: multiple tabs between each word in nowdoc' => [
66+
'length' => 27,
67+
'content' => 'tab between each word
68+
',
69+
'orig_content' => 'tab between each word
70+
',
71+
],
72+
'Inline: mixed spaces/tabs in text string' => [
73+
'length' => 20,
74+
'content' => "'tab separated'",
75+
'orig_content' => "'tab separated'",
76+
],
77+
'Inline: mixed spaces/tabs between each word in text string' => [
78+
'length' => 31,
79+
'content' => '"tab $between each word"',
80+
'orig_content' => '"tab $between each word"',
81+
],
82+
'Inline: tab becomes single space in comment (with tabwidth 4)' => [
83+
'length' => 50,
84+
'content' => '// -123 With tabwidth 4, the tab size should be 1.
85+
',
86+
'orig_content' => '// -123 With tabwidth 4, the tab size should be 1.
87+
',
88+
],
89+
'Inline: tab becomes 2 spaces in comment (with tabwidth 4)' => [
90+
'length' => 52,
91+
'content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
92+
'orig_content' => '/* -12 With tabwidth 4, the tab size should be 2. */',
93+
],
94+
'Inline: tab becomes 3 spaces in doc comment string (with tabwidth 4)' => [
95+
'length' => 45,
96+
'content' => '-1 With tabwidth 4, the tab size should be 3.',
97+
'orig_content' => '-1 With tabwidth 4, the tab size should be 3.',
98+
],
99+
'Inline: tab becomes 4 spaces in comment (with tabwidth 4)' => [
100+
'length' => 47,
101+
'content' => '// - With tabwidth 4, the tab size should be 4.
102+
',
103+
'orig_content' => '// - With tabwidth 4, the tab size should be 4.
104+
',
105+
],
106+
];
107+
108+
}//end getTabReplacementExpected()
109+
110+
111+
}//end class

0 commit comments

Comments
 (0)