Fixed bug #2431 : Use function/const not tokenized as T_STRING when preceded by comment

Author: gsherwood

package.xml

@@ -105,6 +105,7 @@ http://pear.php.net/dtd/package-2.0.xsd">
   - Fixed bug #2393 : The gitmodified filter will infinitely loop when encountering deleted file paths
     -- Thanks to Lucas Manzke for the patch
   - Fixed bug #2396 : Generic.WhiteSpace.ScopeIndent incorrect error when multi-line IF condition mixed with HTML
+  - Fixed bug #2431 : Use function/const not tokenized as T_STRING when preceded by comment
  </notes>
  <contents>
   <dir name="/">

src/Standards/Squiz/Tests/WhiteSpace/FunctionSpacingUnitTest.1.inc

@@ -4,6 +4,12 @@
 */
 use Foo\{function bar, const BAZ};
 
+use /* comment */ function bar\math\{
+    Msin,
+    level\Mcos as BarCos,
+    Mcosh,
+};
+
 include $blah;
 
 

src/Standards/Squiz/Tests/WhiteSpace/FunctionSpacingUnitTest.1.inc.fixed

@@ -4,6 +4,12 @@
 */
 use Foo\{function bar, const BAZ};
 
+use /* comment */ function bar\math\{
+    Msin,
+    level\Mcos as BarCos,
+    Mcosh,
+};
+
 include $blah;
 
 

src/Standards/Squiz/Tests/WhiteSpace/FunctionSpacingUnitTest.php

@@ -30,51 +30,51 @@ public function getErrorList($testFile='')
         switch ($testFile) {
         case 'FunctionSpacingUnitTest.1.inc':
             return [
-                20  => 1,
-                29  => 1,
-                38  => 1,
-                45  => 1,
-                49  => 1,
+                26  => 1,
+                35  => 1,
+                44  => 1,
+                51  => 1,
                 55  => 1,
-                58  => 1,
-                60  => 1,
-                75  => 1,
-                94  => 1,
-                105 => 1,
-                107 => 1,
-                113 => 2,
-                135 => 1,
-                154 => 1,
-                167 => 2,
-                184 => 1,
-                218 => 2,
-                275 => 1,
-                276 => 1,
-                289 => 1,
-                291 => 1,
+                61  => 1,
+                64  => 1,
+                66  => 1,
+                81  => 1,
+                100 => 1,
+                111 => 1,
+                113 => 1,
+                119 => 2,
+                141 => 1,
+                160 => 1,
+                173 => 2,
+                190 => 1,
+                224 => 2,
+                281 => 1,
+                282 => 1,
+                295 => 1,
                 297 => 1,
-                321 => 1,
-                323 => 1,
-                332 => 1,
+                303 => 1,
+                327 => 1,
+                329 => 1,
                 338 => 1,
-                339 => 1,
-                348 => 2,
-                349 => 1,
-                350 => 1,
+                344 => 1,
+                345 => 1,
                 354 => 2,
                 355 => 1,
                 356 => 1,
-                379 => 1,
-                393 => 1,
-                405 => 2,
-                412 => 2,
-                420 => 2,
-                426 => 1,
-                431 => 1,
+                360 => 2,
+                361 => 1,
+                362 => 1,
+                385 => 1,
+                399 => 1,
+                411 => 2,
+                418 => 2,
+                426 => 2,
                 432 => 1,
-                436 => 2,
+                437 => 1,
                 438 => 1,
-                443 => 1,
+                442 => 2,
+                444 => 1,
+                449 => 1,
             ];
 
         case 'FunctionSpacingUnitTest.2.inc':

src/Tokenizers/PHP.php

@@ -504,7 +504,9 @@ protected function tokenize($string)
                 echo ": $type => $content";
             }//end if
 
-            if ($newStackPtr > 0 && $finalTokens[($newStackPtr - 1)]['code'] !== T_WHITESPACE) {
+            if ($newStackPtr > 0
+                && isset(Util\Tokens::$emptyTokens[$finalTokens[($newStackPtr - 1)]['code']]) === false
+            ) {
                 $lastNotEmptyToken = ($newStackPtr - 1);
             }
 
@@ -1281,6 +1283,7 @@ function return types. We want to keep the parenthesis map clean,
                         T_NAMESPACE            => true,
                         T_PAAMAYIM_NEKUDOTAYIM => true,
                     ];
+
                     if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
                         // Special case for syntax like: return new self
                         // where self should not be a string.