Fixed T_FN tokenizing when using yield (ref #2706, #2523)

Author: gsherwood

src/Tokenizers/PHP.php

@@ -1717,7 +1717,6 @@ protected function processAdditional()
                         $endTokens = [
                             T_COLON                => true,
                             T_COMMA                => true,
-                            T_DOUBLE_ARROW         => true,
                             T_SEMICOLON            => true,
                             T_CLOSE_PARENTHESIS    => true,
                             T_CLOSE_SQUARE_BRACKET => true,

tests/Core/Tokenizer/BackfillFnTokenTest.inc

@@ -51,3 +51,6 @@ fn&($x) => $x;
 $a = [
     'a' => fn() => return 1,
 ];
+
+/* testYield */
+$a = fn($x) => yield 'k' => $x;

tests/Core/Tokenizer/BackfillFnTokenTest.php

@@ -403,6 +403,34 @@ public function testArrayValue()
     }//end testArrayValue()
 
 
+    /**
+     * Test arrow functions that use the yield keyword.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
+     *
+     * @return void
+     */
+    public function testYield()
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $token = $this->getTargetToken('/* testYield */', T_FN);
+        $this->backfillHelper($token);
+
+        $this->assertSame($tokens[$token]['scope_opener'], ($token + 5), 'Scope opener is not the arrow token');
+        $this->assertSame($tokens[$token]['scope_closer'], ($token + 14), 'Scope closer is not the semicolon token');
+
+        $opener = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$opener]['scope_opener'], ($token + 5), 'Opener scope opener is not the arrow token');
+        $this->assertSame($tokens[$opener]['scope_closer'], ($token + 14), 'Opener scope closer is not the semicolon token');
+
+        $closer = $tokens[$token]['scope_opener'];
+        $this->assertSame($tokens[$closer]['scope_opener'], ($token + 5), 'Closer scope opener is not the arrow token');
+        $this->assertSame($tokens[$closer]['scope_closer'], ($token + 14), 'Closer scope closer is not the semicolon token');
+
+    }//end testYield()
+
+
     /**
      * Test that anonymous class tokens without parenthesis do not get assigned a parenthesis owner.
      *