MAGETWO-33062: Update dictionary search tool to support \Magento\Framework\Phrase

Author: Yuri Kovsher

dev/tools/Magento/Tools/I18n/Parser/Adapter/Php/Tokenizer.php

@@ -50,6 +50,42 @@ public function parse($filePath)
         $this->_tokensCount = count($this->_tokens);
     }
 
+    /**
+     * Checks if the next set of tokens is a valid class identifier and it matches passed class name
+     *
+     * @param string $className
+     * @return bool
+     */
+    public function isMatchingClass($className)
+    {
+        /*
+         * 1 - beginning, class identifier can begin either with identifier or with namespace separator
+         * 2 - after namespace separator, identifier is expected
+         * 3 - after identifier, namespace separator or open brace is expected
+         * 4 - ending, tells that class identifier is valid
+         */
+        $state = 1;
+        $classString = '';
+        while ($token = $this->getNextRealToken()) {
+            if ($token->isNamespaceSeparator() && ($state != 2)) {
+                $classString .= $token->getValue();
+                $state = 2;
+            } elseif ($token->isIdentifier() && ($state != 3)) {
+                $classString .= $token->getValue();
+                $state = 3;
+            } elseif ($token->isOpenBrace() && $state == 3) {
+                $state = 4;
+                break;
+            } else {
+                return false;
+            }
+        }
+        if ($state == 4 && $className == substr($classString, -strlen($className))) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Get arguments tokens of function
      *
@@ -139,13 +175,26 @@ public function getNextToken()
     }
 
     /**
-     * Check is it last token
+     * Get next token skipping all whitespaces
+     *
+     * @return \Magento\Tools\I18n\Parser\Adapter\Php\Tokenizer\Token|false
+     */
+    public function getNextRealToken()
+    {
+        do {
+            $token = $this->getNextToken();
+        } while ($token && $token->isWhitespace());
+        return $token;
+    }
+
+    /**
+     * Check if it is end of loop
      *
      * @return bool
      */
-    public function isLastToken()
+    public function isEndOfLoop()
     {
-        return 0 == $this->_tokensCount || key($this->_tokens) + 1 == $this->_tokensCount;
+        return is_null(key($this->_tokens));
     }
 
     /**

dev/tools/Magento/Tools/I18n/Parser/Adapter/Php/Tokenizer/PhraseCollector.php

@@ -62,7 +62,7 @@ public function parse($file)
         $this->_phrases = [];
         $this->_file = $file;
         $this->_tokenizer->parse($file);
-        while (!$this->_tokenizer->isLastToken()) {
+        while (!$this->_tokenizer->isEndOfLoop()) {
             $this->_extractPhrases();
         }
     }
@@ -74,12 +74,22 @@ public function parse($file)
      */
     protected function _extractPhrases()
     {
-        $phraseStartToken = $this->_tokenizer->getNextToken();
-        if ($phraseStartToken->isEqualFunction('__') && $this->_tokenizer->getNextToken()->isOpenBrace()) {
-            $arguments = $this->_tokenizer->getFunctionArgumentsTokens();
-            $phrase = $this->_collectPhrase(array_shift($arguments));
-            if (null !== $phrase) {
-                $this->_addPhrase($phrase, count($arguments), $this->_file, $phraseStartToken->getLine());
+        if ($firstToken = $this->_tokenizer->getNextRealToken()) {
+            if ($firstToken->isEqualFunction('__')) {
+                $secondToken = $this->_tokenizer->getNextRealToken();
+                if ($secondToken && $secondToken->isOpenBrace()) {
+                    $arguments = $this->_tokenizer->getFunctionArgumentsTokens();
+                    $phrase = $this->_collectPhrase(array_shift($arguments));
+                    if (null !== $phrase) {
+                        $this->_addPhrase($phrase, count($arguments), $this->_file, $firstToken->getLine());
+                    }
+                }
+            } elseif ($firstToken->isNew() && $this->_tokenizer->isMatchingClass('Phrase')) {
+                $arguments = $this->_tokenizer->getFunctionArgumentsTokens();
+                $phrase = $this->_collectPhrase(array_shift($arguments));
+                if (null !== $phrase) {
+                    $this->_addPhrase($phrase, count($arguments), $this->_file, $firstToken->getLine());
+                }
             }
         }
     }

dev/tools/Magento/Tools/I18n/Parser/Adapter/Php/Tokenizer/Token.php

@@ -75,6 +75,16 @@ public function getLine()
         return $this->_line;
     }
 
+    /**
+     * Is "new" operator
+     *
+     * @return bool
+     */
+    public function isNew()
+    {
+        return $this->getName() == T_NEW;
+    }
+
     /**
      * Whenever token is equal function
      *
@@ -155,4 +165,24 @@ public function isSemicolon()
     {
         return $this->getValue() == ';';
     }
+
+    /**
+     * Is namespace separator
+     *
+     * @return bool
+     */
+    public function isNamespaceSeparator()
+    {
+        return $this->getName() == T_NS_SEPARATOR;
+    }
+
+    /**
+     * Is identifier
+     *
+     * @return bool
+     */
+    public function isIdentifier()
+    {
+        return $this->getName() == T_STRING;
+    }
 }