diff --git a/composer.json b/composer.json index a8f85ce..faa2957 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,8 @@ "phpstan/phpstan": "^2.0", "phpstan/extension-installer": "^1.4.3", "illuminate/support": "^11.30", - "symfony/process": "^7.2" + "symfony/process": "^7.2", + "rogervila/array-diff-multidimensional": "^2.1" }, "config": { "allow-plugins": { diff --git a/meta/sample.php b/meta/sample.php index a9014b9..7e5bacd 100644 --- a/meta/sample.php +++ b/meta/sample.php @@ -3,10 +3,13 @@ set_time_limit(2); use Phiki\Environment\Environment; +use Phiki\Grammar\DefaultGrammars; use Phiki\Phiki; use Phiki\Theme\Theme; +use Phiki\Token\Token; +use Symfony\Component\Process\Process; -require_once __DIR__.'/../vendor/autoload.php'; +require_once __DIR__ . '/../vendor/autoload.php'; set_error_handler(function ($severity, $message, $file, $line) { throw new ErrorException($message, 0, $severity, $file, $line); @@ -20,10 +23,43 @@ $grammars = $repository->getAllGrammarNames(); natsort($grammars); -$sample = file_get_contents(__DIR__.'/../resources/samples/'.$grammar.'.sample'); +$sample = file_get_contents($samplePath = __DIR__ . '/../resources/samples/' . $grammar . '.sample'); $tokens = (new Phiki($environment))->codeToTokens($sample, $grammar); $html = (new Phiki($environment))->codeToHtml($sample, $grammar, ['light' => Theme::GithubLight, 'dark' => Theme::GithubDark], $withGutter); +$process = new Process( + [ + 'node', + __DIR__ . '/../tests/Fixtures/vscode-textmate-compliance.js', + $samplePath, + array_flip(DefaultGrammars::SCOPES_TO_NAMES)[$grammar], + json_encode(collect(DefaultGrammars::SCOPES_TO_NAMES) + ->mapWithKeys(fn(string $name, string $scope) => [$scope => DefaultGrammars::NAMES_TO_PATHS[$name]]) + ->all()), + ], +); + +$process->run(); + +if (! $process->isSuccessful()) { + throw new RuntimeException($process->getErrorOutput() . ':' . PHP_EOL . $process->getOutput()); +} + +$vscodeTextmateOutput = array_map( + fn(array $lineTokens) => array_map( + fn(array $token) => new Token( + scopes: $token['scopes'], + text: $token['text'], + start: $token['start'], + end: $token['end'], + ), + $lineTokens + ), + json_decode($process->getOutput(), true), +); + +$tokenDiff = array_diff_multidimensional($tokens, $vscodeTextmateOutput, false); + ?> @@ -102,7 +138,22 @@ class="flex items-center gap-x-4"> - +
+
+

Phiki tokens:

+ +
+ +
+

vscode-textmate tokens:

+ +
+ +
+

Differences:

+ +
+
diff --git a/resources/samples/actionscript-3.sample b/resources/samples/actionscript-3.sample index e4f20d7..450544d 100644 --- a/resources/samples/actionscript-3.sample +++ b/resources/samples/actionscript-3.sample @@ -24,4 +24,4 @@ private function createParticles( count ):void{ } } -// From https://code.tutsplus.com/tutorials/actionscript-30-optimization-a-practical-example--active-11295 \ No newline at end of file +// From https://code.tutsplus.com/tutorials/actionscript-30-optimization-a-practical-example--active-11295 diff --git a/src/Generators/HtmlGenerator.php b/src/Generators/HtmlGenerator.php index eab7ca1..ae7fd48 100644 --- a/src/Generators/HtmlGenerator.php +++ b/src/Generators/HtmlGenerator.php @@ -96,7 +96,7 @@ private function buildLine(array $line, int $index): string $output[] = $this->buildToken($token); } - return ''.implode($output).''; + return ''.implode($output)."\n"; } private function buildLineNumber(int $lineNumber): string diff --git a/src/Generators/TerminalGenerator.php b/src/Generators/TerminalGenerator.php index 7bbd7c2..7b05206 100644 --- a/src/Generators/TerminalGenerator.php +++ b/src/Generators/TerminalGenerator.php @@ -28,6 +28,8 @@ public function generate(array $tokens): string $output .= Color::ANSI_RESET; } } + + $output .= PHP_EOL; } return $output; diff --git a/src/Tokenizer.php b/src/Tokenizer.php index 6a116f3..aa73487 100644 --- a/src/Tokenizer.php +++ b/src/Tokenizer.php @@ -53,6 +53,26 @@ public function tokenize(string $input): array $this->state->resetAnchorPositions(); $this->tokenizeLine($line, $lineText."\n"); + + // If the last token added has a newline character, we need to remove it. + if (isset($this->tokens[$line]) && count($this->tokens[$line]) > 0) { + $lastToken = end($this->tokens[$line]); + + // If we have a newline token and the line has more than just that newline token, + // we need to remove it from the tokens array. + if ($lastToken->text === "\n" && count($this->tokens[$line]) > 1) { + array_pop($this->tokens[$line]); + // Otherwise, if the last token is a newline character, we need to remove that from the token text. + } elseif ($lastToken->text === "\n") { + $lastToken->text = ''; + $lastToken->end = $lastToken->start + 1; + // And if the last token ends with a newline character, we need to remove that from the token text + // and update the end position accordingly. + } elseif (str_ends_with($lastToken->text, "\n")) { + $lastToken->text = substr($lastToken->text, 0, -1); + $lastToken->end = $lastToken->start + strlen($lastToken->text) + 1; + } + } } return $this->tokens; @@ -63,7 +83,6 @@ public function tokenizeLine(int $line, string $lineText): void $this->checkWhileConditions($line, $lineText); while ($this->state->getLinePosition() < strlen($lineText)) { - $remainingText = substr($lineText, $this->state->getLinePosition()); $root = $this->state->getPattern(); $matched = $this->match($lineText); $endIsMatched = false; diff --git a/tests/Integration/VscodeTextmateTest.php b/tests/Integration/VscodeTextmateTest.php index 6a90b8c..1b494f4 100644 --- a/tests/Integration/VscodeTextmateTest.php +++ b/tests/Integration/VscodeTextmateTest.php @@ -10,6 +10,6 @@ $expected = vscodeTextmateTokenize($samplePath, $grammar); $actual = (new Phiki)->codeToTokens(file_get_contents($samplePath), $grammar); - expect($actual)->toEqualCanonicalizing($expected); + expect($actual)->toEqualCanonicalizing($expected, 'Phiki produced different tokens than vscode-textmate.'); }) ->with('grammars'); diff --git a/tests/Languages/CppTest.php b/tests/Languages/CppTest.php index a3b7e93..5601957 100644 --- a/tests/Languages/CppTest.php +++ b/tests/Languages/CppTest.php @@ -22,7 +22,6 @@ new Token(['source.cpp', 'comment.block.cpp', 'punctuation.definition.comment.begin.cpp'], '/*', 10, 12), new Token(['source.cpp', 'comment.block.cpp'], ' comment ', 12, 21), new Token(['source.cpp', 'comment.block.cpp', 'punctuation.definition.comment.end.cpp'], '*/', 21, 23), - new Token(['source.cpp'], "\n", 23, 23), ], ]); })->issue(57); diff --git a/tests/Languages/HtmlTest.php b/tests/Languages/HtmlTest.php index 1fbba4e..c7e75af 100644 --- a/tests/Languages/HtmlTest.php +++ b/tests/Languages/HtmlTest.php @@ -15,7 +15,6 @@ new Token(['text.html.basic', 'meta.tag.structure.div.end.html', 'punctuation.definition.tag.begin.html'], '', 10, 11), - new Token(['text.html.basic'], "\n", 11, 11), ], ]); }); @@ -32,7 +31,6 @@ new Token(['text.html.basic', 'meta.tag.structure.h1.end.html', 'punctuation.definition.tag.begin.html'], '', 21, 22), - new Token(['text.html.basic'], "\n", 22, 22), ], ]); }); @@ -50,7 +48,6 @@ new Token(['text.html.basic', 'meta.tag.structure.h1.end.html', 'punctuation.definition.tag.begin.html'], '', 15, 16), - new Token(['text.html.basic'], "\n", 16, 16), ], ]); }); @@ -72,7 +69,6 @@ new Token(['text.html.basic', 'meta.tag.structure.h1.end.html', 'punctuation.definition.tag.begin.html'], '', 20, 21), - new Token(['text.html.basic'], "\n", 21, 21), ], ]); }); diff --git a/tests/Languages/JavascriptTest.php b/tests/Languages/JavascriptTest.php index 6d9cb6d..cfffddd 100644 --- a/tests/Languages/JavascriptTest.php +++ b/tests/Languages/JavascriptTest.php @@ -11,7 +11,6 @@ [ new Token(['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'], '//', 0, 2), new Token(['source.js', 'comment.line.double-slash.js'], ' This is a comment.', 2, 21), - new Token(['source.js'], "\n", 21, 21), ], ]); }); diff --git a/tests/Languages/PhpTest.php b/tests/Languages/PhpTest.php index 24fddd0..3a0c320 100644 --- a/tests/Languages/PhpTest.php +++ b/tests/Languages/PhpTest.php @@ -12,7 +12,6 @@ new Token(['source.php', 'string.quoted.double.php', 'punctuation.definition.string.begin.php'], '"', 0, 1), new Token(['source.php', 'string.quoted.double.php'], 'Hello, world!', 1, 14), new Token(['source.php', 'string.quoted.double.php', 'punctuation.definition.string.end.php'], '"', 14, 15), - new Token(['source.php'], "\n", 15, 15), ], ]); }); @@ -24,7 +23,6 @@ [ new Token(['source.php', 'variable.other.php', 'punctuation.definition.variable.php'], '$', 0, 1), new Token(['source.php', 'variable.other.php'], 'name', 1, 5), - new Token(['source.php'], "\n", 5, 5), ], ]); }); @@ -42,7 +40,6 @@ new Token(['source.php', 'string.quoted.double.php', 'punctuation.definition.variable.php'], '}', 14, 15), new Token(['source.php', 'string.quoted.double.php'], '!', 15, 16), new Token(['source.php', 'string.quoted.double.php', 'punctuation.definition.string.end.php'], '"', 16, 17), - new Token(['source.php'], "\n", 17, 17), ], ]); }); @@ -62,7 +59,6 @@ new Token(['source.php', 'meta.class.php'], ' ', 17, 18), new Token(['source.php', 'meta.class.php', 'punctuation.definition.class.begin.bracket.curly.php'], '{', 18, 19), new Token(['source.php', 'meta.class.php', 'punctuation.definition.class.end.bracket.curly.php'], '}', 19, 20), - new Token(['source.php'], "\n", 20, 20), ], ]); }); @@ -76,7 +72,6 @@ new Token(['source.php', 'meta.use.php'], ' ', 3, 4), new Token(['source.php', 'meta.use.php', 'support.class.php'], 'A', 4, 5), new Token(['source.php', 'punctuation.terminator.expression.php'], ';', 5, 6), - new Token(['source.php'], "\n", 6, 6), ], ]); }); @@ -94,7 +89,6 @@ new Token(['source.php', 'meta.use.php', 'support.other.namespace.php', 'punctuation.separator.inheritance.php'], '\\', 7, 8), new Token(['source.php', 'meta.use.php', 'support.class.php'], 'C', 8, 9), new Token(['source.php', 'punctuation.terminator.expression.php'], ';', 9, 10), - new Token(['source.php'], "\n", 10, 10), ], ]); }); @@ -116,7 +110,6 @@ new Token(['source.php'], ' ', 16, 17), new Token(['source.php', 'punctuation.definition.begin.bracket.curly.php'], '{', 17, 18), new Token(['source.php', 'punctuation.definition.end.bracket.curly.php'], '}', 18, 19), - new Token(['source.php'], "\n", 19, 19), ], ]); }); @@ -140,7 +133,6 @@ new Token(['source.php'], ' ', 18, 19), new Token(['source.php', 'punctuation.definition.begin.bracket.curly.php'], '{', 19, 20), new Token(['source.php', 'punctuation.definition.end.bracket.curly.php'], '}', 20, 21), - new Token(['source.php'], "\n", 21, 21), ], ]); }); @@ -164,7 +156,6 @@ new Token(['source.php'], ' ', 24, 25), new Token(['source.php', 'punctuation.definition.begin.bracket.curly.php'], '{', 25, 26), new Token(['source.php', 'punctuation.definition.end.bracket.curly.php'], '}', 26, 27), - new Token(['source.php'], "\n", 27, 27), ], ]); }); diff --git a/tests/Languages/TomlTest.php b/tests/Languages/TomlTest.php index 9ee91f8..74e853b 100644 --- a/tests/Languages/TomlTest.php +++ b/tests/Languages/TomlTest.php @@ -12,7 +12,6 @@ new Token(['source.toml', 'meta.group.toml', 'punctuation.definition.section.begin.toml'], '[', 0, 1), new Token(['source.toml', 'meta.group.toml', 'entity.name.section.toml'], 'group', 1, 6), new Token(['source.toml', 'meta.group.toml', 'punctuation.definition.section.begin.toml'], ']', 6, 7), - new Token(['source.toml'], "\n", 7, 7), ], ]); }); @@ -27,7 +26,6 @@ new Token(['source.toml', 'meta.group.toml'], '.', 6, 7), new Token(['source.toml', 'meta.group.toml', 'entity.name.section.toml'], 'subgroup', 7, 15), new Token(['source.toml', 'meta.group.toml', 'punctuation.definition.section.begin.toml'], ']', 15, 16), - new Token(['source.toml'], "\n", 16, 16), ], ]); }); diff --git a/tests/Languages/YamlTest.php b/tests/Languages/YamlTest.php index 84a25f2..cafc822 100644 --- a/tests/Languages/YamlTest.php +++ b/tests/Languages/YamlTest.php @@ -17,7 +17,6 @@ new Token(['source.yaml', 'string.quoted.double.yaml', 'punctuation.definition.string.begin.yaml'], '"', 6, 7), new Token(['source.yaml', 'string.quoted.double.yaml'], 'Hello, world', 7, 19), new Token(['source.yaml', 'string.quoted.double.yaml', 'punctuation.definition.string.end.yaml'], '"', 19, 20), - new Token(['source.yaml'], "\n", 20, 20), ], ]); }); diff --git a/tests/Unit/TokenizerTest.php b/tests/Unit/TokenizerTest.php index c1873e0..5556564 100644 --- a/tests/Unit/TokenizerTest.php +++ b/tests/Unit/TokenizerTest.php @@ -23,7 +23,6 @@ new Token(['source.test', 'keyword.control.test'], 'while', 8, 13), new Token(['source.test'], ' ', 13, 14), new Token(['source.test', 'keyword.control.test'], 'end', 14, 17), - new Token(['source.test'], "\n", 17, 17), ], ]); }); @@ -52,7 +51,7 @@ new Token(['source.test', 'meta.function.test', 'storage.type.function.test'], 'function', 0, 8), new Token(['source.test', 'meta.function.test'], ' ', 8, 9), new Token(['source.test', 'meta.function.test', 'entity.name.function.test'], 'foo', 9, 12), - new Token(['source.test'], "() {}\n", 12, 17), + new Token(['source.test'], "() {}", 12, 18), ], ]); }); @@ -87,7 +86,7 @@ new Token(['source.test', 'meta.namespace.test', 'keyword.other.namespace.test'], 'namespace', 0, 9), new Token(['source.test', 'meta.namespace.test'], ' ', 9, 10), new Token(['source.test', 'meta.namespace.test', 'entity.name.type.namespace.test'], 'Foo', 10, 13), - new Token(['source.test'], ";\n", 13, 14), + new Token(['source.test'], ";", 13, 15), ], ]); }); @@ -126,7 +125,7 @@ new Token(['source.test', 'meta.namespace.test', 'entity.name.type.namespace.test'], 'Bar', 14, 17), new Token(['source.test', 'meta.namespace.test', 'entity.name.type.namespace.test', 'punctuation.separator.inheritance.test'], '\\', 17, 18), new Token(['source.test', 'meta.namespace.test', 'entity.name.type.namespace.test'], 'Baz', 18, 21), - new Token(['source.test'], ";\n", 21, 22), + new Token(['source.test'], ";", 21, 23), ], ]); }); @@ -202,7 +201,6 @@ [ new Token(['source.test', 'variable.other.php', 'punctuation.definition.variable.php'], '$', 0, 1), new Token(['source.test', 'variable.other.php'], 'hello', 1, 6), - new Token(['source.test'], "\n", 6, 7), ], ]); }); @@ -226,7 +224,6 @@ new Token(['source.test', 'meta.block.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -258,7 +255,6 @@ new Token(['source.test', 'meta.block.test', 'keyword.control.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test', 'keyword.control.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -285,7 +281,6 @@ new Token(['source.test', 'meta.block.test', 'keyword.control.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test', 'keyword.control.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -318,7 +313,6 @@ new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.begin.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -351,7 +345,6 @@ new Token(['source.test', 'meta.block.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.end.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -388,7 +381,6 @@ new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.begin.test'], 'begin', 0, 5), new Token(['source.test', 'meta.block.test'], ' ', 5, 6), new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.end.test'], 'end', 6, 9), - new Token(['source.test'], "\n", 9, 9), ], ]); }); @@ -418,7 +410,6 @@ new Token(['source.test', 'meta.block.test', 'entity.name.test'], 'foo', 6, 9), new Token(['source.test', 'meta.block.test'], ' ', 9, 10), new Token(['source.test', 'meta.block.test'], 'end', 10, 13), - new Token(['source.test'], "\n", 13, 13), ], ]); }); @@ -453,7 +444,6 @@ new Token(['source.test', 'meta.block.test', 'entity.name.test', 'entity.name.foo.test'], 'foo', 6, 9), new Token(['source.test', 'meta.block.test'], ' ', 9, 10), new Token(['source.test', 'meta.block.test'], 'end', 10, 13), - new Token(['source.test'], "\n", 13, 13), ], ]); }); @@ -483,16 +473,13 @@ expect($tokens)->toEqualCanonicalizing([ [ new Token(['source.test', 'meta.block.test'], 'begin', 0, 5), - new Token(['source.test', 'meta.block.test'], "\n", 5, 5), ], [ new Token(['source.test', 'meta.block.test'], ' ', 0, 4), new Token(['source.test', 'meta.block.test', 'entity.name.test'], 'foo', 4, 7), - new Token(['source.test', 'meta.block.test'], "\n", 7, 7), ], [ new Token(['source.test', 'meta.block.test'], 'end', 0, 3), - new Token(['source.test'], "\n", 3, 3), ], ]); }); @@ -542,16 +529,13 @@ expect($tokens)->toEqualCanonicalizing([ [ new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.begin.test'], 'begin', 0, 5), - new Token(['source.test', 'meta.block.test'], "\n", 5, 5), ], [ new Token(['source.test', 'meta.block.test'], ' ', 0, 4), new Token(['source.test', 'meta.block.test', 'entity.name.test', 'entity.name.foo.test'], 'foo', 4, 7), - new Token(['source.test', 'meta.block.test'], "\n", 7, 7), ], [ new Token(['source.test', 'meta.block.test', 'keyword.control.test', 'keyword.control.end.test'], 'end', 0, 3), - new Token(['source.test'], "\n", 3, 3), ], ]); }); @@ -582,7 +566,6 @@ new Token(['source.test', 'meta.block.test', 'meta.begin.end.block.test', 'entity.name.test'], 'foo', 6, 9), new Token(['source.test', 'meta.block.test', 'meta.begin.end.block.test'], ' ', 9, 10), new Token(['source.test', 'meta.block.test'], 'end', 10, 13), - new Token(['source.test'], "\n", 13, 13), ], ]); }); @@ -608,7 +591,6 @@ expect($tokens)->toEqualCanonicalizing([ [ new Token(['source.test', 'entity.name.test', 'entity.name.foo.test'], 'foo', 0, 3), - new Token(['source.test'], "\n", 3, 3), ], ]); });