diff --git a/composer.json b/composer.json index 653748b..a8f85ce 100644 --- a/composer.json +++ b/composer.json @@ -33,7 +33,8 @@ "laravel/pint": "^1.18.1", "phpstan/phpstan": "^2.0", "phpstan/extension-installer": "^1.4.3", - "illuminate/support": "^11.30" + "illuminate/support": "^11.30", + "symfony/process": "^7.2" }, "config": { "allow-plugins": { @@ -54,7 +55,11 @@ "Composer\\Config::disableProcessTimeout", "@php ./meta/update-sources.php" ], - "test": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1", + "test": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --exclude-group integration/vscode-textmate", + "test:full": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1", + "test:vscode": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/vscode-textmate --bail", + "test:grammars": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/grammars", + "test:themes": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/themes", "lint": "vendor/bin/phpstan" } } diff --git a/tests/Fixtures/vscode-textmate-compliance.js b/tests/Fixtures/vscode-textmate-compliance.js new file mode 100644 index 0000000..b63e541 --- /dev/null +++ b/tests/Fixtures/vscode-textmate-compliance.js @@ -0,0 +1,72 @@ +const process = require('node:process'); +const fs = require('node:fs'); +const path = require('node:path'); +const oniguruma = require('vscode-oniguruma'); +const vsctm = require('vscode-textmate'); + +const sample = process.argv[2]; +const scope = process.argv[3]; +const scopeMap = JSON.parse(process.argv[4]); + +function readFile(path) { + return new Promise((resolve, reject) => { + fs.readFile(path, (error, data) => + error ? reject(error) : resolve(data) + ); + }); +} + +const wasmBin = fs.readFileSync( + path.join(__dirname, "../../node_modules/vscode-oniguruma/release/onig.wasm") +).buffer; + +const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin).then(() => { + return { + createOnigScanner(patterns) { + return new oniguruma.OnigScanner(patterns); + }, + createOnigString(s) { + return new oniguruma.OnigString(s); + }, + }; +}); + +const registry = new vsctm.Registry({ + onigLib: vscodeOnigurumaLib, + loadGrammar: (scopeName) => { + if (! scopeMap[scopeName]) { + console.error(`Unknown scope name: ${scopeName}`); + process.exit(1); + } + + return readFile(scopeMap[scopeName]).then(data => { + return vsctm.parseRawGrammar(data.toString(), scopeMap[scopeName]); + }) + }, +}); + +const tokens = [] + +registry.loadGrammar(scope).then(async grammar => { + const text = await readFile(sample); + const lines = text.toString().split(/\r?\n/); + + let ruleStack = vsctm.INITIAL; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineTokens = grammar.tokenizeLine(line, ruleStack); + + tokens.push( + lineTokens.tokens.map(lineToken => ({ + scopes: lineToken.scopes, + text: line.substring(lineToken.startIndex, lineToken.endIndex), + start: lineToken.startIndex, + end: lineToken.endIndex, + })) + ) + + ruleStack = lineTokens.ruleStack; + } + + console.log(JSON.stringify(tokens)); +}) diff --git a/tests/Integration/GrammarsTest.php b/tests/Integration/GrammarsTest.php index 4a4e7d2..c2308a2 100644 --- a/tests/Integration/GrammarsTest.php +++ b/tests/Integration/GrammarsTest.php @@ -1,8 +1,9 @@ group('integration/grammars'); + describe('Grammars', function () { test('default grammars do not produce warnings or exceptions', function (string $grammar) { $sample = file_get_contents(__DIR__.'/../../resources/samples/'.$grammar.'.sample'); @@ -12,33 +13,3 @@ ->with('grammars') ->throwsNoExceptions(); }); - -dataset('grammars', function () { - $repository = new GrammarRepository; - $grammars = array_filter($repository->getAllGrammarNames(), fn (string $grammar) => ! in_array($grammar, [ - 'astro', - 'haxe', - 'fluent', - 'stylus', - 'viml', - 'sas', - 'git-commit', - 'hxml', - 'groovy', - 'make', - 'shellsession', - // Act as includes, basically. - 'html-derivative', - 'cpp-macro', - 'jinja-html', - // No sample file. - 'git-rebase', - // Empty. - 'txt', - ])); - - sort($grammars, SORT_NATURAL); - - // FIXME: These grammars have known issues and should be skipped. - return array_values($grammars); -}); diff --git a/tests/Integration/VscodeTextmateTest.php b/tests/Integration/VscodeTextmateTest.php new file mode 100644 index 0000000..6a90b8c --- /dev/null +++ b/tests/Integration/VscodeTextmateTest.php @@ -0,0 +1,15 @@ +group('integration/vscode-textmate'); + +test('it produces the same tokens as vscode-textmate', function (string $grammar) { + $samplePath = __DIR__ . "/../../resources/samples/{$grammar}.sample"; + + $expected = vscodeTextmateTokenize($samplePath, $grammar); + $actual = (new Phiki)->codeToTokens(file_get_contents($samplePath), $grammar); + + expect($actual)->toEqualCanonicalizing($expected); +}) + ->with('grammars'); diff --git a/tests/Pest.php b/tests/Pest.php index 0b6a144..469ef6d 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,8 +1,12 @@ tokenize($input); } + +function vscodeTextmateTokenize(string $samplePath, string $grammarName): array +{ + $process = new Process( + [ + 'node', + __DIR__ . '/Fixtures/vscode-textmate-compliance.js', + $samplePath, + array_flip(DefaultGrammars::SCOPES_TO_NAMES)[$grammarName], + json_encode(collect(DefaultGrammars::SCOPES_TO_NAMES) + ->mapWithKeys(fn (string $name, string $scope) => [$scope => DefaultGrammars::NAMES_TO_PATHS[$name]]) + ->all()), + ], + ); + + $process->run(); + + if (! $process->isSuccessful()) { + throw new RuntimeException($process->getErrorOutput() . ':' . PHP_EOL . $process->getOutput()); + } + + $output = json_decode($process->getOutput(), true); + + if (! is_array($output)) { + throw new RuntimeException('Invalid output from process:' . PHP_EOL . $process->getOutput()); + } + + return array_map( + fn (array $lineTokens) => array_map( + fn (array $token) => new Token( + scopes: $token['scopes'], + text: $token['text'], + start: $token['start'], + end: $token['end'], + ), + $lineTokens + ), + $output + ); +} + +dataset('grammars', function () { + $repository = new GrammarRepository; + $grammars = array_filter($repository->getAllGrammarNames(), fn(string $grammar) => ! in_array($grammar, [ + 'astro', + 'haxe', + 'fluent', + 'stylus', + 'viml', + 'sas', + 'git-commit', + 'hxml', + 'groovy', + 'make', + 'shellsession', + // Act as includes, basically. + 'html-derivative', + 'cpp-macro', + 'jinja-html', + // No sample file. + 'git-rebase', + // Empty. + 'txt', + ])); + + sort($grammars, SORT_NATURAL); + + // FIXME: These grammars have known issues and should be skipped. + return array_values($grammars); +});