Add vscode-textmate compliance/comparison suite

composer.json

@@ -33,7 +33,8 @@
         "laravel/pint": "^1.18.1",
         "phpstan/phpstan": "^2.0",
         "phpstan/extension-installer": "^1.4.3",
-        "illuminate/support": "^11.30"
+        "illuminate/support": "^11.30",
+        "symfony/process": "^7.2"
     },
     "config": {
         "allow-plugins": {
@@ -54,7 +55,11 @@
             "Composer\\Config::disableProcessTimeout",
             "@php ./meta/update-sources.php"
         ],
-        "test": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1",
+        "test": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --exclude-group integration/vscode-textmate",
+        "test:full": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1",
+        "test:vscode": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/vscode-textmate --bail",
+        "test:grammars": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/grammars",
+        "test:themes": "@php -dmemory_limit=-1 vendor/bin/pest --enforce-time-limit --default-time-limit=1 --group integration/themes",
         "lint": "vendor/bin/phpstan"
     }
 }

tests/Fixtures/vscode-textmate-compliance.js

@@ -0,0 +1,72 @@
+const process = require('node:process');
+const fs = require('node:fs');
+const path = require('node:path');
+const oniguruma = require('vscode-oniguruma');
+const vsctm = require('vscode-textmate');
+
+const sample = process.argv[2];
+const scope = process.argv[3];
+const scopeMap = JSON.parse(process.argv[4]);
+
+function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (error, data) =>
+            error ? reject(error) : resolve(data)
+        );
+    });
+}
+
+const wasmBin = fs.readFileSync(
+    path.join(__dirname, "../../node_modules/vscode-oniguruma/release/onig.wasm")
+).buffer;
+
+const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin).then(() => {
+    return {
+        createOnigScanner(patterns) {
+            return new oniguruma.OnigScanner(patterns);
+        },
+        createOnigString(s) {
+            return new oniguruma.OnigString(s);
+        },
+    };
+});
+
+const registry = new vsctm.Registry({
+    onigLib: vscodeOnigurumaLib,
+    loadGrammar: (scopeName) => {
+        if (! scopeMap[scopeName]) {
+            console.error(`Unknown scope name: ${scopeName}`);
+            process.exit(1);
+        }
+
+        return readFile(scopeMap[scopeName]).then(data => {
+            return vsctm.parseRawGrammar(data.toString(), scopeMap[scopeName]);
+        })
+    },
+});
+
+const tokens = []
+
+registry.loadGrammar(scope).then(async grammar => {
+    const text = await readFile(sample);
+    const lines = text.toString().split(/\r?\n/);
+
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const lineTokens = grammar.tokenizeLine(line, ruleStack);
+
+        tokens.push(
+            lineTokens.tokens.map(lineToken => ({
+                scopes: lineToken.scopes,
+                text: line.substring(lineToken.startIndex, lineToken.endIndex),
+                start: lineToken.startIndex,
+                end: lineToken.endIndex,
+            }))
+        )
+
+        ruleStack = lineTokens.ruleStack;
+    }
+
+    console.log(JSON.stringify(tokens));
+})

tests/Integration/GrammarsTest.php

@@ -1,8 +1,9 @@
 <?php
 
-use Phiki\Grammar\GrammarRepository;
 use Phiki\Phiki;
 
+pest()->group('integration/grammars');
+
 describe('Grammars', function () {
     test('default grammars do not produce warnings or exceptions', function (string $grammar) {
         $sample = file_get_contents(__DIR__.'/../../resources/samples/'.$grammar.'.sample');
@@ -12,33 +13,3 @@
         ->with('grammars')
         ->throwsNoExceptions();
 });
-
-dataset('grammars', function () {
-    $repository = new GrammarRepository;
-    $grammars = array_filter($repository->getAllGrammarNames(), fn (string $grammar) => ! in_array($grammar, [
-        'astro',
-        'haxe',
-        'fluent',
-        'stylus',
-        'viml',
-        'sas',
-        'git-commit',
-        'hxml',
-        'groovy',
-        'make',
-        'shellsession',
-        // Act as includes, basically.
-        'html-derivative',
-        'cpp-macro',
-        'jinja-html',
-        // No sample file.
-        'git-rebase',
-        // Empty.
-        'txt',
-    ]));
-
-    sort($grammars, SORT_NATURAL);
-
-    // FIXME: These grammars have known issues and should be skipped.
-    return array_values($grammars);
-});

tests/Integration/VscodeTextmateTest.php

@@ -0,0 +1,15 @@
+<?php
+
+use Phiki\Phiki;
+
+pest()->group('integration/vscode-textmate');
+
+test('it produces the same tokens as vscode-textmate', function (string $grammar) {
+    $samplePath = __DIR__ . "/../../resources/samples/{$grammar}.sample";
+
+    $expected = vscodeTextmateTokenize($samplePath, $grammar);
+    $actual = (new Phiki)->codeToTokens(file_get_contents($samplePath), $grammar);
+
+    expect($actual)->toEqualCanonicalizing($expected);
+})
+    ->with('grammars');

tests/Pest.php

@@ -1,8 +1,12 @@
 <?php
 
 use Phiki\Environment\Environment;
+use Phiki\Grammar\DefaultGrammars;
+use Phiki\Grammar\GrammarRepository;
 use Phiki\Grammar\ParsedGrammar;
+use Phiki\Token\Token;
 use Phiki\Tokenizer;
+use Symfony\Component\Process\Process;
 
 function tokenize(string $input, array $grammar): array
 {
@@ -13,3 +17,73 @@ function tokenize(string $input, array $grammar): array
 
     return $tokenizer->tokenize($input);
 }
+
+function vscodeTextmateTokenize(string $samplePath, string $grammarName): array
+{
+    $process = new Process(
+        [
+            'node',
+            __DIR__ . '/Fixtures/vscode-textmate-compliance.js',
+            $samplePath,
+            array_flip(DefaultGrammars::SCOPES_TO_NAMES)[$grammarName],
+            json_encode(collect(DefaultGrammars::SCOPES_TO_NAMES)
+                ->mapWithKeys(fn (string $name, string $scope) => [$scope => DefaultGrammars::NAMES_TO_PATHS[$name]])
+                ->all()),
+        ],
+    );
+
+    $process->run();
+
+    if (! $process->isSuccessful()) {
+        throw new RuntimeException($process->getErrorOutput() . ':' . PHP_EOL . $process->getOutput());
+    }
+
+    $output = json_decode($process->getOutput(), true);
+
+    if (! is_array($output)) {
+        throw new RuntimeException('Invalid output from process:' . PHP_EOL . $process->getOutput());
+    }
+
+    return array_map(
+        fn (array $lineTokens) => array_map(
+            fn (array $token) => new Token(
+                scopes: $token['scopes'],
+                text: $token['text'],
+                start: $token['start'],
+                end: $token['end'],
+            ),
+            $lineTokens
+        ),
+        $output
+    );
+}
+
+dataset('grammars', function () {
+    $repository = new GrammarRepository;
+    $grammars = array_filter($repository->getAllGrammarNames(), fn(string $grammar) => ! in_array($grammar, [
+        'astro',
+        'haxe',
+        'fluent',
+        'stylus',
+        'viml',
+        'sas',
+        'git-commit',
+        'hxml',
+        'groovy',
+        'make',
+        'shellsession',
+        // Act as includes, basically.
+        'html-derivative',
+        'cpp-macro',
+        'jinja-html',
+        // No sample file.
+        'git-rebase',
+        // Empty.
+        'txt',
+    ]));
+
+    sort($grammars, SORT_NATURAL);
+
+    // FIXME: These grammars have known issues and should be skipped.
+    return array_values($grammars);
+});