feat(cli): Adds a 'test' command for targeted testing (#236)

Unlike the 'snapshot' command, this allows for checking the data
in specific occurrences instead of checking ALL the SCIP data in
a given Document. This command can potentially be used by
people working on SCIP indexers.

Co-authored-by: Varun Gandhi <varun.gandhi@sourcegraph.com>

Author: matthewnitschke-wk

cmd/scip/main.go

@@ -23,7 +23,8 @@ func commands() []*cli.Command {
 	print := printCommand()
 	snapshot := snapshotCommand()
 	stats := statsCommand()
-	return []*cli.Command{&lint, &print, &snapshot, &stats}
+	test := testCommand()
+	return []*cli.Command{&lint, &print, &snapshot, &stats, &test}
 }
 
 //go:embed version.txt
@@ -74,6 +75,15 @@ func fromFlag(storage *string) *cli.StringFlag {
 	}
 }
 
+func commentSyntaxFlag(storage *string) *cli.StringFlag {
+	return &cli.StringFlag{
+		Name:        "comment-syntax",
+		Usage:       "Comment syntax to use for snapshot files",
+		Destination: storage,
+		Value:       "//",
+	}
+}
+
 func projectRootFlag(storage *string) *cli.StringFlag {
 	return &cli.StringFlag{
 		Name: "project-root",

cmd/scip/main_test.go

@@ -1,14 +1,17 @@
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/hexops/autogold/v2"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
 
 	"github.com/sourcegraph/scip/bindings/go/scip"
 	"github.com/sourcegraph/scip/bindings/go/scip/testutil"
@@ -101,3 +104,115 @@ func TestSCIPSnapshots(t *testing.T) {
 		return snapshots
 	})
 }
+
+func unwrap[T any](v T, err error) func(*testing.T) T {
+	return func(t *testing.T) T {
+		require.NoError(t, err)
+		return v
+	}
+}
+
+func TestSCIPTests(t *testing.T) {
+	cwd := unwrap(os.Getwd())(t)
+	testDir := filepath.Join(cwd, "tests", "test_cmd")
+	testPaths := unwrap(os.ReadDir(testDir))(t)
+	require.Truef(t, len(testPaths) >= 1, "Expected at least one test case in directory: %v", testDir)
+
+	os.Setenv("NO_COLOR", "1")
+	t.Cleanup(func() {
+		os.Unsetenv("NO_COLOR")
+	})
+
+	type TestCase struct {
+		dir        string
+		passOutput autogold.Value
+		failOutput autogold.Value
+	}
+
+	// To update the snapshot values, run 'go test ./cmd/scip -update'.
+	testCases := []TestCase{
+		{"roles",
+			autogold.Expect("✓ passes.repro (3 assertions)\n"),
+			autogold.Expect(`✗ fails-wrong-role.repro
+    Failure - row: 0, column: 13
+      Expected: 'reference reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().'
+      Actual:
+        - 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().'✗ fails-wrong-symbol.repro
+    Failure - row: 0, column: 13
+      Expected: 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello2().'
+      Actual:
+        - 'definition reprolang repro_manager roles 1.0.0 fails-wrong-symbol.repro/hello().'`),
+		},
+		{"ranges",
+			autogold.Expect("✓ passes.repro (3 assertions)\n"),
+			autogold.Expect(`✗ fails.repro
+    Failure - row: 0, column: 10
+      Expected: 'definition passes.repro/hello().'
+      Actual:
+        - No attributes found`),
+		},
+		{"diagnostics",
+			autogold.Expect("✓ passes.repro (2 assertions)\n"),
+			autogold.Expect(`✗ fails-incorrect-diagnostic.repro
+    Failure - row: 0, column: 11
+      Expected: 'diagnostic Warning:'
+                'THIS IS NOT CORRECT'
+      Actual:
+        - 'definition reprolang repro_manager diagnostics 1.0.0 fails-incorrect-diagnostic.repro/deprecatedMethod.'
+        - 'diagnostic Warning'
+          'deprecated identifier'✗ fails-no-diagnostic.repro
+    Failure - row: 0, column: 11
+      Expected: 'diagnostic Warning:'
+                'deprecated identifier'
+      Actual:
+        - 'definition reprolang repro_manager diagnostics 1.0.0 fails-no-diagnostic.repro/hello().'`),
+		},
+	}
+
+	for _, testPath := range testPaths {
+		require.Truef(t, slices.ContainsFunc(testCases, func(testCase TestCase) bool {
+			return testCase.dir == testPath.Name()
+		}), "Missing entry in testOutputs for %q", testPath.Name())
+	}
+
+	for _, testCase := range testCases {
+		var dirEntry os.DirEntry
+		require.Truef(t, slices.ContainsFunc(testPaths, func(entry os.DirEntry) bool {
+			if entry.Name() == testCase.dir {
+				dirEntry = entry
+				return true
+			}
+			return false
+		}), "Stale entry in testOutputs for %q; did you rename or remove the directory", testCase.dir)
+
+		subtestDir := filepath.Join(testDir, dirEntry.Name())
+		require.Truef(t, dirEntry.IsDir(), "not a directory: %q", subtestDir)
+
+		t.Run(testCase.dir, func(t *testing.T) {
+			sources := unwrap(scip.NewSourcesFromDirectory(subtestDir))(t)
+			index := unwrap(repro.Index("file:/"+subtestDir, dirEntry.Name(), sources, []*repro.Dependency{}))(t)
+
+			var passFiles, failFiles []string
+			testFiles := unwrap(os.ReadDir(subtestDir))(t)
+			for _, testFile := range testFiles {
+				if strings.HasPrefix(testFile.Name(), "passes") {
+					passFiles = append(passFiles, testFile.Name())
+				} else if strings.HasPrefix(testFile.Name(), "fails") {
+					failFiles = append(failFiles, testFile.Name())
+				} else {
+					t.Fatalf("Test files must start with 'passes' or 'fails'. Received %v", testFile.Name())
+				}
+			}
+
+			var passOutput bytes.Buffer
+			err := testMain(subtestDir, passFiles, index, "#", &passOutput)
+			require.NoError(t, err)
+			testCase.passOutput.Equal(t, passOutput.String())
+
+			var failOutput bytes.Buffer
+			err = testMain(subtestDir, failFiles, index, "#", &failOutput)
+			require.Error(t, err)
+			testCase.failOutput.Equal(t, failOutput.String())
+		})
+	}
+}

cmd/scip/snapshot.go

@@ -28,7 +28,12 @@ func snapshotCommand() cli.Command {
 		Description: `The snapshot subcommand generates snapshot files which
 can be use for inspecting the output of an index in a
 visual way. Occurrences are marked with caret signs (^)
-and symbol information.`,
+and symbol information.
+
+For testing a SCIP indexer, you can either use this subcommand
+along with 'git diff' or equivalent, or you can use the dedicated
+'test' subcommand for more targeted checks.
+`,
 		Flags: []cli.Flag{
 			fromFlag(&snapshotFlags.from),
 			&cli.StringFlag{
@@ -44,12 +49,7 @@ and symbol information.`,
 				Destination: &snapshotFlags.strict,
 				Value:       true,
 			},
-			&cli.StringFlag{
-				Name:        "comment-syntax",
-				Usage:       "Comment syntax to use for snapshot files",
-				Destination: &snapshotFlags.commentSyntax,
-				Value:       "//",
-			},
+			commentSyntaxFlag(&snapshotFlags.commentSyntax),
 		},
 		Action: func(c *cli.Context) error {
 			return snapshotMain(snapshotFlags)