Skip to content

Commit 1ffdfa7

Browse files
feat(cli): Adds a 'test' command for targeted testing (#236)
Unlike the 'snapshot' command, this allows for checking the data in specific occurrences instead of checking ALL the SCIP data in a given Document. This command can potentially be used by people working on SCIP indexers. Co-authored-by: Varun Gandhi <varun.gandhi@sourcegraph.com>
1 parent 90220de commit 1ffdfa7

16 files changed

+772
-9
lines changed

cmd/scip/main.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ func commands() []*cli.Command {
2323
print := printCommand()
2424
snapshot := snapshotCommand()
2525
stats := statsCommand()
26-
return []*cli.Command{&lint, &print, &snapshot, &stats}
26+
test := testCommand()
27+
return []*cli.Command{&lint, &print, &snapshot, &stats, &test}
2728
}
2829

2930
//go:embed version.txt
@@ -74,6 +75,15 @@ func fromFlag(storage *string) *cli.StringFlag {
7475
}
7576
}
7677

78+
func commentSyntaxFlag(storage *string) *cli.StringFlag {
79+
return &cli.StringFlag{
80+
Name: "comment-syntax",
81+
Usage: "Comment syntax to use for snapshot files",
82+
Destination: storage,
83+
Value: "//",
84+
}
85+
}
86+
7787
func projectRootFlag(storage *string) *cli.StringFlag {
7888
return &cli.StringFlag{
7989
Name: "project-root",

cmd/scip/main_test.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package main
22

33
import (
4+
"bytes"
45
"flag"
56
"fmt"
67
"os"
78
"path/filepath"
89
"strings"
910
"testing"
1011

12+
"github.com/hexops/autogold/v2"
1113
"github.com/stretchr/testify/require"
14+
"golang.org/x/exp/slices"
1215

1316
"github.com/sourcegraph/scip/bindings/go/scip"
1417
"github.com/sourcegraph/scip/bindings/go/scip/testutil"
@@ -101,3 +104,115 @@ func TestSCIPSnapshots(t *testing.T) {
101104
return snapshots
102105
})
103106
}
107+
108+
func unwrap[T any](v T, err error) func(*testing.T) T {
109+
return func(t *testing.T) T {
110+
require.NoError(t, err)
111+
return v
112+
}
113+
}
114+
115+
func TestSCIPTests(t *testing.T) {
116+
cwd := unwrap(os.Getwd())(t)
117+
testDir := filepath.Join(cwd, "tests", "test_cmd")
118+
testPaths := unwrap(os.ReadDir(testDir))(t)
119+
require.Truef(t, len(testPaths) >= 1, "Expected at least one test case in directory: %v", testDir)
120+
121+
os.Setenv("NO_COLOR", "1")
122+
t.Cleanup(func() {
123+
os.Unsetenv("NO_COLOR")
124+
})
125+
126+
type TestCase struct {
127+
dir string
128+
passOutput autogold.Value
129+
failOutput autogold.Value
130+
}
131+
132+
// To update the snapshot values, run 'go test ./cmd/scip -update'.
133+
testCases := []TestCase{
134+
{"roles",
135+
autogold.Expect("✓ passes.repro (3 assertions)\n"),
136+
autogold.Expect(`✗ fails-wrong-role.repro
137+
Failure - row: 0, column: 13
138+
Expected: 'reference reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().'
139+
Actual:
140+
- 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().'✗ fails-wrong-symbol.repro
141+
Failure - row: 0, column: 13
142+
Expected: 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello2().'
143+
Actual:
144+
- 'definition reprolang repro_manager roles 1.0.0 fails-wrong-symbol.repro/hello().'`),
145+
},
146+
{"ranges",
147+
autogold.Expect("✓ passes.repro (3 assertions)\n"),
148+
autogold.Expect(`✗ fails.repro
149+
Failure - row: 0, column: 10
150+
Expected: 'definition passes.repro/hello().'
151+
Actual:
152+
- No attributes found`),
153+
},
154+
{"diagnostics",
155+
autogold.Expect("✓ passes.repro (2 assertions)\n"),
156+
autogold.Expect(`✗ fails-incorrect-diagnostic.repro
157+
Failure - row: 0, column: 11
158+
Expected: 'diagnostic Warning:'
159+
'THIS IS NOT CORRECT'
160+
Actual:
161+
- 'definition reprolang repro_manager diagnostics 1.0.0 fails-incorrect-diagnostic.repro/deprecatedMethod.'
162+
- 'diagnostic Warning'
163+
'deprecated identifier'✗ fails-no-diagnostic.repro
164+
Failure - row: 0, column: 11
165+
Expected: 'diagnostic Warning:'
166+
'deprecated identifier'
167+
Actual:
168+
- 'definition reprolang repro_manager diagnostics 1.0.0 fails-no-diagnostic.repro/hello().'`),
169+
},
170+
}
171+
172+
for _, testPath := range testPaths {
173+
require.Truef(t, slices.ContainsFunc(testCases, func(testCase TestCase) bool {
174+
return testCase.dir == testPath.Name()
175+
}), "Missing entry in testOutputs for %q", testPath.Name())
176+
}
177+
178+
for _, testCase := range testCases {
179+
var dirEntry os.DirEntry
180+
require.Truef(t, slices.ContainsFunc(testPaths, func(entry os.DirEntry) bool {
181+
if entry.Name() == testCase.dir {
182+
dirEntry = entry
183+
return true
184+
}
185+
return false
186+
}), "Stale entry in testOutputs for %q; did you rename or remove the directory", testCase.dir)
187+
188+
subtestDir := filepath.Join(testDir, dirEntry.Name())
189+
require.Truef(t, dirEntry.IsDir(), "not a directory: %q", subtestDir)
190+
191+
t.Run(testCase.dir, func(t *testing.T) {
192+
sources := unwrap(scip.NewSourcesFromDirectory(subtestDir))(t)
193+
index := unwrap(repro.Index("file:/"+subtestDir, dirEntry.Name(), sources, []*repro.Dependency{}))(t)
194+
195+
var passFiles, failFiles []string
196+
testFiles := unwrap(os.ReadDir(subtestDir))(t)
197+
for _, testFile := range testFiles {
198+
if strings.HasPrefix(testFile.Name(), "passes") {
199+
passFiles = append(passFiles, testFile.Name())
200+
} else if strings.HasPrefix(testFile.Name(), "fails") {
201+
failFiles = append(failFiles, testFile.Name())
202+
} else {
203+
t.Fatalf("Test files must start with 'passes' or 'fails'. Received %v", testFile.Name())
204+
}
205+
}
206+
207+
var passOutput bytes.Buffer
208+
err := testMain(subtestDir, passFiles, index, "#", &passOutput)
209+
require.NoError(t, err)
210+
testCase.passOutput.Equal(t, passOutput.String())
211+
212+
var failOutput bytes.Buffer
213+
err = testMain(subtestDir, failFiles, index, "#", &failOutput)
214+
require.Error(t, err)
215+
testCase.failOutput.Equal(t, failOutput.String())
216+
})
217+
}
218+
}

cmd/scip/snapshot.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ func snapshotCommand() cli.Command {
2828
Description: `The snapshot subcommand generates snapshot files which
2929
can be use for inspecting the output of an index in a
3030
visual way. Occurrences are marked with caret signs (^)
31-
and symbol information.`,
31+
and symbol information.
32+
33+
For testing a SCIP indexer, you can either use this subcommand
34+
along with 'git diff' or equivalent, or you can use the dedicated
35+
'test' subcommand for more targeted checks.
36+
`,
3237
Flags: []cli.Flag{
3338
fromFlag(&snapshotFlags.from),
3439
&cli.StringFlag{
@@ -44,12 +49,7 @@ and symbol information.`,
4449
Destination: &snapshotFlags.strict,
4550
Value: true,
4651
},
47-
&cli.StringFlag{
48-
Name: "comment-syntax",
49-
Usage: "Comment syntax to use for snapshot files",
50-
Destination: &snapshotFlags.commentSyntax,
51-
Value: "//",
52-
},
52+
commentSyntaxFlag(&snapshotFlags.commentSyntax),
5353
},
5454
Action: func(c *cli.Context) error {
5555
return snapshotMain(snapshotFlags)

0 commit comments

Comments
 (0)