feat: add end-to-end tests

Taskfile.yml

@@ -18,14 +18,23 @@ tasks:
     desc: Run unit tests
     run: always
     cmd: go test ./...
+  e2e:
+    desc: Run end-to-end tests
+    summary: |
+      Run tests that mimic how user enters commands and flags.
+      These tests make real requests to the Algolia API.
+      To run them, create a `.env` file with the `ALGOLIA_APPLICATION_ID`
+      and `ALGOLIA_API_KEY` credentials.
+    cmd: go test ./e2e -tags=e2e
+    dotenv: [.env]
   lint:
     desc: Lint code
     cmd: golangci-lint run
   format:
     desc: Format code
     cmds:
-      - gofumpt -w pkg cmd test internal api
-      - golines -w pkg cmd test internal api
+      - gofumpt -w pkg cmd test internal api e2e
+      - golines -w pkg cmd test internal api e2e
   ci:
     desc: Test, lint, and format
     aliases:

devbox.json

@@ -8,5 +8,6 @@
     "golines@latest",
     "gh@latest",
     "curl@latest"
-  ]
+  ],
+  "env_from": ".env"
 }

e2e/README.md

@@ -0,0 +1,66 @@
+# End-to-end tests
+
+These tests run CLI commands like a user would,
+built on top of the [`go-internal/testscript`](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript) package.
+
+They make real API requests,
+so they work best in an empty Algolia application.
+To run these tests,
+you need to set the `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` environment variables.
+If you're using `devbox`, create a `.env` file in the project root directory with these variables.
+If you start a development environment with `devbox shell`,
+the environment variables will be available to you.
+
+## New tests
+
+The tests use a simple format.
+For more information, run `go doc testscript`.
+
+To add a new scenario, create a new directory under the `testscripts` directory,
+and add your files with the extension `txtar`.
+Each test directory can have multiple test files.
+Multiple directories are tested in parallel.
+
+### Example
+
+A simple 'hello world' testscript may look like this:
+
+```txt
+# Test if output is hello
+exec echo 'hello'
+! stderr .
+stdout '^hello\n$'
+```
+
+Read the documentation of the `testscript` package for more information.
+
+To add the new directory to the test suite,
+add a new function to the file `./e2e/e2e_test.go`.
+The function name must begin with `Test`.
+
+```go
+// TestHello is a basic example
+func TestHello(t *testing.T) {
+	RunTestsInDir(t, "testscripts/hello")
+}
+```
+
+## Notes
+
+Since this makes real real requests to the same Algolia application,
+these tests aren't fully isolated from each other.
+
+To make tests interfere less, follow these guidelines:
+
+- Use a unique index name in each `txtar` file.
+  For example, use `test-index` in `indices.txtar` and `test-settings` in `settings.txtar`
+
+- Delete indices at the end of your test with `defer`.
+  For an example, see `indices.txtar`.
+
+- Don't test for number of indices, or empty lists.
+  As other tests might create their own indices and objects,
+  checks that expect a certain number of items might fail.
+  You can ensure that the index with a given name exists or doesn't exist
+  by searching for the index name's pattern in the standard output.
+  Again, see `indices.txtar`.

e2e/e2e_test.go

@@ -0,0 +1,150 @@
+//go:build e2e
+
+package e2e_test
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/algolia/cli/pkg/cmd/root"
+	"github.com/cli/go-internal/testscript"
+)
+
+// algolia runs the root command of the Algolia CLI
+func algolia() int {
+	return int(root.Execute())
+}
+
+// TestMain sets the executable program so that we don't depend on the compiled binary
+func TestMain(m *testing.M) {
+	os.Exit(testscript.RunMain(m, map[string]func() int{
+		"algolia": algolia,
+	}))
+}
+
+// testEnvironment stores the environment variables we need to setup for the tests
+type testEnvironment struct {
+	AppID  string
+	ApiKey string
+}
+
+// getEnv reads the environment variables and prints errors for missing ones
+func (e *testEnvironment) getEnv() error {
+	env := map[string]string{}
+
+	required := []string{
+		// The CLI testing Algolia app
+		"ALGOLIA_APPLICATION_ID",
+		// API key with sufficient permissions to run all tests
+		"ALGOLIA_API_KEY",
+	}
+
+	var missing []string
+
+	for _, envVar := range required {
+		val, ok := os.LookupEnv(envVar)
+		if val == "" || !ok {
+			missing = append(missing, envVar)
+			continue
+		}
+
+		env[envVar] = val
+	}
+
+	if len(missing) > 0 {
+		return fmt.Errorf("missing environment variables: %s", strings.Join(missing, ", "))
+	}
+
+	e.AppID = env["ALGOLIA_APPLICATION_ID"]
+	e.ApiKey = env["ALGOLIA_API_KEY"]
+
+	return nil
+}
+
+// For the `defer` function
+var keyT struct{}
+
+// setupEnv sets up the environment variables for the test
+func setupEnv(testEnv testEnvironment) func(ts *testscript.Env) error {
+	return func(ts *testscript.Env) error {
+		ts.Setenv("ALGOLIA_APPLICATION_ID", testEnv.AppID)
+		ts.Setenv("ALGOLIA_API_KEY", testEnv.ApiKey)
+
+		ts.Values[keyT] = ts.T()
+		return nil
+	}
+}
+
+// setupCmds sets up custom commands we want to make available in the test scripts
+func setupCmds(
+	testEnv testEnvironment,
+) map[string]func(ts *testscript.TestScript, neg bool, args []string) {
+	return map[string]func(ts *testscript.TestScript, neg bool, args []string){
+		"defer": func(ts *testscript.TestScript, neg bool, args []string) {
+			if neg {
+				ts.Fatalf("unsupported ! defer")
+			}
+			tt, ok := ts.Value(keyT).(testscript.T)
+			if !ok {
+				ts.Fatalf("%v is not a testscript.T", ts.Value(keyT))
+			}
+			ts.Defer(func() {
+				if err := ts.Exec(args[0], args[1:]...); err != nil {
+					tt.FailNow()
+				}
+			})
+		},
+	}
+}
+
+// runTestsInDir runs all test scripts from a directory
+func runTestsInDir(t *testing.T, dirName string) {
+	var testEnv testEnvironment
+	if err := testEnv.getEnv(); err != nil {
+		t.Fatal(err)
+	}
+	t.Parallel()
+	t.Log("Running e2e tests in", dirName)
+	testscript.Run(t, testscript.Params{
+		Dir:   dirName,
+		Setup: setupEnv(testEnv),
+		Cmds:  setupCmds(testEnv),
+	})
+}
+
+// TestVersion tests the version option
+func TestVersion(t *testing.T) {
+	runTestsInDir(t, "testscripts/version")
+}
+
+// TestIndices test `algolia indices` commands
+func TestIndices(t *testing.T) {
+	runTestsInDir(t, "testscripts/indices")
+}
+
+// TestSettings tests `algolia settings` commands
+func TestSettings(t *testing.T) {
+	runTestsInDir(t, "testscripts/settings")
+}
+
+// TestObjects tests `algolia objects` commands
+func TestObjects(t *testing.T) {
+	runTestsInDir(t, "testscripts/objects")
+}
+
+// TestSynonyms tests `algolia synonyms` commands
+func TestSynonyms(t *testing.T) {
+	runTestsInDir(t, "testscripts/synonyms")
+}
+
+// TestRules tests `algolia rules` commands
+func TestRules(t *testing.T) {
+	runTestsInDir(t, "testscripts/rules")
+}
+
+// TestSearch tests `algolia search`
+func TestSearch(t *testing.T) {
+	runTestsInDir(t, "testscripts/search")
+}

e2e/testscripts/indices/indices.txtar

@@ -0,0 +1,61 @@
+# Create a new index
+exec algolia settings set test-index --searchableAttributes "foo" --wait
+! stderr .
+
+# Cleanup
+defer algolia indices delete test-index --confirm
+
+# Confirm that the index setting is set
+exec algolia settings get test-index
+stdout -count=1 '"searchableAttributes":\["foo"\]'
+
+# Test that index is listed
+exec algolia indices list
+stdout -count=1 ^test-index
+
+# Copy the index
+exec algolia indices copy test-index test-copy --wait --confirm
+! stderr .
+
+# Confirm that there are 2 indices now
+exec algolia indices list
+stdout -count=1 ^test-index
+stdout -count=1 ^test-copy
+
+# Add replica indices to the copy
+exec algolia settings set test-copy --replicas 'test-replica1,test-replica2' --wait
+! stderr .
+
+# Confirm that there are 4 indices now
+exec algolia indices list
+stdout -count=1 ^test-index
+stdout -count=1 ^test-copy
+stdout -count=1 ^test-replica1
+stdout -count=1 ^test-replica2
+
+# Delete one of the replica indices
+exec algolia indices delete test-replica1 --confirm --wait
+! stderr .
+
+# Confirm that there are 3 indices now
+exec algolia indices list
+stdout -count=1 ^test-index
+stdout -count=1 ^test-copy
+! stdout ^test-replica1
+stdout -count=1 ^test-replica2
+
+# Confirm that the test-copy index still has 1 replica index
+exec algolia settings get test-copy
+stdout -count=1 test-replica2
+! stdout test-replica1
+
+# Delete the copy index including its replicas
+exec algolia indices delete test-copy --include-replicas --confirm --wait
+! stderr .
+
+# Confirm that there is 1 index now
+exec algolia indices list
+stdout -count=1 ^test-index
+! stdout ^test-copy
+! stdout ^test-replica1
+! stdout ^test-replica2

e2e/testscripts/indices/replicas.txtar

@@ -0,0 +1,17 @@
+env INDEX_NAME=test-can-delete-index
+env REPLICA_NAME=test-can-delete-replica
+
+# Create a new index with one replica index
+exec algolia settings set ${INDEX_NAME} --replicas ${REPLICA_NAME} --wait
+! stderr .
+
+# Check that you can delete both manually
+exec algolia index delete ${INDEX_NAME} ${REPLICA_NAME} --confirm
+! stderr .
+! stdout .
+
+# Check that both indices have been deleted
+exec algolia index list
+! stderr .
+! stdout ${INDEX_NAME}
+! stdout ${REPLICA_NAME}

e2e/testscripts/objects/objects.txtar

@@ -0,0 +1,42 @@
+env INDEX_NAME=test-objects
+
+# Import a record without objectID from a file
+! exec algolia objects import ${INDEX_NAME} --file record.jsonl --wait
+! stdout .
+stderr '^missing objectID on line 0$'
+
+# Defer cleanup
+defer algolia index delete ${INDEX_NAME} --confirm
+
+# Import a record with autogenerated objectID
+exec algolia objects import ${INDEX_NAME} --file record.jsonl --wait --auto-generate-object-id-if-not-exist
+! stderr .
+
+# Check that record exists (use aliases)
+exec algolia records list ${INDEX_NAME}
+! stderr .
+stdout -count=1 '"name":"foo"'
+stdout -count=1 'objectID'
+
+# Add another record from stdin with objectID
+stdin objectID.jsonl
+exec algolia records import ${INDEX_NAME} --wait --file -
+! stderr .
+
+# Update a record
+exec algolia objects update ${INDEX_NAME} --file update.jsonl --wait --create-if-not-exists --continue-on-error
+! stderr .
+
+# Check that record has that new attribute
+exec algolia objects browse ${INDEX_NAME}
+! stderr .
+stdout -count=1 '"level":1'
+
+-- record.jsonl --
+{"name": "foo"}
+
+-- objectID.jsonl --
+{"objectID": "test-record-1", "name": "test"}
+
+-- update.jsonl --
+{"objectID": "test-record-1", "level": 1}