From 781486af2b84223489961b1f8efd3db28820ee17 Mon Sep 17 00:00:00 2001 From: George Lesica Date: Mon, 19 Oct 2020 18:02:40 -0600 Subject: [PATCH 1/6] Implement file matcher / assertion --- CHANGELOG.md | 1 + README.md | 1 + commander_unix.yaml | 2 +- integration/unix/commander_test.yaml | 10 +++++++++ pkg/matcher/matcher.go | 33 ++++++++++++++++++++++++++++ pkg/matcher/matcher_test.go | 24 ++++++++++++++++---- pkg/matcher/zoologist.txt | 1 + pkg/matcher/zoom.txt | 1 + pkg/runtime/runtime.go | 1 + pkg/runtime/validator.go | 7 ++++++ pkg/suite/yaml_suite.go | 6 +++++ 11 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 pkg/matcher/zoologist.txt create mode 100644 pkg/matcher/zoom.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 83468b67..a86aa8bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # v2.4.0 - Add ability to test suite from a url + - Add `file` assertion to `stdout` and `stderr` # v2.3.0 diff --git a/README.md b/README.md index 6bba6136..519e7080 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ tests: object.attr: hello # Make assertions on json objects xml: "//book//auhtor": Steven King # Make assertions on xml documents + file: correct-output.txt exit-code: 127 skip: false diff --git a/commander_unix.yaml b/commander_unix.yaml index b5842a21..40fb523d 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -16,7 +16,7 @@ tests: contains: - ✓ [local] it should exit with error code - "- [local] it should skip, was skipped" - line-count: 17 + line-count: 19 exit-code: 0 it should assert that commander will fail: diff --git a/integration/unix/commander_test.yaml b/integration/unix/commander_test.yaml index 3f91b489..534e117f 100644 --- a/integration/unix/commander_test.yaml +++ b/integration/unix/commander_test.yaml @@ -59,6 +59,16 @@ tests: /books/0/author: J. R. R. Tokien /books/1/author: Joanne K. Rowling + it should assert file contents on stdout: + command: cat ./integration/unix/_fixtures/big_out.txt + stdout: + file: ./integration/unix/_fixtures/big_out.txt + + it should assert file contents on stderr: + command: cat ./integration/unix/_fixtures/big_out.txt >&2 + stderr: + file: ./integration/unix/_fixtures/big_out.txt + it should inherit from parent env: config: inherit-env: true diff --git a/pkg/matcher/matcher.go b/pkg/matcher/matcher.go index d01c4147..4dadc9fd 100644 --- a/pkg/matcher/matcher.go +++ b/pkg/matcher/matcher.go @@ -5,6 +5,7 @@ import ( "github.com/antchfx/xmlquery" "github.com/pmezard/go-difflib/difflib" "github.com/tidwall/gjson" + "io/ioutil" "log" "strings" ) @@ -20,6 +21,7 @@ const ( NotContains = "notcontains" JSON = "json" XML = "xml" + File = "file" ) // NewMatcher creates a new matcher by type @@ -37,6 +39,8 @@ func NewMatcher(matcher string) Matcher { return JSONMatcher{} case XML: return XMLMatcher{} + case File: + return FileMatcher{} default: panic(fmt.Sprintf("Validator '%s' does not exist!", matcher)) } @@ -240,3 +244,32 @@ to be equal to return result } + +// FileMatcher matches output captured from stdout or stderr +// against the contents of a file +type FileMatcher struct { +} + +func (m FileMatcher) Match(got interface{}, expected interface{}) MatcherResult { + expectedText, err := ioutil.ReadFile(expected.(string)) + if err != nil { + panic(err.Error()) + } + expectedString := string(expectedText) + + result := got == expectedString + + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(got.(string)), + B: difflib.SplitLines(expectedString), + FromFile: "Got", + ToFile: "Expected", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return MatcherResult{ + Diff: diffText, + Success: result, + } +} diff --git a/pkg/matcher/matcher_test.go b/pkg/matcher/matcher_test.go index 7e8a2cca..5bd933c3 100644 --- a/pkg/matcher/matcher_test.go +++ b/pkg/matcher/matcher_test.go @@ -48,16 +48,16 @@ func TestTextMatcher_ValidateFails(t *testing.T) { func TestEqualMatcher_Validate(t *testing.T) { m := EqualMatcher{} - got := m.Match(1, 1) + got := m.Match(2, 2) assert.True(t, got.Success) } func TestEqualMatcher_ValidateFails(t *testing.T) { m := EqualMatcher{} - got := m.Match(1, 0) + got := m.Match(2, 3) assert.False(t, got.Success) - assert.Contains(t, got.Diff, "+0") - assert.Contains(t, got.Diff, "-1") + assert.Contains(t, got.Diff, "+3") + assert.Contains(t, got.Diff, "-2") } func TestContainsMatcher_Match(t *testing.T) { @@ -183,3 +183,19 @@ another` assert.False(t, r.Success) assert.Equal(t, diff, r.Diff) } + +func TestFileMatcher_Validate(t *testing.T) { + m := FileMatcher{} + got := m.Match("zoom", "zoom.txt") + assert.True(t, got.Success) + assert.Equal(t, "", got.Diff) +} + +func TestFileMatcher_ValidateFails(t *testing.T) { + m := FileMatcher{} + got := m.Match("zoom", "zoologist.txt") + assert.False(t, got.Success) + println(got.Diff) + assert.Contains(t, got.Diff, "+zoologist") + assert.Contains(t, got.Diff, "-zoom") +} diff --git a/pkg/matcher/zoologist.txt b/pkg/matcher/zoologist.txt new file mode 100644 index 00000000..0d9511f6 --- /dev/null +++ b/pkg/matcher/zoologist.txt @@ -0,0 +1 @@ +zoologist \ No newline at end of file diff --git a/pkg/matcher/zoom.txt b/pkg/matcher/zoom.txt new file mode 100644 index 00000000..fe9d4234 --- /dev/null +++ b/pkg/matcher/zoom.txt @@ -0,0 +1 @@ +zoom \ No newline at end of file diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 405088df..5e933211 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -99,6 +99,7 @@ type ExpectedOut struct { NotContains []string `yaml:"not-contains,omitempty"` JSON map[string]string `yaml:"json,omitempty"` XML map[string]string `yaml:"xml,omitempty"` + File string `yaml:"file,omitempty"` } // CommandUnderTest represents the command under test diff --git a/pkg/runtime/validator.go b/pkg/runtime/validator.go index 3ef66254..0c712ded 100644 --- a/pkg/runtime/validator.go +++ b/pkg/runtime/validator.go @@ -118,6 +118,13 @@ func validateExpectedOut(got string, expected ExpectedOut) matcher.MatcherResult } } + if expected.File != "" { + m = matcher.NewMatcher(matcher.File) + if result = m.Match(got, expected.File); !result.Success { + return result + } + } + return result } diff --git a/pkg/suite/yaml_suite.go b/pkg/suite/yaml_suite.go index 720f4bee..618e4076 100644 --- a/pkg/suite/yaml_suite.go +++ b/pkg/suite/yaml_suite.go @@ -222,6 +222,7 @@ func (y *YAMLSuiteConf) convertToExpectedOut(value interface{}) runtime.Expected "lines", "json", "xml", + "file", "not-contains": break default: @@ -242,6 +243,11 @@ func (y *YAMLSuiteConf) convertToExpectedOut(value interface{}) runtime.Expected exp.Exactly = toString(exactly) } + // Parse file key + if file := v["file"]; file != nil { + exp.File = toString(file) + } + //Parse line-count key if lc := v["line-count"]; lc != nil { exp.LineCount = lc.(int) From fe891292c1ee3a7f3cfd54095e7e9905e9b9a99a Mon Sep 17 00:00:00 2001 From: George Lesica Date: Mon, 19 Oct 2020 23:19:10 -0600 Subject: [PATCH 2/6] Add a failing test case and dedicated fixtures --- commander_unix.yaml | 3 ++- integration/unix/_fixtures/file_output_0.txt | 2 ++ integration/unix/_fixtures/file_output_1.txt | 3 +++ integration/unix/failing_suite.yaml | 7 ++++++- 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 integration/unix/_fixtures/file_output_0.txt create mode 100644 integration/unix/_fixtures/file_output_1.txt diff --git a/commander_unix.yaml b/commander_unix.yaml index cac8a91c..3ca0362c 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -23,10 +23,11 @@ tests: command: ./commander test ./integration/unix/failing_suite.yaml stdout: contains: + - ✗ [local] 'file matcher should fail when file and output are different', on property 'Stdout' - ✗ [local] 'it will fail', on property 'ExitCode' - ✗ [local] 'test timeout' could not be executed with error message - Command timed out after 10ms - - "Count: 2, Failed: 2" + - "Count: 3, Failed: 3" exit-code: 1 it should validate a big output: diff --git a/integration/unix/_fixtures/file_output_0.txt b/integration/unix/_fixtures/file_output_0.txt new file mode 100644 index 00000000..9c40573a --- /dev/null +++ b/integration/unix/_fixtures/file_output_0.txt @@ -0,0 +1,2 @@ +first line +second line \ No newline at end of file diff --git a/integration/unix/_fixtures/file_output_1.txt b/integration/unix/_fixtures/file_output_1.txt new file mode 100644 index 00000000..272b2e47 --- /dev/null +++ b/integration/unix/_fixtures/file_output_1.txt @@ -0,0 +1,3 @@ +first line +second line +third line \ No newline at end of file diff --git a/integration/unix/failing_suite.yaml b/integration/unix/failing_suite.yaml index 36eab02f..8ffd2cb3 100644 --- a/integration/unix/failing_suite.yaml +++ b/integration/unix/failing_suite.yaml @@ -7,4 +7,9 @@ tests: command: sleep 1 config: timeout: 10ms - exit-code: 0 \ No newline at end of file + exit-code: 0 + + file matcher should fail when file and output are different: + command: cat ./integration/unix/_fixtures/file_output_1.txt + stdout: + file: ./integration/unix/_fixtures/file_output_0.txt From 2a616dcd7d0c8e35310974e4884570becf224ccd Mon Sep 17 00:00:00 2001 From: George Lesica Date: Tue, 20 Oct 2020 14:03:31 -0600 Subject: [PATCH 3/6] Add validator test and allow fake ReadFile --- pkg/matcher/matcher.go | 6 +++++- pkg/matcher/matcher_test.go | 17 +++++++++++------ pkg/matcher/zoologist.txt | 1 - pkg/matcher/zoom.txt | 1 - pkg/runtime/validator_test.go | 24 ++++++++++++++++++++++++ 5 files changed, 40 insertions(+), 9 deletions(-) delete mode 100644 pkg/matcher/zoologist.txt delete mode 100644 pkg/matcher/zoom.txt diff --git a/pkg/matcher/matcher.go b/pkg/matcher/matcher.go index 4dadc9fd..15d9f3d0 100644 --- a/pkg/matcher/matcher.go +++ b/pkg/matcher/matcher.go @@ -24,6 +24,10 @@ const ( File = "file" ) +// The function used to open files when necessary for matching +// Allows the file IO to be overridden during tests +var ReadFile = ioutil.ReadFile + // NewMatcher creates a new matcher by type func NewMatcher(matcher string) Matcher { switch matcher { @@ -251,7 +255,7 @@ type FileMatcher struct { } func (m FileMatcher) Match(got interface{}, expected interface{}) MatcherResult { - expectedText, err := ioutil.ReadFile(expected.(string)) + expectedText, err := ReadFile(expected.(string)) if err != nil { panic(err.Error()) } diff --git a/pkg/matcher/matcher_test.go b/pkg/matcher/matcher_test.go index 5bd933c3..d085c0ec 100644 --- a/pkg/matcher/matcher_test.go +++ b/pkg/matcher/matcher_test.go @@ -184,18 +184,23 @@ another` assert.Equal(t, diff, r.Diff) } -func TestFileMatcher_Validate(t *testing.T) { +func TestFileMatcher_Match(t *testing.T) { + ReadFile = func(filename string) ([]byte, error) { + return []byte("line one\nline two"), nil + } m := FileMatcher{} - got := m.Match("zoom", "zoom.txt") + got := m.Match("line one\nline two", "fake.txt") assert.True(t, got.Success) assert.Equal(t, "", got.Diff) } func TestFileMatcher_ValidateFails(t *testing.T) { + ReadFile = func(filename string) ([]byte, error) { + return []byte("line one\nline two"), nil + } m := FileMatcher{} - got := m.Match("zoom", "zoologist.txt") + got := m.Match("line one\nline three", "fake.txt") assert.False(t, got.Success) - println(got.Diff) - assert.Contains(t, got.Diff, "+zoologist") - assert.Contains(t, got.Diff, "-zoom") + assert.Contains(t, got.Diff, "+line two") + assert.Contains(t, got.Diff, "-line three") } diff --git a/pkg/matcher/zoologist.txt b/pkg/matcher/zoologist.txt deleted file mode 100644 index 0d9511f6..00000000 --- a/pkg/matcher/zoologist.txt +++ /dev/null @@ -1 +0,0 @@ -zoologist \ No newline at end of file diff --git a/pkg/matcher/zoom.txt b/pkg/matcher/zoom.txt deleted file mode 100644 index fe9d4234..00000000 --- a/pkg/matcher/zoom.txt +++ /dev/null @@ -1 +0,0 @@ -zoom \ No newline at end of file diff --git a/pkg/runtime/validator_test.go b/pkg/runtime/validator_test.go index d09bbc7a..fc632812 100644 --- a/pkg/runtime/validator_test.go +++ b/pkg/runtime/validator_test.go @@ -3,6 +3,7 @@ package runtime import ( "github.com/commander-cli/commander/pkg/matcher" "github.com/stretchr/testify/assert" + "io/ioutil" "testing" ) @@ -192,6 +193,29 @@ test` assert.Equal(t, diff, r.Diff) } +func Test_ValidateExpectedOut_ValidateFile(t *testing.T) { + content := "line one" + matcher.ReadFile = func(filename string) ([]byte, error) { + return []byte(content), nil + } + r := validateExpectedOut(content, ExpectedOut{File: "fake.txt"}) + assert.True(t, r.Success) + assert.Equal(t, "", r.Diff) + + diff := `--- Got ++++ Expected +@@ -1,2 +1 @@ + line one +-line two +` + + r = validateExpectedOut(content+"\nline two", ExpectedOut{File: "fake.txt"}) + assert.False(t, r.Success) + assert.Equal(t, diff, r.Diff) + + matcher.ReadFile = ioutil.ReadFile +} + func Test_ValidateExpectedOut_ValidateXML(t *testing.T) { xml := ` J. R. R. Tolkien From 5b1bcbbbf979de8ca2e88932e38cb6a582704037 Mon Sep 17 00:00:00 2001 From: George Lesica Date: Tue, 20 Oct 2020 18:15:28 -0600 Subject: [PATCH 4/6] Add example and update readme --- README.md | 16 ++++++++++++++++ examples/_fixtures/output.txt | 2 ++ examples/commander.yaml | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 examples/_fixtures/output.txt diff --git a/README.md b/README.md index 557a5fce..fdbbd2ce 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For more information take a look at the [quick start](#quick-start), the [exampl * [line-count](#line-count) * [not-contains](#not-contains) * [xml](#xml) + * [file](#file) - [stderr](#stderr) - [skip](#skip) + [Config](#user-content-config-config) @@ -530,6 +531,21 @@ cat some.xml: ``` +##### file + +`file` is a file path, relative to the working directory that will have +its entire contents matched against the command output. Other than +reading from a file this works the same as [exactly](#exactly). + +The example below will always pass. + +```yaml +output should match file: + command: cat output.txt + stdout: + file: output.txt +``` + #### stderr See [stdout](#stdout) for more information. diff --git a/examples/_fixtures/output.txt b/examples/_fixtures/output.txt new file mode 100644 index 00000000..4c00b399 --- /dev/null +++ b/examples/_fixtures/output.txt @@ -0,0 +1,2 @@ +line one +line two \ No newline at end of file diff --git a/examples/commander.yaml b/examples/commander.yaml index 859c0aa5..282a9a3d 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -24,4 +24,9 @@ tests: it should skip: command: echo "I should be skipped" stdout: I should be skipped - skip: true \ No newline at end of file + skip: true + + it should match file output: + command: printf "line one\nline two" + stdout: + file: _fixtures/output.txt From adeb098b144e4efe1d22a27757b57fa88957e9c0 Mon Sep 17 00:00:00 2001 From: George Lesica Date: Tue, 20 Oct 2020 19:04:06 -0600 Subject: [PATCH 5/6] Fix path --- examples/commander.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/commander.yaml b/examples/commander.yaml index 282a9a3d..cdae1cb2 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -29,4 +29,4 @@ tests: it should match file output: command: printf "line one\nline two" stdout: - file: _fixtures/output.txt + file: examples/_fixtures/output.txt From 571c5e4d7d60ccdc029b1a6f8025d4db5cd0edb6 Mon Sep 17 00:00:00 2001 From: George Lesica Date: Tue, 20 Oct 2020 19:17:56 -0600 Subject: [PATCH 6/6] Oops, fix path again --- examples/commander.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/commander.yaml b/examples/commander.yaml index cdae1cb2..7eafe51f 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -29,4 +29,4 @@ tests: it should match file output: command: printf "line one\nline two" stdout: - file: examples/_fixtures/output.txt + file: ../../examples/_fixtures/output.txt