From 50128d289cdc9cc158aaa07545b188fdd7ad4f28 Mon Sep 17 00:00:00 2001 From: braydonk Date: Mon, 17 Jul 2023 23:35:07 -0400 Subject: [PATCH 1/5] Command Integrations Tests This PR adds integration tests that will actually run a built yamlfmt binary in an isolated test directory. It is the first step to more sophisticated testing for yamlfmt, covering cases that are impossible in unit tests. --- integrationtest/local_test.go | 31 +++++++ integrationtest/testcase.go | 89 +++++++++++++++++++ .../local/include_document_start/after/x.yaml | 3 + .../include_document_start/before/x.yaml | 2 + .../include_document_start/stdout/stdout.txt | 1 + .../testdata/local/path_arg/after/x.yaml | 2 + .../testdata/local/path_arg/before/x.yaml | 2 + .../testdata/local/path_arg/stdout/stdout.txt | 1 + 8 files changed, 131 insertions(+) create mode 100644 integrationtest/local_test.go create mode 100644 integrationtest/testcase.go create mode 100755 integrationtest/testdata/local/include_document_start/after/x.yaml create mode 100644 integrationtest/testdata/local/include_document_start/before/x.yaml create mode 100755 integrationtest/testdata/local/include_document_start/stdout/stdout.txt create mode 100755 integrationtest/testdata/local/path_arg/after/x.yaml create mode 100644 integrationtest/testdata/local/path_arg/before/x.yaml create mode 100755 integrationtest/testdata/local/path_arg/stdout/stdout.txt diff --git a/integrationtest/local_test.go b/integrationtest/local_test.go new file mode 100644 index 0000000..696389a --- /dev/null +++ b/integrationtest/local_test.go @@ -0,0 +1,31 @@ +package integrationtest + +import ( + "flag" + "testing" +) + +// This suite contains tests that can easily be run on a local machine +// and operate entirely within temp directories that are created and +// destroyed per-test. + +var ( + updateFlag = flag.Bool("update", false, "Whether to update the goldens.") +) + +func TestPathArg(t *testing.T) { + tempDirTestCase{ + Dir: "path_arg", + // TODO: Path to command is the last thing to figure out before merge + Command: "/home/braydon/go/bin/yamlfmt x.yaml", + Update: *updateFlag, + }.Run(t) +} + +func TestIncludeDocumentStart(t *testing.T) { + tempDirTestCase{ + Dir: "include_document_start", + Command: "/home/braydon/go/bin/yamlfmt -formatter include_document_start=true x.yaml", + Update: *updateFlag, + }.Run(t) +} diff --git a/integrationtest/testcase.go b/integrationtest/testcase.go new file mode 100644 index 0000000..dc2e39b --- /dev/null +++ b/integrationtest/testcase.go @@ -0,0 +1,89 @@ +package integrationtest + +import ( + "bytes" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/yamlfmt/internal/assert" + "github.com/google/yamlfmt/internal/tempfile" +) + +const ( + stdoutGoldenFile = "stdout.txt" +) + +type tempDirTestCase struct { + Dir string + Command string + Update bool +} + +func (tc tempDirTestCase) Run(t *testing.T) { + // I wanna write on the first indent level lol + t.Run(tc.Dir, tc.run) +} + +func (tc tempDirTestCase) run(t *testing.T) { + // Replicate the "before" directory in the test temp directory. + tempDir := t.TempDir() + paths, err := tempfile.ReplicateDirectory(tc.testFolderBeforePath(), tempDir) + assert.NilErr(t, err) + err = paths.CreateAll() + assert.NilErr(t, err) + + // Run the command for the test in the temp directory. + var stdoutBuf bytes.Buffer + cmd := tc.command(tempDir, &stdoutBuf) + err = cmd.Run() + assert.NilErr(t, err) + + err = tc.goldenStdout(stdoutBuf.Bytes()) + assert.NilErr(t, err) + err = tc.goldenAfter(tempDir) + assert.NilErr(t, err) +} + +func (tc tempDirTestCase) testFolderBeforePath() string { + return tc.testdataDirPath() + "/before" +} + +func (tc tempDirTestCase) command(wd string, stdoutBuf *bytes.Buffer) *exec.Cmd { + cmdParts := strings.Split(tc.Command, " ") + return &exec.Cmd{ + Path: cmdParts[0], // This is just the path to look up the binary + Args: cmdParts, // Args needs to be an array of everything including the command name + Stdout: stdoutBuf, + Dir: wd, + } +} + +func (tc tempDirTestCase) goldenStdout(stdoutResult []byte) error { + goldenCtx := tempfile.GoldenCtx{ + Dir: tc.testFolderStdoutPath(), + Update: tc.Update, + } + return goldenCtx.CompareGoldenFile(stdoutGoldenFile, stdoutResult) +} + +func (tc tempDirTestCase) goldenAfter(wd string) error { + goldenCtx := tempfile.GoldenCtx{ + Dir: tc.testFolderAfterPath(), + Update: tc.Update, + } + return goldenCtx.CompareDirectory(wd) +} + +func (tc tempDirTestCase) testFolderAfterPath() string { + return filepath.Join(tc.testdataDirPath(), "after") +} + +func (tc tempDirTestCase) testFolderStdoutPath() string { + return filepath.Join(tc.testdataDirPath(), "stdout") +} + +func (tc tempDirTestCase) testdataDirPath() string { + return filepath.Join("testdata/local", tc.Dir) +} diff --git a/integrationtest/testdata/local/include_document_start/after/x.yaml b/integrationtest/testdata/local/include_document_start/after/x.yaml new file mode 100755 index 0000000..449551a --- /dev/null +++ b/integrationtest/testdata/local/include_document_start/after/x.yaml @@ -0,0 +1,3 @@ +--- +hello: + world: 1 diff --git a/integrationtest/testdata/local/include_document_start/before/x.yaml b/integrationtest/testdata/local/include_document_start/before/x.yaml new file mode 100644 index 0000000..baf52d2 --- /dev/null +++ b/integrationtest/testdata/local/include_document_start/before/x.yaml @@ -0,0 +1,2 @@ +hello: + world: 1 \ No newline at end of file diff --git a/integrationtest/testdata/local/include_document_start/stdout/stdout.txt b/integrationtest/testdata/local/include_document_start/stdout/stdout.txt new file mode 100755 index 0000000..a042389 --- /dev/null +++ b/integrationtest/testdata/local/include_document_start/stdout/stdout.txt @@ -0,0 +1 @@ +hello world! diff --git a/integrationtest/testdata/local/path_arg/after/x.yaml b/integrationtest/testdata/local/path_arg/after/x.yaml new file mode 100755 index 0000000..85f21a0 --- /dev/null +++ b/integrationtest/testdata/local/path_arg/after/x.yaml @@ -0,0 +1,2 @@ +6tark: + does: 64 diff --git a/integrationtest/testdata/local/path_arg/before/x.yaml b/integrationtest/testdata/local/path_arg/before/x.yaml new file mode 100644 index 0000000..fd33cdb --- /dev/null +++ b/integrationtest/testdata/local/path_arg/before/x.yaml @@ -0,0 +1,2 @@ +6tark: + does: 64 \ No newline at end of file diff --git a/integrationtest/testdata/local/path_arg/stdout/stdout.txt b/integrationtest/testdata/local/path_arg/stdout/stdout.txt new file mode 100755 index 0000000..a042389 --- /dev/null +++ b/integrationtest/testdata/local/path_arg/stdout/stdout.txt @@ -0,0 +1 @@ +hello world! From e3276e1853ba0138fdfad97df55f12355cff3daf Mon Sep 17 00:00:00 2001 From: braydonk Date: Sat, 2 Sep 2023 10:12:31 -0400 Subject: [PATCH 2/5] add tests that work --- Makefile | 18 ++ go.mod | 3 +- go.sum | 2 + integrationtest/local/README.md | 11 ++ integrationtest/local/local_test.go | 46 +++++ integrationtest/{ => local}/testcase.go | 38 ++-- .../include_document_start/after/x.yaml | 0 .../include_document_start/before/x.yaml | 0 .../include_document_start/stdout/stdout.txt | 0 .../testdata}/path_arg/after/x.yaml | 0 .../testdata}/path_arg/before/x.yaml | 0 .../local/testdata/path_arg/stdout/stdout.txt | 0 integrationtest/local_test.go | 31 ---- .../include_document_start/stdout/stdout.txt | 1 - .../testdata/local/path_arg/stdout/stdout.txt | 1 - internal/assert/assert.go | 133 ++++++++++++++ internal/assert/assert_test.go | 171 ++++++++++++++++++ internal/tempfile/golden.go | 124 +++++++++++++ internal/tempfile/path.go | 10 +- 19 files changed, 536 insertions(+), 53 deletions(-) create mode 100644 integrationtest/local/README.md create mode 100644 integrationtest/local/local_test.go rename integrationtest/{ => local}/testcase.go (59%) rename integrationtest/{testdata/local => local/testdata}/include_document_start/after/x.yaml (100%) rename integrationtest/{testdata/local => local/testdata}/include_document_start/before/x.yaml (100%) create mode 100755 integrationtest/local/testdata/include_document_start/stdout/stdout.txt rename integrationtest/{testdata/local => local/testdata}/path_arg/after/x.yaml (100%) rename integrationtest/{testdata/local => local/testdata}/path_arg/before/x.yaml (100%) create mode 100755 integrationtest/local/testdata/path_arg/stdout/stdout.txt delete mode 100644 integrationtest/local_test.go delete mode 100755 integrationtest/testdata/local/include_document_start/stdout/stdout.txt delete mode 100755 integrationtest/testdata/local/path_arg/stdout/stdout.txt create mode 100644 internal/assert/assert.go create mode 100644 internal/assert/assert_test.go create mode 100644 internal/tempfile/golden.go diff --git a/Makefile b/Makefile index 6b52ce2..963835c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +.EXPORT_ALL_VARIABLES: + .PHONY: build build: go build ./cmd/yamlfmt @@ -10,6 +12,22 @@ test: test_v: go test -v ./... +YAMLFMT_BIN ?= $(shell pwd)/yamlfmt +.PHONY: export_yamlfmt_bin +export_yamlfmt_bin: export YAMLFMT_BIN = $(YAMLFMT_BIN) + +.PHONY: integrationtest_local +integrationtest_local: + go test -tags=integration_test ./integrationtest/local -update + +.PHONY: integrationtest_local_v +integrationtest_local_v: + go test -tags=integration_test ./integrationtest/local -update + +.PHONY: integrationtest_local_update +integrationtest_local_update: + YAMLFMT_BIN="$(YAMLFMT_BIN)" go test -tags=integration_test ./integrationtest/local -update + .PHONY: install install: go install ./cmd/yamlfmt diff --git a/go.mod b/go.mod index 3f5f5bc..a2cff1a 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,5 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/braydonk/yaml v0.7.0 github.com/mitchellh/mapstructure v1.5.0 + github.com/google/go-cmp v0.5.9 ) - -require github.com/google/go-cmp v0.5.9 // indirect diff --git a/go.sum b/go.sum index cbcd8f4..f249bef 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/RageCage64/go-assert v0.2.2 h1:wwPA2yibB7XmaQKpw4xk7NfVPdJ1v79GlBvmS8P9ROM= +github.com/RageCage64/go-assert v0.2.2/go.mod h1:YuWzhAJlE4Z2TW2D4msNx1mNU0wA+6lmAL0lP1l7yd4= github.com/RageCage64/multilinediff v0.2.0 h1:yNSpSF5NXIrmo6bRIgO4Q0g7SXqFD4j/WEcBE+BdCFY= github.com/RageCage64/multilinediff v0.2.0/go.mod h1:pKr+KLgP0gvRzA+yv0/IUaYQuBYN1ucWysvsL58aMP0= github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= diff --git a/integrationtest/local/README.md b/integrationtest/local/README.md new file mode 100644 index 0000000..be80ded --- /dev/null +++ b/integrationtest/local/README.md @@ -0,0 +1,11 @@ +# Local Integration Tests + +These are tests that can be run directly on the host machine. + +Each test runs by: +* Creating a temporary directory +* Copying everything from `before` in the testdata folder for the given test into the temp directory +* Run the specified command for the given test with the temp directory as the working directory +* Compare goldens for command output and state of the directory + - If running with a `-update` flag, simply overwrite all golden files + - If running normally, compare the golden files to ensure all the files are the same and the content of each file matches diff --git a/integrationtest/local/local_test.go b/integrationtest/local/local_test.go new file mode 100644 index 0000000..3d73cf2 --- /dev/null +++ b/integrationtest/local/local_test.go @@ -0,0 +1,46 @@ +//go:build integration_test + +package local_test + +import ( + "flag" + "fmt" + "os" + "testing" + + "github.com/google/yamlfmt/integrationtest/local" +) + +var ( + updateFlag *bool = flag.Bool("update", false, "Whether to update the goldens.") + yamlfmtBin string +) + +func init() { + yamlfmtBinVar := os.Getenv("YAMLFMT_BIN") + if yamlfmtBinVar == "" { + fmt.Println("Must provide a YAMLFMT_BIN environment variable.") + os.Exit(1) + } + yamlfmtBin = yamlfmtBinVar +} + +func TestPathArg(t *testing.T) { + local.TestCase{ + Dir: "path_arg", + Command: yamlfmtWithArgs("x.yaml"), + Update: *updateFlag, + }.Run(t) +} + +func TestIncludeDocumentStart(t *testing.T) { + local.TestCase{ + Dir: "include_document_start", + Command: yamlfmtWithArgs("-formatter include_document_start=true x.yaml"), + Update: *updateFlag, + }.Run(t) +} + +func yamlfmtWithArgs(args string) string { + return fmt.Sprintf("%s %s", yamlfmtBin, args) +} diff --git a/integrationtest/testcase.go b/integrationtest/local/testcase.go similarity index 59% rename from integrationtest/testcase.go rename to integrationtest/local/testcase.go index dc2e39b..c1a4574 100644 --- a/integrationtest/testcase.go +++ b/integrationtest/local/testcase.go @@ -1,4 +1,6 @@ -package integrationtest +//go:build integration_test + +package local import ( "bytes" @@ -15,18 +17,18 @@ const ( stdoutGoldenFile = "stdout.txt" ) -type tempDirTestCase struct { +type TestCase struct { Dir string Command string Update bool } -func (tc tempDirTestCase) Run(t *testing.T) { +func (tc TestCase) Run(t *testing.T) { // I wanna write on the first indent level lol t.Run(tc.Dir, tc.run) } -func (tc tempDirTestCase) run(t *testing.T) { +func (tc TestCase) run(t *testing.T) { // Replicate the "before" directory in the test temp directory. tempDir := t.TempDir() paths, err := tempfile.ReplicateDirectory(tc.testFolderBeforePath(), tempDir) @@ -46,21 +48,27 @@ func (tc tempDirTestCase) run(t *testing.T) { assert.NilErr(t, err) } -func (tc tempDirTestCase) testFolderBeforePath() string { +func (tc TestCase) testFolderBeforePath() string { return tc.testdataDirPath() + "/before" } -func (tc tempDirTestCase) command(wd string, stdoutBuf *bytes.Buffer) *exec.Cmd { - cmdParts := strings.Split(tc.Command, " ") +func (tc TestCase) command(wd string, stdoutBuf *bytes.Buffer) *exec.Cmd { + cmdArgs := []string{} + for _, arg := range strings.Split(tc.Command, " ") { + // This is to handle potential typos in args with extra spaces. + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } return &exec.Cmd{ - Path: cmdParts[0], // This is just the path to look up the binary - Args: cmdParts, // Args needs to be an array of everything including the command name + Path: cmdArgs[0], // This is just the path to the command + Args: cmdArgs, // Args needs to be an array of everything including the command Stdout: stdoutBuf, Dir: wd, } } -func (tc tempDirTestCase) goldenStdout(stdoutResult []byte) error { +func (tc TestCase) goldenStdout(stdoutResult []byte) error { goldenCtx := tempfile.GoldenCtx{ Dir: tc.testFolderStdoutPath(), Update: tc.Update, @@ -68,7 +76,7 @@ func (tc tempDirTestCase) goldenStdout(stdoutResult []byte) error { return goldenCtx.CompareGoldenFile(stdoutGoldenFile, stdoutResult) } -func (tc tempDirTestCase) goldenAfter(wd string) error { +func (tc TestCase) goldenAfter(wd string) error { goldenCtx := tempfile.GoldenCtx{ Dir: tc.testFolderAfterPath(), Update: tc.Update, @@ -76,14 +84,14 @@ func (tc tempDirTestCase) goldenAfter(wd string) error { return goldenCtx.CompareDirectory(wd) } -func (tc tempDirTestCase) testFolderAfterPath() string { +func (tc TestCase) testFolderAfterPath() string { return filepath.Join(tc.testdataDirPath(), "after") } -func (tc tempDirTestCase) testFolderStdoutPath() string { +func (tc TestCase) testFolderStdoutPath() string { return filepath.Join(tc.testdataDirPath(), "stdout") } -func (tc tempDirTestCase) testdataDirPath() string { - return filepath.Join("testdata/local", tc.Dir) +func (tc TestCase) testdataDirPath() string { + return filepath.Join("testdata/", tc.Dir) } diff --git a/integrationtest/testdata/local/include_document_start/after/x.yaml b/integrationtest/local/testdata/include_document_start/after/x.yaml similarity index 100% rename from integrationtest/testdata/local/include_document_start/after/x.yaml rename to integrationtest/local/testdata/include_document_start/after/x.yaml diff --git a/integrationtest/testdata/local/include_document_start/before/x.yaml b/integrationtest/local/testdata/include_document_start/before/x.yaml similarity index 100% rename from integrationtest/testdata/local/include_document_start/before/x.yaml rename to integrationtest/local/testdata/include_document_start/before/x.yaml diff --git a/integrationtest/local/testdata/include_document_start/stdout/stdout.txt b/integrationtest/local/testdata/include_document_start/stdout/stdout.txt new file mode 100755 index 0000000..e69de29 diff --git a/integrationtest/testdata/local/path_arg/after/x.yaml b/integrationtest/local/testdata/path_arg/after/x.yaml similarity index 100% rename from integrationtest/testdata/local/path_arg/after/x.yaml rename to integrationtest/local/testdata/path_arg/after/x.yaml diff --git a/integrationtest/testdata/local/path_arg/before/x.yaml b/integrationtest/local/testdata/path_arg/before/x.yaml similarity index 100% rename from integrationtest/testdata/local/path_arg/before/x.yaml rename to integrationtest/local/testdata/path_arg/before/x.yaml diff --git a/integrationtest/local/testdata/path_arg/stdout/stdout.txt b/integrationtest/local/testdata/path_arg/stdout/stdout.txt new file mode 100755 index 0000000..e69de29 diff --git a/integrationtest/local_test.go b/integrationtest/local_test.go deleted file mode 100644 index 696389a..0000000 --- a/integrationtest/local_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package integrationtest - -import ( - "flag" - "testing" -) - -// This suite contains tests that can easily be run on a local machine -// and operate entirely within temp directories that are created and -// destroyed per-test. - -var ( - updateFlag = flag.Bool("update", false, "Whether to update the goldens.") -) - -func TestPathArg(t *testing.T) { - tempDirTestCase{ - Dir: "path_arg", - // TODO: Path to command is the last thing to figure out before merge - Command: "/home/braydon/go/bin/yamlfmt x.yaml", - Update: *updateFlag, - }.Run(t) -} - -func TestIncludeDocumentStart(t *testing.T) { - tempDirTestCase{ - Dir: "include_document_start", - Command: "/home/braydon/go/bin/yamlfmt -formatter include_document_start=true x.yaml", - Update: *updateFlag, - }.Run(t) -} diff --git a/integrationtest/testdata/local/include_document_start/stdout/stdout.txt b/integrationtest/testdata/local/include_document_start/stdout/stdout.txt deleted file mode 100755 index a042389..0000000 --- a/integrationtest/testdata/local/include_document_start/stdout/stdout.txt +++ /dev/null @@ -1 +0,0 @@ -hello world! diff --git a/integrationtest/testdata/local/path_arg/stdout/stdout.txt b/integrationtest/testdata/local/path_arg/stdout/stdout.txt deleted file mode 100755 index a042389..0000000 --- a/integrationtest/testdata/local/path_arg/stdout/stdout.txt +++ /dev/null @@ -1 +0,0 @@ -hello world! diff --git a/internal/assert/assert.go b/internal/assert/assert.go new file mode 100644 index 0000000..eb7584e --- /dev/null +++ b/internal/assert/assert.go @@ -0,0 +1,133 @@ +package assert + +var ( + // The failure format string for values not being equal. Formatted with `expected` then `got`. + EqualMessage = "value did not equal expectation.\nexpected: %v\n got: %v" + + // The error format string for one or both pointers being nil. Formatted with `got` then `expected`. + DereferenceEqualErrMsg = "could not dereference nil pointer\ngot %v, expected %v" + + // The failure format string if the err is not nil. Formatted with `err`. + NilErrMessage = "expected no error, got error:\n%v" + + // The failure format string for slices being different sizes. Formatted with `expected` then `got`. + SliceSizeMessage = "slices were different sizes.\nexpected len:%d\n got len:%d\n" + + // The failure format string for slices not matching at some index. Formatted with the mismatched + // index, then `expected`, then `got`. + SliceMismatchMessage = "slices differed at index %d.\nexpected: %v\n got: %v" +) + +// The interface that represents the subset of `testing.T` that this package +// requires. Passing in a `testing.T` satisfies this interface. +type TestingT interface { + Helper() + Fatal(...any) + Fatalf(string, ...any) + Errorf(string, ...any) +} + +// Assert that the passed condition is true. If not, fatally fail with +// `message` and format `args` into it. +func Assert(t TestingT, condition bool, message string, args ...any) { + t.Helper() + + if !condition { + t.Fatalf(message, args...) + } +} + +// Assert that `got` equals `expected`. The types between compared +// arguments must be the same. Uses `assert.EqualMessage`. +func Equal[T comparable](t TestingT, expected T, got T) { + t.Helper() + EqualMsg(t, expected, got, EqualMessage) +} + +// Assert that the value at `got` equals the value at `expected`. Will +// error if either pointer is nil. Uses `assert.DereferenceEqualErrMsg` +// and `assert.EqualMessage`. +func DereferenceEqual[T comparable](t TestingT, expected *T, got *T) { + t.Helper() + DereferenceEqualMsg(t, expected, got, DereferenceEqualErrMsg, EqualMessage) +} + +// Assert that that `err` is nil. Uses `assert.NilErrMessage`. +func NilErr(t TestingT, err error) { + t.Helper() + NilErrMsg(t, err, NilErrMessage) +} + +// Assert that slices `got` and `expected` are equal. Will produce a +// different message if the lengths are different or if any element +// mismatches. Uses `assert.SliceSizeMessage` and +// `assert.SliceMismatchMessage`. +func SliceEqual[T comparable](t TestingT, expected []T, got []T) { + t.Helper() + SliceEqualMsg( + t, + expected, + got, + SliceSizeMessage, + SliceMismatchMessage, + ) +} + +// Assert that `got` equals `expected`. The types between compared +// arguments must be the same. Uses `message`. +func EqualMsg[T comparable](t TestingT, expected T, got T, message string) { + t.Helper() + + if got != expected { + t.Fatalf(message, expected, got) + } +} + +// Assert that the value at `got` equals the value at `expected`. Will +// error if either pointer is nil. Uses `errMessage` and `mismatchMessage`. +func DereferenceEqualMsg[T comparable]( + t TestingT, + expected *T, + got *T, + errMessage, + mismatchMessage string, +) { + t.Helper() + + if got == nil || expected == nil { + t.Errorf(errMessage, expected, got) + } else { + EqualMsg(t, *expected, *got, mismatchMessage) + } +} + +// Assert that that `err` is nil. Uses `message`. +func NilErrMsg(t TestingT, err error, message string) { + t.Helper() + + if err != nil { + t.Fatalf(message, err) + } +} + +// Assert that slices `got` and `expected` are equal. Will produce a +// different message if the lengths are different or if any element +// mismatches. Uses `sizeMessage` and `mismatchMessage`. +func SliceEqualMsg[T comparable]( + t TestingT, + expected []T, + got []T, + sizeMessage, mismatchMessage string, +) { + t.Helper() + + if len(got) != len(expected) { + t.Fatalf(sizeMessage, len(expected), len(got)) + } else { + for i := range got { + if got[i] != expected[i] { + t.Fatalf(mismatchMessage, i, expected[i], got[i]) + } + } + } +} diff --git a/internal/assert/assert_test.go b/internal/assert/assert_test.go new file mode 100644 index 0000000..8c0f8ba --- /dev/null +++ b/internal/assert/assert_test.go @@ -0,0 +1,171 @@ +package assert_test + +import ( + "fmt" + "testing" + + "github.com/google/yamlfmt/internal/assert" +) + +type tMock struct { + logs []string + failed bool + err error +} + +func newTMock() *tMock { + return &tMock{ + logs: []string{}, + } +} + +func (t *tMock) Helper() {} + +func (t *tMock) Fatal(...any) { + t.failed = true +} + +func (t *tMock) Fatalf(msg string, args ...any) { + t.logs = append(t.logs, fmt.Sprintf(msg, args...)) + t.Fatal() +} + +func (t *tMock) Errorf(msg string, args ...any) { + t.failed = true + t.err = fmt.Errorf(msg, args...) +} + +func TestAssertFail(t *testing.T) { + testInstance := newTMock() + failMsg := "expected %d to equal %d" + a := 1 + b := 2 + assert.Assert(testInstance, a == b, failMsg, a, b) + if !testInstance.failed { + t.Fatalf("Assert failed. %v", *testInstance) + } + if len(testInstance.logs) != 1 { + t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) + } + expectedFailLog := fmt.Sprintf(failMsg, a, b) + if testInstance.logs[0] != expectedFailLog { + t.Fatalf( + "Failure log didn't match.\nexpected: %s\ngot: %s", + expectedFailLog, + testInstance.logs[0], + ) + } +} + +func TestEqualFail(t *testing.T) { + testInstance := newTMock() + failMsg := "expected %v to equal %v" + expected := 1 + got := 2 + assert.EqualMsg(testInstance, expected, got, failMsg) + if len(testInstance.logs) != 1 { + t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) + } + expectedFailLog := fmt.Sprintf(failMsg, expected, got) + if testInstance.logs[0] != expectedFailLog { + t.Fatalf( + "Failure log didn't match.\nexpected: %s\ngot: %s", + expectedFailLog, + testInstance.logs[0], + ) + } +} + +func TestDereferenceEqualErr(t *testing.T) { + testInstance := newTMock() + expected := &struct{}{} + errMsg := "nil pointer %v %v" + assert.DereferenceEqualMsg(testInstance, expected, nil, errMsg, "does not matter") + if testInstance.err == nil { + t.Fatalf("DereferenceEqual should have failed") + } + expectedErr := fmt.Errorf(errMsg, expected, nil) + if testInstance.err.Error() != expectedErr.Error() { + t.Fatalf( + "Errors didn't match.\nexpected: %s\ngot: %s", + expectedErr, + testInstance.err, + ) + } +} + +func TestDerefenceEqualFail(t *testing.T) { + testInstance := newTMock() + type x struct { + num int + } + failMsg := "%v not equal %v" + expected := &x{num: 1} + got := &x{num: 2} + assert.DereferenceEqualMsg(testInstance, expected, got, "does not matter", failMsg) + if len(testInstance.logs) != 1 { + t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) + } + expectedFailLog := fmt.Sprintf(failMsg, *expected, *got) + if testInstance.logs[0] != expectedFailLog { + t.Fatalf( + "Failure log didn't match.\nexpected: %s\ngot: %s", + expectedFailLog, + testInstance.logs[0], + ) + } +} + +func TestDereferenceEqualPass(t *testing.T) { + testInstance := newTMock() + type x struct { + num int + } + expected := &x{num: 1} + got := &x{num: 1} + assert.DereferenceEqualMsg(testInstance, expected, got, "doesn't matter", "doesn't matter") + if testInstance.failed { + t.Fatalf("test failed when it should have passed") + } + if len(testInstance.logs) != 0 { + t.Fatalf("test instance had logs when it shouldn't: %v", testInstance.logs) + } +} + +func TestSliceEqualFailDiffSize(t *testing.T) { + testInstance := newTMock() + failSizeMsg := "%v and %v" + expected := []int{1, 2, 3, 4} + got := []int{1, 2, 3} + assert.SliceEqualMsg(testInstance, expected, got, failSizeMsg, "something else") + if len(testInstance.logs) != 1 { + t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) + } + expectedFailLog := fmt.Sprintf(failSizeMsg, len(expected), len(got)) + if testInstance.logs[0] != expectedFailLog { + t.Fatalf( + "Failure log didn't match.\nexpected: %s\ngot: %s", + expectedFailLog, + testInstance.logs[0], + ) + } +} + +func TestSliceEqualMismatch(t *testing.T) { + testInstance := newTMock() + failSizeMsg := "%v and %v" + expected := []int{1, 2, 4} + got := []int{1, 2, 3} + assert.SliceEqualMsg(testInstance, expected, got, failSizeMsg, "something else") + if len(testInstance.logs) != 1 { + t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) + } + expectedFailLog := fmt.Sprintf(failSizeMsg, expected[2], got[2]) + if testInstance.logs[0] != expectedFailLog { + t.Fatalf( + "Failure log didn't match.\nexpected: %s\ngot: %s", + expectedFailLog, + testInstance.logs[0], + ) + } +} diff --git a/internal/tempfile/golden.go b/internal/tempfile/golden.go new file mode 100644 index 0000000..35f9391 --- /dev/null +++ b/internal/tempfile/golden.go @@ -0,0 +1,124 @@ +package tempfile + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/google/go-cmp/cmp" + "github.com/google/yamlfmt/internal/collections" +) + +type GoldenCtx struct { + Dir string + Update bool +} + +func (g GoldenCtx) goldenPath(path string) string { + return filepath.Join(g.Dir, path) +} + +func (g GoldenCtx) CompareGoldenFile(path string, gotContent []byte) error { + // If we are updating, just rewrite the file. + if g.Update { + return os.WriteFile(g.goldenPath(path), gotContent, os.ModePerm) + } + + // If we are not updating, check that the content is the same. + expectedContent, err := os.ReadFile(g.goldenPath(path)) + if err != nil { + return err + } + // Edge case for empty stdout. + if gotContent == nil { + gotContent = []byte{} + } + diff := cmp.Diff(expectedContent, gotContent) + // If there is no diff between the content, nothing to do in either mode. + if diff == "" { + return nil + } + return &GoldenDiffError{path: path, diff: diff} +} + +func (g GoldenCtx) CompareDirectory(resultPath string) error { + // If in update mode, clobber the whole directory and recreate it with + // the result of the test. + if g.Update { + return g.updateGoldenDirectory(resultPath) + } + + // Compare the two directories by reading all paths. + resultPaths, err := readAllPaths(resultPath) + if err != nil { + return err + } + goldenPaths, err := readAllPaths(g.Dir) + if err != nil { + return err + } + + // If the directories differ in paths then the test has failed. + if !resultPaths.Equals(goldenPaths) { + return errors.New("the directories were different") + } + + // Compare each file and gather each error. + compareErrors := collections.Errors{} + for path := range resultPaths { + gotContent, err := os.ReadFile(filepath.Join(resultPath, path)) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + err = g.CompareGoldenFile(path, gotContent) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + // If there are no errors this will be nil, otherwise will be a + // combination of every error that occurred. + return compareErrors.Combine() +} + +func (g GoldenCtx) updateGoldenDirectory(resultPath string) error { + // Clear the golden directory + err := os.RemoveAll(g.Dir) + if err != nil { + return fmt.Errorf("could not clear golden directory %s: %w", g.Dir, err) + } + err = os.Mkdir(g.Dir, os.ModePerm) + if err != nil { + return fmt.Errorf("could not recreate golden directory %s: %w", g.Dir, err) + } + + // Recreate the goldens directory + paths, err := ReplicateDirectory(resultPath, g.Dir) + if err != nil { + return err + } + return paths.CreateAll() +} + +func readAllPaths(dirPath string) (collections.Set[string], error) { + paths := collections.Set[string]{} + allNamesButCurrentDirectory := func(path string, d fs.DirEntry, err error) error { + if path == dirPath { + return nil + } + paths.Add(d.Name()) + return nil + } + err := filepath.WalkDir(dirPath, allNamesButCurrentDirectory) + return paths, err +} + +type GoldenDiffError struct { + path string + diff string +} + +func (e *GoldenDiffError) Error() string { + return fmt.Sprintf("golden: %s differed: %s", e.path, e.diff) +} diff --git a/internal/tempfile/path.go b/internal/tempfile/path.go index 8f129d5..63f82ad 100644 --- a/internal/tempfile/path.go +++ b/internal/tempfile/path.go @@ -42,11 +42,14 @@ func (ps Paths) CreateAll() error { func ReplicateDirectory(dir string, newBase string) (Paths, error) { paths := Paths{} - - err := filepath.Walk(dir, func(path string, info fs.FileInfo, walkErr error) error { + walkAllButCurrentDirectory := func(path string, info fs.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } + // Skip the current directory (basically the . directory) + if path == dir { + return nil + } content := []byte{} if !info.IsDir() { @@ -64,6 +67,7 @@ func ReplicateDirectory(dir string, newBase string) (Paths, error) { Content: content, }) return nil - }) + } + err := filepath.Walk(dir, walkAllButCurrentDirectory) return paths, err } From d9bce2f4d326bdba8057dd8016fdc57eb3e5547d Mon Sep 17 00:00:00 2001 From: braydonk Date: Thu, 7 Sep 2023 20:26:59 -0400 Subject: [PATCH 3/5] finish the tests and make target --- .github/workflows/ci.yaml | 9 ++++--- Makefile | 26 +++++++++---------- integrationtest/{local => command}/README.md | 7 +++-- .../local_test.go => command/command_test.go} | 8 +++--- .../{local => command}/testcase.go | 2 +- .../include_document_start/after/x.yaml | 0 .../include_document_start/before/x.yaml | 0 .../include_document_start/stdout/stdout.txt | 0 .../testdata/path_arg/after/x.yaml | 0 .../testdata/path_arg/before/x.yaml | 0 .../testdata/path_arg/stdout/stdout.txt | 0 11 files changed, 29 insertions(+), 23 deletions(-) rename integrationtest/{local => command}/README.md (59%) rename integrationtest/{local/local_test.go => command/command_test.go} (87%) rename integrationtest/{local => command}/testcase.go (99%) rename integrationtest/{local => command}/testdata/include_document_start/after/x.yaml (100%) rename integrationtest/{local => command}/testdata/include_document_start/before/x.yaml (100%) rename integrationtest/{local => command}/testdata/include_document_start/stdout/stdout.txt (100%) rename integrationtest/{local => command}/testdata/path_arg/after/x.yaml (100%) rename integrationtest/{local => command}/testdata/path_arg/before/x.yaml (100%) rename integrationtest/{local => command}/testdata/path_arg/stdout/stdout.txt (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 69e8820..0973188 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ '1.18', '1.19', '1.20' ] + go: [ '1.18', '1.19', '1.20', '1.21' ] steps: - uses: actions/checkout@v3 @@ -39,11 +39,14 @@ jobs: - name: Test run: go mod tidy + - name: Test + run: go vet + - name: Test run: go test -v ./... - name: Test - run: go vet - + run: make integrationtest + - name: goimports run: test -z "$(set -o pipefail && goimports -l . | tee goimports.out)" || { cat goimports.out && exit 1; } diff --git a/Makefile b/Makefile index 963835c..9237cf7 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ .PHONY: build build: - go build ./cmd/yamlfmt + go build -o dist/yamlfmt ./cmd/yamlfmt .PHONY: test test: @@ -12,21 +12,21 @@ test: test_v: go test -v ./... -YAMLFMT_BIN ?= $(shell pwd)/yamlfmt -.PHONY: export_yamlfmt_bin -export_yamlfmt_bin: export YAMLFMT_BIN = $(YAMLFMT_BIN) +YAMLFMT_BIN ?= $(shell pwd)/dist/yamlfmt +.PHONY: integrationtest +integrationtest: + $(MAKE) build + go test -tags=integration_test ./integrationtest/command -.PHONY: integrationtest_local -integrationtest_local: - go test -tags=integration_test ./integrationtest/local -update - -.PHONY: integrationtest_local_v -integrationtest_local_v: - go test -tags=integration_test ./integrationtest/local -update +.PHONY: integrationtest_v +integrationtest_v: + $(MAKE) build + go test -v -tags=integration_test ./integrationtest/command .PHONY: integrationtest_local_update -integrationtest_local_update: - YAMLFMT_BIN="$(YAMLFMT_BIN)" go test -tags=integration_test ./integrationtest/local -update +integrationtest_update: + $(MAKE) build + go test -tags=integration_test ./integrationtest/command -update .PHONY: install install: diff --git a/integrationtest/local/README.md b/integrationtest/command/README.md similarity index 59% rename from integrationtest/local/README.md rename to integrationtest/command/README.md index be80ded..9e1d428 100644 --- a/integrationtest/local/README.md +++ b/integrationtest/command/README.md @@ -1,11 +1,14 @@ -# Local Integration Tests +# Command Integration Tests -These are tests that can be run directly on the host machine. +These are tests that run a yamlfmt binary with different combos of commands in a temp directory set up by the test data. Each test runs by: +* Accepting the absolute path to a binary in the `YAMLFMT_BIN` environment variable * Creating a temporary directory * Copying everything from `before` in the testdata folder for the given test into the temp directory * Run the specified command for the given test with the temp directory as the working directory * Compare goldens for command output and state of the directory - If running with a `-update` flag, simply overwrite all golden files - If running normally, compare the golden files to ensure all the files are the same and the content of each file matches + +You can run the tests by running `make integrationtest` which will build the binary and run the tests with it. \ No newline at end of file diff --git a/integrationtest/local/local_test.go b/integrationtest/command/command_test.go similarity index 87% rename from integrationtest/local/local_test.go rename to integrationtest/command/command_test.go index 3d73cf2..529e929 100644 --- a/integrationtest/local/local_test.go +++ b/integrationtest/command/command_test.go @@ -1,6 +1,6 @@ //go:build integration_test -package local_test +package command_test import ( "flag" @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/google/yamlfmt/integrationtest/local" + "github.com/google/yamlfmt/integrationtest/command" ) var ( @@ -26,7 +26,7 @@ func init() { } func TestPathArg(t *testing.T) { - local.TestCase{ + command.TestCase{ Dir: "path_arg", Command: yamlfmtWithArgs("x.yaml"), Update: *updateFlag, @@ -34,7 +34,7 @@ func TestPathArg(t *testing.T) { } func TestIncludeDocumentStart(t *testing.T) { - local.TestCase{ + command.TestCase{ Dir: "include_document_start", Command: yamlfmtWithArgs("-formatter include_document_start=true x.yaml"), Update: *updateFlag, diff --git a/integrationtest/local/testcase.go b/integrationtest/command/testcase.go similarity index 99% rename from integrationtest/local/testcase.go rename to integrationtest/command/testcase.go index c1a4574..4081a38 100644 --- a/integrationtest/local/testcase.go +++ b/integrationtest/command/testcase.go @@ -1,6 +1,6 @@ //go:build integration_test -package local +package command import ( "bytes" diff --git a/integrationtest/local/testdata/include_document_start/after/x.yaml b/integrationtest/command/testdata/include_document_start/after/x.yaml similarity index 100% rename from integrationtest/local/testdata/include_document_start/after/x.yaml rename to integrationtest/command/testdata/include_document_start/after/x.yaml diff --git a/integrationtest/local/testdata/include_document_start/before/x.yaml b/integrationtest/command/testdata/include_document_start/before/x.yaml similarity index 100% rename from integrationtest/local/testdata/include_document_start/before/x.yaml rename to integrationtest/command/testdata/include_document_start/before/x.yaml diff --git a/integrationtest/local/testdata/include_document_start/stdout/stdout.txt b/integrationtest/command/testdata/include_document_start/stdout/stdout.txt similarity index 100% rename from integrationtest/local/testdata/include_document_start/stdout/stdout.txt rename to integrationtest/command/testdata/include_document_start/stdout/stdout.txt diff --git a/integrationtest/local/testdata/path_arg/after/x.yaml b/integrationtest/command/testdata/path_arg/after/x.yaml similarity index 100% rename from integrationtest/local/testdata/path_arg/after/x.yaml rename to integrationtest/command/testdata/path_arg/after/x.yaml diff --git a/integrationtest/local/testdata/path_arg/before/x.yaml b/integrationtest/command/testdata/path_arg/before/x.yaml similarity index 100% rename from integrationtest/local/testdata/path_arg/before/x.yaml rename to integrationtest/command/testdata/path_arg/before/x.yaml diff --git a/integrationtest/local/testdata/path_arg/stdout/stdout.txt b/integrationtest/command/testdata/path_arg/stdout/stdout.txt similarity index 100% rename from integrationtest/local/testdata/path_arg/stdout/stdout.txt rename to integrationtest/command/testdata/path_arg/stdout/stdout.txt From 92e4a8f1e1150424f4b5bd431decdf31163838b7 Mon Sep 17 00:00:00 2001 From: braydonk Date: Thu, 7 Sep 2023 20:34:12 -0400 Subject: [PATCH 4/5] fix broken test in assert lib --- internal/assert/assert_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/assert/assert_test.go b/internal/assert/assert_test.go index 8c0f8ba..a16395a 100644 --- a/internal/assert/assert_test.go +++ b/internal/assert/assert_test.go @@ -153,14 +153,14 @@ func TestSliceEqualFailDiffSize(t *testing.T) { func TestSliceEqualMismatch(t *testing.T) { testInstance := newTMock() - failSizeMsg := "%v and %v" + failMismatchMsg := "at index %v: %v and %v" expected := []int{1, 2, 4} got := []int{1, 2, 3} - assert.SliceEqualMsg(testInstance, expected, got, failSizeMsg, "something else") + assert.SliceEqualMsg(testInstance, expected, got, "something else", failMismatchMsg) if len(testInstance.logs) != 1 { t.Fatalf("Found %d logs. %v", len(testInstance.logs), testInstance.logs) } - expectedFailLog := fmt.Sprintf(failSizeMsg, expected[2], got[2]) + expectedFailLog := fmt.Sprintf(failMismatchMsg, 2, expected[2], got[2]) if testInstance.logs[0] != expectedFailLog { t.Fatalf( "Failure log didn't match.\nexpected: %s\ngot: %s", From bdcb5f402f2a8a12c2019e41f6e31c46c01119a4 Mon Sep 17 00:00:00 2001 From: braydonk Date: Thu, 7 Sep 2023 20:37:25 -0400 Subject: [PATCH 5/5] fix the CI workflow to be better named and get rid of goimports --- .github/workflows/ci.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0973188..96be6fa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,20 +33,14 @@ jobs: go-version: ${{ matrix.go }} cache: true - - name: Install goimports - run: go install golang.org/x/tools/cmd/goimports@latest - - - name: Test + - name: Go Mod Tidy run: go mod tidy - - name: Test + - name: Go Vet run: go vet - name: Test run: go test -v ./... - - name: Test + - name: Integration Test run: make integrationtest - - - name: goimports - run: test -z "$(set -o pipefail && goimports -l . | tee goimports.out)" || { cat goimports.out && exit 1; }