Command Integrations Tests (#127)

* Command Integrations Tests

This PR adds integration tests that will actually run a built yamlfmt
binary in an isolated test directory. It is the first step to more
sophisticated testing for yamlfmt, covering cases that are impossible in
unit tests.

* add tests that work

* finish the tests and make target

* fix broken test in assert lib

* fix the CI workflow to be better named and get rid of goimports

.github/workflows/ci.yaml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.18', '1.19', '1.20' ]
+        go: [ '1.18', '1.19', '1.20', '1.21' ]
     steps:
       - uses: actions/checkout@v3
 
@@ -33,17 +33,14 @@ jobs:
           go-version: ${{ matrix.go }}
           cache: true
 
-      - name: Install goimports
-        run: go install golang.org/x/tools/cmd/goimports@latest
-
-      - name: Test
+      - name: Go Mod Tidy
         run: go mod tidy
 
+      - name: Go Vet
+        run: go vet
+
       - name: Test
         run: go test -v ./...
 
-      - name: Test
-        run: go vet
-
-      - name: goimports
-        run: test -z "$(set -o pipefail && goimports -l . | tee goimports.out)" || { cat goimports.out && exit 1; }
+      - name: Integration Test
+        run: make integrationtest

Makefile

@@ -1,6 +1,8 @@
+.EXPORT_ALL_VARIABLES:
+
 .PHONY: build
 build:
-	go build ./cmd/yamlfmt
+	go build -o dist/yamlfmt ./cmd/yamlfmt 
 
 .PHONY: test
 test:
@@ -10,6 +12,22 @@ test:
 test_v:
 	go test -v ./...
 
+YAMLFMT_BIN ?= $(shell pwd)/dist/yamlfmt
+.PHONY: integrationtest
+integrationtest:
+	$(MAKE) build
+	go test -tags=integration_test ./integrationtest/command
+
+.PHONY: integrationtest_v
+integrationtest_v:
+	$(MAKE) build
+	go test -v -tags=integration_test ./integrationtest/command
+
+.PHONY: integrationtest_local_update
+integrationtest_update:
+	$(MAKE) build
+	go test -tags=integration_test ./integrationtest/command -update	
+
 .PHONY: install
 install:
 	go install ./cmd/yamlfmt

go.mod

@@ -7,6 +7,5 @@ require (
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/braydonk/yaml v0.7.0
 	github.com/mitchellh/mapstructure v1.5.0
+	github.com/google/go-cmp v0.5.9 
 )
-
-require github.com/google/go-cmp v0.5.9 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/RageCage64/go-assert v0.2.2 h1:wwPA2yibB7XmaQKpw4xk7NfVPdJ1v79GlBvmS8P9ROM=
+github.com/RageCage64/go-assert v0.2.2/go.mod h1:YuWzhAJlE4Z2TW2D4msNx1mNU0wA+6lmAL0lP1l7yd4=
 github.com/RageCage64/multilinediff v0.2.0 h1:yNSpSF5NXIrmo6bRIgO4Q0g7SXqFD4j/WEcBE+BdCFY=
 github.com/RageCage64/multilinediff v0.2.0/go.mod h1:pKr+KLgP0gvRzA+yv0/IUaYQuBYN1ucWysvsL58aMP0=
 github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=

integrationtest/command/README.md

@@ -0,0 +1,14 @@
+# Command Integration Tests
+
+These are tests that run a yamlfmt binary with different combos of commands in a temp directory set up by the test data. 
+
+Each test runs by:
+* Accepting the absolute path to a binary in the `YAMLFMT_BIN` environment variable
+* Creating a temporary directory
+* Copying everything from `before` in the testdata folder for the given test into the temp directory
+* Run the specified command for the given test with the temp directory as the working directory
+* Compare goldens for command output and state of the directory
+    - If running with a `-update` flag, simply overwrite all golden files
+    - If running normally, compare the golden files to ensure all the files are the same and the content of each file matches
+
+You can run the tests by running `make integrationtest` which will build the binary and run the tests with it.

integrationtest/command/command_test.go

@@ -0,0 +1,46 @@
+//go:build integration_test
+
+package command_test
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/yamlfmt/integrationtest/command"
+)
+
+var (
+	updateFlag *bool = flag.Bool("update", false, "Whether to update the goldens.")
+	yamlfmtBin string
+)
+
+func init() {
+	yamlfmtBinVar := os.Getenv("YAMLFMT_BIN")
+	if yamlfmtBinVar == "" {
+		fmt.Println("Must provide a YAMLFMT_BIN environment variable.")
+		os.Exit(1)
+	}
+	yamlfmtBin = yamlfmtBinVar
+}
+
+func TestPathArg(t *testing.T) {
+	command.TestCase{
+		Dir:     "path_arg",
+		Command: yamlfmtWithArgs("x.yaml"),
+		Update:  *updateFlag,
+	}.Run(t)
+}
+
+func TestIncludeDocumentStart(t *testing.T) {
+	command.TestCase{
+		Dir:     "include_document_start",
+		Command: yamlfmtWithArgs("-formatter include_document_start=true x.yaml"),
+		Update:  *updateFlag,
+	}.Run(t)
+}
+
+func yamlfmtWithArgs(args string) string {
+	return fmt.Sprintf("%s %s", yamlfmtBin, args)
+}

integrationtest/command/testcase.go

@@ -0,0 +1,97 @@
+//go:build integration_test
+
+package command
+
+import (
+	"bytes"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/yamlfmt/internal/assert"
+	"github.com/google/yamlfmt/internal/tempfile"
+)
+
+const (
+	stdoutGoldenFile = "stdout.txt"
+)
+
+type TestCase struct {
+	Dir     string
+	Command string
+	Update  bool
+}
+
+func (tc TestCase) Run(t *testing.T) {
+	// I wanna write on the first indent level lol
+	t.Run(tc.Dir, tc.run)
+}
+
+func (tc TestCase) run(t *testing.T) {
+	// Replicate the "before" directory in the test temp directory.
+	tempDir := t.TempDir()
+	paths, err := tempfile.ReplicateDirectory(tc.testFolderBeforePath(), tempDir)
+	assert.NilErr(t, err)
+	err = paths.CreateAll()
+	assert.NilErr(t, err)
+
+	// Run the command for the test in the temp directory.
+	var stdoutBuf bytes.Buffer
+	cmd := tc.command(tempDir, &stdoutBuf)
+	err = cmd.Run()
+	assert.NilErr(t, err)
+
+	err = tc.goldenStdout(stdoutBuf.Bytes())
+	assert.NilErr(t, err)
+	err = tc.goldenAfter(tempDir)
+	assert.NilErr(t, err)
+}
+
+func (tc TestCase) testFolderBeforePath() string {
+	return tc.testdataDirPath() + "/before"
+}
+
+func (tc TestCase) command(wd string, stdoutBuf *bytes.Buffer) *exec.Cmd {
+	cmdArgs := []string{}
+	for _, arg := range strings.Split(tc.Command, " ") {
+		// This is to handle potential typos in args with extra spaces.
+		if arg != "" {
+			cmdArgs = append(cmdArgs, arg)
+		}
+	}
+	return &exec.Cmd{
+		Path:   cmdArgs[0], // This is just the path to the command
+		Args:   cmdArgs,    // Args needs to be an array of everything including the command
+		Stdout: stdoutBuf,
+		Dir:    wd,
+	}
+}
+
+func (tc TestCase) goldenStdout(stdoutResult []byte) error {
+	goldenCtx := tempfile.GoldenCtx{
+		Dir:    tc.testFolderStdoutPath(),
+		Update: tc.Update,
+	}
+	return goldenCtx.CompareGoldenFile(stdoutGoldenFile, stdoutResult)
+}
+
+func (tc TestCase) goldenAfter(wd string) error {
+	goldenCtx := tempfile.GoldenCtx{
+		Dir:    tc.testFolderAfterPath(),
+		Update: tc.Update,
+	}
+	return goldenCtx.CompareDirectory(wd)
+}
+
+func (tc TestCase) testFolderAfterPath() string {
+	return filepath.Join(tc.testdataDirPath(), "after")
+}
+
+func (tc TestCase) testFolderStdoutPath() string {
+	return filepath.Join(tc.testdataDirPath(), "stdout")
+}
+
+func (tc TestCase) testdataDirPath() string {
+	return filepath.Join("testdata/", tc.Dir)
+}

integrationtest/command/testdata/include_document_start/after/x.yaml

@@ -0,0 +1,3 @@
+---
+hello:
+  world: 1

integrationtest/command/testdata/include_document_start/before/x.yaml

@@ -0,0 +1,2 @@
+hello:
+ world: 1

integrationtest/command/testdata/path_arg/after/x.yaml

@@ -0,0 +1,2 @@
+6tark:
+  does: 64

integrationtest/command/testdata/path_arg/before/x.yaml

@@ -0,0 +1,2 @@
+6tark:
+ does: 64

internal/assert/assert.go

@@ -0,0 +1,133 @@
+package assert
+
+var (
+	// The failure format string for values not being equal. Formatted with `expected` then `got`.
+	EqualMessage = "value did not equal expectation.\nexpected: %v\n     got: %v"
+
+	// The error format string for one or both pointers being nil. Formatted with `got` then `expected`.
+	DereferenceEqualErrMsg = "could not dereference nil pointer\ngot %v, expected %v"
+
+	// The failure format string if the err is not nil. Formatted with `err`.
+	NilErrMessage = "expected no error, got error:\n%v"
+
+	// The failure format string for slices being different sizes. Formatted with `expected` then `got`.
+	SliceSizeMessage = "slices were different sizes.\nexpected len:%d\n     got len:%d\n"
+
+	// The failure format string for slices not matching at some index. Formatted with the mismatched
+	// index, then `expected`, then `got`.
+	SliceMismatchMessage = "slices differed at index %d.\nexpected: %v\n     got: %v"
+)
+
+// The interface that represents the subset of `testing.T` that this package
+// requires. Passing in a `testing.T` satisfies this interface.
+type TestingT interface {
+	Helper()
+	Fatal(...any)
+	Fatalf(string, ...any)
+	Errorf(string, ...any)
+}
+
+// Assert that the passed condition is true. If not, fatally fail with
+// `message` and format `args` into it.
+func Assert(t TestingT, condition bool, message string, args ...any) {
+	t.Helper()
+
+	if !condition {
+		t.Fatalf(message, args...)
+	}
+}
+
+// Assert that `got` equals `expected`. The types between compared
+// arguments must be the same. Uses `assert.EqualMessage`.
+func Equal[T comparable](t TestingT, expected T, got T) {
+	t.Helper()
+	EqualMsg(t, expected, got, EqualMessage)
+}
+
+// Assert that the value at `got` equals the value at `expected`. Will
+// error if either pointer is nil. Uses `assert.DereferenceEqualErrMsg`
+// and `assert.EqualMessage`.
+func DereferenceEqual[T comparable](t TestingT, expected *T, got *T) {
+	t.Helper()
+	DereferenceEqualMsg(t, expected, got, DereferenceEqualErrMsg, EqualMessage)
+}
+
+// Assert that that `err` is nil. Uses `assert.NilErrMessage`.
+func NilErr(t TestingT, err error) {
+	t.Helper()
+	NilErrMsg(t, err, NilErrMessage)
+}
+
+// Assert that slices `got` and `expected` are equal. Will produce a
+// different message if the lengths are different or if any element
+// mismatches. Uses `assert.SliceSizeMessage` and
+// `assert.SliceMismatchMessage`.
+func SliceEqual[T comparable](t TestingT, expected []T, got []T) {
+	t.Helper()
+	SliceEqualMsg(
+		t,
+		expected,
+		got,
+		SliceSizeMessage,
+		SliceMismatchMessage,
+	)
+}
+
+// Assert that `got` equals `expected`. The types between compared
+// arguments must be the same. Uses `message`.
+func EqualMsg[T comparable](t TestingT, expected T, got T, message string) {
+	t.Helper()
+
+	if got != expected {
+		t.Fatalf(message, expected, got)
+	}
+}
+
+// Assert that the value at `got` equals the value at `expected`. Will
+// error if either pointer is nil. Uses `errMessage` and `mismatchMessage`.
+func DereferenceEqualMsg[T comparable](
+	t TestingT,
+	expected *T,
+	got *T,
+	errMessage,
+	mismatchMessage string,
+) {
+	t.Helper()
+
+	if got == nil || expected == nil {
+		t.Errorf(errMessage, expected, got)
+	} else {
+		EqualMsg(t, *expected, *got, mismatchMessage)
+	}
+}
+
+// Assert that that `err` is nil. Uses `message`.
+func NilErrMsg(t TestingT, err error, message string) {
+	t.Helper()
+
+	if err != nil {
+		t.Fatalf(message, err)
+	}
+}
+
+// Assert that slices `got` and `expected` are equal. Will produce a
+// different message if the lengths are different or if any element
+// mismatches. Uses `sizeMessage` and `mismatchMessage`.
+func SliceEqualMsg[T comparable](
+	t TestingT,
+	expected []T,
+	got []T,
+	sizeMessage, mismatchMessage string,
+) {
+	t.Helper()
+
+	if len(got) != len(expected) {
+		t.Fatalf(sizeMessage, len(expected), len(got))
+	} else {
+		for i := range got {
+			if got[i] != expected[i] {
+				t.Fatalf(mismatchMessage, i, expected[i], got[i])
+			}
+		}
+	}
+}