make unit tests faster

by enabling parallel tests.
Made unit tests run 30% faster

Also added some parallelism to Datastore
tests, but opt-in. MemDB and datastore proxy
tests are now parallel.

Also changed CRDB test entry points to run in
parallel, since the introduction of integrity tests
meant double the tests to run, and sequentially

internal/datastore/crdb/crdb_test.go

@@ -54,6 +54,7 @@ func (cds *crdbDatastore) ExampleRetryableError() error {
 }
 
 func TestCRDBDatastoreWithoutIntegrity(t *testing.T) {
+	t.Parallel()
 	b := testdatastore.RunCRDBForTesting(t, "")
 	test.All(t, test.DatastoreTesterFunc(func(revisionQuantization, gcInterval, gcWindow time.Duration, watchBufferLength uint16) (datastore.Datastore, error) {
 		ctx := context.Background()
@@ -72,10 +73,11 @@ func TestCRDBDatastoreWithoutIntegrity(t *testing.T) {
 		})
 
 		return ds, nil
-	}))
+	}), false)
 }
 
 func TestCRDBDatastoreWithFollowerReads(t *testing.T) {
+	t.Parallel()
 	followerReadDelay := time.Duration(4.8 * float64(time.Second))
 	gcWindow := 100 * time.Second
 
@@ -136,6 +138,7 @@ var defaultKeyForTesting = proxy.KeyConfig{
 }
 
 func TestCRDBDatastoreWithIntegrity(t *testing.T) {
+	t.Parallel()
 	b := testdatastore.RunCRDBForTesting(t, "")
 
 	test.All(t, test.DatastoreTesterFunc(func(revisionQuantization, gcInterval, gcWindow time.Duration, watchBufferLength uint16) (datastore.Datastore, error) {
@@ -159,7 +162,7 @@ func TestCRDBDatastoreWithIntegrity(t *testing.T) {
 		})
 
 		return ds, nil
-	}))
+	}), false)
 
 	unwrappedTester := test.DatastoreTesterFunc(func(revisionQuantization, gcInterval, gcWindow time.Duration, watchBufferLength uint16) (datastore.Datastore, error) {
 		ctx := context.Background()
@@ -187,6 +190,7 @@ func TestCRDBDatastoreWithIntegrity(t *testing.T) {
 }
 
 func TestWatchFeatureDetection(t *testing.T) {
+	t.Parallel()
 	pool, err := dockertest.NewPool("")
 	require.NoError(t, err)
 	cases := []struct {
@@ -229,7 +233,9 @@ func TestWatchFeatureDetection(t *testing.T) {
 		},
 	}
 	for _, tt := range cases {
+		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
 			ctx, cancel := context.WithCancel(context.Background())
 			t.Cleanup(cancel)
 			adminConn, connStrings := newCRDBWithUser(t, pool)

internal/datastore/memdb/memdb_test.go

@@ -26,10 +26,12 @@ func (mdbt memDBTest) New(revisionQuantization, _, gcWindow time.Duration, watch
 }
 
 func TestMemdbDatastore(t *testing.T) {
-	test.All(t, memDBTest{})
+	t.Parallel()
+	test.All(t, memDBTest{}, true)
 }
 
 func TestConcurrentWritePanic(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	ds, err := NewMemdbDatastore(0, 1*time.Hour, 1*time.Hour)
@@ -81,6 +83,7 @@ func TestConcurrentWritePanic(t *testing.T) {
 }
 
 func TestConcurrentWriteRelsError(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	ds, err := NewMemdbDatastore(0, 1*time.Hour, 1*time.Hour)

internal/datastore/mysql/datastore_test.go

@@ -100,7 +100,7 @@ func TestMySQLDatastoreDSNWithoutParseTime(t *testing.T) {
 func TestMySQL8Datastore(t *testing.T) {
 	b := testdatastore.RunMySQLForTestingWithOptions(t, testdatastore.MySQLTesterOptions{MigrateForNewDatastore: true}, "")
 	dst := datastoreTester{b: b, t: t}
-	test.AllWithExceptions(t, test.DatastoreTesterFunc(dst.createDatastore), test.WithCategories(test.WatchSchemaCategory, test.WatchCheckpointsCategory))
+	test.AllWithExceptions(t, test.DatastoreTesterFunc(dst.createDatastore), test.WithCategories(test.WatchSchemaCategory, test.WatchCheckpointsCategory), false)
 	additionalMySQLTests(t, b)
 }
 

internal/datastore/postgres/postgres_shared_test.go

@@ -101,7 +101,7 @@ func testPostgresDatastore(t *testing.T, pc []postgresConfig) {
 					return ds
 				})
 				return ds, nil
-			}))
+			}), false)
 
 			t.Run("TransactionTimestamps", createDatastoreTest(
 				b,
@@ -236,7 +236,7 @@ func testPostgresDatastoreWithoutCommitTimestamps(t *testing.T, pc []postgresCon
 					return ds
 				})
 				return ds, nil
-			}), test.WithCategories(test.WatchCategory))
+			}), test.WithCategories(test.WatchCategory), false)
 		})
 	}
 }

internal/datastore/proxy/observable_test.go

@@ -20,7 +20,7 @@ func (obs observableTest) New(revisionQuantization, _, gcWindow time.Duration, w
 }
 
 func TestObservableProxy(t *testing.T) {
-	test.All(t, observableTest{})
+	test.All(t, observableTest{}, true)
 }
 
 func (p *observableProxy) ExampleRetryableError() error {

internal/datastore/proxy/schemacaching/estimatedsize_test.go

@@ -91,6 +91,8 @@ func TestEstimatedDefinitionSizes(t *testing.T) {
 			for _, caveatDef := range fullyResolved.CaveatDefinitions {
 				caveatDef := caveatDef
 				t.Run("caveat "+caveatDef.Name, func(t *testing.T) {
+					t.Parallel()
+
 					serialized, _ := caveatDef.MarshalVT()
 					sizevt := caveatDef.SizeVT()
 					estimated := estimatedCaveatDefinitionSize(sizevt)

internal/datastore/spanner/spanner_test.go

@@ -43,7 +43,7 @@ func TestSpannerDatastore(t *testing.T) {
 			return ds
 		})
 		return ds, nil
-	}), test.WithCategories(test.GCCategory, test.WatchCategory, test.StatsCategory))
+	}), test.WithCategories(test.GCCategory, test.WatchCategory, test.StatsCategory), false)
 
 	t.Run("TestFakeStats", createDatastoreTest(
 		b,

internal/dispatch/graph/check_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
 
 	"github.com/authzed/spicedb/internal/datastore/common"
 	"github.com/authzed/spicedb/internal/datastore/memdb"
@@ -30,7 +29,7 @@ import (
 var ONR = tuple.ObjectAndRelation
 
 func TestSimpleCheck(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	type expected struct {
 		relation string
@@ -119,6 +118,7 @@ func TestSimpleCheck(t *testing.T) {
 				userset := userset
 				expected := expected
 				t.Run(name, func(t *testing.T) {
+					t.Parallel()
 					require := require.New(t)
 
 					ctx, dispatch, revision := newLocalDispatcher(t)
@@ -150,6 +150,7 @@ func TestSimpleCheck(t *testing.T) {
 }
 
 func TestMaxDepth(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	rawDS, err := memdb.NewMemdbDatastore(0, 0, memdb.DisableGC)
@@ -182,6 +183,7 @@ func TestMaxDepth(t *testing.T) {
 }
 
 func TestCheckMetadata(t *testing.T) {
+	t.Parallel()
 	type expected struct {
 		relation              string
 		isMember              bool
@@ -287,6 +289,7 @@ func TestCheckMetadata(t *testing.T) {
 }
 
 func TestCheckPermissionOverSchema(t *testing.T) {
+	t.Parallel()
 	testCases := []struct {
 		name                      string
 		schema                    string
@@ -1370,6 +1373,7 @@ func addFrame(trace *v1.CheckDebugTrace, foundFrames *mapz.Set[string]) {
 }
 
 func TestCheckDebugging(t *testing.T) {
+	t.Parallel()
 	type expectedFrame struct {
 		resourceType *core.RelationReference
 		resourceIDs  []string
@@ -1482,6 +1486,7 @@ func TestCheckDebugging(t *testing.T) {
 }
 
 func TestCheckWithHints(t *testing.T) {
+	t.Parallel()
 	testCases := []struct {
 		name                   string
 		schema                 string
@@ -1853,6 +1858,7 @@ func TestCheckWithHints(t *testing.T) {
 }
 
 func TestCheckHintsPartialApplication(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	dispatcher := NewLocalOnlyDispatcher(10, 100)
@@ -1898,6 +1904,7 @@ func TestCheckHintsPartialApplication(t *testing.T) {
 }
 
 func TestCheckHintsPartialApplicationOverArrow(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	dispatcher := NewLocalOnlyDispatcher(10, 100)

internal/dispatch/graph/dispatch_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 func TestDispatchChunking(t *testing.T) {
+	t.Parallel()
 	schema := `
 		definition user {
 			relation self: user
@@ -35,6 +36,7 @@ func TestDispatchChunking(t *testing.T) {
 	ctx, dispatcher, revision := newLocalDispatcherWithSchemaAndRels(t, schema, append(enabled, resources...))
 
 	t.Run("check", func(t *testing.T) {
+		t.Parallel()
 		for _, tpl := range resources[:1] {
 			checkResult, err := dispatcher.DispatchCheck(ctx, &v1.DispatchCheckRequest{
 				ResourceRelation: RR(tpl.ResourceAndRelation.Namespace, "view"),
@@ -55,6 +57,8 @@ func TestDispatchChunking(t *testing.T) {
 	})
 
 	t.Run("lookup-resources", func(t *testing.T) {
+		t.Parallel()
+
 		for _, tpl := range resources[:1] {
 			stream := dispatch.NewCollectingDispatchStream[*v1.DispatchLookupResourcesResponse](ctx)
 			err := dispatcher.DispatchLookupResources(&v1.DispatchLookupResourcesRequest{
@@ -75,6 +79,8 @@ func TestDispatchChunking(t *testing.T) {
 	})
 
 	t.Run("lookup-subjects", func(t *testing.T) {
+		t.Parallel()
+
 		for _, tpl := range resources[:1] {
 			stream := dispatch.NewCollectingDispatchStream[*v1.DispatchLookupSubjectsResponse](ctx)
 

internal/dispatch/graph/expand_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
 	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/testing/protocmp"
 
@@ -137,7 +136,7 @@ var (
 )
 
 func TestExpand(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	testCases := []struct {
 		start                 *core.ObjectAndRelation
@@ -275,7 +274,7 @@ func onrExpr(onr *core.ObjectAndRelation) ast.Expr {
 }
 
 func TestMaxDepthExpand(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	require := require.New(t)
 
@@ -306,7 +305,7 @@ func TestMaxDepthExpand(t *testing.T) {
 }
 
 func TestExpandOverSchema(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	testCases := []struct {
 		name          string

internal/dispatch/graph/graph_test.go

@@ -13,11 +13,14 @@ import (
 )
 
 func TestUnwrapStatusError(t *testing.T) {
+	t.Parallel()
+
 	err := rewriteError(context.Background(), graph.NewCheckFailureErr(status.Error(codes.Canceled, "canceled")))
 	grpcutil.RequireStatus(t, codes.Canceled, err)
 }
 
 func TestConcurrencyLimitsWithOverallDefaultLimit(t *testing.T) {
+	t.Parallel()
 	cl := ConcurrencyLimits{}
 	require.Equal(t, uint16(0), cl.Check)
 	require.Equal(t, uint16(0), cl.LookupResources)
@@ -38,6 +41,7 @@ func TestConcurrencyLimitsWithOverallDefaultLimit(t *testing.T) {
 }
 
 func TestSharedConcurrencyLimits(t *testing.T) {
+	t.Parallel()
 	cl := SharedConcurrencyLimits(42)
 	require.Equal(t, uint16(42), cl.Check)
 	require.Equal(t, uint16(42), cl.LookupResources)

internal/dispatch/graph/lookupresources2_test.go

@@ -21,13 +21,11 @@ import (
 	"github.com/authzed/spicedb/pkg/genutil/mapz"
 	core "github.com/authzed/spicedb/pkg/proto/core/v1"
 	v1 "github.com/authzed/spicedb/pkg/proto/dispatch/v1"
-	"github.com/authzed/spicedb/pkg/testutil"
 	"github.com/authzed/spicedb/pkg/tuple"
 )
 
 func TestSimpleLookupResources2(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
-
+	// FIXME marking this parallel makes goleak detect a leaked goroutine
 	testCases := []struct {
 		start                 *core.RelationReference
 		target                *core.ObjectAndRelation
@@ -159,7 +157,7 @@ func TestSimpleLookupResources2(t *testing.T) {
 }
 
 func TestSimpleLookupResourcesWithCursor2(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	for _, tc := range []struct {
 		subject        string
@@ -242,7 +240,7 @@ func TestSimpleLookupResourcesWithCursor2(t *testing.T) {
 }
 
 func TestLookupResourcesCursorStability2(t *testing.T) {
-	defer goleak.VerifyNone(t, append(testutil.GoLeakIgnores(), goleak.IgnoreCurrent())...)
+	t.Parallel()
 
 	require := require.New(t)
 	ctx, dispatcher, revision := newLocalDispatcher(t)
@@ -309,6 +307,7 @@ func processResults2(stream *dispatch.CollectingDispatchStream[*v1.DispatchLooku
 }
 
 func TestMaxDepthLookup2(t *testing.T) {
+	t.Parallel()
 	require := require.New(t)
 
 	rawDS, err := memdb.NewMemdbDatastore(0, 0, memdb.DisableGC)
@@ -338,6 +337,7 @@ func TestMaxDepthLookup2(t *testing.T) {
 }
 
 func TestLookupResources2OverSchemaWithCursors(t *testing.T) {
+	t.Parallel()
 	testCases := []struct {
 		name                string
 		schema              string
@@ -682,9 +682,11 @@ func TestLookupResources2OverSchemaWithCursors(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
 			for _, pageSize := range []int{0, 104, 1023} {
 				pageSize := pageSize
 				t.Run(fmt.Sprintf("ps-%d_", pageSize), func(t *testing.T) {
+					t.Parallel()
 					require := require.New(t)
 
 					dispatcher := NewLocalOnlyDispatcher(10, 100)
@@ -752,7 +754,7 @@ func TestLookupResources2OverSchemaWithCursors(t *testing.T) {
 }
 
 func TestLookupResources2ImmediateTimeout(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+	t.Parallel()
 
 	require := require.New(t)
 
@@ -787,7 +789,7 @@ func TestLookupResources2ImmediateTimeout(t *testing.T) {
 }
 
 func TestLookupResources2WithError(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+	t.Parallel()
 
 	require := require.New(t)
 
@@ -822,7 +824,7 @@ func TestLookupResources2WithError(t *testing.T) {
 }
 
 func TestLookupResources2EnsureCheckHints(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+	t.Parallel()
 
 	tcs := []struct {
 		name          string