pulumiTest Integration tests (#2052)

This refactors the cross-tests to expose an integration test module
`pulCheck` for use outside of cross-tests.

It allows us to easily write integration tests where we specify a TF
schema, which gets bridged and use the pulumiTest framework for the test
itself.

---------

Co-authored-by: Ian Wahbe <ian@wahbe.com>

pkg/tests/cross-tests/assert.go

@@ -0,0 +1,30 @@
+package crosstests
+
+import (
+	"github.com/hashicorp/go-cty/cty"
+	"github.com/stretchr/testify/require"
+)
+
+func FailNotEqual(t T, name string, tfVal, pulVal any) {
+	t.Logf(name + " not equal!")
+	t.Logf("TF value %s", tfVal)
+	t.Logf("PU value %s", pulVal)
+	t.Fail()
+}
+
+func assertCtyValEqual(t T, name string, tfVal, pulVal cty.Value) {
+	if !tfVal.RawEquals(pulVal) {
+		FailNotEqual(t, name, tfVal.GoString(), pulVal.GoString())
+	}
+}
+
+func assertValEqual(t T, name string, tfVal, pulVal any) {
+	// usually plugin-sdk schema types
+	if hasEqualTfVal, ok := tfVal.(interface{ Equal(interface{}) bool }); ok {
+		if !hasEqualTfVal.Equal(pulVal) {
+			FailNotEqual(t, name, tfVal, pulVal)
+		}
+	} else {
+		require.Equal(t, tfVal, pulVal, "Values for key %s do not match", name)
+	}
+}

pkg/tests/cross-tests/cross_test.go

@@ -97,8 +97,7 @@ func TestSetReordering(t *testing.T) {
 				Type:     schema.TypeSet,
 				Optional: true,
 				Elem: &schema.Schema{
-					Type:     schema.TypeString,
-					Optional: true,
+					Type: schema.TypeString,
 				},
 			},
 		},

pkg/tests/cross-tests/defaults.go

@@ -0,0 +1,9 @@
+package crosstests
+
+const (
+	defProviderShortName = "crossprovider"
+	defRtype             = "crossprovider_test_res"
+	defRtok              = "TestRes"
+	defRtoken            = defProviderShortName + ":index:" + defRtok
+	defProviderVer       = "0.0.1"
+)

pkg/tests/cross-tests/diff_check.go

@@ -15,14 +15,12 @@
 package crosstests
 
 import (
-	"context"
+	"os"
+	"path/filepath"
 
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-	"github.com/pulumi/providertest/providers"
-	"github.com/pulumi/providertest/pulumitest"
-	"github.com/pulumi/providertest/pulumitest/opttest"
-	shimv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/internal/pulcheck"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 	"github.com/stretchr/testify/assert"
@@ -47,58 +45,30 @@ type diffTestCase struct {
 }
 
 func runDiffCheck(t T, tc diffTestCase) {
-	var (
-		providerShortName = "crossprovider"
-		rtype             = "crossprovider_testres"
-		rtok              = "TestRes"
-		rtoken            = providerShortName + ":index:" + rtok
-		providerVer       = "0.0.1"
-	)
-
 	tfwd := t.TempDir()
 
-	tfd := newTfDriver(t, tfwd, providerShortName, rtype, tc.Resource)
-	_ = tfd.writePlanApply(t, tc.Resource.Schema, rtype, "example", tc.Config1)
-	tfDiffPlan := tfd.writePlanApply(t, tc.Resource.Schema, rtype, "example", tc.Config2)
-
-	tfp := &schema.Provider{
-		ResourcesMap: map[string]*schema.Resource{
-			rtype: tc.Resource,
-		},
-	}
+	tfd := newTfDriver(t, tfwd, defProviderShortName, defRtype, tc.Resource)
+	_ = tfd.writePlanApply(t, tc.Resource.Schema, defRtype, "example", tc.Config1)
+	tfDiffPlan := tfd.writePlanApply(t, tc.Resource.Schema, defRtype, "example", tc.Config2)
 
-	shimProvider := shimv2.NewProvider(tfp, shimv2.WithPlanResourceChange(
-		func(tfResourceType string) bool { return true },
-	))
+	resMap := map[string]*schema.Resource{defRtype: tc.Resource}
+	bridgedProvider := pulcheck.BridgedProvider(t, defProviderShortName, resMap)
 
 	pd := &pulumiDriver{
-		name:                providerShortName,
-		version:             providerVer,
-		shimProvider:        shimProvider,
-		pulumiResourceToken: rtoken,
-		tfResourceName:      rtype,
+		name:                defProviderShortName,
+		pulumiResourceToken: defRtoken,
+		tfResourceName:      defRtype,
 		objectType:          nil,
 	}
-
-	puwd := t.TempDir()
-	pd.writeYAML(t, puwd, tc.Config1)
-
-	pt := pulumitest.NewPulumiTest(t, puwd,
-		opttest.TestInPlace(),
-		opttest.SkipInstall(),
-		opttest.AttachProvider(
-			providerShortName,
-			func(ctx context.Context, pt providers.PulumiTest) (providers.Port, error) {
-				handle, err := pd.startPulumiProvider(ctx)
-				require.NoError(t, err)
-				return providers.Port(handle.Port), nil
-			},
-		),
-	)
+	yamlProgram := pd.generateYAML(t, bridgedProvider.P.ResourcesMap(), tc.Config1)
+	pt := pulcheck.PulCheck(t, bridgedProvider, string(yamlProgram))
 
 	pt.Up()
 
-	pd.writeYAML(t, puwd, tc.Config2)
+	yamlProgram = pd.generateYAML(t, bridgedProvider.P.ResourcesMap(), tc.Config2)
+	p := filepath.Join(pt.CurrentStack().Workspace().WorkDir(), "Pulumi.yaml")
+	err := os.WriteFile(p, yamlProgram, 0o600)
+	require.NoErrorf(t, err, "writing Pulumi.yaml")
 	x := pt.Up()
 
 	tfAction := tfd.parseChangesFromTFPlan(*tfDiffPlan)

pkg/tests/cross-tests/input_check.go

@@ -15,15 +15,10 @@ package crosstests
 import (
 	"context"
 
-	"github.com/hashicorp/go-cty/cty"
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-	"github.com/pulumi/providertest/providers"
-	"github.com/pulumi/providertest/pulumitest"
-	"github.com/pulumi/providertest/pulumitest/opttest"
-	shimv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2"
-	"github.com/stretchr/testify/require"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/internal/pulcheck"
 )
 
 // Adapted from diff_check.go
@@ -35,46 +30,9 @@ type inputTestCase struct {
 	ObjectType *tftypes.Object
 }
 
-func FailNotEqual(t T, name string, tfVal, pulVal any) {
-	t.Logf(name + " not equal!")
-	t.Logf("TF value %s", tfVal)
-	t.Logf("PU value %s", pulVal)
-	t.Fail()
-}
-
-func assertCtyValEqual(t T, name string, tfVal, pulVal cty.Value) {
-	if !tfVal.RawEquals(pulVal) {
-		FailNotEqual(t, name, tfVal.GoString(), pulVal.GoString())
-	}
-}
-
-func assertValEqual(t T, name string, tfVal, pulVal any) {
-	// usually plugin-sdk schema types
-	if hasEqualTfVal, ok := tfVal.(interface{ Equal(interface{}) bool }); ok {
-		if !hasEqualTfVal.Equal(pulVal) {
-			FailNotEqual(t, name, tfVal, pulVal)
-		}
-	} else {
-		require.Equal(t, tfVal, pulVal, "Values for key %s do not match", name)
-	}
-}
-
-func ensureProviderValid(t T, tfp *schema.Provider) {
-	for _, r := range tfp.ResourcesMap {
-		//nolint:staticcheck
-		if r.Read == nil && r.ReadContext == nil {
-			r.ReadContext = func(_ context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics {
-				return nil
-			}
-		}
-	}
-	require.NoError(t, tfp.InternalValidate())
-}
-
 // Adapted from diff_check.go
 func runCreateInputCheck(t T, tc inputTestCase) {
-	//nolint:staticcheck
-	if tc.Resource.CreateContext != nil || tc.Resource.Create != nil {
+	if tc.Resource.CreateContext != nil {
 		t.Errorf("Create methods should not be set for these tests!")
 	}
 
@@ -89,55 +47,23 @@ func runCreateInputCheck(t T, tc inputTestCase) {
 		rd.SetId("someid") // CreateContext must pick an ID
 		return make(diag.Diagnostics, 0)
 	}
-	var (
-		providerShortName = "crossprovider"
-		rtype             = "crossprovider_testres"
-		rtok              = "TestRes"
-		rtoken            = providerShortName + ":index:" + rtok
-		providerVer       = "0.0.1"
-	)
 
 	tfwd := t.TempDir()
 
-	tfd := newTfDriver(t, tfwd, providerShortName, rtype, tc.Resource)
-	tfd.writePlanApply(t, tc.Resource.Schema, rtype, "example", tc.Config)
-
-	tfp := &schema.Provider{
-		ResourcesMap: map[string]*schema.Resource{
-			rtype: tc.Resource,
-		},
-	}
-	ensureProviderValid(t, tfp)
-
-	shimProvider := shimv2.NewProvider(tfp, shimv2.WithPlanResourceChange(
-		func(tfResourceType string) bool { return true },
-	))
+	tfd := newTfDriver(t, tfwd, defProviderShortName, defRtype, tc.Resource)
+	tfd.writePlanApply(t, tc.Resource.Schema, defRtype, "example", tc.Config)
 
+	resMap := map[string]*schema.Resource{defRtype: tc.Resource}
+	bridgedProvider := pulcheck.BridgedProvider(t, defProviderShortName, resMap)
 	pd := &pulumiDriver{
-		name:                providerShortName,
-		version:             providerVer,
-		shimProvider:        shimProvider,
-		pulumiResourceToken: rtoken,
-		tfResourceName:      rtype,
+		name:                defProviderShortName,
+		pulumiResourceToken: defRtoken,
+		tfResourceName:      defRtype,
 		objectType:          tc.ObjectType,
 	}
+	yamlProgram := pd.generateYAML(t, bridgedProvider.P.ResourcesMap(), tc.Config)
 
-	puwd := t.TempDir()
-	pd.writeYAML(t, puwd, tc.Config)
-
-	pt := pulumitest.NewPulumiTest(t, puwd,
-		opttest.TestInPlace(),
-		opttest.SkipInstall(),
-		opttest.AttachProvider(
-			providerShortName,
-			func(ctx context.Context, pt providers.PulumiTest) (providers.Port, error) {
-				handle, err := pd.startPulumiProvider(ctx)
-				require.NoError(t, err)
-				return providers.Port(handle.Port), nil
-			},
-		),
-		opttest.Env("DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true"),
-	)
+	pt := pulcheck.PulCheck(t, bridgedProvider, string(yamlProgram))
 
 	pt.Up()
 

pkg/tests/cross-tests/pu_driver.go

@@ -15,83 +15,21 @@
 package crosstests
 
 import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
-	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
-	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
-	pulumidiag "github.com/pulumi/pulumi/sdk/v3/go/common/diag"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
-	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
 	"github.com/stretchr/testify/require"
-	"google.golang.org/grpc"
 	"gopkg.in/yaml.v3"
 )
 
 type pulumiDriver struct {
 	name                string
-	version             string
-	shimProvider        shim.Provider
 	pulumiResourceToken string
 	tfResourceName      string
 	objectType          *tftypes.Object
 }
 
-func (pd *pulumiDriver) providerInfo() tfbridge.ProviderInfo {
-	return tfbridge.ProviderInfo{
-		Name: pd.name,
-		P:    pd.shimProvider,
-
-		Resources: map[string]*tfbridge.ResourceInfo{
-			pd.tfResourceName: {
-				Tok: tokens.Type(pd.pulumiResourceToken),
-			},
-		},
-	}
-}
-
-func (pd *pulumiDriver) startPulumiProvider(ctx context.Context) (*rpcutil.ServeHandle, error) {
-	info := pd.providerInfo()
-
-	sink := pulumidiag.DefaultSink(io.Discard, io.Discard, pulumidiag.FormatOptions{
-		Color: colors.Never,
-	})
-
-	schema, err := tfgen.GenerateSchema(info, sink)
-	if err != nil {
-		return nil, fmt.Errorf("tfgen.GenerateSchema failed: %w", err)
-	}
-
-	schemaBytes, err := json.MarshalIndent(schema, "", " ")
-	if err != nil {
-		return nil, fmt.Errorf("json.MarshalIndent(schema, ..) failed: %w", err)
-	}
-
-	prov := tfbridge.NewProvider(ctx, nil, pd.name, pd.version, info.P, info, schemaBytes)
-
-	handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
-		Init: func(srv *grpc.Server) error {
-			pulumirpc.RegisterResourceProviderServer(srv, prov)
-			return nil
-		},
-	})
-	if err != nil {
-		return nil, fmt.Errorf("rpcutil.ServeWithOptions failed: %w", err)
-	}
-
-	return &handle, nil
-}
-
-func (pd *pulumiDriver) writeYAML(t T, workdir string, tfConfig any) {
-	res := pd.shimProvider.ResourcesMap().Get(pd.tfResourceName)
+func (pd *pulumiDriver) generateYAML(t T, resMap shim.ResourceMap, tfConfig any) []byte {
+	res := resMap.Get(pd.tfResourceName)
 	schema := res.Schema()
 
 	data, err := generateYaml(schema, pd.pulumiResourceToken, pd.objectType, tfConfig)
@@ -100,7 +38,5 @@ func (pd *pulumiDriver) writeYAML(t T, workdir string, tfConfig any) {
 	b, err := yaml.Marshal(data)
 	require.NoErrorf(t, err, "marshaling Pulumi.yaml")
 	t.Logf("\n\n%s", b)
-	p := filepath.Join(workdir, "Pulumi.yaml")
-	err = os.WriteFile(p, b, 0o600)
-	require.NoErrorf(t, err, "writing Pulumi.yaml")
+	return b
 }

pkg/tests/cross-tests/t.go

@@ -24,6 +24,7 @@ import (
 type T interface {
 	Logf(string, ...any)
 	TempDir() string
+	Skip(...any)
 	require.TestingT
 	assert.TestingT
 	pulumitest.PT