Merge pull request #504 from weaveworks/add-output-typeinfo

enhance outputs with type information

controllers/tc000061_vars_hcl_input_test.go

@@ -137,28 +137,34 @@ func Test_000061_vars_hcl_input_test(t *testing.T) {
 			return -1, err
 		}
 		return len(outputSecret.Data), nil
-	}, timeout, interval).Should(Equal(4))
+	}, timeout, interval).Should(Equal(7))
 
 	By("checking that the TF output secrets contains the correct output provisioned by the TF hello world")
 	// Value is a JSON representation of TF's OutputMeta
 	expectedOutputValue := map[string]interface{}{
 		"Name":      "tf-output-" + terraformName,
 		"Namespace": "flux-system",
 		"Values": map[string]string{
-			"cluster_id": "eu-test-1:stg:winter-squirrel",
-			"active":     "true",
-			"node_count": "10",
-			"azs":        `["eu-test-1a","eu-test-1b","eu-test-1c"]`,
+			"cluster_id":      "eu-test-1:stg:winter-squirrel",
+			"active":          "true",
+			"active.type":     `"bool"`,
+			"node_count":      "10",
+			"node_count.type": `"number"`,
+			"azs":             "[\n      \"eu-test-1a\",\n      \"eu-test-1b\",\n      \"eu-test-1c\"\n    ]",
+			"azs.type":        "[\n      \"list\",\n      \"string\"\n    ]",
 		},
 		"OwnerRef[0]": string(createdHelloWorldTF.UID),
 	}
 	g.Eventually(func() (map[string]interface{}, error) {
 		err := k8sClient.Get(ctx, outputKey, &outputSecret)
 		values := map[string]string{
-			"cluster_id": string(outputSecret.Data["cluster_id"]),
-			"active":     string(outputSecret.Data["active"]),
-			"node_count": string(outputSecret.Data["node_count"]),
-			"azs":        string(outputSecret.Data["azs"]),
+			"cluster_id":      string(outputSecret.Data["cluster_id"]),
+			"active":          string(outputSecret.Data["active"]),
+			"active.type":     string(outputSecret.Data["active.type"]),
+			"node_count":      string(outputSecret.Data["node_count"]),
+			"node_count.type": string(outputSecret.Data["node_count.type"]),
+			"azs":             string(outputSecret.Data["azs"]),
+			"azs.type":        string(outputSecret.Data["azs.type"]),
 		}
 		return map[string]interface{}{
 			"Name":        outputSecret.Name,

controllers/tf_controller_outputs.go

@@ -2,7 +2,6 @@ package controllers
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -75,6 +74,9 @@ func (r *TerraformReconciler) processOutputs(ctx context.Context, runnerClient r
 	log := ctrl.LoggerFrom(ctx)
 	objectKey := types.NamespacedName{Namespace: terraform.Namespace, Name: terraform.Name}
 
+	// OutputMeta has
+	// 1. type
+	// 2. value
 	outputs := map[string]tfexec.OutputMeta{}
 	var err error
 	terraform, err = r.obtainOutputs(ctx, terraform, tfInstance, runnerClient, revision, &outputs)
@@ -133,81 +135,37 @@ func (r *TerraformReconciler) writeOutput(ctx context.Context, terraform infrav1
 	wots := terraform.Spec.WriteOutputsToSecret
 	data := map[string][]byte{}
 
-	// if not specified .spec.writeOutputsToSecret.outputs,
-	// then it means export all outputs
+	var filteredOutputs map[string]tfexec.OutputMeta
 	if len(wots.Outputs) == 0 {
-		for output, v := range outputs {
-			ct, err := ctyjson.UnmarshalType(v.Type)
-			if err != nil {
-				return terraform, err
-			}
-			// if it's a string, we can embed it directly into Secret's data
-			switch ct {
-			case cty.String:
-				cv, err := ctyjson.Unmarshal(v.Value, ct)
-				if err != nil {
-					return terraform, err
-				}
-				data[output] = []byte(cv.AsString())
-			// there's no need to unmarshal and convert to []byte
-			// we'll just pass the []byte directly from OutputMeta Value
-			case cty.Number, cty.Bool:
-				data[output] = v.Value
-			default:
-				outputBytes, err := json.Marshal(v.Value)
-				if err != nil {
-					return terraform, err
-				}
-				data[output] = outputBytes
-			}
-		}
+		filteredOutputs = outputs
 	} else {
-		// filter only defined output
-		// output maybe contain mapping output:mapped_name
-		for _, outputMapping := range wots.Outputs {
-			parts := strings.SplitN(outputMapping, ":", 2)
-			var output string
-			var mappedTo string
-			if len(parts) == 1 {
-				output = parts[0]
-				mappedTo = parts[0]
-				// no mapping
-			} else if len(parts) == 2 {
-				output = parts[0]
-				mappedTo = parts[1]
-			} else {
-				log.Error(fmt.Errorf("invalid mapping format"), outputMapping)
-				continue
-			}
+		if result, err := filterOutputs(outputs, wots.Outputs); err != nil {
+			return infrav1.TerraformNotReady(
+				terraform,
+				revision,
+				infrav1.OutputsWritingFailedReason,
+				err.Error(),
+			), err
+		} else {
+			filteredOutputs = result
+		}
+	}
 
-			v, exist := outputs[output]
-			if !exist {
-				log.Error(fmt.Errorf("output not found"), output)
-				continue
-			}
+	for outputOrAlias, outputMeta := range filteredOutputs {
+		ct, err := ctyjson.UnmarshalType(outputMeta.Type)
+		if err != nil {
+			return terraform, err
+		}
 
-			ct, err := ctyjson.UnmarshalType(v.Type)
+		if ct == cty.String {
+			cv, err := ctyjson.Unmarshal(outputMeta.Value, ct)
 			if err != nil {
 				return terraform, err
 			}
-			switch ct {
-			case cty.String:
-				cv, err := ctyjson.Unmarshal(v.Value, ct)
-				if err != nil {
-					return terraform, err
-				}
-				data[mappedTo] = []byte(cv.AsString())
-			// there's no need to unmarshal and convert to []byte
-			// we'll just pass the []byte directly from OutputMeta Value
-			case cty.Number, cty.Bool:
-				data[mappedTo] = v.Value
-			default:
-				outputBytes, err := json.Marshal(v.Value)
-				if err != nil {
-					return terraform, err
-				}
-				data[mappedTo] = outputBytes
-			}
+			data[outputOrAlias] = []byte(cv.AsString())
+		} else {
+			data[outputOrAlias] = outputMeta.Value
+			data[outputOrAlias+".type"] = outputMeta.Type
 		}
 	}
 
@@ -243,3 +201,41 @@ func (r *TerraformReconciler) writeOutput(ctx context.Context, terraform infrav1
 
 	return infrav1.TerraformOutputsWritten(terraform, revision, "Outputs written"), nil
 }
+
+func filterOutputs(outputs map[string]tfexec.OutputMeta, outputsToWrite []string) (map[string]tfexec.OutputMeta, error) {
+	if outputs == nil || outputsToWrite == nil {
+		return nil, fmt.Errorf("input maps or outputsToWrite slice cannot be nil")
+	}
+
+	filteredOutputs := make(map[string]tfexec.OutputMeta)
+	for _, outputMapping := range outputsToWrite {
+		if len(outputMapping) == 0 {
+			return nil, fmt.Errorf("output mapping cannot be empty")
+		}
+
+		// parse output mapping (output[:alias])
+		parts := strings.SplitN(outputMapping, ":", 2)
+		var (
+			output string
+			alias  string
+		)
+		if len(parts) == 1 {
+			output = parts[0]
+			alias = parts[0]
+		} else if len(parts) == 2 {
+			output = parts[0]
+			alias = parts[1]
+		} else {
+			return nil, fmt.Errorf("invalid output mapping format: %s", outputMapping)
+		}
+
+		outputMeta, exist := outputs[output]
+		if !exist {
+			return nil, fmt.Errorf("output not found: %s", output)
+		}
+
+		filteredOutputs[alias] = outputMeta
+	}
+
+	return filteredOutputs, nil
+}

docs/References/terraform.md

@@ -1485,7 +1485,7 @@ int32
 </td>
 <td>
 <em>(Optional)</em>
-<p>Parallelism limits the number of concurrent operations of Terraform apply step.</p>
+<p>Parallelism limits the number of concurrent operations of Terraform apply step. Zero (0) means using the default value.</p>
 </td>
 </tr>
 <tr>
@@ -1969,7 +1969,7 @@ int32
 </td>
 <td>
 <em>(Optional)</em>
-<p>Parallelism limits the number of concurrent operations of Terraform apply step.</p>
+<p>Parallelism limits the number of concurrent operations of Terraform apply step. Zero (0) means using the default value.</p>
 </td>
 </tr>
 <tr>

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-retryablehttp v0.7.1
+	github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80
 	github.com/hashicorp/terraform-exec v0.16.1
 	github.com/hashicorp/terraform-json v0.13.0
 	github.com/onsi/gomega v1.24.0
@@ -60,6 +61,9 @@ require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
+	github.com/apparentlymart/go-textseg v1.0.0 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 // indirect
@@ -116,7 +120,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
@@ -149,6 +153,7 @@ require (
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/theckman/yacspin v0.13.12 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect