Resource type attribute changes and stability fixes for yaml parsing …

…and other config changes APE-16695, APE-16692, APE-16734 (#1673) * fix for crash in lambda function cft yaml if code key is not present * removed duplicate tag image tag * bump up the version to 1.18.5 * added feature to parse cft template for ssm parameter * bumped the version to 1.18.7 * bump up version to v1.18.8 * Sanitize the cft template file for aws:: words as it causes parser to fail * Sanitize the cft template file for aws:: words as it causes parser to fail * increment version * APE-16518 - improve the stability of yaml to json parsing * Fixed indirect resource id reference and rds-cluster and restapi for gatewaystage * add resource references for json format * terraform resource type names added * terraform res type names added for reference * fixed lint errors for comments * tags name added as per terraform structure * lint fixes * add dynamodb BillingMode property * fixed lint error * added CrossZoneLoadBalancing for lb * load balancer lb added * fixing lint errors * APE-16777 APE-16773 fixed attribute as per normalize json * fix lint * Update ecs-service.go * Update ecs-service.go * Update ecs-service.go * sonar issue resolved * sonar issue resolved --------- Co-authored-by: pankaj rai <parai@tenable.com> Co-authored-by: prai80 <106154428+prai80@users.noreply.github.com>
tenable · Mar 7, 2024 · 4422eb5 · 4422eb5
1 parent 2e75f8f
commit 4422eb5
Show file tree

Hide file tree

Showing 122 changed files with 978 additions and 184 deletions.
diff --git a/pkg/iac-providers/cft/v1/load-file.go b/pkg/iac-providers/cft/v1/load-file.go
@@ -132,7 +132,7 @@ func (a *CFTV1) extractTemplate(file string, data *[]byte) (*cloudformation.Temp
 	}
 
 	zap.S().Debug("sanitizing cft template file", zap.String("file", file))
-	sanitized, err := a.sanitizeCftTemplate(*data, isYaml)
+	sanitized, err := a.sanitizeCftTemplate(file, *data, isYaml)
 	if err != nil {
 		zap.S().Debug("failed to sanitize cft template file", zap.String("file", file), zap.Error(err))
 		return nil, err
@@ -219,7 +219,7 @@ func (*CFTV1) getFileType(file string, data *[]byte) string {
 		if isJSON(string(*data)) {
 			return JSONExtension
 		}
-		return YAMLExtension
+		return TXTExtension
 	}
 	return UnknownExtension
 }

diff --git a/pkg/iac-providers/cft/v1/sanitize-cft-template.go b/pkg/iac-providers/cft/v1/sanitize-cft-template.go
@@ -36,31 +36,58 @@ const PARAMETERS = "Parameters"
 // RESOURCES is a constant to fetch Resources from CFT
 const RESOURCES = "Resources"
 
-func (a *CFTV1) sanitizeCftTemplate(data []byte, isYAML bool) (map[string]interface{}, error) {
+func (a *CFTV1) sanitizeCftTemplate(fileName string, data []byte, isYAML bool) (map[string]interface{}, error) {
 	var (
 		intrinsified []byte
 		err          error
 	)
-
 	if isYAML {
-		data, err = removeRefAnchors(data)
-		if err != nil {
-			return nil, err
-		}
+		fallbackToDefaultProcessing := false
 
-		// Process all AWS CloudFormation intrinsic functions (e.g. Fn::Join)
-		intrinsified, err = intrinsics.ProcessYAML(data, nil)
-		if err != nil {
-			return nil, fmt.Errorf("error while resolving intrinsic functions, error %w", err)
+		for i := 0; i < 1; i++ {
+			// convert the yaml into json
+			jsonData, err := a.ReadYAMLFileIntoJSON(fileName)
+			if err == nil {
+				jsonData, err := a.resolveResourceIDs(jsonData)
+				if err != nil {
+					zap.S().Debug(fmt.Sprintf("error while resolving Resource IDs, error %s", err.Error()))
+					fallbackToDefaultProcessing = true
+					break
+				}
+				intrinsified, err = intrinsics.ProcessJSON(jsonData, nil)
+				if err != nil {
+					zap.S().Debug(fmt.Sprintf("error while resolving Resource IDs, error %s", err.Error()))
+					fallbackToDefaultProcessing = true
+					break
+				}
+			}
+		}
+		if fallbackToDefaultProcessing || len(intrinsified) == 0 { // fallback to default behaviour of yaml processing
+			data, err = removeRefAnchors(data)
+			if err != nil {
+				return nil, err
+			}
+			// Process all AWS CloudFormation intrinsic functions (e.g. Fn::Join)
+			intrinsified, err = intrinsics.ProcessYAML(data, nil)
+			if err != nil {
+				return nil, fmt.Errorf("error while resolving intrinsic functions, error %w", err)
+			}
 		}
 	} else {
-		// Process all AWS CloudFormation intrinsic functions (e.g. Fn::Join)
-		intrinsified, err = intrinsics.ProcessJSON(data, nil)
-		if err != nil {
-			return nil, fmt.Errorf("error while resolving intrinsic functions, error %w", err)
+		jsonData, err := a.resolveResourceIDs(data)
+		if err == nil {
+			intrinsified, err = intrinsics.ProcessJSON(jsonData, nil)
+			if err != nil {
+				return nil, fmt.Errorf("error while resolving intrinsic functions, error %w", err)
+			}
+		} else {
+			// Process all AWS CloudFormation intrinsic functions (e.g. Fn::Join)
+			intrinsified, err = intrinsics.ProcessJSON(data, nil)
+			if err != nil {
+				return nil, fmt.Errorf("error while resolving intrinsic functions, error %w", err)
+			}
 		}
 	}
-
 	templateFileMap := make(map[string]interface{})
 
 	err = json.Unmarshal(intrinsified, &templateFileMap)
@@ -510,3 +537,117 @@ func findKeyAndReplace(obj interface{}, propValues map[string]interface{}) (inte
 	}
 	return nil, false
 }
+
+// ReadYAMLFileIntoJSON converts the given file into JSON string
+func (a *CFTV1) ReadYAMLFileIntoJSON(fileName string) ([]byte, error) {
+	templateSample, err := a.File(fileName)
+	if err != nil {
+		return nil, err
+	}
+	gostruct, err := templateSample.Map()
+	if err != nil {
+		zap.S().Errorf("failed to map yaml to json. error : %s in file %s", err.Error(), fileName)
+		return nil, err
+	}
+	jsonData, err := json.Marshal(gostruct)
+	if err != nil {
+		zap.S().Errorf("failed to convert yaml to json. error : %s", err.Error())
+		return nil, err
+	}
+	return jsonData, nil
+}
+func (a *CFTV1) getMapOfResourceIds(allData interface{}) map[string]string {
+	mapOfresourceIds := make(map[string]string)
+	mapOfParameters := make(map[string]interface{})
+	if templateFileMap, ok := allData.(map[string]interface{}); ok {
+		r, ok := templateFileMap[PARAMETERS]
+		if ok {
+			rMap, ok := r.(map[string]interface{})
+			if ok {
+				for rName, val := range rMap {
+					zap.S().Debug("inspecting resource", zap.String("Parameters Name", rName))
+					if val1, ok := val.(map[string]interface{}); ok {
+						mapOfParameters[rName] = val1["Default"]
+					}
+				}
+			}
+		}
+	}
+
+	if templateFileMap, ok := allData.(map[string]interface{}); ok {
+		r, ok := templateFileMap[RESOURCES]
+		if ok {
+			rMap, ok := r.(map[string]interface{})
+			if ok {
+				for rName := range rMap {
+					zap.S().Debug("inspecting resource", zap.String("Resource Name", rName))
+					if _, ok := mapOfParameters[rName]; !ok {
+						mapOfresourceIds[rName] = rName
+					}
+				}
+			}
+		}
+	}
+	return mapOfresourceIds
+}
+
+// resolveResourceIDs resolves the indirect resource to resource references
+func (a *CFTV1) resolveResourceIDs(jsonData []byte) ([]byte, error) {
+	var unmarshalled interface{}
+	if err := json.Unmarshal(jsonData, &unmarshalled); err != nil {
+		return nil, fmt.Errorf("invalid JSON: %s", err)
+	}
+	mapOfparentReferences := a.getMapOfResourceIds(unmarshalled)
+	unmarshalledResult := a.resolveIndirectReferences(nil, unmarshalled, "", mapOfparentReferences)
+	resultBytes, err := json.Marshal(unmarshalledResult)
+	if err != nil {
+		return nil, fmt.Errorf("invalid JSON: %s", err)
+	}
+	jsonData = resultBytes
+	return jsonData, nil
+}
+
+// resolveIndirectReferences finds if the references are of the Name of the resource and replace it with actual Name
+func (a *CFTV1) resolveIndirectReferences(parent interface{}, input interface{}, parentKey string, mapOfReferences map[string]string) interface{} {
+
+	switch value := input.(type) {
+
+	case map[string]interface{}:
+		processed := map[string]interface{}{}
+		for key, val := range value {
+			if key == "Ref" && parentKey != "" {
+				valuStr := fmt.Sprintf("%v", val)
+				if _, ok := mapOfReferences[valuStr]; ok {
+					if parentVal, ok := parent.(map[string]interface{}); ok {
+						parentVal[parentKey] = valuStr
+						val = valuStr
+						return val
+					}
+				}
+			}
+			processed[key] = a.resolveIndirectReferences(value, val, key, mapOfReferences)
+		}
+		return processed
+
+	case []interface{}:
+
+		// We found an array in the JSON - recurse through it's elements looking for intrinsic functions
+		processed := []interface{}{}
+		for _, val := range value {
+			processed = append(processed, a.resolveIndirectReferences(parent, val, parentKey, mapOfReferences))
+		}
+		return processed
+
+	case nil:
+		return value
+	case bool:
+		return value
+	case float64:
+		return value
+	case string:
+		return value
+	default:
+		return nil
+
+	}
+}
diff --git a/pkg/iac-providers/cft/v1/sanitize-cft-template_test.go b/pkg/iac-providers/cft/v1/sanitize-cft-template_test.go
@@ -66,7 +66,7 @@ func TestCFTV1_sanitizeCftTemplate(t *testing.T) {
 				t.Error("CFTV1.sanitizeCftTemplate() got no error, expected parsing error")
 			}
 
-			templateMap, err := a.sanitizeCftTemplate(data, tt.args.isYAML)
+			templateMap, err := a.sanitizeCftTemplate(tt.inputFile, data, tt.args.isYAML)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CFTV1.sanitizeCftTemplate() error = %v, wantErr %v", err, tt.wantErr)
 				return

diff --git a/pkg/iac-providers/cft/v1/testdata/deploy.yaml b/pkg/iac-providers/cft/v1/testdata/deploy.yaml
@@ -26,5 +26,5 @@ Resources:
         Statement:
           - Effect: Allow
             Action: *
-            Principal: '*'
+            Principal: 'star'