From e3de2b5ad7de6e9bfd1f4b84958a25a24ffcf460 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Tue, 10 Oct 2023 15:13:47 -0700 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=20terraform.resources=20{=20related?= =?UTF-8?q?=20}=20support=20(#2159)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discover all related resources of a given terraform resource. Additionally overhaul the internal discovery of all terraform blocks and reduce the amount of repetition drastically. Signed-off-by: Dominik Richter --- providers/terraform/resources/hcl.go | 205 +++++++++++++++--- providers/terraform/resources/terraform.lr | 2 + providers/terraform/resources/terraform.lr.go | 33 ++- .../resources/terraform.lr.manifest.yaml | 2 + 4 files changed, 207 insertions(+), 35 deletions(-) diff --git a/providers/terraform/resources/hcl.go b/providers/terraform/resources/hcl.go index b3ad865065..8fda929bc0 100644 --- a/providers/terraform/resources/hcl.go +++ b/providers/terraform/resources/hcl.go @@ -9,6 +9,7 @@ import ( "fmt" "strconv" "strings" + "sync" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/ext/typeexpr" @@ -23,8 +24,12 @@ import ( "go.mondoo.com/cnquery/v9/types" ) -func (t *mqlTerraform) id() (string, error) { - return "terraform", nil +type mqlTerraformInternal struct { + blocksByName map[string]*mqlTerraformBlock + relatedBlocks map[string][]*mqlTerraformBlock + // these are blocks with the type set to "terraform", used for settings + terraformBlocks []*mqlTerraformBlock + lock sync.Mutex } func (t *mqlTerraform) files() ([]interface{}, error) { @@ -90,56 +95,159 @@ func (t *mqlTerraform) blocks() ([]interface{}, error) { } mqlHclBlocks = append(mqlHclBlocks, blocks...) } - return mqlHclBlocks, nil + + return mqlHclBlocks, t.refreshCache(mqlHclBlocks) } -func filterBlockByType(runtime *plugin.Runtime, filterType string) ([]interface{}, error) { - conn := runtime.Connection.(*connection.Connection) - parsed := conn.Parser() - if parsed == nil { - // no results, because this is not a regular parsed HCL - return []interface{}{}, nil +func (t *mqlTerraform) refreshCache(blocks []interface{}) error { + if blocks == nil { + raw := t.GetBlocks() + return raw.Error } - files := parsed.Files() + t.lock.Lock() + if t.blocksByName != nil { + return nil + } + defer t.lock.Unlock() + + t.blocksByName = map[string]*mqlTerraformBlock{} + t.relatedBlocks = map[string][]*mqlTerraformBlock{} + + t.Providers.State = plugin.StateIsSet + t.Providers.Data = []interface{}{} + t.Datasources.State = plugin.StateIsSet + t.Datasources.Data = []interface{}{} + t.Resources.State = plugin.StateIsSet + t.Resources.Data = []interface{}{} + t.Variables.State = plugin.StateIsSet + t.Variables.Data = []interface{}{} + t.Outputs.State = plugin.StateIsSet + t.Outputs.Data = []interface{}{} + t.terraformBlocks = []*mqlTerraformBlock{} + + for i := range blocks { + block := blocks[i].(*mqlTerraformBlock) + + // type must be pre-initialized + typ := block.Type.Data + + switch typ { + case "provider": + t.Providers.Data = append(t.Providers.Data, block) + case "data": + t.Datasources.Data = append(t.Providers.Data, block) + case "resource": + t.Resources.Data = append(t.Resources.Data, block) + case "variable": + t.Variables.Data = append(t.Variables.Data, block) + case "output": + t.Outputs.Data = append(t.Outputs.Data, block) + case "terraform": + t.terraformBlocks = append(t.terraformBlocks, block) + default: + // Note: we don't do anything with these blocks yet. + // They might be worth looking into. + } - var mqlHclBlocks []interface{} - for k := range files { - f := files[k] - blocks, err := listHclBlocks(runtime, f.Body, f) + // labels must be pre-initialized + name := block.terraformID() + t.blocksByName[name] = block + } + + // We need blocks by name before we can jump into + // related blocks, because we need to access them + // via their name + for i := range blocks { + block := blocks[i].(*mqlTerraformBlock) + name := block.terraformID() + + rel, err := listRelatedBlocks(t, block.block.Data.Body) if err != nil { - return nil, err + return err } - for i := range blocks { - b := blocks[i].(*mqlTerraformBlock) - blockType := b.Type.Data - if blockType == filterType { - mqlHclBlocks = append(mqlHclBlocks, b) + // connect from this block to all related blocks... + t.relatedBlocks[name] = append(t.relatedBlocks[name], rel...) + // ... and also connect the inverse (related to this block) + for i := range rel { + relBlock := rel[i] + relName := relBlock.terraformID() + t.relatedBlocks[relName] = append(t.relatedBlocks[relName], block) + } + } + + for k, v := range t.relatedBlocks { + block, ok := t.blocksByName[k] + if !ok { + return errors.New("cannot find terraform block by name: " + k) + } + + vi := make([]interface{}, len(v)) + for i := range v { + vi[i] = v[i] + } + + block.Related = plugin.TValue[[]interface{}]{ + State: plugin.StateIsSet, + Data: vi, + } + } + + return nil +} + +func (g *mqlTerraformBlock) terraformID() string { + labels := g.Labels.Data + var namearr []string + for i := range labels { + if s, ok := labels[i].(string); ok { + namearr = append(namearr, s) + } + } + return strings.Join(namearr, "\x00") +} + +func listRelatedBlocks(t *mqlTerraform, rawBody interface{}) ([]*mqlTerraformBlock, error) { + var res []*mqlTerraformBlock + switch body := rawBody.(type) { + case *hclsyntax.Body: + for _, v := range body.Attributes { + refs := getReferences(v.Expr, &hcl.EvalContext{ + Functions: hclFunctions(), + }) + // we need the resource name and its ID at least + if len(refs) < 2 { + continue } + + refName := strings.Join(refs[0:2], "\x00") + res = append(res, t.blocksByName[refName]) } + case hcl.Body: + return nil, errors.New("cannot yet list related blocks for regular hcl Body") } - return mqlHclBlocks, nil + return res, nil } func (t *mqlTerraform) providers() ([]interface{}, error) { - return filterBlockByType(t.MqlRuntime, "provider") + return nil, t.refreshCache(nil) } func (t *mqlTerraform) datasources() ([]interface{}, error) { - return filterBlockByType(t.MqlRuntime, "data") + return nil, t.refreshCache(nil) } func (t *mqlTerraform) resources() ([]interface{}, error) { - return filterBlockByType(t.MqlRuntime, "resource") + return nil, t.refreshCache(nil) } func (t *mqlTerraform) variables() ([]interface{}, error) { - return filterBlockByType(t.MqlRuntime, "variable") + return nil, t.refreshCache(nil) } func (t *mqlTerraform) outputs() ([]interface{}, error) { - return filterBlockByType(t.MqlRuntime, "output") + return nil, t.refreshCache(nil) } func extractHclCodeSnippet(file *hcl.File, fileRange hcl.Range) string { @@ -399,6 +507,28 @@ func getCtyValue(expr hcl.Expression, ctx *hcl.EvalContext) interface{} { } } +func getReferences(expr hcl.Expression, ctx *hcl.EvalContext) []string { + switch t := expr.(type) { + case *hclsyntax.ScopeTraversalExpr: + traversal := t.Variables() + res := []string{} + for i := range traversal { + tr := traversal[i] + for j := range tr { + switch v := tr[j].(type) { + case hcl.TraverseRoot: + res = append(res, v.Name) + case hcl.TraverseAttr: + res = append(res, v.Name) + } + } + } + return res + default: + return nil + } +} + func GetKeyString(key interface{}) string { switch v := key.(type) { case []string: @@ -465,6 +595,19 @@ func listHclBlocks(runtime *plugin.Runtime, rawBody interface{}, file *hcl.File) return mqlHclBlocks, nil } +func (g *mqlTerraformBlock) related() ([]interface{}, error) { + // This field should be default be set by the Terraform routine that + // initializes all blocks. If we land here from a recording or other + // path, re-run it. + o, err := CreateResource(g.MqlRuntime, "terraform", map[string]*llx.RawData{}) + if err != nil { + return nil, err + } + + terraform := o.(*mqlTerraform) + return nil, terraform.refreshCache(nil) +} + func newFilePosRange(runtime *plugin.Runtime, r hcl.Range) (plugin.Resource, plugin.Resource, error) { start, err := CreateResource(runtime, "terraform.fileposition", map[string]*llx.RawData{ "path": llx.StringData(r.Filename), @@ -545,11 +688,17 @@ func (t *mqlTerraformModule) block() (*mqlTerraformBlock, error) { } func initTerraformSettings(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) { - blocks, err := filterBlockByType(runtime, "terraform") + o, err := CreateResource(runtime, "terraform", map[string]*llx.RawData{}) if err != nil { return nil, nil, err } + terraform := o.(*mqlTerraform) + if err := terraform.refreshCache(nil); err != nil { + return nil, nil, err + } + + blocks := terraform.terraformBlocks if len(blocks) != 1 { // no terraform settings block found, this is ok for terraform and not an error // TODO: return modified arguments to load from recording @@ -560,7 +709,7 @@ func initTerraformSettings(runtime *plugin.Runtime, args map[string]*llx.RawData }, nil } - settingsBlock := blocks[0].(*mqlTerraformBlock) + settingsBlock := blocks[0] args["block"] = llx.ResourceData(settingsBlock, "terraform.block") args["requiredProviders"] = llx.DictData(map[string]interface{}{}) args["backend"] = llx.DictData(map[string]interface{}{}) diff --git a/providers/terraform/resources/terraform.lr b/providers/terraform/resources/terraform.lr index 8003b211a0..2a21722be0 100644 --- a/providers/terraform/resources/terraform.lr +++ b/providers/terraform/resources/terraform.lr @@ -63,6 +63,8 @@ terraform.block @defaults("type labels") { attributes() dict // Child Blocks blocks() []terraform.block + // Related blocks + related() []terraform.block // Block Snippet snippet string } diff --git a/providers/terraform/resources/terraform.lr.go b/providers/terraform/resources/terraform.lr.go index c392ea3291..eaab04ad6e 100644 --- a/providers/terraform/resources/terraform.lr.go +++ b/providers/terraform/resources/terraform.lr.go @@ -210,6 +210,9 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "terraform.block.blocks": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlTerraformBlock).GetBlocks()).ToDataRes(types.Array(types.Resource("terraform.block"))) }, + "terraform.block.related": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlTerraformBlock).GetRelated()).ToDataRes(types.Array(types.Resource("terraform.block"))) + }, "terraform.block.snippet": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlTerraformBlock).GetSnippet()).ToDataRes(types.String) }, @@ -495,6 +498,10 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlTerraformBlock).Blocks, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) return }, + "terraform.block.related": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlTerraformBlock).Related, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, "terraform.block.snippet": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlTerraformBlock).Snippet, ok = plugin.RawToTValue[string](v.Value, v.Error) return @@ -783,7 +790,7 @@ func SetAllData(resource plugin.Resource, args map[string]*llx.RawData) error { type mqlTerraform struct { MqlRuntime *plugin.Runtime __id string - // optional: if you define mqlTerraformInternal it will be used here + mqlTerraformInternal Files plugin.TValue[[]interface{}] Tfvars plugin.TValue[interface{}] Modules plugin.TValue[[]interface{}] @@ -806,12 +813,7 @@ func createTerraform(runtime *plugin.Runtime, args map[string]*llx.RawData) (plu return res, err } - if res.__id == "" { - res.__id, err = res.id() - if err != nil { - return nil, err - } - } + // to override __id implement: id() (string, error) if runtime.HasRecording { args, err = runtime.ResourceFromRecording("terraform", res.__id) @@ -1109,6 +1111,7 @@ type mqlTerraformBlock struct { Arguments plugin.TValue[interface{}] Attributes plugin.TValue[interface{}] Blocks plugin.TValue[[]interface{}] + Related plugin.TValue[[]interface{}] Snippet plugin.TValue[string] } @@ -1199,6 +1202,22 @@ func (c *mqlTerraformBlock) GetBlocks() *plugin.TValue[[]interface{}] { }) } +func (c *mqlTerraformBlock) GetRelated() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.Related, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("terraform.block", c.__id, "related") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.related() + }) +} + func (c *mqlTerraformBlock) GetSnippet() *plugin.TValue[string] { return &c.Snippet } diff --git a/providers/terraform/resources/terraform.lr.manifest.yaml b/providers/terraform/resources/terraform.lr.manifest.yaml index 1480fe773e..f2ea3a0363 100755 --- a/providers/terraform/resources/terraform.lr.manifest.yaml +++ b/providers/terraform/resources/terraform.lr.manifest.yaml @@ -33,6 +33,8 @@ resources: end: {} labels: {} nameLabel: {} + related: + min_mondoo_version: 9.0.12 snippet: {} start: {} type: {}