⭐️ gitlab terraform discovery (#1755)

This PR adds the ability to the gitlab provider to detect terraform
inside of the git repo.

```bash
cnquery shell gitlab --group example --project example-gitlab
→ loaded configuration from /Users/chris/.config/mondoo/mondoo.yml using source default
→ discover related assets for 1 asset(s)
→ resolved assets resolved-assets=2

    Available assets                               
                                                   
  > 1. example / Example Gitlab (gitlab-project)
    2. example / Example Gitlab (terraform-hcl) 
 ```

apps/cnquery/cmd/builder/common/common.go

@@ -690,13 +690,15 @@ func GitlabProviderCmd(commonCmdFlags CommonFlagsFn, preRun CommonPreRunFn, runF
  PreRun: func(cmd *cobra.Command, args []string) {
  viper.BindPFlag("token", cmd.Flags().Lookup("token"))
  viper.BindPFlag("group", cmd.Flags().Lookup("group"))
+ viper.BindPFlag("project", cmd.Flags().Lookup("project"))
  preRun(cmd, args)
  },
  Run: runFn,
  }
  commonCmdFlags(cmd)
- cmd.Flags().String("group", "", "a GitLab group to scan")
+ cmd.Flags().String("group", "", "a GitLab group")
  cmd.MarkFlagRequired("group")
+ cmd.Flags().String("project", "", "a GitLab project")
  cmd.Flags().String("token", "", "GitLab personal access token")
  return cmd
 }

apps/cnquery/cmd/builder/parse.go

@@ -640,6 +640,12 @@ func ParseTargetAsset(cmd *cobra.Command, args []string, providerType providers.
  connection.Options["group"] = x
  }
 
+ if x, err := cmd.Flags().GetString("project"); err != nil {
+ log.Fatal().Err(err).Msg("cannot parse --project value")
+ } else if x != "" {
+ connection.Options["project"] = x
+ }
+
  if x, err := cmd.Flags().GetString("token"); err != nil {
  log.Fatal().Err(err).Msg("cannot parse --token value")
  } else if x != "" {

motor/discovery/gitlab/gitlab.go

@@ -3,8 +3,13 @@ package gitlab
 import (
  "context"
  "errors"
+ "github.com/rs/zerolog/log"
+ terraform_resolver "go.mondoo.com/cnquery/motor/discovery/terraform"
+ "os"
+ "strings"
 
  "github.com/xanzy/go-gitlab"
+ gitlab_lib "github.com/xanzy/go-gitlab"
  "go.mondoo.com/cnquery/motor/asset"
  "go.mondoo.com/cnquery/motor/discovery/common"
  "go.mondoo.com/cnquery/motor/providers"
@@ -14,8 +19,9 @@ import (
 )
 
 const (
- DiscoveryGroup = "group"
- DiscoveryProject = "project"
+ DiscoveryGroup = "group"
+ DiscoveryProject = "project"
+ DiscoveryTerraform = "terraform"
 )
 
 type Resolver struct{}
@@ -58,20 +64,61 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
  case "gitlab-project":
  if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) {
  name := defaultName
+ project, _ := p.Project()
+ grp, _ := p.Group()
+
  if name == "" {
- project, _ := p.Project()
  if project != nil {
  name = project.NameWithNamespace
  }
  }
 
- list = append(list, &asset.Asset{
+ projectAsset := &asset.Asset{
  PlatformIds: []string{identifier},
  Name: name,
  Platform: pf,
  Connections: []*providers.Config{pCfg}, // pass-in the current config
  State: asset.State_STATE_ONLINE,
- })
+ }
+
+ list = append(list, projectAsset)
+
+ if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
+ terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
+ if err != nil {
+ log.Error().Err(err).Msg("error discovering terraform")
+ } else if len(terraformFiles) > 0 {
+ terraformCfg := pCfg.Clone()
+ terraformCfg.Backend = providers.ProviderType_TERRAFORM
+
+ terraformCfg.Options = map[string]string{
+ "asset-type": "hcl",
+ "path": "git+" + project.HTTPURLToRepo,
+ }
+
+ if pCfg.Credentials == nil {
+ token := os.Getenv("GITLAB_TOKEN")
+ terraformCfg.Credentials = []*vault.Credential{{
+ Type: vault.CredentialType_password,
+ User: "oauth2",
+ Secret: []byte(token),
+ }}
+ } else {
+ // add oauth2 user to the credentials
+ for i := range pCfg.Credentials {
+ cred := pCfg.Credentials[i]
+ if cred.Type == vault.CredentialType_password {
+ cred.User = "oauth2"
+ }
+ }
+ }
+
+ assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
+ if err == nil && len(assets) > 0 {
+ list = append(list, assets...)
+ }
+ }
+ }
  }
  case "gitlab-group":
  var grp *gitlab.Group
@@ -103,19 +150,69 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
  if clonedConfig.Options == nil {
  clonedConfig.Options = map[string]string{}
  }
- clonedConfig.Options["group"] = grp.Name
- clonedConfig.Options["project"] = project.Name
+ clonedConfig.Options["group"] = grp.Path
+ clonedConfig.Options["project"] = project.Path
 
- list = append(list, &asset.Asset{
+ projectAsset := &asset.Asset{
  PlatformIds: []string{identifier},
  Name: project.NameWithNamespace,
  Platform: gitlab_provider.GitLabProjectPlatform,
  Connections: []*providers.Config{clonedConfig}, // pass-in the current config
  State: asset.State_STATE_ONLINE,
- })
+ }
+ list = append(list, projectAsset)
+
+ if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
+ terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
+ if err == nil && len(terraformFiles) > 0 {
+ terraformCfg := pCfg.Clone()
+
+ assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
+ if err == nil && len(assets) > 0 {
+ list = append(list, assets...)
+ }
+ }
+ }
  }
  }
  }
  }
  return list, nil
 }
+
+// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset
+func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group string, project string) ([]string, error) {
+ opts := &gitlab_lib.ListTreeOptions{
+ ListOptions: gitlab_lib.ListOptions{
+ PerPage: 100,
+ },
+ Recursive: gitlab_lib.Bool(true),
+ }
+
+ nodes := []*gitlab_lib.TreeNode{}
+ for {
+ data, resp, err := client.Repositories.ListTree(group+"/"+project, opts)
+ if err != nil {
+ return nil, err
+ }
+ nodes = append(nodes, data...)
+
+ // Exit the loop when we've seen all pages.
+ if resp.NextPage == 0 {
+ break
+ }
+
+ // Update the page number to get the next page.
+ opts.Page = resp.NextPage
+ }
+
+ terraformFiles := []string{}
+ for i := range nodes {
+ node := nodes[i]
+ if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") {
+ terraformFiles = append(terraformFiles, node.Path)
+ }
+ }
+
+ return terraformFiles, nil
+}

motor/providers/terraform/provider.go

@@ -5,6 +5,7 @@ import (
  "encoding/hex"
  "encoding/json"
  "fmt"
+
  "io/fs"
  "os"
  "path/filepath"
@@ -17,6 +18,7 @@ import (
  "github.com/hashicorp/hcl/v2/hclparse"
  "github.com/rs/zerolog/log"
  "go.mondoo.com/cnquery/motor/providers"
+ "go.mondoo.com/cnquery/motor/vault"
 )
 
 var (
@@ -44,7 +46,7 @@ func New(tc *providers.Config) (*Provider, error) {
  var closer func()
  path := tc.Options["path"]
  if strings.HasPrefix(path, "git+https://") || strings.HasPrefix(path, "git+ssh://") {
- path, closer, err = processGitForTerraform(path)
+ path, closer, err = processGitForTerraform(path, tc.Credentials)
  if err != nil {
  return nil, err
  }
@@ -239,20 +241,55 @@ func (p *Provider) Plan() (*Plan, error) {
 
 var reGitHttps = regexp.MustCompile(`git\+https://([^/]+)/(.*)`)
 
-func processGitForTerraform(url string) (string, func(), error) {
+// gitCloneUrl returns a git clone url from a git+https url
+// If a token is provided, it will be used to clone the repo
+// gitlab: git clone https://oauth2:ACCESS_TOKEN@somegitlab.com/vendor/package.git
+func gitCloneUrl(url string, credentials []*vault.Credential) (string, error) {
+
+ user := ""
+ token := ""
+ for i := range credentials {
+ cred := credentials[i]
+ if cred.Type == vault.CredentialType_password {
+ user = cred.User
+ token = string(cred.Secret)
+ if token == "" && cred.Password != "" {
+ token = string(cred.Password)
+ }
+ }
+ }
+
  m := reGitHttps.FindStringSubmatch(url)
  if len(m) == 3 {
  if strings.Contains(m[1], ":") {
- return "", nil, errors.New("url cannot contain a port! (" + m[1] + ")")
+ return "", errors.New("url cannot contain a port! (" + m[1] + ")")
+ }
+
+ if user != "" && token != "" {
+ // e.g. used by GitLab
+ url = "https://" + user + ":" + token + "@" + m[1] + "/" + m[2]
+ } else if token != "" {
+ // e.g. used by GitHub
+ url = "https://" + token + "@" + m[1] + "/" + m[2]
+ } else {
+ url = "git@" + m[1] + ":" + m[2]
  }
- url = "git@" + m[1] + ":" + m[2]
  }
  // url = strings.ReplaceAll(url, "git+https://gitlab.com/", "git@gitlab.com:")
  url = strings.ReplaceAll(url, "git+ssh://", "")
 
  if !strings.HasSuffix(url, ".git") {
  url += ".git"
  }
+ return url, nil
+}
+
+// processGitForTerraform clones a git repo and returns the path to the clone
+func processGitForTerraform(url string, credentials []*vault.Credential) (string, func(), error) {
+ cloneUrl, err := gitCloneUrl(url, credentials)
+ if err != nil {
+ return "", nil, errors.Wrap(err, "failed to parse git clone url "+url)
+ }
 
  cloneDir, err := os.MkdirTemp(os.TempDir(), "gitClone")
  if err != nil {
@@ -269,7 +306,7 @@ func processGitForTerraform(url string) (string, func(), error) {
 
  log.Info().Str("url", url).Str("path", cloneDir).Msg("git clone")
  repo, err := git.PlainClone(cloneDir, false, &git.CloneOptions{
- URL: url,
+ URL: cloneUrl,
  Progress: os.Stderr,
  Depth: 1,
  RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,

motor/providers/terraform/provider_test.go

@@ -1,6 +1,7 @@
 package terraform
 
 import (
+ "go.mondoo.com/cnquery/motor/vault"
  "testing"
 
  "github.com/stretchr/testify/assert"
@@ -32,3 +33,17 @@ func TestModuleManifestIssue676(t *testing.T) {
  require.NotNil(t, p.modulesManifest)
  require.Len(t, p.modulesManifest.Records, 3)
 }
+
+func TestGitCloneUrl(t *testing.T) {
+ cloneUrl, err := gitCloneUrl("git+https://somegitlab.com/vendor/package.git", nil)
+ require.NoError(t, err)
+ assert.Equal(t, "git@somegitlab.com:vendor/package.git", cloneUrl)
+
+ cloneUrl, err = gitCloneUrl("git+https://somegitlab.com/vendor/package.git", []*vault.Credential{{
+ Type: vault.CredentialType_password,
+ User: "oauth2",
+ Password: "ACCESS_TOKEN",
+ }})
+ require.NoError(t, err)
+ assert.Equal(t, "https://oauth2:ACCESS_TOKEN@somegitlab.com/vendor/package.git", cloneUrl)
+}