feat: connection from scraper

[skip ci]

.gitignore

@@ -19,3 +19,4 @@ coverprofile.out
 junit-report.xml
 nohup.out
 .envrc
+.creds

Makefile

@@ -99,7 +99,7 @@ gen-schemas:
 	go run ./main.go
 
 .PHONY: manifests
-manifests: controller-gen generate gen-schemas ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+manifests: controller-gen generate #gen-schemas ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) crd paths="./api/..." output:crd:artifacts:config=config/crds
 
 .PHONY: generate

api/v1/playbook_actions.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/flanksource/commons/duration"
 	"github.com/flanksource/commons/utils"
+	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"k8s.io/client-go/kubernetes"
@@ -237,8 +238,8 @@ func (git GitCheckout) GetCertificate() types.EnvVar {
 type ExecAction struct {
 	// Script can be an inline script or a path to a script that needs to be executed
 	// On windows executed via powershell and in darwin and linux executed using bash
-	Script      string          `yaml:"script" json:"script" template:"true"`
-	Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"`
+	Script      string                     `yaml:"script" json:"script" template:"true"`
+	Connections connection.ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty" template:"true"`
 	// Artifacts to save
 	Artifacts []Artifact `yaml:"artifacts,omitempty" json:"artifacts,omitempty" template:"true"`
 	// EnvVars are the environment variables that are accessible to exec processes
@@ -247,12 +248,6 @@ type ExecAction struct {
 	Checkout *GitCheckout `yaml:"checkout,omitempty" json:"checkout,omitempty"`
 }
 
-type ExecConnections struct {
-	AWS   *AWSConnection   `yaml:"aws,omitempty" json:"aws,omitempty"`
-	GCP   *GCPConnection   `yaml:"gcp,omitempty" json:"gcp,omitempty"`
-	Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
-}
-
 type connectionContext interface {
 	gocontext.Context
 	HydrateConnectionByURL(connectionName string) (*models.Connection, error)

config/crds/mission-control.flanksource.com_playbooks.yaml

@@ -341,17 +341,15 @@ spec:
                                           type: string
                                       type: object
                                   type: object
+                                assumeRole:
+                                  type: string
                                 connection:
                                   description: ConnectionName of the connection. It'll
                                     be used to populate the endpoint, accessKey and
                                     secretKey.
                                   type: string
                                 endpoint:
                                   type: string
-                                objectPath:
-                                  description: glob path to restrict matches to a
-                                    subset
-                                  type: string
                                 region:
                                   type: string
                                 secretKey:
@@ -448,10 +446,6 @@ spec:
                                   description: Skip TLS verify when connecting to
                                     aws
                                   type: boolean
-                                usePathStyle:
-                                  description: 'Use path style path: http://s3.amazonaws.com/BUCKET/KEY
-                                    instead of http://BUCKET.s3.amazonaws.com/KEY'
-                                  type: boolean
                               type: object
                             azure:
                               properties:
@@ -550,6 +544,8 @@ spec:
                                 tenantID:
                                   type: string
                               type: object
+                            fromConfigItem:
+                              type: string
                             gcp:
                               properties:
                                 connection:
@@ -603,6 +599,59 @@ spec:
                                   type: object
                                 endpoint:
                                   type: string
+                                skipTLSVerify:
+                                  description: Skip TLS verify
+                                  type: boolean
+                              type: object
+                            kubernetes:
+                              properties:
+                                connection:
+                                  type: string
+                                kubeconfig:
+                                  properties:
+                                    name:
+                                      type: string
+                                    value:
+                                      type: string
+                                    valueFrom:
+                                      properties:
+                                        configMapKeyRef:
+                                          properties:
+                                            key:
+                                              type: string
+                                            name:
+                                              type: string
+                                          required:
+                                          - key
+                                          type: object
+                                        helmRef:
+                                          properties:
+                                            key:
+                                              description: Key is a JSONPath expression
+                                                used to fetch the key from the merged
+                                                JSON.
+                                              type: string
+                                            name:
+                                              type: string
+                                          required:
+                                          - key
+                                          type: object
+                                        secretKeyRef:
+                                          properties:
+                                            key:
+                                              type: string
+                                            name:
+                                              type: string
+                                          required:
+                                          - key
+                                          type: object
+                                        serviceAccount:
+                                          description: ServiceAccount specifies the
+                                            service account whose token should be
+                                            fetched
+                                          type: string
+                                      type: object
+                                  type: object
                               type: object
                           type: object
                         env:

fixtures/playbooks/connection-from-scraper.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: mission-control.flanksource.com/v1
+kind: Playbook
+metadata:
+  name: kubernetes-connection-from-scraper
+  namespace: mc
+spec:
+  configs:
+    - types:
+        - Kubernetes::Deployment
+  actions:
+    - exec:
+        script: "kubectl get deployments"
+        connections:
+          fromConfigItem: "$(.config.id)"
+      name: list 
+  category: Echoer
+  description: Lists all deployments 

playbook/actions/exec.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/flanksource/commons/hash"
 	"github.com/flanksource/commons/utils"
+	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
 	v1 "github.com/flanksource/incident-commander/api/v1"
@@ -73,70 +74,15 @@ func (c *ExecAction) Run(ctx context.Context, exec v1.ExecAction) (*ExecDetails,
 		cmd.Dir = envParams.mountPoint
 	}
 
-	if err := setupConnection(ctx, exec, cmd); err != nil {
+	if cleanup, err := connection.SetupConnection(ctx, exec.Connections, cmd); err != nil {
 		return nil, ctx.Oops().Wrapf(err, "failed to setup connection")
+	} else {
+		defer cleanup()
 	}
 
 	return runCmd(ctx, cmd, exec.Artifacts...)
 }
 
-func setupConnection(ctx context.Context, check v1.ExecAction, cmd *osExec.Cmd) error {
-	if check.Connections.AWS != nil {
-		if err := check.Connections.AWS.Populate(ctx, ctx.Kubernetes(), ctx.GetNamespace()); err != nil {
-			return fmt.Errorf("failed to hydrate aws connection: %w", err)
-		}
-
-		configPath, err := saveConfig(awsConfigTemplate, check.Connections.AWS)
-		defer os.RemoveAll(filepath.Dir(configPath))
-		if err != nil {
-			return fmt.Errorf("failed to store AWS credentials: %w", err)
-		}
-
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151
-		cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configPath))
-		if check.Connections.AWS.Region != "" {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", check.Connections.AWS.Region))
-		}
-	}
-
-	if check.Connections.Azure != nil {
-		if err := check.Connections.Azure.HydrateConnection(ctx); err != nil {
-			return fmt.Errorf("failed to hydrate connection %w", err)
-		}
-
-		// login with service principal
-		runCmd := osExec.Command("az", "login", "--service-principal", "--username", check.Connections.Azure.ClientID.ValueStatic, "--password", check.Connections.Azure.ClientSecret.ValueStatic, "--tenant", check.Connections.Azure.TenantID)
-		if err := runCmd.Run(); err != nil {
-			return fmt.Errorf("failed to login: %w", err)
-		}
-	}
-
-	if check.Connections.GCP != nil {
-		if err := check.Connections.GCP.HydrateConnection(ctx); err != nil {
-			return fmt.Errorf("failed to hydrate connection %w", err)
-		}
-
-		configPath, err := saveConfig(gcloudConfigTemplate, check.Connections.GCP)
-		defer os.RemoveAll(filepath.Dir(configPath))
-		if err != nil {
-			return fmt.Errorf("failed to store gcloud credentials: %w", err)
-		}
-
-		// to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS,
-		// we need to explicitly activate it
-		runCmd := osExec.Command("gcloud", "auth", "activate-service-account", "--key-file", configPath)
-		if err := runCmd.Run(); err != nil {
-			return fmt.Errorf("failed to activate GCP service account: %w", err)
-		}
-
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configPath))
-	}
-
-	return nil
-}
-
 func runCmd(ctx context.Context, cmd *osExec.Cmd, artifactConfigs ...v1.Artifact) (*ExecDetails, error) {
 	var (
 		result ExecDetails

playbook/runner/runner.go

@@ -261,7 +261,6 @@ func ExecuteAndSaveAction(ctx context.Context, playbookID any, action *models.Pl
 
 }
 
-// TemplateAndExecuteAction executes the given playbook action after templating it.
 func RunAction(ctx context.Context, run *models.PlaybookRun, action *models.PlaybookRunAction) error {
 	playbook, err := action.GetPlaybook(ctx.DB())
 	if err != nil {

playbook/runner/template.go

@@ -186,5 +186,18 @@ func TemplateEnv(ctx context.Context, env actions.TemplateEnv, template string)
 // TemplateAction all the go templates in the action
 func TemplateAction(ctx context.Context, actionSpec *v1.PlaybookAction, env actions.TemplateEnv) error {
 	templater := ctx.NewStructTemplater(env.AsMap(), "template", getGomplateFuncs(ctx, env))
-	return templater.Walk(&actionSpec)
+	if err := templater.Walk(&actionSpec); err != nil {
+		return err
+	}
+
+	// TODO: make this work with template.Walk()
+	if actionSpec.Exec.Connections.FromConfigItem != nil {
+		if v, err := ctx.RunTemplate(gomplate.Template{Template: *actionSpec.Exec.Connections.FromConfigItem}, env.AsMap()); err != nil {
+			return err
+		} else {
+			actionSpec.Exec.Connections.FromConfigItem = &v
+		}
+	}
+
+	return nil
 }