From db547366aa42388f6d02674664cde2666b7a7cb8 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 7 Nov 2024 14:39:19 +0545 Subject: [PATCH] feat: connection from scraper [skip ci] --- .gitignore | 1 + Makefile | 2 +- api/v1/playbook_actions.go | 11 +--- api/v1/zz_generated.deepcopy.go | 30 --------- ...ion-control.flanksource.com_playbooks.yaml | 65 ++++++++++++++++--- .../playbooks/connection-from-scraper.yaml | 18 +++++ playbook/actions/exec.go | 62 ++---------------- playbook/runner/runner.go | 1 - playbook/runner/template.go | 15 ++++- 9 files changed, 98 insertions(+), 107 deletions(-) create mode 100644 fixtures/playbooks/connection-from-scraper.yaml diff --git a/.gitignore b/.gitignore index b224bb3b0..442ac7ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ coverprofile.out junit-report.xml nohup.out .envrc +.creds diff --git a/Makefile b/Makefile index 1ef3c9d7f..39c1977c1 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ gen-schemas: go run ./main.go .PHONY: manifests -manifests: controller-gen generate gen-schemas ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +manifests: controller-gen generate #gen-schemas ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) crd paths="./api/..." output:crd:artifacts:config=config/crds .PHONY: generate diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index a629b52bb..bf600d2f7 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -8,6 +8,7 @@ import ( "github.com/flanksource/commons/duration" "github.com/flanksource/commons/utils" + "github.com/flanksource/duty/connection" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "k8s.io/client-go/kubernetes" @@ -237,8 +238,8 @@ func (git GitCheckout) GetCertificate() types.EnvVar { type ExecAction struct { // Script can be an inline script or a path to a script that needs to be executed // On windows executed via powershell and in darwin and linux executed using bash - Script string `yaml:"script" json:"script" template:"true"` - Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"` + Script string `yaml:"script" json:"script" template:"true"` + Connections connection.ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty" template:"true"` // Artifacts to save Artifacts []Artifact `yaml:"artifacts,omitempty" json:"artifacts,omitempty" template:"true"` // EnvVars are the environment variables that are accessible to exec processes @@ -247,12 +248,6 @@ type ExecAction struct { Checkout *GitCheckout `yaml:"checkout,omitempty" json:"checkout,omitempty"` } -type ExecConnections struct { - AWS *AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"` - GCP *GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"` - Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"` -} - type connectionContext interface { gocontext.Context HydrateConnectionByURL(connectionName string) (*models.Connection, error) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index e3fa09670..7b9163bdc 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -882,36 +882,6 @@ func (in *ExecAction) DeepCopy() *ExecAction { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExecConnections) DeepCopyInto(out *ExecConnections) { - *out = *in - if in.AWS != nil { - in, out := &in.AWS, &out.AWS - *out = new(AWSConnection) - (*in).DeepCopyInto(*out) - } - if in.GCP != nil { - in, out := &in.GCP, &out.GCP - *out = new(GCPConnection) - (*in).DeepCopyInto(*out) - } - if in.Azure != nil { - in, out := &in.Azure, &out.Azure - *out = new(AzureConnection) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecConnections. -func (in *ExecConnections) DeepCopy() *ExecConnections { - if in == nil { - return nil - } - out := new(ExecConnections) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPConnection) DeepCopyInto(out *GCPConnection) { *out = *in diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index c83d32385..de6ad25fd 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -341,6 +341,8 @@ spec: type: string type: object type: object + assumeRole: + type: string connection: description: ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and @@ -348,10 +350,6 @@ spec: type: string endpoint: type: string - objectPath: - description: glob path to restrict matches to a - subset - type: string region: type: string secretKey: @@ -448,10 +446,6 @@ spec: description: Skip TLS verify when connecting to aws type: boolean - usePathStyle: - description: 'Use path style path: http://s3.amazonaws.com/BUCKET/KEY - instead of http://BUCKET.s3.amazonaws.com/KEY' - type: boolean type: object azure: properties: @@ -550,6 +544,8 @@ spec: tenantID: type: string type: object + fromConfigItem: + type: string gcp: properties: connection: @@ -603,6 +599,59 @@ spec: type: object endpoint: type: string + skipTLSVerify: + description: Skip TLS verify + type: boolean + type: object + kubernetes: + properties: + connection: + type: string + kubeconfig: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + helmRef: + properties: + key: + description: Key is a JSONPath expression + used to fetch the key from the merged + JSON. + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + serviceAccount: + description: ServiceAccount specifies the + service account whose token should be + fetched + type: string + type: object + type: object type: object type: object env: diff --git a/fixtures/playbooks/connection-from-scraper.yaml b/fixtures/playbooks/connection-from-scraper.yaml new file mode 100644 index 000000000..5451f8d98 --- /dev/null +++ b/fixtures/playbooks/connection-from-scraper.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: kubernetes-connection-from-scraper + namespace: mc +spec: + configs: + - types: + - Kubernetes::Deployment + actions: + - exec: + script: "kubectl get deployments" + connections: + fromConfigItem: "$(.config.id)" + name: list + category: Echoer + description: Lists all deployments diff --git a/playbook/actions/exec.go b/playbook/actions/exec.go index 08ba5ffc4..08953cf9d 100644 --- a/playbook/actions/exec.go +++ b/playbook/actions/exec.go @@ -18,6 +18,7 @@ import ( "github.com/flanksource/commons/hash" "github.com/flanksource/commons/utils" + "github.com/flanksource/duty/connection" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" v1 "github.com/flanksource/incident-commander/api/v1" @@ -73,70 +74,15 @@ func (c *ExecAction) Run(ctx context.Context, exec v1.ExecAction) (*ExecDetails, cmd.Dir = envParams.mountPoint } - if err := setupConnection(ctx, exec, cmd); err != nil { + if cleanup, err := connection.SetupConnection(ctx, exec.Connections, cmd); err != nil { return nil, ctx.Oops().Wrapf(err, "failed to setup connection") + } else { + defer cleanup() } return runCmd(ctx, cmd, exec.Artifacts...) } -func setupConnection(ctx context.Context, check v1.ExecAction, cmd *osExec.Cmd) error { - if check.Connections.AWS != nil { - if err := check.Connections.AWS.Populate(ctx, ctx.Kubernetes(), ctx.GetNamespace()); err != nil { - return fmt.Errorf("failed to hydrate aws connection: %w", err) - } - - configPath, err := saveConfig(awsConfigTemplate, check.Connections.AWS) - defer os.RemoveAll(filepath.Dir(configPath)) - if err != nil { - return fmt.Errorf("failed to store AWS credentials: %w", err) - } - - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 - cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configPath)) - if check.Connections.AWS.Region != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", check.Connections.AWS.Region)) - } - } - - if check.Connections.Azure != nil { - if err := check.Connections.Azure.HydrateConnection(ctx); err != nil { - return fmt.Errorf("failed to hydrate connection %w", err) - } - - // login with service principal - runCmd := osExec.Command("az", "login", "--service-principal", "--username", check.Connections.Azure.ClientID.ValueStatic, "--password", check.Connections.Azure.ClientSecret.ValueStatic, "--tenant", check.Connections.Azure.TenantID) - if err := runCmd.Run(); err != nil { - return fmt.Errorf("failed to login: %w", err) - } - } - - if check.Connections.GCP != nil { - if err := check.Connections.GCP.HydrateConnection(ctx); err != nil { - return fmt.Errorf("failed to hydrate connection %w", err) - } - - configPath, err := saveConfig(gcloudConfigTemplate, check.Connections.GCP) - defer os.RemoveAll(filepath.Dir(configPath)) - if err != nil { - return fmt.Errorf("failed to store gcloud credentials: %w", err) - } - - // to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS, - // we need to explicitly activate it - runCmd := osExec.Command("gcloud", "auth", "activate-service-account", "--key-file", configPath) - if err := runCmd.Run(); err != nil { - return fmt.Errorf("failed to activate GCP service account: %w", err) - } - - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configPath)) - } - - return nil -} - func runCmd(ctx context.Context, cmd *osExec.Cmd, artifactConfigs ...v1.Artifact) (*ExecDetails, error) { var ( result ExecDetails diff --git a/playbook/runner/runner.go b/playbook/runner/runner.go index 2e4868d68..4c2321baf 100644 --- a/playbook/runner/runner.go +++ b/playbook/runner/runner.go @@ -261,7 +261,6 @@ func ExecuteAndSaveAction(ctx context.Context, playbookID any, action *models.Pl } -// TemplateAndExecuteAction executes the given playbook action after templating it. func RunAction(ctx context.Context, run *models.PlaybookRun, action *models.PlaybookRunAction) error { playbook, err := action.GetPlaybook(ctx.DB()) if err != nil { diff --git a/playbook/runner/template.go b/playbook/runner/template.go index 966d3b468..f0d1a4b7b 100644 --- a/playbook/runner/template.go +++ b/playbook/runner/template.go @@ -186,5 +186,18 @@ func TemplateEnv(ctx context.Context, env actions.TemplateEnv, template string) // TemplateAction all the go templates in the action func TemplateAction(ctx context.Context, actionSpec *v1.PlaybookAction, env actions.TemplateEnv) error { templater := ctx.NewStructTemplater(env.AsMap(), "template", getGomplateFuncs(ctx, env)) - return templater.Walk(&actionSpec) + if err := templater.Walk(&actionSpec); err != nil { + return err + } + + // TODO: make this work with template.Walk() + if actionSpec.Exec.Connections.FromConfigItem != nil { + if v, err := ctx.RunTemplate(gomplate.Template{Template: *actionSpec.Exec.Connections.FromConfigItem}, env.AsMap()); err != nil { + return err + } else { + actionSpec.Exec.Connections.FromConfigItem = &v + } + } + + return nil }