From d0c0b6f55590f2007964781b94c6d0cd4c8cea99 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 15 Aug 2023 15:20:05 +0545 Subject: [PATCH 01/39] init: define spec for playbooks --- api/v1/playbook_actions.go | 80 ++++ api/v1/playbook_types.go | 48 +++ api/v1/zz_generated.deepcopy.go | 343 ++++++++++++++++ ...ion-control.flanksource.com_playbooks.yaml | 379 ++++++++++++++++++ 4 files changed, 850 insertions(+) create mode 100644 api/v1/playbook_actions.go create mode 100644 api/v1/playbook_types.go create mode 100644 config/crds/mission-control.flanksource.com_playbooks.yaml diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go new file mode 100644 index 000000000..265081aae --- /dev/null +++ b/api/v1/playbook_actions.go @@ -0,0 +1,80 @@ +package v1 + +import "github.com/flanksource/duty/types" + +type Labels map[string]string + +type Description struct { + // Description for the check + Description string `yaml:"description,omitempty" json:"description,omitempty" template:"true"` + // Name of the check + Name string `yaml:"name" json:"name" template:"true"` + // Icon for overwriting default icon on the dashboard + Icon string `yaml:"icon,omitempty" json:"icon,omitempty" template:"true"` + // Labels for the check + Labels Labels `yaml:"labels,omitempty" json:"labels,omitempty"` + // Transformed checks have a delete strategy on deletion they can either be marked healthy, unhealthy or left as is + TransformDeleteStrategy string `yaml:"transformDeleteStrategy,omitempty" json:"transformDeleteStrategy,omitempty"` +} + +type Template struct { + Template string `yaml:"template,omitempty" json:"template,omitempty"` + JSONPath string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"` + Expression string `yaml:"expr,omitempty" json:"expr,omitempty"` + Javascript string `yaml:"javascript,omitempty" json:"javascript,omitempty"` +} + +type Templatable struct { + Test Template `yaml:"test,omitempty" json:"test,omitempty"` + Display Template `yaml:"display,omitempty" json:"display,omitempty"` + Transform Template `yaml:"transform,omitempty" json:"transform,omitempty"` +} + +type ExecAction struct { + Description `yaml:",inline" json:",inline"` + Templatable `yaml:",inline" json:",inline"` + // Script can be a inline script or a path to a script that needs to be executed + // On windows executed via powershell and in darwin and linux executed using bash + Script string `yaml:"script" json:"script"` + Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"` +} + +type ExecConnections struct { + AWS *AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"` + GCP *GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"` + Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"` +} + +type GCPConnection struct { + // ConnectionName of the connection. It'll be used to populate the endpoint and credentials. + ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` + Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"` + Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"` +} + +type AzureConnection struct { + ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` + ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` + ClientSecret *types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` + TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"` +} + +type AWSConnection struct { + // ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey. + ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` + AccessKey types.EnvVar `yaml:"accessKey" json:"accessKey,omitempty"` + SecretKey types.EnvVar `yaml:"secretKey" json:"secretKey,omitempty"` + SessionToken types.EnvVar `yaml:"sessionToken,omitempty" json:"sessionToken,omitempty"` + Region string `yaml:"region,omitempty" json:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // Skip TLS verify when connecting to aws + SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"` + // glob path to restrict matches to a subset + ObjectPath string `yaml:"objectPath,omitempty" json:"objectPath,omitempty"` + // Use path style path: http://s3.amazonaws.com/BUCKET/KEY instead of http://BUCKET.s3.amazonaws.com/KEY + UsePathStyle bool `yaml:"usePathStyle,omitempty" json:"usePathStyle,omitempty"` +} + +type PlaybookAction struct { + Exec ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` +} diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go new file mode 100644 index 000000000..582d0f568 --- /dev/null +++ b/api/v1/playbook_types.go @@ -0,0 +1,48 @@ +package v1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type Permission struct { + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Team string `json:"team,omitempty" yaml:"team,omitempty"` + Ref string `json:"ref,omitempty" yaml:"ref,omitempty"` +} + +// PlaybookResourceFilter defines a filter that decides whether a resource (config or a component) +// is permitted be run on the Playbook. +type PlaybookResourceFilter struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Tags map[string]string `json:"tags,omitempty" yaml:"tags,omitempty"` +} + +// PlaybookParameter defines a parameter that a playbook needs to run. +type PlaybookParameter struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Label string `json:"label,omitempty" yaml:"label,omitempty"` +} + +type PlaybookSpec struct { + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Permissions []Permission `json:"permissions,omitempty" yaml:"permissions,omitempty"` + Configs []PlaybookResourceFilter `json:"configs,omitempty" yaml:"configs,omitempty"` + Components []PlaybookResourceFilter `json:"components,omitempty" yaml:"components,omitempty"` + Parameters []PlaybookParameter `json:"parameters,omitempty" yaml:"parameters,omitempty"` + Actions []PlaybookAction `json:"actions" yaml:"actions"` +} + +// PlaybookStatus defines the observed state of Playbook +type PlaybookStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// Playbook is the schema for the Playbooks API +type Playbook struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PlaybookSpec `json:"spec,omitempty"` + Status PlaybookStatus `json:"status,omitempty"` +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 8e1405378..5b4480c0e 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -10,6 +10,49 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSConnection) DeepCopyInto(out *AWSConnection) { + *out = *in + in.AccessKey.DeepCopyInto(&out.AccessKey) + in.SecretKey.DeepCopyInto(&out.SecretKey) + in.SessionToken.DeepCopyInto(&out.SessionToken) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSConnection. +func (in *AWSConnection) DeepCopy() *AWSConnection { + if in == nil { + return nil + } + out := new(AWSConnection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureConnection) DeepCopyInto(out *AzureConnection) { + *out = *in + if in.ClientID != nil { + in, out := &in.ClientID, &out.ClientID + *out = new(types.EnvVar) + (*in).DeepCopyInto(*out) + } + if in.ClientSecret != nil { + in, out := &in.ClientSecret, &out.ClientSecret + *out = new(types.EnvVar) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureConnection. +func (in *AzureConnection) DeepCopy() *AzureConnection { + if in == nil { + return nil + } + out := new(AzureConnection) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Connection) DeepCopyInto(out *Connection) { *out = *in @@ -111,6 +154,96 @@ func (in *ConnectionStatus) DeepCopy() *ConnectionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Description) DeepCopyInto(out *Description) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(Labels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Description. +func (in *Description) DeepCopy() *Description { + if in == nil { + return nil + } + out := new(Description) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecAction) DeepCopyInto(out *ExecAction) { + *out = *in + in.Description.DeepCopyInto(&out.Description) + out.Templatable = in.Templatable + in.Connections.DeepCopyInto(&out.Connections) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecAction. +func (in *ExecAction) DeepCopy() *ExecAction { + if in == nil { + return nil + } + out := new(ExecAction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecConnections) DeepCopyInto(out *ExecConnections) { + *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(AWSConnection) + (*in).DeepCopyInto(*out) + } + if in.GCP != nil { + in, out := &in.GCP, &out.GCP + *out = new(GCPConnection) + (*in).DeepCopyInto(*out) + } + if in.Azure != nil { + in, out := &in.Azure, &out.Azure + *out = new(AzureConnection) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecConnections. +func (in *ExecConnections) DeepCopy() *ExecConnections { + if in == nil { + return nil + } + out := new(ExecConnections) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCPConnection) DeepCopyInto(out *GCPConnection) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(types.EnvVar) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPConnection. +func (in *GCPConnection) DeepCopy() *GCPConnection { + if in == nil { + return nil + } + out := new(GCPConnection) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IncidentRule) DeepCopyInto(out *IncidentRule) { *out = *in @@ -184,3 +317,213 @@ func (in *IncidentRuleStatus) DeepCopy() *IncidentRuleStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Labels) DeepCopyInto(out *Labels) { + { + in := &in + *out = make(Labels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Labels. +func (in Labels) DeepCopy() Labels { + if in == nil { + return nil + } + out := new(Labels) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Permission) DeepCopyInto(out *Permission) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Permission. +func (in *Permission) DeepCopy() *Permission { + if in == nil { + return nil + } + out := new(Permission) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Playbook) DeepCopyInto(out *Playbook) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Playbook. +func (in *Playbook) DeepCopy() *Playbook { + if in == nil { + return nil + } + out := new(Playbook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Playbook) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookAction) DeepCopyInto(out *PlaybookAction) { + *out = *in + in.Exec.DeepCopyInto(&out.Exec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookAction. +func (in *PlaybookAction) DeepCopy() *PlaybookAction { + if in == nil { + return nil + } + out := new(PlaybookAction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookParameter) DeepCopyInto(out *PlaybookParameter) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookParameter. +func (in *PlaybookParameter) DeepCopy() *PlaybookParameter { + if in == nil { + return nil + } + out := new(PlaybookParameter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookResourceFilter) DeepCopyInto(out *PlaybookResourceFilter) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookResourceFilter. +func (in *PlaybookResourceFilter) DeepCopy() *PlaybookResourceFilter { + if in == nil { + return nil + } + out := new(PlaybookResourceFilter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookSpec) DeepCopyInto(out *PlaybookSpec) { + *out = *in + if in.Permissions != nil { + in, out := &in.Permissions, &out.Permissions + *out = make([]Permission, len(*in)) + copy(*out, *in) + } + if in.Configs != nil { + in, out := &in.Configs, &out.Configs + *out = make([]PlaybookResourceFilter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make([]PlaybookResourceFilter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make([]PlaybookParameter, len(*in)) + copy(*out, *in) + } + if in.Actions != nil { + in, out := &in.Actions, &out.Actions + *out = make([]PlaybookAction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookSpec. +func (in *PlaybookSpec) DeepCopy() *PlaybookSpec { + if in == nil { + return nil + } + out := new(PlaybookSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookStatus) DeepCopyInto(out *PlaybookStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookStatus. +func (in *PlaybookStatus) DeepCopy() *PlaybookStatus { + if in == nil { + return nil + } + out := new(PlaybookStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Templatable) DeepCopyInto(out *Templatable) { + *out = *in + out.Test = in.Test + out.Display = in.Display + out.Transform = in.Transform +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Templatable. +func (in *Templatable) DeepCopy() *Templatable { + if in == nil { + return nil + } + out := new(Templatable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Template) DeepCopyInto(out *Template) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Template. +func (in *Template) DeepCopy() *Template { + if in == nil { + return nil + } + out := new(Template) + in.DeepCopyInto(out) + return out +} diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml new file mode 100644 index 000000000..7772acea2 --- /dev/null +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -0,0 +1,379 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: playbooks.mission-control.flanksource.com +spec: + group: mission-control.flanksource.com + names: + kind: Playbook + listKind: PlaybookList + plural: playbooks + singular: playbook + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Playbook is the schema for the Playbooks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + actions: + items: + properties: + exec: + properties: + connections: + properties: + aws: + properties: + accessKey: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + connection: + description: ConnectionName of the connection. It'll + be used to populate the endpoint, accessKey and + secretKey. + type: string + endpoint: + type: string + objectPath: + description: glob path to restrict matches to a + subset + type: string + region: + type: string + secretKey: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + sessionToken: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + skipTLSVerify: + description: Skip TLS verify when connecting to + aws + type: boolean + usePathStyle: + description: 'Use path style path: http://s3.amazonaws.com/BUCKET/KEY + instead of http://BUCKET.s3.amazonaws.com/KEY' + type: boolean + type: object + azure: + properties: + clientID: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + clientSecret: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + connection: + type: string + tenantID: + type: string + type: object + gcp: + properties: + connection: + description: ConnectionName of the connection. It'll + be used to populate the endpoint and credentials. + type: string + credentials: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + type: object + type: object + type: object + endpoint: + type: string + type: object + type: object + description: + description: Description for the check + type: string + display: + properties: + expr: + type: string + javascript: + type: string + jsonPath: + type: string + template: + type: string + type: object + icon: + description: Icon for overwriting default icon on the dashboard + type: string + labels: + additionalProperties: + type: string + description: Labels for the check + type: object + name: + description: Name of the check + type: string + script: + description: Script can be a inline script or a path to + a script that needs to be executed On windows executed + via powershell and in darwin and linux executed using + bash + type: string + test: + properties: + expr: + type: string + javascript: + type: string + jsonPath: + type: string + template: + type: string + type: object + transform: + properties: + expr: + type: string + javascript: + type: string + jsonPath: + type: string + template: + type: string + type: object + transformDeleteStrategy: + description: Transformed checks have a delete strategy on + deletion they can either be marked healthy, unhealthy + or left as is + type: string + required: + - name + - script + type: object + type: object + type: array + components: + items: + description: PlaybookResourceFilter defines a filter that decides + whether a resource (config or a component) is permitted be run + on the Playbook. + properties: + tags: + additionalProperties: + type: string + type: object + type: + type: string + type: object + type: array + configs: + items: + description: PlaybookResourceFilter defines a filter that decides + whether a resource (config or a component) is permitted be run + on the Playbook. + properties: + tags: + additionalProperties: + type: string + type: object + type: + type: string + type: object + type: array + description: + type: string + parameters: + items: + description: PlaybookParameter defines a parameter that a playbook + needs to run. + properties: + label: + type: string + name: + type: string + type: object + type: array + permissions: + items: + properties: + ref: + type: string + role: + type: string + team: + type: string + type: object + type: array + required: + - actions + type: object + status: + description: PlaybookStatus defines the observed state of Playbook + type: object + type: object + served: true + storage: true + subresources: + status: {} From 6d314849d10fb5d3384bf2cd8244bc01a77456a3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 15 Aug 2023 16:37:28 +0545 Subject: [PATCH 02/39] feat: add a new /playbook/run endpoint [skip ci] --- cmd/server.go | 4 +++ db/playbooks.go | 23 +++++++++++++++ go.mod | 3 ++ n | 1 + playbook/controllers.go | 62 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+) create mode 100644 db/playbooks.go create mode 100644 n create mode 100644 playbook/controllers.go diff --git a/cmd/server.go b/cmd/server.go index 94ae9a8c8..ba47d3d36 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -26,6 +26,7 @@ import ( "github.com/flanksource/incident-commander/events" "github.com/flanksource/incident-commander/jobs" "github.com/flanksource/incident-commander/logs" + "github.com/flanksource/incident-commander/playbook" "github.com/flanksource/incident-commander/rbac" "github.com/flanksource/incident-commander/snapshot" "github.com/flanksource/incident-commander/upstream" @@ -147,6 +148,9 @@ func createHTTPServer(gormDB *gorm.DB) *echo.Echo { upstreamGroup.GET("/canary/pull/:agent_name", canary.Pull) upstreamGroup.GET("/status/:agent_name", upstream.Status) + playbookGroup := e.Group("/playbook") + playbookGroup.POST("/run", playbook.Run) + forward(e, "/config", configDb) forward(e, "/canary", api.CanaryCheckerPath) forward(e, "/kratos", kratosAPI) diff --git a/db/playbooks.go b/db/playbooks.go new file mode 100644 index 000000000..1073c49a5 --- /dev/null +++ b/db/playbooks.go @@ -0,0 +1,23 @@ +package db + +import ( + "errors" + + "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" + "github.com/google/uuid" + "gorm.io/gorm" +) + +func FindPlaybook(ctx *api.Context, id uuid.UUID) (*models.Playbook, error) { + var p models.Playbook + if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + + return nil, err + } + + return &p, nil +} diff --git a/go.mod b/go.mod index c08583dd8..abc6c9968 100644 --- a/go.mod +++ b/go.mod @@ -223,3 +223,6 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + + +replace "github.com/flanksource/duty" => ../duty \ No newline at end of file diff --git a/n b/n new file mode 100644 index 000000000..800632e18 --- /dev/null +++ b/n @@ -0,0 +1 @@ +{"error":"either config_id or component_id is required","message":"invalid request"} diff --git a/playbook/controllers.go b/playbook/controllers.go new file mode 100644 index 000000000..f94065c9f --- /dev/null +++ b/playbook/controllers.go @@ -0,0 +1,62 @@ +package playbook + +import ( + "fmt" + "net/http" + + "github.com/flanksource/incident-commander/api" + "github.com/flanksource/incident-commander/db" + "github.com/google/uuid" + "github.com/labstack/echo/v4" +) + +type RunParams struct { + ID uuid.UUID `json:"id"` + ConfigID uuid.UUID `json:"config_id"` + ComponentID uuid.UUID `json:"component_id"` + Params map[string]string `json:"params"` +} + +func (r *RunParams) Valid() error { + if r.ID == uuid.Nil { + return fmt.Errorf("playbook id is required") + } + + if r.ConfigID == uuid.Nil && r.ComponentID == uuid.Nil { + return fmt.Errorf("either config_id or component_id is required") + } + + if r.ConfigID != uuid.Nil && r.ComponentID != uuid.Nil { + return fmt.Errorf("either config_id or component_id is required") + } + + return nil +} + +// Run runs the requested playbook with the provided parameters. +func Run(c echo.Context) error { + ctx := c.(*api.Context) + + var req RunParams + if err := c.Bind(&req); err != nil { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid request"}) + } + + if err := req.Valid(); err != nil { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid request"}) + } + + playbook, err := db.FindPlaybook(ctx, req.ID) + if err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to get playbook"}) + } else if playbook == nil { + return c.JSON(http.StatusNotFound, api.HTTPError{Error: "not found", Message: fmt.Sprintf("playbook(id=%s) not found", req.ID)}) + } + + // TODO: Run the playbook + // Return the playbook run ID and exit without waiting for the run to finish. + // The user will query the status of the playbook run via the another endpoint using + // the playbook run ID. + + return nil +} From 2a1c400c03549e2ad886a58dbba8a4a867bf9ce3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 15 Aug 2023 17:32:25 +0545 Subject: [PATCH 03/39] chore: register runs and return runs [skip ci] --- api/v1/playbook_actions.go | 3 ++- cmd/server.go | 3 ++- db/playbooks.go | 13 +++++++++++++ n | 1 - playbook/controllers.go | 35 +++++++++++++++++++++++++++++++---- playbook/runner.go | 22 ++++++++++++++++++++++ 6 files changed, 70 insertions(+), 7 deletions(-) delete mode 100644 n create mode 100644 playbook/runner.go diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index 265081aae..a832b6406 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -76,5 +76,6 @@ type AWSConnection struct { } type PlaybookAction struct { - Exec ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` + TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"` + Exec ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` } diff --git a/cmd/server.go b/cmd/server.go index ba47d3d36..088d4ca89 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -149,7 +149,8 @@ func createHTTPServer(gormDB *gorm.DB) *echo.Echo { upstreamGroup.GET("/status/:agent_name", upstream.Status) playbookGroup := e.Group("/playbook") - playbookGroup.POST("/run", playbook.Run) + playbookGroup.POST("/run", playbook.HandlePlaybookRun) + playbookGroup.GET("/run/:id", playbook.HandlePlaybookRunStatus) forward(e, "/config", configDb) forward(e, "/canary", api.CanaryCheckerPath) diff --git a/db/playbooks.go b/db/playbooks.go index 1073c49a5..7b35b115b 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -21,3 +21,16 @@ func FindPlaybook(ctx *api.Context, id uuid.UUID) (*models.Playbook, error) { return &p, nil } + +func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { + var p models.PlaybookRun + if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + + return nil, err + } + + return &p, nil +} diff --git a/n b/n deleted file mode 100644 index 800632e18..000000000 --- a/n +++ /dev/null @@ -1 +0,0 @@ -{"error":"either config_id or component_id is required","message":"invalid request"} diff --git a/playbook/controllers.go b/playbook/controllers.go index f94065c9f..62efd42b3 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -3,6 +3,7 @@ package playbook import ( "fmt" "net/http" + "time" "github.com/flanksource/incident-commander/api" "github.com/flanksource/incident-commander/db" @@ -10,6 +11,11 @@ import ( "github.com/labstack/echo/v4" ) +type RunResponse struct { + RunID string `json:"run_id"` + CreatedAt string `json:"created_at"` +} + type RunParams struct { ID uuid.UUID `json:"id"` ConfigID uuid.UUID `json:"config_id"` @@ -33,8 +39,8 @@ func (r *RunParams) Valid() error { return nil } -// Run runs the requested playbook with the provided parameters. -func Run(c echo.Context) error { +// HandlePlaybookRun handles playbook run requests. +func HandlePlaybookRun(c echo.Context) error { ctx := c.(*api.Context) var req RunParams @@ -55,8 +61,29 @@ func Run(c echo.Context) error { // TODO: Run the playbook // Return the playbook run ID and exit without waiting for the run to finish. - // The user will query the status of the playbook run via the another endpoint using + // The user will query the status of the playbook run via another endpoint using // the playbook run ID. + run, err := Run(ctx, *playbook, req) + if err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to register playbook run"}) + } - return nil + return c.JSON(http.StatusCreated, RunResponse{ + RunID: run.ID.String(), + CreatedAt: run.CreatedAt.Format(time.RFC3339), + }) +} + +func HandlePlaybookRunStatus(c echo.Context) error { + ctx := c.(*api.Context) + id := c.Param("id") + + run, err := db.FindPlaybookRun(ctx, id) + if err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to get playbook run"}) + } else if run == nil { + return c.JSON(http.StatusNotFound, api.HTTPError{Error: "not found", Message: fmt.Sprintf("playbook run(id=%s) not found", id)}) + } + + return c.JSON(http.StatusOK, run) } diff --git a/playbook/runner.go b/playbook/runner.go new file mode 100644 index 000000000..adf185209 --- /dev/null +++ b/playbook/runner.go @@ -0,0 +1,22 @@ +package playbook + +import ( + "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" +) + +// Run runs the requested playbook with the provided parameters. +func Run(ctx *api.Context, playbook models.Playbook, req RunParams) (*models.PlaybookRun, error) { + run := models.PlaybookRun{ + PlaybookID: playbook.ID, + // CreatedBy: ctx.User().ID, // TODO: Add user id to the context + } + if err := ctx.DB().Create(&run).Error; err != nil { + return nil, err + } + + // For now run in go routine. + // Might need to implement a runner. + + return &run, nil +} From a1fb1439ffb59983512a5f3afe775412a2dc7a6f Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 09:45:53 +0545 Subject: [PATCH 04/39] chore: add a basic playbook run queue processor [skip ci] --- db/playbooks.go | 9 ++++++++ jobs/jobs.go | 7 ++++++ playbook/controllers.go | 32 +++++++++++++++++---------- playbook/runner.go | 48 ++++++++++++++++++++++++++++++++--------- 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/db/playbooks.go b/db/playbooks.go index 7b35b115b..0dca0c6dc 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -34,3 +34,12 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } + +func GetSchedulePlaybookRuns(ctx *api.Context) ([]models.PlaybookRun, error) { + var runs []models.PlaybookRun + if err := ctx.DB().Where("start_date <= NOW()").Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { + return nil, err + } + + return runs, nil +} diff --git a/jobs/jobs.go b/jobs/jobs.go index 1cfdfe89c..4b5e11d95 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -6,6 +6,7 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/incident-commander/api" + "github.com/flanksource/incident-commander/playbook" "github.com/flanksource/incident-commander/responder" "github.com/flanksource/incident-commander/rules" "github.com/flanksource/incident-commander/upstream" @@ -19,6 +20,7 @@ const ( ResponderConfigSyncSchedule = "@every 1h" CleanupJobHistoryTableSchedule = "@every 24h" PushAgentReconcileSchedule = "@every 30m" + ProcessPlaybookRunQueueSchedule = "@every 5s" ) var FuncScheduler = cron.New() @@ -65,6 +67,11 @@ func Start() { } } + job := newFuncJob(playbook.ProcessRunQueue, withName("process playbook-run queue"), withRunNow(true)) + if err := job.schedule(FuncScheduler, ProcessPlaybookRunQueueSchedule); err != nil { + logger.Errorf("Failed to schedule 'playbookRun queue processor' job: %v", err) + } + incidentRulesSchedule := fmt.Sprintf("@every %s", rules.Period.String()) logger.Infof("IncidentRulesSchedule %s", incidentRulesSchedule) if _, err := ScheduleFunc(incidentRulesSchedule, func() { diff --git a/playbook/controllers.go b/playbook/controllers.go index 62efd42b3..ee98181de 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -5,6 +5,7 @@ import ( "net/http" "time" + "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" "github.com/flanksource/incident-commander/db" "github.com/google/uuid" @@ -12,8 +13,8 @@ import ( ) type RunResponse struct { - RunID string `json:"run_id"` - CreatedAt string `json:"created_at"` + RunID string `json:"run_id"` + StartsAt string `json:"starts_at"` } type RunParams struct { @@ -59,18 +60,27 @@ func HandlePlaybookRun(c echo.Context) error { return c.JSON(http.StatusNotFound, api.HTTPError{Error: "not found", Message: fmt.Sprintf("playbook(id=%s) not found", req.ID)}) } - // TODO: Run the playbook - // Return the playbook run ID and exit without waiting for the run to finish. - // The user will query the status of the playbook run via another endpoint using - // the playbook run ID. - run, err := Run(ctx, *playbook, req) - if err != nil { - return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to register playbook run"}) + run := models.PlaybookRun{ + PlaybookID: playbook.ID, + Status: models.PlaybookRunStatusScheduled, + // CreatedBy: ctx.User().ID, // TODO: Add user id to the context + } + + if req.ComponentID != uuid.Nil { + run.ComponentID = &req.ComponentID + } + + if req.ConfigID != uuid.Nil { + run.ConfigID = &req.ConfigID + } + + if err := ctx.DB().Create(&run).Error; err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to create playbook run"}) } return c.JSON(http.StatusCreated, RunResponse{ - RunID: run.ID.String(), - CreatedAt: run.CreatedAt.Format(time.RFC3339), + RunID: run.ID.String(), + StartsAt: run.StartDate.Format(time.RFC3339), }) } diff --git a/playbook/runner.go b/playbook/runner.go index adf185209..2758fcefe 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -1,22 +1,50 @@ package playbook import ( + "fmt" + "time" + + "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" + "github.com/flanksource/incident-commander/db" ) -// Run runs the requested playbook with the provided parameters. -func Run(ctx *api.Context, playbook models.Playbook, req RunParams) (*models.PlaybookRun, error) { - run := models.PlaybookRun{ - PlaybookID: playbook.ID, - // CreatedBy: ctx.User().ID, // TODO: Add user id to the context +func ProcessRunQueue(ctx *api.Context) error { + runs, err := db.GetSchedulePlaybookRuns(ctx) + if err != nil { + return fmt.Errorf("failed to get playbook runs: %w", err) + } + + if len(runs) == 0 { + return nil } - if err := ctx.DB().Create(&run).Error; err != nil { - return nil, err + + logger.Infof("Processing %d playbook runs", len(runs)) + + for _, r := range runs { + logger.Infof("running %v", r) + go func(run models.PlaybookRun) { + if err := executeRun(ctx, run); err != nil { + logger.Errorf("failed to execute playbook run: %v", err) + } + }(r) } - // For now run in go routine. - // Might need to implement a runner. + return nil +} + +func executeRun(ctx *api.Context, run models.PlaybookRun) error { + if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumn("status", models.PlaybookRunStatusRunning).Error; err != nil { + return err + } + + // The actual job here + time.Sleep(time.Second * 3) + + if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumn("status", models.PlaybookRunStatusCompleted).Error; err != nil { + return err + } - return &run, nil + return nil } From 7be19087493bed089a13f6877f894ec19443cbc1 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 10:33:42 +0545 Subject: [PATCH 05/39] feat: add parameters validation & update playbook run on completion [skip ci] --- db/playbooks.go | 4 ++-- playbook/controllers.go | 43 ++++++++++++++++++++++++++++++---- playbook/runner.go | 51 ++++++++++++++++++++++++++++++----------- 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/db/playbooks.go b/db/playbooks.go index 0dca0c6dc..d671d6a11 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -35,9 +35,9 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } -func GetSchedulePlaybookRuns(ctx *api.Context) ([]models.PlaybookRun, error) { +func GetScheduledPlaybookRuns(ctx *api.Context) ([]models.PlaybookRun, error) { var runs []models.PlaybookRun - if err := ctx.DB().Where("start_date <= NOW()").Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { + if err := ctx.DB().Where("start_time <= NOW()").Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { return nil, err } diff --git a/playbook/controllers.go b/playbook/controllers.go index ee98181de..d8ad680f6 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -1,12 +1,15 @@ package playbook import ( + "encoding/json" "fmt" "net/http" "time" "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" "github.com/google/uuid" "github.com/labstack/echo/v4" @@ -24,7 +27,7 @@ type RunParams struct { Params map[string]string `json:"params"` } -func (r *RunParams) Valid() error { +func (r *RunParams) valid() error { if r.ID == uuid.Nil { return fmt.Errorf("playbook id is required") } @@ -40,6 +43,28 @@ func (r *RunParams) Valid() error { return nil } +func (r *RunParams) validateParams(params []v1.PlaybookParameter) error { + if len(params) != len(r.Params) { + return fmt.Errorf("invalid number of parameters. expected %d, got %d", len(params), len(r.Params)) + } + + for k := range r.Params { + var ok bool + for _, p := range params { + if k == p.Name { + ok = true + break + } + } + + if !ok { + return fmt.Errorf("unknown parameter %s", k) + } + } + + return nil +} + // HandlePlaybookRun handles playbook run requests. func HandlePlaybookRun(c echo.Context) error { ctx := c.(*api.Context) @@ -49,7 +74,7 @@ func HandlePlaybookRun(c echo.Context) error { return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid request"}) } - if err := req.Valid(); err != nil { + if err := req.valid(); err != nil { return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid request"}) } @@ -60,10 +85,20 @@ func HandlePlaybookRun(c echo.Context) error { return c.JSON(http.StatusNotFound, api.HTTPError{Error: "not found", Message: fmt.Sprintf("playbook(id=%s) not found", req.ID)}) } + var spec v1.PlaybookSpec + if err := json.Unmarshal(playbook.Spec, &spec); err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to unmarshal playbook spec"}) + } + + if err := req.validateParams(spec.Parameters); err != nil { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid parameters"}) + } + run := models.PlaybookRun{ PlaybookID: playbook.ID, Status: models.PlaybookRunStatusScheduled, - // CreatedBy: ctx.User().ID, // TODO: Add user id to the context + Parameters: types.JSONStringMap(req.Params), + // CreatedBy: ctx.User().ID, // TODO: Add user id to the context from a middleware } if req.ComponentID != uuid.Nil { @@ -80,7 +115,7 @@ func HandlePlaybookRun(c echo.Context) error { return c.JSON(http.StatusCreated, RunResponse{ RunID: run.ID.String(), - StartsAt: run.StartDate.Format(time.RFC3339), + StartsAt: run.StartTime.Format(time.RFC3339), }) } diff --git a/playbook/runner.go b/playbook/runner.go index 2758fcefe..8fcfafc6c 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -1,7 +1,9 @@ package playbook import ( + "errors" "fmt" + "math/rand" "time" "github.com/flanksource/commons/logger" @@ -11,7 +13,7 @@ import ( ) func ProcessRunQueue(ctx *api.Context) error { - runs, err := db.GetSchedulePlaybookRuns(ctx) + runs, err := db.GetScheduledPlaybookRuns(ctx) if err != nil { return fmt.Errorf("failed to get playbook runs: %w", err) } @@ -20,31 +22,54 @@ func ProcessRunQueue(ctx *api.Context) error { return nil } - logger.Infof("Processing %d playbook runs", len(runs)) + logger.Infof("Starting to execute %d playbook runs", len(runs)) for _, r := range runs { - logger.Infof("running %v", r) - go func(run models.PlaybookRun) { - if err := executeRun(ctx, run); err != nil { - logger.Errorf("failed to execute playbook run: %v", err) - } - }(r) + go ExecuteRun(ctx, r) } return nil } -func executeRun(ctx *api.Context, run models.PlaybookRun) error { +func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { + logger.Infof("Executing playbook run: %s", run.ID) + if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumn("status", models.PlaybookRunStatusRunning).Error; err != nil { + logger.Errorf("failed to update playbook run status: %v", err) + return + } + + start := time.Now() + columnUpdates := map[string]any{ + "end_time": "NOW()", + } + + if err := executeRun(ctx, run); err != nil { + logger.Errorf("failed to execute playbook run: %v", err) + columnUpdates["status"] = models.PlaybookRunStatusFailed + columnUpdates["result"] = err.Error() + } else { + columnUpdates["status"] = models.PlaybookRunStatusCompleted + } + + columnUpdates["duration"] = time.Since(start).Milliseconds() + if err := ctx.DB().Debug().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumns(&columnUpdates).Error; err != nil { + logger.Errorf("failed to update playbook run status: %v", err) + } +} + +func executeRun(ctx *api.Context, run models.PlaybookRun) error { + var playbook models.Playbook + if err := ctx.DB().Where("id = ?", run.PlaybookID).First(&playbook).Error; err != nil { return err } // The actual job here - time.Sleep(time.Second * 3) + time.Sleep(time.Second * time.Duration(rand.Intn(4))) - if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumn("status", models.PlaybookRunStatusCompleted).Error; err != nil { - return err + if rand.Intn(4) != 0 { // 75% chance of failing + return nil } - return nil + return errors.New("dummy error") } From 6c2404d0641707d6cd7b193b466ea39a4c1f6ff2 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 12:56:16 +0545 Subject: [PATCH 06/39] feat: implement exec action [skip ci] --- api/global.go | 4 + api/v1/playbook_actions.go | 97 +++++++++- api/v1/playbook_types.go | 26 ++- api/v1/zz_generated.deepcopy.go | 6 +- ...ion-control.flanksource.com_playbooks.yaml | 2 + playbook/actions.go | 173 ++++++++++++++++++ playbook/runner.go | 44 +++-- 7 files changed, 333 insertions(+), 19 deletions(-) create mode 100644 playbook/actions.go diff --git a/api/global.go b/api/global.go index d429ed6fb..25c49a711 100644 --- a/api/global.go +++ b/api/global.go @@ -61,6 +61,10 @@ func (c *Context) GetEnvVarValue(input types.EnvVar) (string, error) { return duty.GetEnvValueFromCache(c.Kubernetes, input, c.Namespace) } +func (ctx *Context) GetEnvValueFromCache(env types.EnvVar) (string, error) { + return duty.GetEnvValueFromCache(ctx.Kubernetes, env, ctx.Namespace) +} + func (c *Context) HydrateConnection(connectionName string) (*models.Connection, error) { if connectionName == "" || !strings.HasPrefix(connectionName, "connection://") { return nil, nil diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index a832b6406..39ac2346b 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -1,6 +1,14 @@ package v1 -import "github.com/flanksource/duty/types" +import ( + "context" + "fmt" + + "github.com/flanksource/duty" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" + "k8s.io/client-go/kubernetes" +) type Labels map[string]string @@ -45,6 +53,12 @@ type ExecConnections struct { Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"` } +type connectionContext interface { + context.Context + HydrateConnection(connectionName string) (*models.Connection, error) + GetEnvValueFromCache(env types.EnvVar) (string, error) +} + type GCPConnection struct { // ConnectionName of the connection. It'll be used to populate the endpoint and credentials. ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` @@ -52,6 +66,22 @@ type GCPConnection struct { Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"` } +// HydrateConnection attempts to find the connection by name +// and populate the endpoint and credentials. +func (g *GCPConnection) HydrateConnection(ctx connectionContext) error { + connection, err := ctx.HydrateConnection(g.ConnectionName) + if err != nil { + return err + } + + if connection != nil { + g.Credentials = &types.EnvVar{ValueStatic: connection.Certificate} + g.Endpoint = connection.URL + } + + return nil +} + type AzureConnection struct { ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` @@ -59,6 +89,23 @@ type AzureConnection struct { TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"` } +// HydrateConnection attempts to find the connection by name +// and populate the endpoint and credentials. +func (g *AzureConnection) HydrateConnection(ctx connectionContext) error { + connection, err := ctx.HydrateConnection(g.ConnectionName) + if err != nil { + return err + } + + if connection != nil { + g.ClientID = &types.EnvVar{ValueStatic: connection.Username} + g.ClientSecret = &types.EnvVar{ValueStatic: connection.Password} + g.TenantID = connection.Properties["tenantID"] + } + + return nil +} + type AWSConnection struct { // ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey. ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` @@ -75,7 +122,51 @@ type AWSConnection struct { UsePathStyle bool `yaml:"usePathStyle,omitempty" json:"usePathStyle,omitempty"` } +// Populate populates an AWSConnection with credentials and other information. +// If a connection name is specified, it'll be used to populate the endpoint, accessKey and secretKey. +func (t *AWSConnection) Populate(ctx connectionContext, k8s kubernetes.Interface, namespace string) error { + if t.ConnectionName != "" { + connection, err := ctx.HydrateConnection(t.ConnectionName) + if err != nil { + return fmt.Errorf("could not parse EC2 access key: %v", err) + } + + t.AccessKey.ValueStatic = connection.Username + t.SecretKey.ValueStatic = connection.Password + if t.Endpoint == "" { + t.Endpoint = connection.URL + } + + t.SkipTLSVerify = connection.InsecureTLS + if t.Region == "" { + if region, ok := connection.Properties["region"]; ok { + t.Region = region + } + } + } + + if accessKey, err := duty.GetEnvValueFromCache(k8s, t.AccessKey, namespace); err != nil { + return fmt.Errorf("could not parse AWS access key id: %v", err) + } else { + t.AccessKey.ValueStatic = accessKey + } + + if secretKey, err := duty.GetEnvValueFromCache(k8s, t.SecretKey, namespace); err != nil { + return fmt.Errorf(fmt.Sprintf("Could not parse AWS secret access key: %v", err)) + } else { + t.SecretKey.ValueStatic = secretKey + } + + if sessionToken, err := duty.GetEnvValueFromCache(k8s, t.SessionToken, namespace); err != nil { + return fmt.Errorf(fmt.Sprintf("Could not parse AWS session token: %v", err)) + } else { + t.SessionToken.ValueStatic = sessionToken + } + + return nil +} + type PlaybookAction struct { - TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"` - Exec ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` + TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"` + Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` } diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index 582d0f568..fb1617360 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -1,6 +1,12 @@ package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + "encoding/json" + + "github.com/flanksource/duty/models" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) type Permission struct { Role string `json:"role,omitempty" yaml:"role,omitempty"` @@ -46,3 +52,21 @@ type Playbook struct { Spec PlaybookSpec `json:"spec,omitempty"` Status PlaybookStatus `json:"status,omitempty"` } + +func PlaybookFromModel(p models.Playbook) (Playbook, error) { + var spec PlaybookSpec + if err := json.Unmarshal(p.Spec, &spec); err != nil { + return Playbook{}, nil + } + + out := Playbook{ + ObjectMeta: metav1.ObjectMeta{ + Name: p.Name, + UID: types.UID(p.ID.String()), + CreationTimestamp: metav1.Time{Time: p.CreatedAt}, + }, + Spec: spec, + } + + return out, nil +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 5b4480c0e..22c742d3b 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -384,7 +384,11 @@ func (in *Playbook) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlaybookAction) DeepCopyInto(out *PlaybookAction) { *out = *in - in.Exec.DeepCopyInto(&out.Exec) + if in.Exec != nil { + in, out := &in.Exec, &out.Exec + *out = new(ExecAction) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookAction. diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index 7772acea2..35402c633 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -312,6 +312,8 @@ spec: - name - script type: object + timeout-minutes: + type: string type: object type: array components: diff --git a/playbook/actions.go b/playbook/actions.go new file mode 100644 index 000000000..eb4866eab --- /dev/null +++ b/playbook/actions.go @@ -0,0 +1,173 @@ +package playbook + +import ( + "bytes" + "fmt" + "math/rand" + "os" + osExec "os/exec" + "path/filepath" + "runtime" + "strings" + textTemplate "text/template" + + "github.com/flanksource/commons/logger" + "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" +) + +type ExecAction struct { +} + +type ExecDetails struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode int `json:"exitCode"` +} + +func (c *ExecAction) Run(ctx *api.Context, exec v1.ExecAction) (*ExecDetails, error) { + switch runtime.GOOS { + case "windows": + return execPowershell(exec, ctx) + default: + return execBash(exec, ctx) + } +} + +func execPowershell(check v1.ExecAction, ctx *api.Context) (*ExecDetails, error) { + ps, err := osExec.LookPath("powershell.exe") + if err != nil { + return nil, err + } + args := []string{check.Script} + cmd := osExec.Command(ps, args...) + return runCmd(cmd) +} + +func setupConnection(ctx *api.Context, check v1.ExecAction, cmd *osExec.Cmd) error { + if check.Connections.AWS != nil { + if err := check.Connections.AWS.Populate(ctx, ctx.Kubernetes, ctx.Namespace); err != nil { + return fmt.Errorf("failed to hydrate aws connection: %w", err) + } + + configPath, err := saveConfig(awsConfigTemplate, check.Connections.AWS) + defer os.RemoveAll(filepath.Dir(configPath)) + if err != nil { + return fmt.Errorf("failed to store AWS credentials: %w", err) + } + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 + cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configPath)) + if check.Connections.AWS.Region != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", check.Connections.AWS.Region)) + } + } + + if check.Connections.Azure != nil { + if err := check.Connections.Azure.HydrateConnection(ctx); err != nil { + return fmt.Errorf("failed to hydrate connection %w", err) + } + + // login with service principal + runCmd := osExec.Command("az", "login", "--service-principal", "--username", check.Connections.Azure.ClientID.ValueStatic, "--password", check.Connections.Azure.ClientSecret.ValueStatic, "--tenant", check.Connections.Azure.TenantID) + if err := runCmd.Run(); err != nil { + return fmt.Errorf("failed to login: %w", err) + } + } + + if check.Connections.GCP != nil { + if err := check.Connections.GCP.HydrateConnection(ctx); err != nil { + return fmt.Errorf("failed to hydrate connection %w", err) + } + + configPath, err := saveConfig(gcloudConfigTemplate, check.Connections.GCP) + defer os.RemoveAll(filepath.Dir(configPath)) + if err != nil { + return fmt.Errorf("failed to store gcloud credentials: %w", err) + } + + // to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS, + // we need to explicitly activate it + runCmd := osExec.Command("gcloud", "auth", "activate-service-account", "--key-file", configPath) + if err := runCmd.Run(); err != nil { + return fmt.Errorf("failed to activate GCP service account: %w", err) + } + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configPath)) + } + + return nil +} + +func execBash(check v1.ExecAction, ctx *api.Context) (*ExecDetails, error) { + if len(check.Script) == 0 { + return nil, fmt.Errorf("no script provided") + } + + cmd := osExec.Command("bash", "-c", check.Script) + if err := setupConnection(ctx, check, cmd); err != nil { + return nil, fmt.Errorf("failed to setup connection: %w", err) + } + + return runCmd(cmd) +} + +func runCmd(cmd *osExec.Cmd) (*ExecDetails, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to run command: %w", err) + } + + details := ExecDetails{ + Stdout: strings.TrimSpace(stdout.String()), + Stderr: strings.TrimSpace(stderr.String()), + ExitCode: cmd.ProcessState.ExitCode(), + } + if details.ExitCode != 0 { + return nil, fmt.Errorf("non-zero exit-code: %d. (stdout=%s) (stderr=%s)", details.ExitCode, details.Stdout, details.Stderr) + } + + return &details, nil +} + +func saveConfig(configTemplate *textTemplate.Template, view any) (string, error) { + dirPath := filepath.Join(".creds", fmt.Sprintf("cred-%d", rand.Intn(10000000))) + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", err + } + + configPath := fmt.Sprintf("%s/credentials", dirPath) + logger.Tracef("Creating credentials file: %s", configPath) + + file, err := os.Create(configPath) + if err != nil { + return configPath, err + } + defer file.Close() + + if err := configTemplate.Execute(file, view); err != nil { + return configPath, err + } + + return configPath, nil +} + +var ( + awsConfigTemplate *textTemplate.Template + gcloudConfigTemplate *textTemplate.Template +) + +func init() { + awsConfigTemplate = textTemplate.Must(textTemplate.New("").Parse(`[default] +aws_access_key_id = {{.AccessKey.ValueStatic}} +aws_secret_access_key = {{.SecretKey.ValueStatic}} +{{if .SessionToken.ValueStatic}}aws_session_token={{.SessionToken.ValueStatic}}{{end}} +`)) + + gcloudConfigTemplate = textTemplate.Must(textTemplate.New("").Parse(`{{.Credentials}}`)) +} diff --git a/playbook/runner.go b/playbook/runner.go index 8fcfafc6c..80ac1dace 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -1,14 +1,14 @@ package playbook import ( - "errors" + "encoding/json" "fmt" - "math/rand" "time" "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" ) @@ -44,32 +44,48 @@ func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { "end_time": "NOW()", } - if err := executeRun(ctx, run); err != nil { + if result, err := executeRun(ctx, run); err != nil { logger.Errorf("failed to execute playbook run: %v", err) columnUpdates["status"] = models.PlaybookRunStatusFailed - columnUpdates["result"] = err.Error() + columnUpdates["error"] = err.Error() } else { columnUpdates["status"] = models.PlaybookRunStatusCompleted + + resultJson, _ := json.Marshal(result) + columnUpdates["result"] = resultJson } columnUpdates["duration"] = time.Since(start).Milliseconds() - if err := ctx.DB().Debug().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumns(&columnUpdates).Error; err != nil { + if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumns(&columnUpdates).Error; err != nil { logger.Errorf("failed to update playbook run status: %v", err) } } -func executeRun(ctx *api.Context, run models.PlaybookRun) error { - var playbook models.Playbook - if err := ctx.DB().Where("id = ?", run.PlaybookID).First(&playbook).Error; err != nil { - return err +func executeRun(ctx *api.Context, run models.PlaybookRun) ([]map[string]any, error) { + var playbookModel models.Playbook + if err := ctx.DB().Where("id = ?", run.PlaybookID).First(&playbookModel).Error; err != nil { + return nil, err } - // The actual job here - time.Sleep(time.Second * time.Duration(rand.Intn(4))) + playbook, err := v1.PlaybookFromModel(playbookModel) + if err != nil { + return nil, err + } - if rand.Intn(4) != 0 { // 75% chance of failing - return nil + result := make([]map[string]any, 0, len(playbook.Spec.Actions)) + for _, action := range playbook.Spec.Actions { + if action.Exec != nil { + e := ExecAction{} + res, err := e.Run(ctx, *action.Exec) + if err != nil { + return nil, err + } + + result = append(result, map[string]any{ + "exec": res.Stdout, + }) + } } - return errors.New("dummy error") + return result, nil } From e04cd31dbbb6573f9167cab0a2279409acb1e634 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 15:14:52 +0545 Subject: [PATCH 07/39] feat: add playbook actions [skip ci] --- api/v1/playbook_actions.go | 1 + playbook/actions.go | 2 +- playbook/runner.go | 75 ++++++++++++++++++++++++++++---------- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index 39ac2346b..3766e3010 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -167,6 +167,7 @@ func (t *AWSConnection) Populate(ctx connectionContext, k8s kubernetes.Interface } type PlaybookAction struct { + Name string `yaml:"name" json:"name"` TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"` Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` } diff --git a/playbook/actions.go b/playbook/actions.go index eb4866eab..b98c277fd 100644 --- a/playbook/actions.go +++ b/playbook/actions.go @@ -120,7 +120,7 @@ func runCmd(cmd *osExec.Cmd) (*ExecDetails, error) { cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("failed to run command: %w", err) + return nil, fmt.Errorf("%s: %w", strings.TrimSpace(stderr.String()), err) } details := ExecDetails{ diff --git a/playbook/runner.go b/playbook/runner.go index 80ac1dace..f642a07da 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -44,15 +44,11 @@ func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { "end_time": "NOW()", } - if result, err := executeRun(ctx, run); err != nil { + if err := executeRun(ctx, run); err != nil { logger.Errorf("failed to execute playbook run: %v", err) columnUpdates["status"] = models.PlaybookRunStatusFailed - columnUpdates["error"] = err.Error() } else { columnUpdates["status"] = models.PlaybookRunStatusCompleted - - resultJson, _ := json.Marshal(result) - columnUpdates["result"] = resultJson } columnUpdates["duration"] = time.Since(start).Milliseconds() @@ -61,31 +57,70 @@ func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { } } -func executeRun(ctx *api.Context, run models.PlaybookRun) ([]map[string]any, error) { +func executeRun(ctx *api.Context, run models.PlaybookRun) error { var playbookModel models.Playbook if err := ctx.DB().Where("id = ?", run.PlaybookID).First(&playbookModel).Error; err != nil { - return nil, err + return err } playbook, err := v1.PlaybookFromModel(playbookModel) if err != nil { - return nil, err + return err } - result := make([]map[string]any, 0, len(playbook.Spec.Actions)) for _, action := range playbook.Spec.Actions { - if action.Exec != nil { - e := ExecAction{} - res, err := e.Run(ctx, *action.Exec) - if err != nil { - return nil, err - } - - result = append(result, map[string]any{ - "exec": res.Stdout, - }) + logger.Infof("Executing playbook run: %s", run.ID) + + runAction := models.PlaybookRunAction{ + PlaybookRunID: run.ID, + Name: action.Name, + Status: models.PlaybookRunStatusRunning, + } + + if err := ctx.DB().Create(&runAction).Error; err != nil { + logger.Errorf("failed to create playbook run action: %v", err) + return err + } + + start := time.Now() + columnUpdates := map[string]any{ + "end_time": "NOW()", + } + + result, err := executeAction(ctx, run, action) + if err != nil { + logger.Errorf("failed to execute action: %v", err) + columnUpdates["status"] = models.PlaybookRunStatusFailed + columnUpdates["error"] = err.Error() + } else { + columnUpdates["status"] = models.PlaybookRunStatusCompleted + columnUpdates["result"] = result + } + + columnUpdates["duration"] = time.Since(start).Milliseconds() + if err := ctx.DB().Model(&models.PlaybookRunAction{}).Where("id = ?", runAction.ID).UpdateColumns(&columnUpdates).Error; err != nil { + logger.Errorf("failed to update playbook run action status: %v", err) } + + // Even if a single action fails, we stop the execution and mark the Run as failed + if err != nil { + return fmt.Errorf("action %s failed: %w", action.Name, err) + } + } + + return nil +} + +func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookAction) ([]byte, error) { + if action.Exec != nil { + e := ExecAction{} + res, err := e.Run(ctx, *action.Exec) + if err != nil { + return nil, err + } + + return json.Marshal(res.Stdout) } - return result, nil + return nil, nil } From 0ed9909fefaa657d00870de381d9f12281f47d97 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 15:39:06 +0545 Subject: [PATCH 08/39] feat: enable templating of the scripts [skip ci] --- playbook/runner.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/playbook/runner.go b/playbook/runner.go index f642a07da..4c968a784 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -7,11 +7,25 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" + "github.com/flanksource/gomplate/v3" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" ) +// ActionParam defines the config and component passed to a playbook run action. +type ActionParam struct { + Config *models.ConfigItem `json:"config,omitempty"` + Component *models.Component `json:"component,omitempty"` +} + +func (t *ActionParam) AsMap() map[string]any { + return map[string]any{ + "config": t.Config, + "component": t.Component, + } +} + func ProcessRunQueue(ctx *api.Context) error { runs, err := db.GetScheduledPlaybookRuns(ctx) if err != nil { @@ -68,6 +82,17 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return err } + var actionParam ActionParam + if run.ComponentID != nil { + if err := ctx.DB().Where("id = ?", run.ComponentID).First(&actionParam.Component).Error; err != nil { + return err + } + } else if run.ConfigID != nil { + if err := ctx.DB().Where("id = ?", run.ConfigID).First(&actionParam.Config).Error; err != nil { + return err + } + } + for _, action := range playbook.Spec.Actions { logger.Infof("Executing playbook run: %s", run.ID) @@ -87,7 +112,7 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { "end_time": "NOW()", } - result, err := executeAction(ctx, run, action) + result, err := executeAction(ctx, run, action, actionParam) if err != nil { logger.Errorf("failed to execute action: %v", err) columnUpdates["status"] = models.PlaybookRunStatusFailed @@ -111,8 +136,14 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return nil } -func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookAction) ([]byte, error) { +func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookAction, actionParam ActionParam) ([]byte, error) { if action.Exec != nil { + script, err := gomplate.RunTemplate(actionParam.AsMap(), gomplate.Template{Template: action.Exec.Script}) + if err != nil { + return nil, err + } + action.Exec.Script = script + e := ExecAction{} res, err := e.Run(ctx, *action.Exec) if err != nil { From dc757c2a3a00440185d19d9760dbb69197907afa Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 20:14:59 +0545 Subject: [PATCH 09/39] chore: do not store duration & move action to separate package [skip ci] --- playbook/{actions.go => actions/exec.go} | 2 +- playbook/runner.go | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) rename playbook/{actions.go => actions/exec.go} (99%) diff --git a/playbook/actions.go b/playbook/actions/exec.go similarity index 99% rename from playbook/actions.go rename to playbook/actions/exec.go index b98c277fd..24d940506 100644 --- a/playbook/actions.go +++ b/playbook/actions/exec.go @@ -1,4 +1,4 @@ -package playbook +package actions import ( "bytes" diff --git a/playbook/runner.go b/playbook/runner.go index 4c968a784..c2362d290 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -3,7 +3,6 @@ package playbook import ( "encoding/json" "fmt" - "time" "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" @@ -11,6 +10,7 @@ import ( "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" + "github.com/flanksource/incident-commander/playbook/actions" ) // ActionParam defines the config and component passed to a playbook run action. @@ -53,7 +53,6 @@ func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { return } - start := time.Now() columnUpdates := map[string]any{ "end_time": "NOW()", } @@ -65,7 +64,6 @@ func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { columnUpdates["status"] = models.PlaybookRunStatusCompleted } - columnUpdates["duration"] = time.Since(start).Milliseconds() if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumns(&columnUpdates).Error; err != nil { logger.Errorf("failed to update playbook run status: %v", err) } @@ -107,7 +105,6 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return err } - start := time.Now() columnUpdates := map[string]any{ "end_time": "NOW()", } @@ -122,7 +119,6 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { columnUpdates["result"] = result } - columnUpdates["duration"] = time.Since(start).Milliseconds() if err := ctx.DB().Model(&models.PlaybookRunAction{}).Where("id = ?", runAction.ID).UpdateColumns(&columnUpdates).Error; err != nil { logger.Errorf("failed to update playbook run action status: %v", err) } @@ -144,7 +140,7 @@ func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookA } action.Exec.Script = script - e := ExecAction{} + e := actions.ExecAction{} res, err := e.Run(ctx, *action.Exec) if err != nil { return nil, err From ec7df03c0fd4febb3de6ccb4f2558e324f0a4732 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 16 Aug 2023 22:11:11 +0545 Subject: [PATCH 10/39] feat: add playbook run consumer and remove cron job [skip ci] --- cmd/server.go | 3 ++ db/playbooks.go | 4 +- events/event_consumer.go | 49 ++----------------- jobs/jobs.go | 7 --- playbook/queue_consumer.go | 96 ++++++++++++++++++++++++++++++++++++++ playbook/runner.go | 22 +-------- utils/pg_notify.go | 52 +++++++++++++++++++++ 7 files changed, 157 insertions(+), 76 deletions(-) create mode 100644 playbook/queue_consumer.go create mode 100644 utils/pg_notify.go diff --git a/cmd/server.go b/cmd/server.go index 088d4ca89..54fed97a0 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -224,6 +224,9 @@ var Serve = &cobra.Command{ UpstreamPush: api.UpstreamConf, }) + playbookRunConsumer := playbook.NewQueueConsumer(db.Gorm, db.Pool) + go playbookRunConsumer.Listen() + go launchKopper() e := createHTTPServer(db.Gorm) diff --git a/db/playbooks.go b/db/playbooks.go index d671d6a11..730b8d9aa 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -35,9 +35,9 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } -func GetScheduledPlaybookRuns(ctx *api.Context) ([]models.PlaybookRun, error) { +func GetScheduledPlaybookRuns(ctx *api.Context, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { var runs []models.PlaybookRun - if err := ctx.DB().Where("start_time <= NOW()").Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { + if err := ctx.DB().Not(exceptions).Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { return nil, err } diff --git a/events/event_consumer.go b/events/event_consumer.go index 217e6ff71..b20bb15f6 100644 --- a/events/event_consumer.go +++ b/events/event_consumer.go @@ -1,17 +1,15 @@ package events import ( - "context" "errors" "fmt" "sync" "time" - "github.com/sethvargo/go-retry" - "github.com/flanksource/commons/logger" "github.com/flanksource/incident-commander/api" "github.com/flanksource/incident-commander/db" + "github.com/flanksource/incident-commander/utils" "gorm.io/gorm" ) @@ -123,47 +121,6 @@ func (t *EventConsumer) ConsumeEventsUntilEmpty() { wg.Wait() } -// listenToPostgresNotify listens to postgres notifications -// and will retry on failure for dbReconnectMaxAttempt times -func (e *EventConsumer) listenToPostgresNotify(pgNotify chan bool) { - var listen = func(ctx context.Context, pgNotify chan bool) error { - conn, err := db.Pool.Acquire(ctx) - if err != nil { - return fmt.Errorf("error acquiring database connection: %v", err) - } - defer conn.Release() - - _, err = conn.Exec(ctx, "LISTEN event_queue_updates") - if err != nil { - return fmt.Errorf("error listening to database notifications: %v", err) - } - logger.Debugf("listening to database notifications") - - for { - _, err := conn.Conn().WaitForNotification(ctx) - if err != nil { - return fmt.Errorf("error listening to database notifications: %v", err) - } - - pgNotify <- true - } - } - - // retry on failure. - for { - backoff := retry.WithMaxDuration(dbReconnectMaxDuration, retry.NewExponential(dbReconnectBackoffBaseDuration)) - err := retry.Do(context.TODO(), backoff, func(ctx context.Context) error { - if err := listen(ctx, pgNotify); err != nil { - return retry.RetryableError(err) - } - - return nil - }) - - logger.Errorf("failed to connect to database: %v", err) - } -} - func (e *EventConsumer) Listen() { logger.Infof("Started listening for database notify events: %v", e.WatchEvents) @@ -175,8 +132,8 @@ func (e *EventConsumer) Listen() { // Consume pending events e.ConsumeEventsUntilEmpty() - pgNotify := make(chan bool) - go e.listenToPostgresNotify(pgNotify) + pgNotify := make(chan struct{}) + go utils.ListenToPostgresNotify(db.Pool, "event_queue_updates", dbReconnectMaxDuration, dbReconnectBackoffBaseDuration, pgNotify) for { select { diff --git a/jobs/jobs.go b/jobs/jobs.go index 4b5e11d95..1cfdfe89c 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -6,7 +6,6 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/incident-commander/api" - "github.com/flanksource/incident-commander/playbook" "github.com/flanksource/incident-commander/responder" "github.com/flanksource/incident-commander/rules" "github.com/flanksource/incident-commander/upstream" @@ -20,7 +19,6 @@ const ( ResponderConfigSyncSchedule = "@every 1h" CleanupJobHistoryTableSchedule = "@every 24h" PushAgentReconcileSchedule = "@every 30m" - ProcessPlaybookRunQueueSchedule = "@every 5s" ) var FuncScheduler = cron.New() @@ -67,11 +65,6 @@ func Start() { } } - job := newFuncJob(playbook.ProcessRunQueue, withName("process playbook-run queue"), withRunNow(true)) - if err := job.schedule(FuncScheduler, ProcessPlaybookRunQueueSchedule); err != nil { - logger.Errorf("Failed to schedule 'playbookRun queue processor' job: %v", err) - } - incidentRulesSchedule := fmt.Sprintf("@every %s", rules.Period.String()) logger.Infof("IncidentRulesSchedule %s", incidentRulesSchedule) if _, err := ScheduleFunc(incidentRulesSchedule, func() { diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go new file mode 100644 index 000000000..0a2efb324 --- /dev/null +++ b/playbook/queue_consumer.go @@ -0,0 +1,96 @@ +package playbook + +import ( + "fmt" + "sync" + "time" + + "github.com/flanksource/commons/logger" + "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" + "github.com/flanksource/incident-commander/db" + "github.com/flanksource/incident-commander/utils" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "gorm.io/gorm" +) + +type queueConsumer struct { + pool *pgxpool.Pool + db *gorm.DB + tickInterval time.Duration + dbReconnectMaxDuration time.Duration + dbReconnectBackoffBaseDuration time.Duration + + // registry stores the list of playbook run IDs + // that are currently being executed. + registry sync.Map +} + +func NewQueueConsumer(db *gorm.DB, pool *pgxpool.Pool) *queueConsumer { + return &queueConsumer{ + db: db, + pool: pool, + tickInterval: time.Minute, + dbReconnectMaxDuration: time.Minute, + dbReconnectBackoffBaseDuration: time.Second, + registry: sync.Map{}, + } +} + +func (t *queueConsumer) Listen() error { + pgNotify := make(chan struct{}) + go utils.ListenToPostgresNotify(db.Pool, "playbook_run_updates", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotify) + + ctx := api.NewContext(t.db, nil) + for { + select { + case <-pgNotify: + if err := t.consumeAll(ctx); err != nil { + logger.Errorf("%v", err) + } + + case <-time.After(t.tickInterval): + if err := t.consumeAll(ctx); err != nil { + logger.Errorf("%v", err) + } + } + } +} + +func (t *queueConsumer) consumeAll(ctx *api.Context) error { + runs, err := db.GetScheduledPlaybookRuns(ctx, t.getRunIDsInRegistry()...) + if err != nil { + return fmt.Errorf("failed to get playbook runs: %w", err) + } + + if len(runs) == 0 { + return nil + } + + for _, r := range runs { + go func(run models.PlaybookRun) { + if _, loaded := t.registry.LoadOrStore(run.ID, nil); !loaded { + if !run.StartTime.After(time.Now()) { + time.Sleep(time.Until(run.StartTime)) + } + + ExecuteRun(ctx, run) + } + + t.registry.Delete(run.ID) + }(r) + } + + return nil +} + +func (t *queueConsumer) getRunIDsInRegistry() []uuid.UUID { + var ids []uuid.UUID + t.registry.Range(func(k any, val any) bool { + ids = append(ids, k.(uuid.UUID)) + return true + }) + + return ids +} diff --git a/playbook/runner.go b/playbook/runner.go index c2362d290..04a05f49d 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -9,7 +9,6 @@ import ( "github.com/flanksource/gomplate/v3" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" - "github.com/flanksource/incident-commander/db" "github.com/flanksource/incident-commander/playbook/actions" ) @@ -26,25 +25,6 @@ func (t *ActionParam) AsMap() map[string]any { } } -func ProcessRunQueue(ctx *api.Context) error { - runs, err := db.GetScheduledPlaybookRuns(ctx) - if err != nil { - return fmt.Errorf("failed to get playbook runs: %w", err) - } - - if len(runs) == 0 { - return nil - } - - logger.Infof("Starting to execute %d playbook runs", len(runs)) - - for _, r := range runs { - go ExecuteRun(ctx, r) - } - - return nil -} - func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { logger.Infof("Executing playbook run: %s", run.ID) @@ -92,7 +72,7 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { } for _, action := range playbook.Spec.Actions { - logger.Infof("Executing playbook run: %s", run.ID) + logger.WithValues("runID", run.ID).Infof("Executing action: %s", action.Name) runAction := models.PlaybookRunAction{ PlaybookRunID: run.ID, diff --git a/utils/pg_notify.go b/utils/pg_notify.go new file mode 100644 index 000000000..59ba19f66 --- /dev/null +++ b/utils/pg_notify.go @@ -0,0 +1,52 @@ +package utils + +import ( + "context" + "fmt" + "time" + + "github.com/flanksource/commons/logger" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/sethvargo/go-retry" +) + +// listenToPostgresNotify listens to postgres notifications +// and will retry on failure for dbReconnectMaxAttempt times +func ListenToPostgresNotify(pool *pgxpool.Pool, channel string, dbReconnectMaxDuration, dbReconnectBackoffBaseDuration time.Duration, pgNotify chan struct{}) { + var listen = func(ctx context.Context, pgNotify chan<- struct{}) error { + conn, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("error acquiring database connection: %v", err) + } + defer conn.Release() + + _, err = conn.Exec(ctx, fmt.Sprintf("LISTEN %s", channel)) + if err != nil { + return fmt.Errorf("error listening to database notifications: %v", err) + } + logger.Debugf("listening to database notifications") + + for { + _, err := conn.Conn().WaitForNotification(ctx) + if err != nil { + return fmt.Errorf("error listening to database notifications: %v", err) + } + + pgNotify <- struct{}{} + } + } + + // retry on failure. + for { + backoff := retry.WithMaxDuration(dbReconnectMaxDuration, retry.NewExponential(dbReconnectBackoffBaseDuration)) + err := retry.Do(context.TODO(), backoff, func(ctx context.Context) error { + if err := listen(ctx, pgNotify); err != nil { + return retry.RetryableError(err) + } + + return nil + }) + + logger.Errorf("failed to connect to database: %v", err) + } +} From 1200a7758ed65097777bfabc8858fa89395f486f Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 11:22:58 +0545 Subject: [PATCH 11/39] feat: add k8s operator [skip ci] --- api/v1/playbook_types.go | 4 ++++ cmd/server.go | 10 ++++++++++ db/playbooks.go | 23 +++++++++++++++++++++++ fixtures/playbooks/ls.yaml | 9 +++++++++ playbook/controllers.go | 7 +++++++ 5 files changed, 53 insertions(+) create mode 100644 fixtures/playbooks/ls.yaml diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index fb1617360..b153af4f1 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -70,3 +70,7 @@ func PlaybookFromModel(p models.Playbook) (Playbook, error) { return out, nil } + +func init() { + SchemeBuilder.Register(&Playbook{}) +} diff --git a/cmd/server.go b/cmd/server.go index 54fed97a0..577ad1281 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -151,6 +151,7 @@ func createHTTPServer(gormDB *gorm.DB) *echo.Echo { playbookGroup := e.Group("/playbook") playbookGroup.POST("/run", playbook.HandlePlaybookRun) playbookGroup.GET("/run/:id", playbook.HandlePlaybookRunStatus) + playbookGroup.GET("/list", playbook.HandlePlaybookRunStatus) forward(e, "/config", configDb) forward(e, "/canary", api.CanaryCheckerPath) @@ -187,6 +188,15 @@ func launchKopper() { logger.Fatalf("Unable to create controller for IncidentRule: %v", err) } + if err = kopper.SetupReconciler( + mgr, + db.PersistPlaybookFromCRD, + db.DeletePlaybook, + "playbook.mission-control.flanksource.com", + ); err != nil { + logger.Fatalf("Unable to create controller for Playbook: %v", err) + } + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { logger.Fatalf("error running manager: %v", err) } diff --git a/db/playbooks.go b/db/playbooks.go index 730b8d9aa..5b08ced88 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -1,10 +1,12 @@ package db import ( + "encoding/json" "errors" "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" "github.com/google/uuid" "gorm.io/gorm" ) @@ -43,3 +45,24 @@ func GetScheduledPlaybookRuns(ctx *api.Context, exceptions ...uuid.UUID) ([]mode return runs, nil } + +func PersistPlaybookFromCRD(obj *v1.Playbook) error { + specJSON, err := json.Marshal(obj.Spec) + if err != nil { + return err + } + + dbObj := models.Playbook{ + ID: uuid.MustParse(string(obj.GetUID())), + Name: obj.Name, + Spec: specJSON, + Source: models.SourceCRD, + CreatedBy: api.SystemUserID, + } + + return Gorm.Save(&dbObj).Error +} + +func DeletePlaybook(id string) error { + return Gorm.Delete(&models.Playbook{}, id).Error +} diff --git a/fixtures/playbooks/ls.yaml b/fixtures/playbooks/ls.yaml new file mode 100644 index 000000000..70335ebd0 --- /dev/null +++ b/fixtures/playbooks/ls.yaml @@ -0,0 +1,9 @@ +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: list temp +spec: + actions: + - exec: + name: "lister" + script: ls /tmp diff --git a/playbook/controllers.go b/playbook/controllers.go index d8ad680f6..de8dac366 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -132,3 +132,10 @@ func HandlePlaybookRunStatus(c echo.Context) error { return c.JSON(http.StatusOK, run) } + +// Takes config id or component id as a query param +// and returns all the available playbook that supports +// the given component or config. +func HandlePlaybookList(c echo.Context) error { + return nil +} From 2058b80d4b791fd65fbb53329fd09d52f518afcc Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 12:14:55 +0545 Subject: [PATCH 12/39] fix: playbook schema for k8 & pass params to the template renderer [skip ci] --- api/v1/playbook_actions.go | 2 +- api/v1/playbook_types.go | 11 ++++++- api/v1/zz_generated.deepcopy.go | 32 +++++++++++++++++++ ...ion-control.flanksource.com_playbooks.yaml | 5 ++- db/playbooks.go | 2 +- fixtures/playbooks/ec2.yaml | 17 ++++++++++ fixtures/playbooks/ls.yaml | 9 ------ fixtures/playbooks/scale-deployment.yaml | 15 +++++++++ playbook/runner.go | 6 +++- 9 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 fixtures/playbooks/ec2.yaml delete mode 100644 fixtures/playbooks/ls.yaml create mode 100644 fixtures/playbooks/scale-deployment.yaml diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index 3766e3010..e32ef39f5 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -16,7 +16,7 @@ type Description struct { // Description for the check Description string `yaml:"description,omitempty" json:"description,omitempty" template:"true"` // Name of the check - Name string `yaml:"name" json:"name" template:"true"` + Name string `yaml:"name,omitempty" json:"name,omitempty" template:"true"` // Icon for overwriting default icon on the dashboard Icon string `yaml:"icon,omitempty" json:"icon,omitempty" template:"true"` // Labels for the check diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index b153af4f1..3d4cf969d 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -71,6 +71,15 @@ func PlaybookFromModel(p models.Playbook) (Playbook, error) { return out, nil } +// +kubebuilder:object:root=true + +// PlaybookList contains a list of Playbook +type PlaybookList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Playbook `json:"items"` +} + func init() { - SchemeBuilder.Register(&Playbook{}) + SchemeBuilder.Register(&Playbook{}, &PlaybookList{}) } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 22c742d3b..8031158e3 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -401,6 +401,38 @@ func (in *PlaybookAction) DeepCopy() *PlaybookAction { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookList) DeepCopyInto(out *PlaybookList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Playbook, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookList. +func (in *PlaybookList) DeepCopy() *PlaybookList { + if in == nil { + return nil + } + out := new(PlaybookList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PlaybookList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlaybookParameter) DeepCopyInto(out *PlaybookParameter) { *out = *in diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index 35402c633..22fabf5be 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -309,11 +309,14 @@ spec: or left as is type: string required: - - name - script type: object + name: + type: string timeout-minutes: type: string + required: + - name type: object type: array components: diff --git a/db/playbooks.go b/db/playbooks.go index 5b08ced88..04a5a1e1f 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -64,5 +64,5 @@ func PersistPlaybookFromCRD(obj *v1.Playbook) error { } func DeletePlaybook(id string) error { - return Gorm.Delete(&models.Playbook{}, id).Error + return Gorm.Delete(&models.Playbook{}, "id = ?", id).Error } diff --git a/fixtures/playbooks/ec2.yaml b/fixtures/playbooks/ec2.yaml new file mode 100644 index 000000000..18d83b290 --- /dev/null +++ b/fixtures/playbooks/ec2.yaml @@ -0,0 +1,17 @@ +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: ec2-restart +spec: + description: Unconventional EC2 restart + configs: # playbooks can run on configs + - type: EC2 Instance + tags: # can filter non-prod vs prod instances + a: b + actions: # 1 or more actions + - name: 'Stop EC2 instance' + exec: + script: aws ec2 stop-instance --instance-id {{.config.instanceId}} + - name: 'Start again' + exec: + script: aws ec2 start-instance --instance-id {{.config.instanceId}} diff --git a/fixtures/playbooks/ls.yaml b/fixtures/playbooks/ls.yaml deleted file mode 100644 index 70335ebd0..000000000 --- a/fixtures/playbooks/ls.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: mission-control.flanksource.com/v1 -kind: Playbook -metadata: - name: list temp -spec: - actions: - - exec: - name: "lister" - script: ls /tmp diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml new file mode 100644 index 000000000..56374b0d5 --- /dev/null +++ b/fixtures/playbooks/scale-deployment.yaml @@ -0,0 +1,15 @@ +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: scale-deployment +spec: + description: Scale deployment + configs: + - type: Kubernetes::Deployment + parameters: + - name: replicas + label: The new desired number of replicas. + actions: + - name: 'scale deployment' + exec: + script: kubectl scale --replicas={{.params.replicas}} deployment {{.config.name}} diff --git a/playbook/runner.go b/playbook/runner.go index 04a05f49d..d949d6378 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -16,12 +16,14 @@ import ( type ActionParam struct { Config *models.ConfigItem `json:"config,omitempty"` Component *models.Component `json:"component,omitempty"` + Params map[string]string `json:"params,omitempty"` } func (t *ActionParam) AsMap() map[string]any { return map[string]any{ "config": t.Config, "component": t.Component, + "params": t.Params, } } @@ -60,7 +62,9 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return err } - var actionParam ActionParam + actionParam := ActionParam{ + Params: run.Parameters, + } if run.ComponentID != nil { if err := ctx.DB().Where("id = ?", run.ComponentID).First(&actionParam.Component).Error; err != nil { return err From 4bb814eaccad2e92fbb360f5f95f0d83261263b5 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 14:28:39 +0545 Subject: [PATCH 13/39] feat: only fetch scheduled runs that should start in the next 10 minutes. * Improve error message on incorrect params [skip ci] --- db/playbooks.go | 7 +++++-- playbook/controllers.go | 18 ++++++++++++++++-- playbook/queue_consumer.go | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/db/playbooks.go b/db/playbooks.go index 04a5a1e1f..9243ee87b 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "errors" + "time" "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" @@ -37,9 +38,11 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } -func GetScheduledPlaybookRuns(ctx *api.Context, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { +// GetScheduledPlaybookRuns returns all the scheduled playbook runs that should be started +// before X duration from now. +func GetScheduledPlaybookRuns(ctx *api.Context, startingBefore time.Duration, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { var runs []models.PlaybookRun - if err := ctx.DB().Not(exceptions).Where("status = ?", models.PlaybookRunStatusScheduled).Find(&runs).Error; err != nil { + if err := ctx.DB().Debug().Not(exceptions).Where("start_time <= NOW() + ?", startingBefore).Where("status = ?", models.PlaybookRunStatusScheduled).Order("start_time").Find(&runs).Error; err != nil { return nil, err } diff --git a/playbook/controllers.go b/playbook/controllers.go index de8dac366..f6e17f802 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -43,9 +43,23 @@ func (r *RunParams) valid() error { return nil } +func paramStr(params []v1.PlaybookParameter) string { + if len(params) == 0 { + return " no params expected." + } + + out := " supported params: " + for _, p := range params { + out += fmt.Sprintf("(%s=%s), ", p.Name, p.Label) + } + + out = out[:len(out)-1] + return out +} + func (r *RunParams) validateParams(params []v1.PlaybookParameter) error { if len(params) != len(r.Params) { - return fmt.Errorf("invalid number of parameters. expected %d, got %d", len(params), len(r.Params)) + return fmt.Errorf("invalid number of parameters. expected %d, got %d.%s", len(params), len(r.Params), paramStr(params)) } for k := range r.Params { @@ -58,7 +72,7 @@ func (r *RunParams) validateParams(params []v1.PlaybookParameter) error { } if !ok { - return fmt.Errorf("unknown parameter %s", k) + return fmt.Errorf("unknown parameter %s.%s", k, paramStr(params)) } } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index 0a2efb324..4c056f70d 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -59,7 +59,7 @@ func (t *queueConsumer) Listen() error { } func (t *queueConsumer) consumeAll(ctx *api.Context) error { - runs, err := db.GetScheduledPlaybookRuns(ctx, t.getRunIDsInRegistry()...) + runs, err := db.GetScheduledPlaybookRuns(ctx, time.Minute*10, t.getRunIDsInRegistry()...) if err != nil { return fmt.Errorf("failed to get playbook runs: %w", err) } From 2959f9686f9d4fb726013555a84da42e60391d28 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 15:02:05 +0545 Subject: [PATCH 14/39] chore: clean up --- cmd/server.go | 5 +---- db/playbooks.go | 2 +- playbook/controllers.go | 8 ++++++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 577ad1281..9513953da 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -148,10 +148,7 @@ func createHTTPServer(gormDB *gorm.DB) *echo.Echo { upstreamGroup.GET("/canary/pull/:agent_name", canary.Pull) upstreamGroup.GET("/status/:agent_name", upstream.Status) - playbookGroup := e.Group("/playbook") - playbookGroup.POST("/run", playbook.HandlePlaybookRun) - playbookGroup.GET("/run/:id", playbook.HandlePlaybookRunStatus) - playbookGroup.GET("/list", playbook.HandlePlaybookRunStatus) + playbook.RegisterRoutes(e, "playbook") forward(e, "/config", configDb) forward(e, "/canary", api.CanaryCheckerPath) diff --git a/db/playbooks.go b/db/playbooks.go index 9243ee87b..12d248cd8 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -42,7 +42,7 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { // before X duration from now. func GetScheduledPlaybookRuns(ctx *api.Context, startingBefore time.Duration, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { var runs []models.PlaybookRun - if err := ctx.DB().Debug().Not(exceptions).Where("start_time <= NOW() + ?", startingBefore).Where("status = ?", models.PlaybookRunStatusScheduled).Order("start_time").Find(&runs).Error; err != nil { + if err := ctx.DB().Not(exceptions).Where("start_time <= NOW() + ?", startingBefore).Where("status = ?", models.PlaybookRunStatusScheduled).Order("start_time").Find(&runs).Error; err != nil { return nil, err } diff --git a/playbook/controllers.go b/playbook/controllers.go index f6e17f802..d2fe0fd3f 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -153,3 +153,11 @@ func HandlePlaybookRunStatus(c echo.Context) error { func HandlePlaybookList(c echo.Context) error { return nil } + +func RegisterRoutes(e *echo.Echo, prefix string) *echo.Group { + playbookGroup := e.Group(fmt.Sprintf("/%s", prefix)) + playbookGroup.POST("/run", HandlePlaybookRun) + playbookGroup.GET("/run/:id", HandlePlaybookRunStatus) + playbookGroup.GET("/list", HandlePlaybookRunStatus) + return playbookGroup +} From 4cdc4d9d00eff1e4895726274a1c660758b25aa2 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 15:52:03 +0545 Subject: [PATCH 15/39] test: playbook runs [skip ci] --- playbook/playbook_test.go | 121 ++++++++++++++++++++++++++++++++++++++ playbook/suite_test.go | 93 +++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 playbook/playbook_test.go create mode 100644 playbook/suite_test.go diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go new file mode 100644 index 000000000..e58151477 --- /dev/null +++ b/playbook/playbook_test.go @@ -0,0 +1,121 @@ +package playbook + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/flanksource/duty/fixtures/dummy" + "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" + ginkgo "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { + playbookSpec := v1.PlaybookSpec{ + Description: "write config name to file", + Parameters: []v1.PlaybookParameter{ + {Name: "path", Label: "path of the file"}, + }, + Actions: []v1.PlaybookAction{ + { + Name: "write config name to a file", + Exec: &v1.ExecAction{ + Script: "printf {{.config.config_class}} > {{.params.path}}", + }, + }, + }, + } + + var ( + playbook models.Playbook + runResp RunResponse + ) + ginkgo.It("should create a new playbook", func() { + spec, err := json.Marshal(playbookSpec) + Expect(err).NotTo(HaveOccurred()) + + playbook = models.Playbook{ + Name: "config name saver", + Spec: spec, + } + + err = testDB.Create(&playbook).Error + Expect(err).NotTo(HaveOccurred()) + }) + + ginkgo.It("should store dummy data", func() { + err := dummy.PopulateDBWithDummyModels(testDB) + Expect(err).NotTo(HaveOccurred()) + }) + + ginkgo.It("should store playbook run via API", func() { + run := RunParams{ + ID: playbook.ID, + ConfigID: dummy.EKSCluster.ID, + Params: map[string]string{ + "path": tempPath, + }, + } + + bodyJSON, err := json.Marshal(run) + Expect(err).NotTo(HaveOccurred()) + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:%d/playbook/run", echoServerPort), bytes.NewBuffer(bodyJSON)) + Expect(err).NotTo(HaveOccurred()) + + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.SetBasicAuth("admin@local", "admin") + + client := http.Client{} + resp, err := client.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + b, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + + fmt.Println(string(b)) + } + + Expect(resp.StatusCode).To(Equal(http.StatusCreated)) + + err = json.NewDecoder(resp.Body).Decode(&runResp) + Expect(err).NotTo(HaveOccurred()) + + var savedRun models.PlaybookRun + err = testDB.Where("id = ? ", runResp.RunID).First(&savedRun).Error + Expect(err).NotTo(HaveOccurred()) + + Expect(savedRun.PlaybookID).To(Equal(playbook.ID)) + }) + + ginkgo.It("should execute playbook", func() { + consumer := NewQueueConsumer(testDB, testDBPool) + + ctx := api.NewContext(testDB, nil) + err := consumer.consumeAll(ctx) + Expect(err).NotTo(HaveOccurred()) + + // Wait for the action to complete + time.Sleep(time.Second * 5) + + var updatedRun models.PlaybookRun + err = testDB.Where("id = ? ", runResp.RunID).First(&updatedRun).Error + Expect(err).NotTo(HaveOccurred()) + + Expect(updatedRun.Status).To(Equal(models.PlaybookRunStatusCompleted)) + + f, err := os.ReadFile(tempPath) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(f)).To(Equal(dummy.EKSCluster.ConfigClass)) + }) +}) diff --git a/playbook/suite_test.go b/playbook/suite_test.go new file mode 100644 index 000000000..83d34b86c --- /dev/null +++ b/playbook/suite_test.go @@ -0,0 +1,93 @@ +package playbook + +import ( + "context" + "fmt" + "net/http" + "testing" + + embeddedPG "github.com/fergusstrange/embedded-postgres" + "github.com/flanksource/commons/logger" + "github.com/flanksource/duty" + "github.com/flanksource/duty/testutils" + "github.com/flanksource/incident-commander/api" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/labstack/echo/v4" + ginkgo "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +func TestPlaybook(t *testing.T) { + tempPath = fmt.Sprintf("%s/config-class.txt", t.TempDir()) + RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "Playbook") +} + +var ( + // postgres server shared by both agent and upstream + postgresServer *embeddedPG.EmbeddedPostgres + pgServerPort = 9884 + + // tempPath is used to store the result of the action for this test. + tempPath string + + echoServerPort = 11007 + echoServer *echo.Echo + + testDB *gorm.DB + testDBPool *pgxpool.Pool +) + +var _ = ginkgo.BeforeSuite(func() { + config, connection := testutils.GetEmbeddedPGConfig("test", pgServerPort) + postgresServer = embeddedPG.NewDatabase(config) + if err := postgresServer.Start(); err != nil { + ginkgo.Fail(err.Error()) + } + logger.Infof("Started postgres on port: %d", pgServerPort) + + var err error + if testDB, testDBPool, err = duty.SetupDB(connection, nil); err != nil { + ginkgo.Fail(err.Error()) + } + + setupUpstreamHTTPServer() +}) + +var _ = ginkgo.AfterSuite(func() { + logger.Infof("Stopping upstream echo server") + if err := echoServer.Shutdown(context.Background()); err != nil { + ginkgo.Fail(err.Error()) + } + + logger.Infof("Stopping postgres") + if err := postgresServer.Stop(); err != nil { + ginkgo.Fail(err.Error()) + } +}) + +func setupUpstreamHTTPServer() { + echoServer = echo.New() + echoServer.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + cc := api.NewContext(testDB, c) + return next(cc) + } + }) + + RegisterRoutes(echoServer, "playbook") + + listenAddr := fmt.Sprintf(":%d", echoServerPort) + + go func() { + defer ginkgo.GinkgoRecover() // Required by ginkgo, if an assertion is made in a goroutine. + if err := echoServer.Start(listenAddr); err != nil { + if err == http.ErrServerClosed { + logger.Infof("Server closed") + } else { + ginkgo.Fail(fmt.Sprintf("Failed to start test server: %v", err)) + } + } + }() +} From a9eb8fca5f597a9b83e446506746a05b7fb7bbab Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 17 Aug 2023 16:08:21 +0545 Subject: [PATCH 16/39] improved test [skip ci] --- playbook/playbook_test.go | 34 +++++++++++++++++++++++++++++----- playbook/suite_test.go | 3 +-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index e58151477..5b5eaff85 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -13,6 +13,7 @@ import ( "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" + "github.com/google/uuid" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -25,9 +26,15 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { }, Actions: []v1.PlaybookAction{ { - Name: "write config name to a file", + Name: "write config id to a file", Exec: &v1.ExecAction{ - Script: "printf {{.config.config_class}} > {{.params.path}}", + Script: "printf {{.config.id}} > {{.params.path}}", + }, + }, + { + Name: "append config class to the same file ", + Exec: &v1.ExecAction{ + Script: "printf {{.config.config_class}} >> {{.params.path}}", }, }, }, @@ -104,8 +111,19 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { err := consumer.consumeAll(ctx) Expect(err).NotTo(HaveOccurred()) - // Wait for the action to complete - time.Sleep(time.Second * 5) + // Wait until all the runs are processed + var attempts int + for { + time.Sleep(time.Second) // need to wait initially before trying. + if _, ok := consumer.registry.Load(uuid.MustParse(runResp.RunID)); !ok { + break + } + + attempts += 1 + if attempts > 5 { + ginkgo.Fail("Timed out waiting for run to complete") + } + } var updatedRun models.PlaybookRun err = testDB.Where("id = ? ", runResp.RunID).First(&updatedRun).Error @@ -113,9 +131,15 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Expect(updatedRun.Status).To(Equal(models.PlaybookRunStatusCompleted)) + var runActions []models.PlaybookRunAction + err = testDB.Where("playbook_run_id = ?", updatedRun.ID).Find(&runActions).Error + Expect(err).NotTo(HaveOccurred()) + + Expect(len(runActions)).To(Equal(2)) + f, err := os.ReadFile(tempPath) Expect(err).NotTo(HaveOccurred()) - Expect(string(f)).To(Equal(dummy.EKSCluster.ConfigClass)) + Expect(string(f)).To(Equal(fmt.Sprintf("%s%s", dummy.EKSCluster.ID, dummy.EKSCluster.ConfigClass))) }) }) diff --git a/playbook/suite_test.go b/playbook/suite_test.go index 83d34b86c..68b19deec 100644 --- a/playbook/suite_test.go +++ b/playbook/suite_test.go @@ -25,9 +25,8 @@ func TestPlaybook(t *testing.T) { } var ( - // postgres server shared by both agent and upstream postgresServer *embeddedPG.EmbeddedPostgres - pgServerPort = 9884 + pgServerPort = 9885 // tempPath is used to store the result of the action for this test. tempPath string From 147be3b48d09a54bd2760abbdcff00b5cd8eec1b Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 18 Aug 2023 09:16:55 +0545 Subject: [PATCH 17/39] feat: add user ID to the context. It's used to save the creator in a playbook run. [skip ci] --- api/global.go | 14 ++++++++++++++ auth/clerk_client.go | 5 ++++- auth/middleware.go | 6 +++++- playbook/controllers.go | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/api/global.go b/api/global.go index 25c49a711..fc6602778 100644 --- a/api/global.go +++ b/api/global.go @@ -15,6 +15,10 @@ import ( "k8s.io/client-go/kubernetes" ) +type ContextKey string + +const UserIDContextKey ContextKey = "User-ID" + var SystemUserID *uuid.UUID var CanaryCheckerPath string var ApmHubPath string @@ -57,6 +61,16 @@ func (c *Context) DB() *gorm.DB { return c.db.WithContext(c.Context) } +func (c *Context) UserID() *uuid.UUID { + id := c.Context.Value(UserIDContextKey).(string) + u, err := uuid.Parse(id) + if err != nil { + return nil + } + + return &u +} + func (c *Context) GetEnvVarValue(input types.EnvVar) (string, error) { return duty.GetEnvValueFromCache(c.Kubernetes, input, c.Namespace) } diff --git a/auth/clerk_client.go b/auth/clerk_client.go index 55441a6af..6a2dd9bf9 100644 --- a/auth/clerk_client.go +++ b/auth/clerk_client.go @@ -1,6 +1,7 @@ package auth import ( + "context" "fmt" "net/http" "strings" @@ -84,7 +85,9 @@ func (h ClerkHandler) Session(next echo.HandlerFunc) echo.HandlerFunc { c.Request().Header.Set(echo.HeaderAuthorization, fmt.Sprintf("Bearer %s", token)) c.Request().Header.Set(UserIDHeaderKey, *user.ExternalID) - return next(c) + + ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, *user.ExternalID) + return next(ctx) } } diff --git a/auth/middleware.go b/auth/middleware.go index b9a12fa10..df833b0c1 100644 --- a/auth/middleware.go +++ b/auth/middleware.go @@ -10,6 +10,7 @@ import ( "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" + "github.com/flanksource/incident-commander/api" "github.com/flanksource/incident-commander/utils" "github.com/golang-jwt/jwt/v4" "github.com/labstack/echo/v4" @@ -72,7 +73,10 @@ func (k *kratosMiddleware) Session(next echo.HandlerFunc) echo.HandlerFunc { c.Request().Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) c.Request().Header.Set(UserIDHeaderKey, session.Identity.GetId()) - return next(c) + ctx := c.(*api.Context) + ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, session.Identity.GetId()) + + return next(ctx) } } diff --git a/playbook/controllers.go b/playbook/controllers.go index d2fe0fd3f..38c4c5460 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -112,7 +112,7 @@ func HandlePlaybookRun(c echo.Context) error { PlaybookID: playbook.ID, Status: models.PlaybookRunStatusScheduled, Parameters: types.JSONStringMap(req.Params), - // CreatedBy: ctx.User().ID, // TODO: Add user id to the context from a middleware + CreatedBy: ctx.UserID(), } if req.ComponentID != uuid.Nil { From e033eb75d7c643842c35c02f0b0c1091a5dfa7f8 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 21 Aug 2023 10:31:24 +0545 Subject: [PATCH 18/39] feat: show a list of playbooks that can be applied to a config --- db/playbooks.go | 20 ++++++++++++++++++++ playbook/controllers.go | 31 +++++++++++++++++++++++++++++-- playbook/playbook.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 playbook/playbook.go diff --git a/db/playbooks.go b/db/playbooks.go index 12d248cd8..677167d7b 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -6,6 +6,7 @@ import ( "time" "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/google/uuid" @@ -38,6 +39,25 @@ func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } +// FindPlaybooksByTypeAndTags returns all the playbooks that match the given type and tags. +func FindPlaybooksByTypeAndTags(ctx *api.Context, configType string, tags map[string]string) ([]models.Playbook, error) { + joinQuery := `JOIN LATERAL jsonb_array_elements(playbooks."spec"->'configs') AS configs(config) ON 1=1` + if tags != nil { + joinQuery += " AND (?::jsonb) @> (configs.config->'tags')" + } + if configType != "" { + joinQuery += " AND configs.config->>'type' = ?" + } + + query := ctx.DB(). + Select("DISTINCT playbooks.*"). + Joins(joinQuery, types.JSONStringMap(tags), configType) + + var playbooks []models.Playbook + err := query.Find(&playbooks).Error + return playbooks, err +} + // GetScheduledPlaybookRuns returns all the scheduled playbook runs that should be started // before X duration from now. func GetScheduledPlaybookRuns(ctx *api.Context, startingBefore time.Duration, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { diff --git a/playbook/controllers.go b/playbook/controllers.go index 38c4c5460..df7a22f69 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -151,13 +151,40 @@ func HandlePlaybookRunStatus(c echo.Context) error { // and returns all the available playbook that supports // the given component or config. func HandlePlaybookList(c echo.Context) error { - return nil + ctx := c.(*api.Context) + + var ( + configID = c.QueryParam("config_id") + componentID = c.QueryParam("component_id") + ) + + if configID == "" && componentID == "" { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: "either config_id or component_id is required", Message: "invalid request"}) + } else if configID != "" && componentID != "" { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: "only of either config_id or component_id is required", Message: "invalid request"}) + } + + var playbooks []models.Playbook + var err error + if configID != "" { + playbooks, err = ListPlaybooksOfConfig(ctx, configID) + if err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to list playbooks"}) + } + } else if componentID != "" { + playbooks, err = ListPlaybooksOfComponent(ctx, componentID) + if err != nil { + return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to list playbooks"}) + } + } + + return c.JSON(http.StatusOK, playbooks) } func RegisterRoutes(e *echo.Echo, prefix string) *echo.Group { playbookGroup := e.Group(fmt.Sprintf("/%s", prefix)) playbookGroup.POST("/run", HandlePlaybookRun) playbookGroup.GET("/run/:id", HandlePlaybookRunStatus) - playbookGroup.GET("/list", HandlePlaybookRunStatus) + playbookGroup.GET("/list", HandlePlaybookList) return playbookGroup } diff --git a/playbook/playbook.go b/playbook/playbook.go new file mode 100644 index 000000000..497a4a9a8 --- /dev/null +++ b/playbook/playbook.go @@ -0,0 +1,32 @@ +package playbook + +import ( + "fmt" + + "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" + "github.com/flanksource/incident-commander/db" + "github.com/google/uuid" +) + +func ListPlaybooksOfConfig(ctx *api.Context, id string) ([]models.Playbook, error) { + var config models.ConfigItem + if err := ctx.DB().Where("id = ?", id).Find(&config).Error; err != nil { + return nil, err + } else if config.ID == uuid.Nil { + return nil, fmt.Errorf("config(id=%s) not found", id) + } + + return db.FindPlaybooksByTypeAndTags(ctx, *config.Type, *config.Tags) +} + +func ListPlaybooksOfComponent(ctx *api.Context, id string) ([]models.Playbook, error) { + var component models.Component + if err := ctx.DB().Where("id = ?", id).Find(&component).Error; err != nil { + return nil, err + } else if component.ID == uuid.Nil { + return nil, fmt.Errorf("component(id=%s) not found", id) + } + + return db.FindPlaybooksByTypeAndTags(ctx, component.Type, component.Labels) +} From 94248355490ce63abb4448370bd29e56177ee2a3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 21 Aug 2023 10:39:15 +0545 Subject: [PATCH 19/39] chore: update fixtures [skip ci] --- fixtures/playbooks/deleting-configmap.yaml | 15 +++++++++++++++ fixtures/playbooks/scale-deployment.yaml | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 fixtures/playbooks/deleting-configmap.yaml diff --git a/fixtures/playbooks/deleting-configmap.yaml b/fixtures/playbooks/deleting-configmap.yaml new file mode 100644 index 000000000..4e636d257 --- /dev/null +++ b/fixtures/playbooks/deleting-configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: delete-kubernetes-configmap +spec: + description: Delete Kubernetes ConfigMap + configs: + - type: Kubernetes::ConfigMap + tags: + namespace: default + cluster: local-kind-cluster + actions: + - name: 'Delete ConfigMap' + exec: + script: kubectl delete configmap {{.config.name}} diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index 56374b0d5..c7a1549d0 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -6,6 +6,9 @@ spec: description: Scale deployment configs: - type: Kubernetes::Deployment + tags: + namespace: default + cluster: local-kind-cluster parameters: - name: replicas label: The new desired number of replicas. From 6dacf80fdfb24e2b3ee94c85765938851b5f5703 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 21 Aug 2023 17:07:03 +0545 Subject: [PATCH 20/39] chore: make use of the payload sent by pg_notify --- events/event_consumer.go | 2 +- utils/pg_notify.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/events/event_consumer.go b/events/event_consumer.go index b20bb15f6..d03ffece2 100644 --- a/events/event_consumer.go +++ b/events/event_consumer.go @@ -132,7 +132,7 @@ func (e *EventConsumer) Listen() { // Consume pending events e.ConsumeEventsUntilEmpty() - pgNotify := make(chan struct{}) + pgNotify := make(chan string) go utils.ListenToPostgresNotify(db.Pool, "event_queue_updates", dbReconnectMaxDuration, dbReconnectBackoffBaseDuration, pgNotify) for { diff --git a/utils/pg_notify.go b/utils/pg_notify.go index 59ba19f66..9fe2c2a84 100644 --- a/utils/pg_notify.go +++ b/utils/pg_notify.go @@ -12,8 +12,8 @@ import ( // listenToPostgresNotify listens to postgres notifications // and will retry on failure for dbReconnectMaxAttempt times -func ListenToPostgresNotify(pool *pgxpool.Pool, channel string, dbReconnectMaxDuration, dbReconnectBackoffBaseDuration time.Duration, pgNotify chan struct{}) { - var listen = func(ctx context.Context, pgNotify chan<- struct{}) error { +func ListenToPostgresNotify(pool *pgxpool.Pool, channel string, dbReconnectMaxDuration, dbReconnectBackoffBaseDuration time.Duration, pgNotify chan<- string) { + var listen = func(ctx context.Context, pgNotify chan<- string) error { conn, err := pool.Acquire(ctx) if err != nil { return fmt.Errorf("error acquiring database connection: %v", err) @@ -27,12 +27,12 @@ func ListenToPostgresNotify(pool *pgxpool.Pool, channel string, dbReconnectMaxDu logger.Debugf("listening to database notifications") for { - _, err := conn.Conn().WaitForNotification(ctx) + n, err := conn.Conn().WaitForNotification(ctx) if err != nil { return fmt.Errorf("error listening to database notifications: %v", err) } - pgNotify <- struct{}{} + pgNotify <- n.Payload } } From 995e14189805db06926fcf2080c3bc34a5ebb410 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 21 Aug 2023 17:07:43 +0545 Subject: [PATCH 21/39] feat: implement playbook approvals [skip ci] --- api/v1/playbook_types.go | 29 +++++++++ api/v1/zz_generated.deepcopy.go | 46 +++++++++++++ ...ion-control.flanksource.com_playbooks.yaml | 16 +++++ db/playbooks.go | 16 +++++ fixtures/playbooks/scale-deployment.yaml | 7 ++ playbook/controllers.go | 6 +- playbook/queue_consumer.go | 65 ++++++++++++++++++- 7 files changed, 183 insertions(+), 2 deletions(-) diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index 3d4cf969d..d032d040d 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -27,6 +27,34 @@ type PlaybookParameter struct { Label string `json:"label,omitempty" yaml:"label,omitempty"` } +type PlaybookApprovers struct { + People []string `json:"people,omitempty" yaml:"people,omitempty"` + Teams []string `json:"teams,omitempty" yaml:"teams,omitempty"` +} + +func (t *PlaybookApprovers) Empty() bool { + return len(t.People) == 0 && len(t.Teams) == 0 +} + +func (t *PlaybookApprovers) IDs() []string { + return append(t.People, t.Teams...) +} + +type PlaybookApprovalType string + +const ( + // PlaybookApprovalTypeAny means just a single approval can suffice. + PlaybookApprovalTypeAny PlaybookApprovalType = "any" + + // PlaybookApprovalTypeAll means all approvals are required + PlaybookApprovalTypeAll PlaybookApprovalType = "all" +) + +type PlaybookApproval struct { + Type PlaybookApprovalType `json:"type,omitempty" yaml:"type,omitempty"` + Approvers PlaybookApprovers `json:"approvers,omitempty" yaml:"approvers,omitempty"` +} + type PlaybookSpec struct { Description string `json:"description,omitempty" yaml:"description,omitempty"` Permissions []Permission `json:"permissions,omitempty" yaml:"permissions,omitempty"` @@ -34,6 +62,7 @@ type PlaybookSpec struct { Components []PlaybookResourceFilter `json:"components,omitempty" yaml:"components,omitempty"` Parameters []PlaybookParameter `json:"parameters,omitempty" yaml:"parameters,omitempty"` Actions []PlaybookAction `json:"actions" yaml:"actions"` + Approval *PlaybookApproval `json:"approval,omitempty" yaml:"approval,omitempty"` } // PlaybookStatus defines the observed state of Playbook diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 8031158e3..20ec9dcdc 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -401,6 +401,47 @@ func (in *PlaybookAction) DeepCopy() *PlaybookAction { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookApproval) DeepCopyInto(out *PlaybookApproval) { + *out = *in + in.Approvers.DeepCopyInto(&out.Approvers) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookApproval. +func (in *PlaybookApproval) DeepCopy() *PlaybookApproval { + if in == nil { + return nil + } + out := new(PlaybookApproval) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookApprovers) DeepCopyInto(out *PlaybookApprovers) { + *out = *in + if in.People != nil { + in, out := &in.People, &out.People + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Teams != nil { + in, out := &in.Teams, &out.Teams + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookApprovers. +func (in *PlaybookApprovers) DeepCopy() *PlaybookApprovers { + if in == nil { + return nil + } + out := new(PlaybookApprovers) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlaybookList) DeepCopyInto(out *PlaybookList) { *out = *in @@ -504,6 +545,11 @@ func (in *PlaybookSpec) DeepCopyInto(out *PlaybookSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Approval != nil { + in, out := &in.Approval, &out.Approval + *out = new(PlaybookApproval) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookSpec. diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index 22fabf5be..02b3e8764 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -319,6 +319,22 @@ spec: - name type: object type: array + approval: + properties: + approvers: + properties: + people: + items: + type: string + type: array + teams: + items: + type: string + type: array + type: object + type: + type: string + type: object components: items: description: PlaybookResourceFilter defines a filter that decides diff --git a/db/playbooks.go b/db/playbooks.go index 677167d7b..c1bf508f6 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -10,6 +10,7 @@ import ( "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/google/uuid" + "github.com/lib/pq" "gorm.io/gorm" ) @@ -89,3 +90,18 @@ func PersistPlaybookFromCRD(obj *v1.Playbook) error { func DeletePlaybook(id string) error { return Gorm.Delete(&models.Playbook{}, "id = ?", id).Error } + +func UpdateApprovedPlaybookRuns(ctx *api.Context, playbookID string, approverIDs []string) error { + query := ` + WITH run_approvals AS ( + SELECT run_id, ARRAY_AGG(COALESCE(person_id, team_id)) AS ids + FROM playbook_approvals + GROUP BY run_id + ) + UPDATE playbook_runs SET status = ? WHERE + status = ? + AND playbook_id = ? + AND id IN (SELECT run_id FROM run_approvals WHERE ids @> ?)` + + return ctx.DB().Debug().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approverIDs)).Error +} diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index c7a1549d0..9820106ce 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -12,6 +12,13 @@ spec: parameters: - name: replicas label: The new desired number of replicas. + approval: + type: any + approvers: + people: + - 2611d086-926b-4d49-b616-845e61fd6fb2 + teams: + - 018770c4-4b73-5d44-8bb5-0e849d62e461 actions: - name: 'scale deployment' exec: diff --git a/playbook/controllers.go b/playbook/controllers.go index df7a22f69..69f009a06 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -110,11 +110,15 @@ func HandlePlaybookRun(c echo.Context) error { run := models.PlaybookRun{ PlaybookID: playbook.ID, - Status: models.PlaybookRunStatusScheduled, + Status: models.PlaybookRunStatusPending, Parameters: types.JSONStringMap(req.Params), CreatedBy: ctx.UserID(), } + if spec.Approval.Approvers.Empty() { + run.Status = models.PlaybookRunStatusScheduled + } + if req.ComponentID != uuid.Nil { run.ComponentID = &req.ComponentID } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index 4c056f70d..ec193b954 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -1,6 +1,7 @@ package playbook import ( + "encoding/json" "fmt" "sync" "time" @@ -8,6 +9,7 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" "github.com/flanksource/incident-commander/utils" "github.com/google/uuid" @@ -39,9 +41,15 @@ func NewQueueConsumer(db *gorm.DB, pool *pgxpool.Pool) *queueConsumer { } func (t *queueConsumer) Listen() error { - pgNotify := make(chan struct{}) + pgNotify := make(chan string) go utils.ListenToPostgresNotify(db.Pool, "playbook_run_updates", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotify) + pgNotifyPlaybookSpecApprovalUpdated := make(chan string) + go utils.ListenToPostgresNotify(db.Pool, "playbook_spec_approval_updated", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookSpecApprovalUpdated) + + pgNotifyPlaybookApprovalsInserted := make(chan string) + go utils.ListenToPostgresNotify(db.Pool, "playbook_approval_inserted", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookApprovalsInserted) + ctx := api.NewContext(t.db, nil) for { select { @@ -50,6 +58,16 @@ func (t *queueConsumer) Listen() error { logger.Errorf("%v", err) } + case id := <-pgNotifyPlaybookSpecApprovalUpdated: + if err := t.onPlaybookSpecApprovalUpdated(ctx, id); err != nil { + logger.Errorf("%v", err) + } + + case id := <-pgNotifyPlaybookApprovalsInserted: + if err := t.onPlaybookRunNewApproval(ctx, id); err != nil { + logger.Errorf("%v", err) + } + case <-time.After(t.tickInterval): if err := t.consumeAll(ctx); err != nil { logger.Errorf("%v", err) @@ -58,6 +76,51 @@ func (t *queueConsumer) Listen() error { } } +func (t *queueConsumer) onPlaybookSpecApprovalUpdated(ctx *api.Context, playbookID string) error { + var playbook models.Playbook + if err := ctx.DB().Where("id = ?", playbookID).First(&playbook).Error; err != nil { + return err + } + + var spec v1.PlaybookSpec + if err := json.Unmarshal(playbook.Spec, &spec); err != nil { + return err + } + + if spec.Approval == nil { + return nil + } + + return db.UpdateApprovedPlaybookRuns(ctx, playbookID, spec.Approval.Approvers.IDs()) +} + +func (t *queueConsumer) onPlaybookRunNewApproval(ctx *api.Context, runID string) error { + var run models.PlaybookRun + if err := ctx.DB().Where("id = ?", runID).First(&run).Error; err != nil { + return err + } + + if run.Status != models.PlaybookRunStatusPending { + return nil + } + + var playbook models.Playbook + if err := ctx.DB().Where("id = ?", run.PlaybookID).First(&playbook).Error; err != nil { + return err + } + + var spec v1.PlaybookSpec + if err := json.Unmarshal(playbook.Spec, &spec); err != nil { + return err + } + + if spec.Approval == nil { + return nil + } + + return db.UpdateApprovedPlaybookRuns(ctx, playbook.ID.String(), spec.Approval.Approvers.IDs()) +} + func (t *queueConsumer) consumeAll(ctx *api.Context) error { runs, err := db.GetScheduledPlaybookRuns(ctx, time.Minute*10, t.getRunIDsInRegistry()...) if err != nil { From 024b9cfe3e644ca73c459a084ed70c0346080352 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 22 Aug 2023 14:29:38 +0545 Subject: [PATCH 22/39] chore: fix test and context userid error [skip ci] --- api/global.go | 6 +++++- playbook/playbook_test.go | 8 +++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/global.go b/api/global.go index fc6602778..e49b8d020 100644 --- a/api/global.go +++ b/api/global.go @@ -62,7 +62,11 @@ func (c *Context) DB() *gorm.DB { } func (c *Context) UserID() *uuid.UUID { - id := c.Context.Value(UserIDContextKey).(string) + id, ok := c.Context.Value(UserIDContextKey).(string) + if !ok { + return nil + } + u, err := uuid.Parse(id) if err != nil { return nil diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index 5b5eaff85..bc8d0ca96 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -49,8 +49,9 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Expect(err).NotTo(HaveOccurred()) playbook = models.Playbook{ - Name: "config name saver", - Spec: spec, + Name: "config name saver", + Spec: spec, + Source: models.SourceConfigFile, } err = testDB.Create(&playbook).Error @@ -58,7 +59,8 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { }) ginkgo.It("should store dummy data", func() { - err := dummy.PopulateDBWithDummyModels(testDB) + dataset := dummy.GetStaticDummyData() + err := dataset.Populate(testDB) Expect(err).NotTo(HaveOccurred()) }) From 6bdb6d014c07074a231819b9922f938e2eb4332c Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 22 Aug 2023 15:30:31 +0545 Subject: [PATCH 23/39] feat: impl approval endpoint and better error handling [skip ci] --- api/errors.go | 86 +++++++++++++++++++++++++++++++++++++++++ api/http.go | 37 +++++++++++++++++- db/playbooks.go | 10 +++++ playbook/approval.go | 37 ++++++++++++++++++ playbook/controllers.go | 40 +++++++++++++++++-- 5 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 api/errors.go create mode 100644 playbook/approval.go diff --git a/api/errors.go b/api/errors.go new file mode 100644 index 000000000..96db3d10a --- /dev/null +++ b/api/errors.go @@ -0,0 +1,86 @@ +package api + +import ( + "errors" + "fmt" +) + +// Application error codes. +// +// These are meant to be generic and they map well to HTTP error codes. +const ( + ECONFLICT = "conflict" + EFORBIDDEN = "forbidden" + EINTERNAL = "internal" + EINVALID = "invalid" + ENOTFOUND = "not_found" + ENOTIMPLEMENTED = "not_implemented" + EUNAUTHORIZED = "unauthorized" +) + +// Error represents an application-specific error. +type Error struct { + // Machine-readable error code. + Code string + + // Human-readable error message. + Message string + + // DebugInfo contains low-level internal error details that should only be logged. + // End-users should never see this. + DebugInfo string +} + +// Error implements the error interface. Not used by the application otherwise. +func (e *Error) Error() string { + return fmt.Sprintf("error: code=%s message=%s", e.Code, e.Message) +} + +// WithDebugInfo wraps an application error with a debug message. +func (e *Error) WithDebugInfo(msg string, args ...any) *Error { + e.DebugInfo = fmt.Sprintf(msg, args...) + return e +} + +// ErrorCode unwraps an application error and returns its code. +// Non-application errors always return EINTERNAL. +func ErrorCode(err error) string { + var e *Error + if err == nil { + return "" + } else if errors.As(err, &e) { + return e.Code + } + return EINTERNAL +} + +// ErrorMessage unwraps an application error and returns its message. +// Non-application errors always return "Internal error". +func ErrorMessage(err error) string { + var e *Error + if err == nil { + return "" + } else if errors.As(err, &e) { + return e.Message + } + return "Internal error." +} + +// ErrorDebugInfo unwraps an application error and returns its debug message. +func ErrorDebugInfo(err error) string { + var e *Error + if err == nil { + return "" + } else if errors.As(err, &e) { + return e.DebugInfo + } + return "" +} + +// Errorf is a helper function to return an Error with a given code and formatted message. +func Errorf(code string, format string, args ...any) *Error { + return &Error{ + Code: code, + Message: fmt.Sprintf(format, args...), + } +} diff --git a/api/http.go b/api/http.go index 2a035e019..41f2b3e84 100644 --- a/api/http.go +++ b/api/http.go @@ -1,11 +1,46 @@ package api +import ( + "net/http" + + "github.com/flanksource/commons/logger" + "github.com/labstack/echo/v4" +) + type HTTPError struct { Error string `json:"error"` - Message string `json:"message"` + Message string `json:"message,omitempty"` } type HTTPSuccess struct { Message string `json:"message"` Payload any `json:"payload,omitempty"` } + +func WriteError(c echo.Context, err error) error { + code, message := ErrorCode(err), ErrorMessage(err) + + if debugInfo := ErrorDebugInfo(err); debugInfo != "" { + logger.WithValues("code", code, "error", message).Errorf(debugInfo) + } + + return c.JSON(ErrorStatusCode(code), &HTTPError{Error: message}) +} + +// lookup of application error codes to HTTP status codes. +var codes = map[string]int{ + ECONFLICT: http.StatusConflict, + EINVALID: http.StatusBadRequest, + ENOTFOUND: http.StatusNotFound, + ENOTIMPLEMENTED: http.StatusNotImplemented, + EUNAUTHORIZED: http.StatusUnauthorized, + EINTERNAL: http.StatusInternalServerError, +} + +// ErrorStatusCode returns the associated HTTP status code for an application error code. +func ErrorStatusCode(code string) int { + if v, ok := codes[code]; ok { + return v + } + return http.StatusInternalServerError +} diff --git a/db/playbooks.go b/db/playbooks.go index c1bf508f6..dc894f96b 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -105,3 +105,13 @@ func UpdateApprovedPlaybookRuns(ctx *api.Context, playbookID string, approverIDs return ctx.DB().Debug().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approverIDs)).Error } + +func ApprovePlaybookRun(ctx *api.Context, runID uuid.UUID, personID, teamID *uuid.UUID) error { + playbookApproval := models.PlaybookApproval{ + RunID: runID, + PersonID: personID, + TeamID: teamID, + } + + return ctx.DB().Create(&playbookApproval).Error +} diff --git a/playbook/approval.go b/playbook/approval.go new file mode 100644 index 000000000..055521ae5 --- /dev/null +++ b/playbook/approval.go @@ -0,0 +1,37 @@ +package playbook + +import ( + "github.com/flanksource/commons/collections" + "github.com/flanksource/incident-commander/api" + v1 "github.com/flanksource/incident-commander/api/v1" + "github.com/flanksource/incident-commander/db" + "github.com/google/uuid" +) + +func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error { + playbook, err := db.FindPlaybook(ctx, playbookID) + if err != nil { + return api.Errorf(api.EINTERNAL, "something went wrong while finding playbook(id=%s)", playbookID).WithDebugInfo("db.FindPlaybook(id=%s): %v", playbookID, err) + } else if playbook == nil { + return api.Errorf(api.ENOTFOUND, "playbook(id=%s) not found", playbookID) + } + + playbookV1, err := v1.PlaybookFromModel(*playbook) + if err != nil { + return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("v1.PlaybookFromModel: %v", err) + } + + if playbookV1.Spec.Approval == nil || playbookV1.Spec.Approval.Approvers.Empty() { + return api.Errorf(api.EINVALID, "this playbook does not require approval") + } + + if !collections.Contains(playbookV1.Spec.Approval.Approvers.IDs(), approverID.String()) { + return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook") + } + + if err := db.ApprovePlaybookRun(ctx, runID, &approverID, nil); err != nil { + return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approverID, err) + } + + return nil +} diff --git a/playbook/controllers.go b/playbook/controllers.go index 69f009a06..572e478d2 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -115,7 +115,7 @@ func HandlePlaybookRun(c echo.Context) error { CreatedBy: ctx.UserID(), } - if spec.Approval.Approvers.Empty() { + if spec.Approval == nil || spec.Approval.Approvers.Empty() { run.Status = models.PlaybookRunStatusScheduled } @@ -185,10 +185,44 @@ func HandlePlaybookList(c echo.Context) error { return c.JSON(http.StatusOK, playbooks) } +func HandlePlaybookRunApproval(c echo.Context) error { + ctx := c.(*api.Context) + + var ( + userID = ctx.UserID() + playbookID = c.Param("playbook_id") + runID = c.Param("run_id") + ) + + if userID == nil { + return c.JSON(http.StatusUnauthorized, api.HTTPError{Error: "user id is required"}) + } + + playbookUUID, err := uuid.Parse(playbookID) + if err != nil { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid playbook id"}) + } + + runUUID, err := uuid.Parse(runID) + if err != nil { + return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid run id"}) + } + + if err := ApproveRun(ctx, *userID, playbookUUID, runUUID); err != nil { + return api.WriteError(c, err) + } + + return c.JSON(http.StatusOK, api.HTTPSuccess{Message: "playbook run approved"}) +} + func RegisterRoutes(e *echo.Echo, prefix string) *echo.Group { playbookGroup := e.Group(fmt.Sprintf("/%s", prefix)) - playbookGroup.POST("/run", HandlePlaybookRun) - playbookGroup.GET("/run/:id", HandlePlaybookRunStatus) playbookGroup.GET("/list", HandlePlaybookList) + + runGroup := playbookGroup.Group("/run") + runGroup.POST("", HandlePlaybookRun) + runGroup.GET(":id", HandlePlaybookRunStatus) + runGroup.POST("/approve/:playbook_id/:run_id", HandlePlaybookRunApproval) + return playbookGroup } From 3e3aea10ccf5b8ee302fd8ef03a505382e2d5a04 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 22 Aug 2023 16:44:10 +0545 Subject: [PATCH 24/39] feat: support different approval types [skip ci] --- db/playbooks.go | 22 +++++++++++++++++----- events/upstream_test.go | 2 +- fixtures/playbooks/scale-deployment.yaml | 7 ++++--- go.mod | 5 ++--- go.sum | 2 -- jobs/event_queue.go | 4 ++-- playbook/approval.go | 7 +++++++ playbook/queue_consumer.go | 4 ++-- 8 files changed, 35 insertions(+), 18 deletions(-) diff --git a/db/playbooks.go b/db/playbooks.go index dc894f96b..c3bdea761 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "errors" + "fmt" "time" "github.com/flanksource/duty/models" @@ -91,19 +92,30 @@ func DeletePlaybook(id string) error { return Gorm.Delete(&models.Playbook{}, "id = ?", id).Error } -func UpdateApprovedPlaybookRuns(ctx *api.Context, playbookID string, approverIDs []string) error { - query := ` +// UpdatePlaybookRunStatusIfApproved updates the status of the playbook run to "pending" +// if all the approvers have approved it. +func UpdatePlaybookRunStatusIfApproved(ctx *api.Context, playbookID string, approval v1.PlaybookApproval) error { + if approval.Approvers.Empty() { + return nil + } + + subQuery := `SELECT run_id FROM run_approvals WHERE approvers @> ?` + if approval.Type == v1.PlaybookApprovalTypeAny { + subQuery = `SELECT run_id FROM run_approvals WHERE approvers && ?` + } + + query := fmt.Sprintf(` WITH run_approvals AS ( - SELECT run_id, ARRAY_AGG(COALESCE(person_id, team_id)) AS ids + SELECT run_id, ARRAY_AGG(COALESCE(person_id, team_id)) AS approvers FROM playbook_approvals GROUP BY run_id ) UPDATE playbook_runs SET status = ? WHERE status = ? AND playbook_id = ? - AND id IN (SELECT run_id FROM run_approvals WHERE ids @> ?)` + AND id IN (%s)`, subQuery) - return ctx.DB().Debug().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approverIDs)).Error + return ctx.DB().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approval.Approvers.IDs())).Error } func ApprovePlaybookRun(ctx *api.Context, runID uuid.UUID, personID, teamID *uuid.UUID) error { diff --git a/events/upstream_test.go b/events/upstream_test.go index b009838df..84f4c7ba9 100644 --- a/events/upstream_test.go +++ b/events/upstream_test.go @@ -348,7 +348,7 @@ func compareEntities[T any](table string, upstreamDB *gorm.DB, agent agentWrappe // - and the order of the items when fetching from upstream and agent db is identitcal for the comparison to work switch table { case "check_statuses": - err = upstreamDB.Debug().Joins("LEFT JOIN checks ON checks.id = check_statuses.check_id").Where("checks.agent_id = ?", agent.id).Order("check_id, time").Find(&upstream).Error + err = upstreamDB.Joins("LEFT JOIN checks ON checks.id = check_statuses.check_id").Where("checks.agent_id = ?", agent.id).Order("check_id, time").Find(&upstream).Error agentErr = agent.db.Order("check_id, time").Find(&downstream).Error case "config_analysis": diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index 9820106ce..df3c4be19 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -13,12 +13,13 @@ spec: - name: replicas label: The new desired number of replicas. approval: - type: any + type: all approvers: people: - 2611d086-926b-4d49-b616-845e61fd6fb2 - teams: - - 018770c4-4b73-5d44-8bb5-0e849d62e461 + - d87243c9-3183-4ab9-9df9-c77c8278df11 + # teams: + # - 018770c4-4b73-5d44-8bb5-0e849d62e461 actions: - name: 'scale deployment' exec: diff --git a/go.mod b/go.mod index 7b7c15850..c93c88d61 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/fergusstrange/embedded-postgres v1.23.0 github.com/flanksource/commons v1.11.0 github.com/flanksource/duty v1.0.157 + github.com/flanksource/gomplate/v3 v3.20.9 github.com/flanksource/kopper v1.0.6 github.com/google/cel-go v0.17.2 github.com/google/go-cmp v0.5.9 @@ -52,7 +53,6 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/flanksource/gomplate/v3 v3.20.9 // indirect github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37 // indirect github.com/flanksource/mapstructure v1.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -223,5 +223,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace "github.com/flanksource/duty" => ../duty \ No newline at end of file +replace github.com/flanksource/duty => ../duty diff --git a/go.sum b/go.sum index 16c98df0b..c04e885c2 100644 --- a/go.sum +++ b/go.sum @@ -765,8 +765,6 @@ github.com/fergusstrange/embedded-postgres v1.23.0 h1:ZYRD89nammxQDWDi6taJE2CYjD github.com/fergusstrange/embedded-postgres v1.23.0/go.mod h1:wL562t1V+iuFwq0UcgMi2e9rp8CROY9wxWZEfP8Y874= github.com/flanksource/commons v1.11.0 h1:ThP3hnX4Xh4thxVl2GjQ92WvQ93jq5VqzJh46jbW23A= github.com/flanksource/commons v1.11.0/go.mod h1:zYEhi6E2+diQ+loVcROUHo/Bgv+Tn61W2NYmrb5MgVI= -github.com/flanksource/duty v1.0.157 h1:WxOxI+3NGgMXOv6G4t6GXalZgYhIiORBs06FYqg/CLw= -github.com/flanksource/duty v1.0.157/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.20.9 h1:I3H/l1FUDepe6IuG8Nj51QNX9ocdU2EGL4GWz31sZdk= github.com/flanksource/gomplate/v3 v3.20.9/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo= diff --git a/jobs/event_queue.go b/jobs/event_queue.go index 04cabf2b6..285b2ccfb 100644 --- a/jobs/event_queue.go +++ b/jobs/event_queue.go @@ -32,7 +32,7 @@ func CleanupEventQueue() { } for table, age := range pushQueueSchedule { - result := db.Gorm.Debug().Exec("DELETE FROM event_queue WHERE name = 'push_queue.create' AND properties->>'table' = ? AND NOW() - created_at > ?", table, age) + result := db.Gorm.Exec("DELETE FROM event_queue WHERE name = 'push_queue.create' AND properties->>'table' = ? AND NOW() - created_at > ?", table, age) if result.Error != nil { logger.Errorf("Error cleaning up push_queue events for table=%s: %v", table, result.Error) jobHistory.AddError(result.Error.Error()) @@ -43,7 +43,7 @@ func CleanupEventQueue() { } defaultAge := time.Hour * 24 * 30 - result := db.Gorm.Debug().Exec("DELETE FROM event_queue WHERE name != 'push_queue.create' AND NOW() - created_at > ?", defaultAge) + result := db.Gorm.Exec("DELETE FROM event_queue WHERE name != 'push_queue.create' AND NOW() - created_at > ?", defaultAge) if result.Error != nil { logger.Errorf("Error cleaning up events (!push_queue.create): %v", result.Error) jobHistory.AddError(result.Error.Error()) diff --git a/playbook/approval.go b/playbook/approval.go index 055521ae5..2024732e7 100644 --- a/playbook/approval.go +++ b/playbook/approval.go @@ -29,6 +29,13 @@ func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook") } + run, err := db.FindPlaybookRun(ctx, runID.String()) + if err != nil { + return api.Errorf(api.EINTERNAL, "something went wrong while finding playbook run(id=%s)", runID).WithDebugInfo("db.FindPlaybookRun(id=%s): %v", runID, err) + } else if run == nil { + return api.Errorf(api.ENOTFOUND, "playbook run(id=%s) not found", runID) + } + if err := db.ApprovePlaybookRun(ctx, runID, &approverID, nil); err != nil { return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approverID, err) } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index ec193b954..09eb8db8a 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -91,7 +91,7 @@ func (t *queueConsumer) onPlaybookSpecApprovalUpdated(ctx *api.Context, playbook return nil } - return db.UpdateApprovedPlaybookRuns(ctx, playbookID, spec.Approval.Approvers.IDs()) + return db.UpdatePlaybookRunStatusIfApproved(ctx, playbookID, *spec.Approval) } func (t *queueConsumer) onPlaybookRunNewApproval(ctx *api.Context, runID string) error { @@ -118,7 +118,7 @@ func (t *queueConsumer) onPlaybookRunNewApproval(ctx *api.Context, runID string) return nil } - return db.UpdateApprovedPlaybookRuns(ctx, playbook.ID.String(), spec.Approval.Approvers.IDs()) + return db.UpdatePlaybookRunStatusIfApproved(ctx, playbook.ID.String(), *spec.Approval) } func (t *queueConsumer) consumeAll(ctx *api.Context) error { From 4085c06dde95ed94b6d28bdaa6cc4cfa8abb82be Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 22 Aug 2023 17:36:41 +0545 Subject: [PATCH 25/39] feat: implement team approval [skip ci] --- api/http.go | 2 ++ db/people.go | 6 +++++ db/playbooks.go | 16 ++++------- fixtures/playbooks/scale-deployment.yaml | 10 +++---- playbook/approval.go | 34 ++++++++++++++++++------ playbook/controllers.go | 14 +++++----- playbook/playbook.go | 6 ++--- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/api/http.go b/api/http.go index 41f2b3e84..a0d4e8e21 100644 --- a/api/http.go +++ b/api/http.go @@ -32,6 +32,7 @@ var codes = map[string]int{ ECONFLICT: http.StatusConflict, EINVALID: http.StatusBadRequest, ENOTFOUND: http.StatusNotFound, + EFORBIDDEN: http.StatusForbidden, ENOTIMPLEMENTED: http.StatusNotImplemented, EUNAUTHORIZED: http.StatusUnauthorized, EINTERNAL: http.StatusInternalServerError, @@ -42,5 +43,6 @@ func ErrorStatusCode(code string) int { if v, ok := codes[code]; ok { return v } + return http.StatusInternalServerError } diff --git a/db/people.go b/db/people.go index 18dd49882..0f0edcb2e 100644 --- a/db/people.go +++ b/db/people.go @@ -38,6 +38,12 @@ func GetUserByID(ctx *api.Context, id string) (api.Person, error) { return user, err } +func GetTeamIDsForUser(ctx *api.Context, id string) ([]uuid.UUID, error) { + var teamIDs []uuid.UUID + err := ctx.DB().Raw("SELECT team_id FROM team_members WHERE person_id = ?", id).Scan(&teamIDs).Error + return teamIDs, err +} + func GetUserByExternalID(ctx *api.Context, id string) (api.Person, error) { var user api.Person err := ctx.DB().Table("people").Where("external_id = ?", id).First(&user).Error diff --git a/db/playbooks.go b/db/playbooks.go index c3bdea761..f3e3fb7b7 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -28,14 +28,14 @@ func FindPlaybook(ctx *api.Context, id uuid.UUID) (*models.Playbook, error) { return &p, nil } -func FindPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { +func GetPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { var p models.PlaybookRun if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil + return nil, api.Errorf(api.ENOTFOUND, "playbook run(id=%s) not found", id) } - return nil, err + return nil, api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetPlaybookRun(id=%s): %v", id, err) } return &p, nil @@ -118,12 +118,6 @@ func UpdatePlaybookRunStatusIfApproved(ctx *api.Context, playbookID string, appr return ctx.DB().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approval.Approvers.IDs())).Error } -func ApprovePlaybookRun(ctx *api.Context, runID uuid.UUID, personID, teamID *uuid.UUID) error { - playbookApproval := models.PlaybookApproval{ - RunID: runID, - PersonID: personID, - TeamID: teamID, - } - - return ctx.DB().Create(&playbookApproval).Error +func SavePlaybookRunApproval(ctx *api.Context, approval models.PlaybookApproval) error { + return ctx.DB().Create(&approval).Error } diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index df3c4be19..0f88106ba 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -13,13 +13,13 @@ spec: - name: replicas label: The new desired number of replicas. approval: - type: all + type: any approvers: people: - - 2611d086-926b-4d49-b616-845e61fd6fb2 - - d87243c9-3183-4ab9-9df9-c77c8278df11 - # teams: - # - 018770c4-4b73-5d44-8bb5-0e849d62e461 + # - 2611d086-926b-4d49-b616-845e61fd6fb2 + - d87243c9-3183-4ab9-9df9-c77c8278df11 # admin + teams: + - 018770c4-4b73-5d44-8bb5-0e849d62e461 actions: - name: 'scale deployment' exec: diff --git a/playbook/approval.go b/playbook/approval.go index 2024732e7..628ff8ec6 100644 --- a/playbook/approval.go +++ b/playbook/approval.go @@ -2,6 +2,7 @@ package playbook import ( "github.com/flanksource/commons/collections" + "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/db" @@ -25,18 +26,35 @@ func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error return api.Errorf(api.EINVALID, "this playbook does not require approval") } - if !collections.Contains(playbookV1.Spec.Approval.Approvers.IDs(), approverID.String()) { - return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook") + approval := models.PlaybookApproval{ + RunID: runID, } - run, err := db.FindPlaybookRun(ctx, runID.String()) - if err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong while finding playbook run(id=%s)", runID).WithDebugInfo("db.FindPlaybookRun(id=%s): %v", runID, err) - } else if run == nil { - return api.Errorf(api.ENOTFOUND, "playbook run(id=%s) not found", runID) + if collections.Contains(playbookV1.Spec.Approval.Approvers.People, approverID.String()) { + approval.PersonID = &approverID + } else { + teamIDs, err := db.GetTeamIDsForUser(ctx, approverID.String()) + if err != nil { + return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamIDsForUser(id=%s): %v", approverID, err) + } + + for _, teamID := range teamIDs { + if collections.Contains(playbookV1.Spec.Approval.Approvers.Teams, teamID.String()) { + approval.TeamID = &teamID + break + } + } + + if approval.TeamID == nil { + return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook run") + } + } + + if _, err := db.GetPlaybookRun(ctx, runID.String()); err != nil { + return err } - if err := db.ApprovePlaybookRun(ctx, runID, &approverID, nil); err != nil { + if err := db.SavePlaybookRunApproval(ctx, approval); err != nil { return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approverID, err) } diff --git a/playbook/controllers.go b/playbook/controllers.go index 572e478d2..bfa1dc09d 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -137,15 +137,13 @@ func HandlePlaybookRun(c echo.Context) error { }) } -func HandlePlaybookRunStatus(c echo.Context) error { +func HandleGetPlaybookRun(c echo.Context) error { ctx := c.(*api.Context) id := c.Param("id") - run, err := db.FindPlaybookRun(ctx, id) + run, err := db.GetPlaybookRun(ctx, id) if err != nil { - return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to get playbook run"}) - } else if run == nil { - return c.JSON(http.StatusNotFound, api.HTTPError{Error: "not found", Message: fmt.Sprintf("playbook run(id=%s) not found", id)}) + return api.WriteError(c, err) } return c.JSON(http.StatusOK, run) @@ -173,12 +171,12 @@ func HandlePlaybookList(c echo.Context) error { if configID != "" { playbooks, err = ListPlaybooksOfConfig(ctx, configID) if err != nil { - return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to list playbooks"}) + return api.WriteError(c, err) } } else if componentID != "" { playbooks, err = ListPlaybooksOfComponent(ctx, componentID) if err != nil { - return c.JSON(http.StatusInternalServerError, api.HTTPError{Error: err.Error(), Message: "failed to list playbooks"}) + return api.WriteError(c, err) } } @@ -221,7 +219,7 @@ func RegisterRoutes(e *echo.Echo, prefix string) *echo.Group { runGroup := playbookGroup.Group("/run") runGroup.POST("", HandlePlaybookRun) - runGroup.GET(":id", HandlePlaybookRunStatus) + runGroup.GET("/:id", HandleGetPlaybookRun) runGroup.POST("/approve/:playbook_id/:run_id", HandlePlaybookRunApproval) return playbookGroup diff --git a/playbook/playbook.go b/playbook/playbook.go index 497a4a9a8..0584ce31f 100644 --- a/playbook/playbook.go +++ b/playbook/playbook.go @@ -1,8 +1,6 @@ package playbook import ( - "fmt" - "github.com/flanksource/duty/models" "github.com/flanksource/incident-commander/api" "github.com/flanksource/incident-commander/db" @@ -14,7 +12,7 @@ func ListPlaybooksOfConfig(ctx *api.Context, id string) ([]models.Playbook, erro if err := ctx.DB().Where("id = ?", id).Find(&config).Error; err != nil { return nil, err } else if config.ID == uuid.Nil { - return nil, fmt.Errorf("config(id=%s) not found", id) + return nil, api.Errorf(api.ENOTFOUND, "config(id=%s) not found", id) } return db.FindPlaybooksByTypeAndTags(ctx, *config.Type, *config.Tags) @@ -25,7 +23,7 @@ func ListPlaybooksOfComponent(ctx *api.Context, id string) ([]models.Playbook, e if err := ctx.DB().Where("id = ?", id).Find(&component).Error; err != nil { return nil, err } else if component.ID == uuid.Nil { - return nil, fmt.Errorf("component(id=%s) not found", id) + return nil, api.Errorf(api.ENOTFOUND, "component(id=%s) not found", id) } return db.FindPlaybooksByTypeAndTags(ctx, component.Type, component.Labels) From 50edc22d850800f601cf04a7cdc1ff9beb940800 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 22 Aug 2023 19:21:50 +0545 Subject: [PATCH 26/39] feat: add test for playbook approval [skip ci] --- playbook/playbook_test.go | 142 +++++++++++++++++++++++++------------ playbook/queue_consumer.go | 6 +- playbook/suite_test.go | 25 +++++++ 3 files changed, 126 insertions(+), 47 deletions(-) diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index bc8d0ca96..1eeaba783 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -11,40 +11,63 @@ import ( "github.com/flanksource/duty/fixtures/dummy" "github.com/flanksource/duty/models" - "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" - "github.com/google/uuid" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { - playbookSpec := v1.PlaybookSpec{ - Description: "write config name to file", - Parameters: []v1.PlaybookParameter{ - {Name: "path", Label: "path of the file"}, - }, - Actions: []v1.PlaybookAction{ - { - Name: "write config id to a file", - Exec: &v1.ExecAction{ - Script: "printf {{.config.id}} > {{.params.path}}", - }, - }, - { - Name: "append config class to the same file ", - Exec: &v1.ExecAction{ - Script: "printf {{.config.config_class}} >> {{.params.path}}", - }, - }, - }, - } - var ( playbook models.Playbook runResp RunResponse + consumer *queueConsumer ) + + ginkgo.It("should store dummy data", func() { + dataset := dummy.GetStaticDummyData() + err := dataset.Populate(testDB) + Expect(err).NotTo(HaveOccurred()) + }) + + ginkgo.It("start the queue consumer in background", func() { + consumer = NewQueueConsumer(testDB, testDBPool) + go func() { + err := consumer.Listen() + Expect(err).NotTo(HaveOccurred()) + }() + }) + ginkgo.It("should create a new playbook", func() { + playbookSpec := v1.PlaybookSpec{ + Description: "write config name to file", + Parameters: []v1.PlaybookParameter{ + {Name: "path", Label: "path of the file"}, + }, + Approval: &v1.PlaybookApproval{ + Type: v1.PlaybookApprovalTypeAny, // We have two approvers (John Doe & John Wick) and just a single approval is sufficient + Approvers: v1.PlaybookApprovers{ + People: []string{ + dummy.JohnDoe.ID.String(), + dummy.JohnWick.ID.String(), + }, + }, + }, + Actions: []v1.PlaybookAction{ + { + Name: "write config id to a file", + Exec: &v1.ExecAction{ + Script: "printf {{.config.id}} > {{.params.path}}", + }, + }, + { + Name: "append config class to the same file ", + Exec: &v1.ExecAction{ + Script: "printf {{.config.config_class}} >> {{.params.path}}", + }, + }, + }, + } + spec, err := json.Marshal(playbookSpec) Expect(err).NotTo(HaveOccurred()) @@ -58,12 +81,6 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - ginkgo.It("should store dummy data", func() { - dataset := dummy.GetStaticDummyData() - err := dataset.Populate(testDB) - Expect(err).NotTo(HaveOccurred()) - }) - ginkgo.It("should store playbook run via API", func() { run := RunParams{ ID: playbook.ID, @@ -80,7 +97,7 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Expect(err).NotTo(HaveOccurred()) req.Header.Set("Content-Type", "application/json; charset=UTF-8") - req.SetBasicAuth("admin@local", "admin") + req.SetBasicAuth(dummy.JohnDoe.Name, "admin") client := http.Client{} resp, err := client.Do(req) @@ -103,38 +120,75 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { err = testDB.Where("id = ? ", runResp.RunID).First(&savedRun).Error Expect(err).NotTo(HaveOccurred()) - Expect(savedRun.PlaybookID).To(Equal(playbook.ID)) + Expect(savedRun.PlaybookID).To(Equal(playbook.ID), "run should have been created for the correct playbook") + Expect(savedRun.Status).To(Equal(models.PlaybookRunStatusPending), "run should be in pending status because it has approvers") + Expect(*savedRun.CreatedBy).To(Equal(dummy.JohnDoe.ID), "run should have been created by the authenticated person") }) - ginkgo.It("should execute playbook", func() { - consumer := NewQueueConsumer(testDB, testDBPool) + ginkgo.It("should approve the playbook run via API", func() { + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:%d/playbook/run/approve/%s/%s", echoServerPort, playbook.ID.String(), runResp.RunID), nil) + Expect(err).NotTo(HaveOccurred()) - ctx := api.NewContext(testDB, nil) - err := consumer.consumeAll(ctx) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.SetBasicAuth(dummy.JohnWick.Name, "admin") // approve John Wick (who is an approver but not a creator of the playbook) + + client := http.Client{} + resp, err := client.Do(req) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) - // Wait until all the runs are processed + fmt.Println(string(b)) + } + + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + // Wait until all run is marked as scheduled var attempts int for { - time.Sleep(time.Second) // need to wait initially before trying. - if _, ok := consumer.registry.Load(uuid.MustParse(runResp.RunID)); !ok { + time.Sleep(time.Millisecond * 100) + + var savedRun models.PlaybookRun + err = testDB.Where("id = ? ", runResp.RunID).First(&savedRun).Error + Expect(err).NotTo(HaveOccurred()) + + if savedRun.Status == models.PlaybookRunStatusScheduled || savedRun.Status == models.PlaybookRunStatusCompleted { break } attempts += 1 - if attempts > 5 { - ginkgo.Fail("Timed out waiting for run to complete") + if attempts > 20 { // wait for 2 seconds + ginkgo.Fail(fmt.Sprintf("Timed out waiting for run to be scheduled. Status = %s", savedRun.Status)) } } + }) + ginkgo.It("should execute playbook", func() { var updatedRun models.PlaybookRun - err = testDB.Where("id = ? ", runResp.RunID).First(&updatedRun).Error - Expect(err).NotTo(HaveOccurred()) - Expect(updatedRun.Status).To(Equal(models.PlaybookRunStatusCompleted)) + // Wait until all the runs is marked as completed + var attempts int + for { + time.Sleep(time.Millisecond * 100) + + err := testDB.Where("id = ? ", runResp.RunID).First(&updatedRun).Error + Expect(err).NotTo(HaveOccurred()) + + if updatedRun.Status == models.PlaybookRunStatusCompleted { + break + } + + attempts += 1 + if attempts > 20 { // wait for 2 seconds + ginkgo.Fail("Timed out waiting for run to complete") + } + } var runActions []models.PlaybookRunAction - err = testDB.Where("playbook_run_id = ?", updatedRun.ID).Find(&runActions).Error + err := testDB.Where("playbook_run_id = ?", updatedRun.ID).Find(&runActions).Error Expect(err).NotTo(HaveOccurred()) Expect(len(runActions)).To(Equal(2)) diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index 09eb8db8a..e65899be6 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -42,13 +42,13 @@ func NewQueueConsumer(db *gorm.DB, pool *pgxpool.Pool) *queueConsumer { func (t *queueConsumer) Listen() error { pgNotify := make(chan string) - go utils.ListenToPostgresNotify(db.Pool, "playbook_run_updates", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotify) + go utils.ListenToPostgresNotify(t.pool, "playbook_run_updates", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotify) pgNotifyPlaybookSpecApprovalUpdated := make(chan string) - go utils.ListenToPostgresNotify(db.Pool, "playbook_spec_approval_updated", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookSpecApprovalUpdated) + go utils.ListenToPostgresNotify(t.pool, "playbook_spec_approval_updated", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookSpecApprovalUpdated) pgNotifyPlaybookApprovalsInserted := make(chan string) - go utils.ListenToPostgresNotify(db.Pool, "playbook_approval_inserted", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookApprovalsInserted) + go utils.ListenToPostgresNotify(t.pool, "playbook_approval_inserted", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotifyPlaybookApprovalsInserted) ctx := api.NewContext(t.db, nil) for { diff --git a/playbook/suite_test.go b/playbook/suite_test.go index 68b19deec..f2d98bac7 100644 --- a/playbook/suite_test.go +++ b/playbook/suite_test.go @@ -9,6 +9,7 @@ import ( embeddedPG "github.com/fergusstrange/embedded-postgres" "github.com/flanksource/commons/logger" "github.com/flanksource/duty" + "github.com/flanksource/duty/models" "github.com/flanksource/duty/testutils" "github.com/flanksource/incident-commander/api" "github.com/jackc/pgx/v5/pgxpool" @@ -75,6 +76,8 @@ func setupUpstreamHTTPServer() { } }) + echoServer.Use(mockAuthMiddleware) + RegisterRoutes(echoServer, "playbook") listenAddr := fmt.Sprintf(":%d", echoServerPort) @@ -90,3 +93,25 @@ func setupUpstreamHTTPServer() { } }() } + +// mockAuthMiddleware doesn't actually authenticate since we never store auth data. +// It simply ensures that the requested user exists in the DB and then attaches the +// users's ID to the context. +func mockAuthMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + name, _, ok := c.Request().BasicAuth() + if !ok { + return c.String(http.StatusUnauthorized, "Unauthorized") + } + + var person models.Person + if err := testDB.Where("name = ?", name).First(&person).Error; err != nil { + return c.String(http.StatusUnauthorized, "Unauthorized") + } + + ctx := c.(*api.Context) + ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, person.ID.String()) + + return next(c) + } +} From 9e01e4d43d4be14a59b61230550d30f598c2f295 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 23 Aug 2023 11:47:11 +0545 Subject: [PATCH 27/39] chore: update spec [skip ci] --- api/v1/playbook_actions.go | 5 ++--- api/v1/playbook_types.go | 4 ++-- config/crds/mission-control.flanksource.com_playbooks.yaml | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index e32ef39f5..704ddd08a 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -167,7 +167,6 @@ func (t *AWSConnection) Populate(ctx connectionContext, k8s kubernetes.Interface } type PlaybookAction struct { - Name string `yaml:"name" json:"name"` - TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"` - Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` + Name string `yaml:"name" json:"name"` + Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"` } diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index d032d040d..3911a1e29 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -23,8 +23,8 @@ type PlaybookResourceFilter struct { // PlaybookParameter defines a parameter that a playbook needs to run. type PlaybookParameter struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Label string `json:"label,omitempty" yaml:"label,omitempty"` + Name string `json:"name" yaml:"name"` + Label string `json:"label" yaml:"label"` } type PlaybookApprovers struct { diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index 02b3e8764..2f4146939 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -313,8 +313,6 @@ spec: type: object name: type: string - timeout-minutes: - type: string required: - name type: object @@ -374,6 +372,9 @@ spec: type: string name: type: string + required: + - label + - name type: object type: array permissions: From e9dc4419fa02b9402b05b589b24f7da0f1067341 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 24 Aug 2023 11:20:02 +0545 Subject: [PATCH 28/39] chore: remove Description and Templatable for exec action [skip ci] --- api/v1/playbook_actions.go | 30 ------- api/v1/zz_generated.deepcopy.go | 78 ------------------- ...ion-control.flanksource.com_playbooks.yaml | 52 ------------- 3 files changed, 160 deletions(-) diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go index 704ddd08a..67f8b3a45 100644 --- a/api/v1/playbook_actions.go +++ b/api/v1/playbook_actions.go @@ -10,37 +10,7 @@ import ( "k8s.io/client-go/kubernetes" ) -type Labels map[string]string - -type Description struct { - // Description for the check - Description string `yaml:"description,omitempty" json:"description,omitempty" template:"true"` - // Name of the check - Name string `yaml:"name,omitempty" json:"name,omitempty" template:"true"` - // Icon for overwriting default icon on the dashboard - Icon string `yaml:"icon,omitempty" json:"icon,omitempty" template:"true"` - // Labels for the check - Labels Labels `yaml:"labels,omitempty" json:"labels,omitempty"` - // Transformed checks have a delete strategy on deletion they can either be marked healthy, unhealthy or left as is - TransformDeleteStrategy string `yaml:"transformDeleteStrategy,omitempty" json:"transformDeleteStrategy,omitempty"` -} - -type Template struct { - Template string `yaml:"template,omitempty" json:"template,omitempty"` - JSONPath string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"` - Expression string `yaml:"expr,omitempty" json:"expr,omitempty"` - Javascript string `yaml:"javascript,omitempty" json:"javascript,omitempty"` -} - -type Templatable struct { - Test Template `yaml:"test,omitempty" json:"test,omitempty"` - Display Template `yaml:"display,omitempty" json:"display,omitempty"` - Transform Template `yaml:"transform,omitempty" json:"transform,omitempty"` -} - type ExecAction struct { - Description `yaml:",inline" json:",inline"` - Templatable `yaml:",inline" json:",inline"` // Script can be a inline script or a path to a script that needs to be executed // On windows executed via powershell and in darwin and linux executed using bash Script string `yaml:"script" json:"script"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 20ec9dcdc..34333dbc6 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -154,33 +154,9 @@ func (in *ConnectionStatus) DeepCopy() *ConnectionStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Description) DeepCopyInto(out *Description) { - *out = *in - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(Labels, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Description. -func (in *Description) DeepCopy() *Description { - if in == nil { - return nil - } - out := new(Description) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExecAction) DeepCopyInto(out *ExecAction) { *out = *in - in.Description.DeepCopyInto(&out.Description) - out.Templatable = in.Templatable in.Connections.DeepCopyInto(&out.Connections) } @@ -318,27 +294,6 @@ func (in *IncidentRuleStatus) DeepCopy() *IncidentRuleStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in Labels) DeepCopyInto(out *Labels) { - { - in := &in - *out = make(Labels, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Labels. -func (in Labels) DeepCopy() Labels { - if in == nil { - return nil - } - out := new(Labels) - in.DeepCopyInto(out) - return *out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Permission) DeepCopyInto(out *Permission) { *out = *in @@ -576,36 +531,3 @@ func (in *PlaybookStatus) DeepCopy() *PlaybookStatus { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Templatable) DeepCopyInto(out *Templatable) { - *out = *in - out.Test = in.Test - out.Display = in.Display - out.Transform = in.Transform -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Templatable. -func (in *Templatable) DeepCopy() *Templatable { - if in == nil { - return nil - } - out := new(Templatable) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Template) DeepCopyInto(out *Template) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Template. -func (in *Template) DeepCopy() *Template { - if in == nil { - return nil - } - out := new(Template) - in.DeepCopyInto(out) - return out -} diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index 2f4146939..b75a75e86 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -250,64 +250,12 @@ spec: type: string type: object type: object - description: - description: Description for the check - type: string - display: - properties: - expr: - type: string - javascript: - type: string - jsonPath: - type: string - template: - type: string - type: object - icon: - description: Icon for overwriting default icon on the dashboard - type: string - labels: - additionalProperties: - type: string - description: Labels for the check - type: object - name: - description: Name of the check - type: string script: description: Script can be a inline script or a path to a script that needs to be executed On windows executed via powershell and in darwin and linux executed using bash type: string - test: - properties: - expr: - type: string - javascript: - type: string - jsonPath: - type: string - template: - type: string - type: object - transform: - properties: - expr: - type: string - javascript: - type: string - jsonPath: - type: string - template: - type: string - type: object - transformDeleteStrategy: - description: Transformed checks have a delete strategy on - deletion they can either be marked healthy, unhealthy - or left as is - type: string required: - script type: object From 34c63315c0efe8d7312ad339090f7f4675dd169e Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 24 Aug 2023 14:45:28 +0545 Subject: [PATCH 29/39] chore: bump duty --- go.mod | 18 ++++++++---------- go.sum | 30 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 890297d72..296a645cc 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,10 @@ require ( github.com/containrrr/shoutrrr v0.7.1 github.com/fergusstrange/embedded-postgres v1.23.0 github.com/flanksource/commons v1.11.0 - github.com/flanksource/duty v1.0.157 - github.com/flanksource/gomplate/v3 v3.20.9 + github.com/flanksource/duty v1.0.159 + github.com/flanksource/gomplate/v3 v3.20.11 github.com/flanksource/kopper v1.0.6 - github.com/google/cel-go v0.17.2 + github.com/google/cel-go v0.17.6 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.1 github.com/jackc/pgx/v5 v5.4.3 @@ -110,8 +110,8 @@ require ( golang.org/x/term v0.11.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/flanksource/yaml.v3 v3.2.3 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect @@ -141,8 +141,8 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/TomOnTime/utfutil v0.0.0-20230223141146-125e65197b36 - github.com/antonmedv/expr v1.14.2 // indirect - github.com/aws/aws-sdk-go v1.44.327 // indirect + github.com/antonmedv/expr v1.14.3 // indirect + github.com/aws/aws-sdk-go v1.44.330 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/cjlapao/common-go v0.0.39 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -212,7 +212,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.138.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.57.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -221,5 +221,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/flanksource/duty => ../duty diff --git a/go.sum b/go.sum index 80715de99..f689c9678 100644 --- a/go.sum +++ b/go.sum @@ -651,8 +651,8 @@ github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIi github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= -github.com/antonmedv/expr v1.14.2 h1:3gSOv3dGHPjou5yXNTM85KPEGMj0rAf2GpsMD4H7Js0= -github.com/antonmedv/expr v1.14.2/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= +github.com/antonmedv/expr v1.14.3 h1:GPrP7xKPWkFaLANPS7tPrgkNs7FMHpZdL72Dc5kFykg= +github.com/antonmedv/expr v1.14.3/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -667,8 +667,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= -github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.330 h1:kO41s8I4hRYtWSIuMc/O053wmEGfMTT8D4KtPSojUkA= +github.com/aws/aws-sdk-go v1.44.330/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -764,9 +764,11 @@ github.com/fergusstrange/embedded-postgres v1.23.0 h1:ZYRD89nammxQDWDi6taJE2CYjD github.com/fergusstrange/embedded-postgres v1.23.0/go.mod h1:wL562t1V+iuFwq0UcgMi2e9rp8CROY9wxWZEfP8Y874= github.com/flanksource/commons v1.11.0 h1:ThP3hnX4Xh4thxVl2GjQ92WvQ93jq5VqzJh46jbW23A= github.com/flanksource/commons v1.11.0/go.mod h1:zYEhi6E2+diQ+loVcROUHo/Bgv+Tn61W2NYmrb5MgVI= +github.com/flanksource/duty v1.0.159 h1:CkSKDZ4HYQ7cSEy8ufg1WODqcghkWsiAlk2o4I4JkqY= +github.com/flanksource/duty v1.0.159/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= -github.com/flanksource/gomplate/v3 v3.20.9 h1:I3H/l1FUDepe6IuG8Nj51QNX9ocdU2EGL4GWz31sZdk= -github.com/flanksource/gomplate/v3 v3.20.9/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo= +github.com/flanksource/gomplate/v3 v3.20.11 h1:B83fXI0dqlOii2QE+qPNI0blPlcZTmfnPPU8zwVGUtA= +github.com/flanksource/gomplate/v3 v3.20.11/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo= github.com/flanksource/is-healthy v0.0.0-20230705092916-3b4cf510c5fc/go.mod h1:4pQhmF+TnVqJroQKY8wSnSp+T18oLson6YQ2M0qPHfQ= github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37 h1:MHXg2Vo/oHB0rGLgsI0tkU9MGV7aDwqvO1lrbX7/shY= github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37/go.mod h1:BH5gh9JyEAuuWVP6Q5y9h43VozS0RfKyjNpM9L4v4hw= @@ -898,8 +900,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cel-go v0.16.0/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= -github.com/google/cel-go v0.17.2 h1:KXsWCtQuLRIHubUs/IGxJdijOmexk9YKf5HB3da5E5k= -github.com/google/cel-go v0.17.2/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/cel-go v0.17.6 h1:QDvHTIJunIsbgN8yVukx0HGnsqVLSY6xGqo+17IjIyM= +github.com/google/cel-go v0.17.6/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -2071,16 +2073,16 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 h1:Iveh6tGCJkHAjJgEqUQYGDGgbwmhjoAOz8kO/ajxefY= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 h1:WGq4lvB/mlicysM/dUT3SBvijH4D3sm/Ny1A4wmt2CI= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 76508d1dd2cb91d5e3c6884bbe943a1cf8a2393d Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 24 Aug 2023 14:54:54 +0545 Subject: [PATCH 30/39] chore: save exec result as an object --- playbook/actions/exec.go | 6 +++--- playbook/playbook_test.go | 7 ++----- playbook/queue_consumer.go | 7 ++++--- playbook/runner.go | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/playbook/actions/exec.go b/playbook/actions/exec.go index 24d940506..158500f21 100644 --- a/playbook/actions/exec.go +++ b/playbook/actions/exec.go @@ -20,9 +20,9 @@ type ExecAction struct { } type ExecDetails struct { - Stdout string `json:"stdout"` - Stderr string `json:"stderr"` - ExitCode int `json:"exitCode"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + ExitCode int `json:"exitCode,omitempty"` } func (c *ExecAction) Run(ctx *api.Context, exec v1.ExecAction) (*ExecDetails, error) { diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index 1eeaba783..80c75e3a1 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -31,10 +31,7 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { ginkgo.It("start the queue consumer in background", func() { consumer = NewQueueConsumer(testDB, testDBPool) - go func() { - err := consumer.Listen() - Expect(err).NotTo(HaveOccurred()) - }() + go consumer.Listen() }) ginkgo.It("should create a new playbook", func() { @@ -183,7 +180,7 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { attempts += 1 if attempts > 20 { // wait for 2 seconds - ginkgo.Fail("Timed out waiting for run to complete") + ginkgo.Fail(fmt.Sprintf("Timed out waiting for run to complete. Run status: %s", updatedRun.Status)) } } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index e65899be6..0c9a977e1 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -40,7 +40,7 @@ func NewQueueConsumer(db *gorm.DB, pool *pgxpool.Pool) *queueConsumer { } } -func (t *queueConsumer) Listen() error { +func (t *queueConsumer) Listen() { pgNotify := make(chan string) go utils.ListenToPostgresNotify(t.pool, "playbook_run_updates", t.dbReconnectMaxDuration, t.dbReconnectBackoffBaseDuration, pgNotify) @@ -59,7 +59,7 @@ func (t *queueConsumer) Listen() error { } case id := <-pgNotifyPlaybookSpecApprovalUpdated: - if err := t.onPlaybookSpecApprovalUpdated(ctx, id); err != nil { + if err := t.onApprovalUpdated(ctx, id); err != nil { logger.Errorf("%v", err) } @@ -76,7 +76,8 @@ func (t *queueConsumer) Listen() error { } } -func (t *queueConsumer) onPlaybookSpecApprovalUpdated(ctx *api.Context, playbookID string) error { +// onApprovalUpdated is called when the playbook spec approval is updated +func (t *queueConsumer) onApprovalUpdated(ctx *api.Context, playbookID string) error { var playbook models.Playbook if err := ctx.DB().Where("id = ?", playbookID).First(&playbook).Error; err != nil { return err diff --git a/playbook/runner.go b/playbook/runner.go index d949d6378..84296c7a4 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -130,7 +130,7 @@ func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookA return nil, err } - return json.Marshal(res.Stdout) + return json.Marshal(res) } return nil, nil From c8331b44aab1c63701ca01b1eacc9c2db39f4fac Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 24 Aug 2023 17:00:18 +0545 Subject: [PATCH 31/39] fix: listing of playbooks for a component [skip ci] --- api/global.go | 7 +++---- db/playbooks.go | 23 +++++++++++++++++++++-- go.mod | 2 ++ playbook/controllers.go | 4 ++-- playbook/playbook.go | 8 ++++---- playbook/playbook_test.go | 34 +++++++++++++++++++++++++++++++++- 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/api/global.go b/api/global.go index 4ce2015af..14992b2bd 100644 --- a/api/global.go +++ b/api/global.go @@ -17,10 +17,9 @@ import ( type ContextKey string -const ( - UserIDContextKey ContextKey = "User-ID" - UserIDHeaderKey = "X-User-ID" -) +const UserIDContextKey ContextKey = "User-ID" + +const UserIDHeaderKey = "X-User-ID" var SystemUserID *uuid.UUID var CanaryCheckerPath string diff --git a/db/playbooks.go b/db/playbooks.go index f3e3fb7b7..0f59c5a89 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -41,8 +41,8 @@ func GetPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { return &p, nil } -// FindPlaybooksByTypeAndTags returns all the playbooks that match the given type and tags. -func FindPlaybooksByTypeAndTags(ctx *api.Context, configType string, tags map[string]string) ([]models.Playbook, error) { +// FindPlaybooksForConfig returns all the playbooks that match the given config type and tags. +func FindPlaybooksForConfig(ctx *api.Context, configType string, tags map[string]string) ([]models.Playbook, error) { joinQuery := `JOIN LATERAL jsonb_array_elements(playbooks."spec"->'configs') AS configs(config) ON 1=1` if tags != nil { joinQuery += " AND (?::jsonb) @> (configs.config->'tags')" @@ -60,6 +60,25 @@ func FindPlaybooksByTypeAndTags(ctx *api.Context, configType string, tags map[st return playbooks, err } +// FindPlaybooksForComponent returns all the playbooks that match the given component type and tags. +func FindPlaybooksForComponent(ctx *api.Context, configType string, tags map[string]string) ([]models.Playbook, error) { + joinQuery := `JOIN LATERAL jsonb_array_elements(playbooks."spec"->'components') AS components(component) ON 1=1` + if tags != nil { + joinQuery += " AND (?::jsonb) @> (components.component->'tags')" + } + if configType != "" { + joinQuery += " AND components.component->>'type' = ?" + } + + query := ctx.DB(). + Select("DISTINCT playbooks.*"). + Joins(joinQuery, types.JSONStringMap(tags), configType) + + var playbooks []models.Playbook + err := query.Find(&playbooks).Error + return playbooks, err +} + // GetScheduledPlaybookRuns returns all the scheduled playbook runs that should be started // before X duration from now. func GetScheduledPlaybookRuns(ctx *api.Context, startingBefore time.Duration, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { diff --git a/go.mod b/go.mod index 296a645cc..53e6a4a2e 100644 --- a/go.mod +++ b/go.mod @@ -221,3 +221,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/flanksource/duty => ../duty diff --git a/playbook/controllers.go b/playbook/controllers.go index bfa1dc09d..21d194c6b 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -169,12 +169,12 @@ func HandlePlaybookList(c echo.Context) error { var playbooks []models.Playbook var err error if configID != "" { - playbooks, err = ListPlaybooksOfConfig(ctx, configID) + playbooks, err = ListPlaybooksForConfig(ctx, configID) if err != nil { return api.WriteError(c, err) } } else if componentID != "" { - playbooks, err = ListPlaybooksOfComponent(ctx, componentID) + playbooks, err = ListPlaybooksForComponent(ctx, componentID) if err != nil { return api.WriteError(c, err) } diff --git a/playbook/playbook.go b/playbook/playbook.go index 0584ce31f..75a8f8413 100644 --- a/playbook/playbook.go +++ b/playbook/playbook.go @@ -7,7 +7,7 @@ import ( "github.com/google/uuid" ) -func ListPlaybooksOfConfig(ctx *api.Context, id string) ([]models.Playbook, error) { +func ListPlaybooksForConfig(ctx *api.Context, id string) ([]models.Playbook, error) { var config models.ConfigItem if err := ctx.DB().Where("id = ?", id).Find(&config).Error; err != nil { return nil, err @@ -15,10 +15,10 @@ func ListPlaybooksOfConfig(ctx *api.Context, id string) ([]models.Playbook, erro return nil, api.Errorf(api.ENOTFOUND, "config(id=%s) not found", id) } - return db.FindPlaybooksByTypeAndTags(ctx, *config.Type, *config.Tags) + return db.FindPlaybooksForConfig(ctx, *config.Type, *config.Tags) } -func ListPlaybooksOfComponent(ctx *api.Context, id string) ([]models.Playbook, error) { +func ListPlaybooksForComponent(ctx *api.Context, id string) ([]models.Playbook, error) { var component models.Component if err := ctx.DB().Where("id = ?", id).Find(&component).Error; err != nil { return nil, err @@ -26,5 +26,5 @@ func ListPlaybooksOfComponent(ctx *api.Context, id string) ([]models.Playbook, e return nil, api.Errorf(api.ENOTFOUND, "component(id=%s) not found", id) } - return db.FindPlaybooksByTypeAndTags(ctx, component.Type, component.Labels) + return db.FindPlaybooksForComponent(ctx, component.Type, component.Labels) } diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index 80c75e3a1..f09fa39bd 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -11,9 +11,11 @@ import ( "github.com/flanksource/duty/fixtures/dummy" "github.com/flanksource/duty/models" + "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/gorm/clause" ) var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { @@ -40,6 +42,12 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Parameters: []v1.PlaybookParameter{ {Name: "path", Label: "path of the file"}, }, + Configs: []v1.PlaybookResourceFilter{ + {Type: *dummy.EKSCluster.Type, Tags: map[string]string{"environment": "production"}}, + }, + Components: []v1.PlaybookResourceFilter{ + {Type: dummy.Logistics.Type, Tags: map[string]string{"telemetry": "enabled"}}, + }, Approval: &v1.PlaybookApproval{ Type: v1.PlaybookApprovalTypeAny, // We have two approvers (John Doe & John Wick) and just a single approval is sufficient Approvers: v1.PlaybookApprovers{ @@ -74,8 +82,32 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Source: models.SourceConfigFile, } - err = testDB.Create(&playbook).Error + err = testDB.Clauses(clause.Returning{}).Create(&playbook).Error + Expect(err).NotTo(HaveOccurred()) + }) + + ginkgo.It("Should fetch the suitable playbook for configs", func() { + ctx := api.NewContext(testDB, nil) + playbooks, err := ListPlaybooksForConfig(ctx, dummy.EKSCluster.ID.String()) + Expect(err).NotTo(HaveOccurred()) + Expect(len(playbooks)).To(Equal(1)) + Expect(playbooks).To(Equal([]models.Playbook{playbook})) + + playbooks, err = ListPlaybooksForConfig(ctx, dummy.KubernetesCluster.ID.String()) + Expect(err).NotTo(HaveOccurred()) + Expect(len(playbooks)).To(Equal(0)) + }) + + ginkgo.It("Should fetch the suitable playbook for components", func() { + ctx := api.NewContext(testDB, nil) + playbooks, err := ListPlaybooksForComponent(ctx, dummy.Logistics.ID.String()) + Expect(err).NotTo(HaveOccurred()) + Expect(len(playbooks)).To(Equal(1)) + Expect(playbooks).To(Equal([]models.Playbook{playbook})) + + playbooks, err = ListPlaybooksForComponent(ctx, dummy.LogisticsUI.ID.String()) Expect(err).NotTo(HaveOccurred()) + Expect(len(playbooks)).To(Equal(0)) }) ginkgo.It("should store playbook run via API", func() { From cb72e403d3c0ed84e865a960980b5d4d96b69ce8 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 08:41:25 +0545 Subject: [PATCH 32/39] chore: update fixtures --- fixtures/playbooks/deleting-configmap.yaml | 5 +---- fixtures/playbooks/ec2.yaml | 8 ++++---- fixtures/playbooks/scale-deployment.yaml | 8 +++----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/fixtures/playbooks/deleting-configmap.yaml b/fixtures/playbooks/deleting-configmap.yaml index 4e636d257..794060e4e 100644 --- a/fixtures/playbooks/deleting-configmap.yaml +++ b/fixtures/playbooks/deleting-configmap.yaml @@ -6,10 +6,7 @@ spec: description: Delete Kubernetes ConfigMap configs: - type: Kubernetes::ConfigMap - tags: - namespace: default - cluster: local-kind-cluster actions: - name: 'Delete ConfigMap' exec: - script: kubectl delete configmap {{.config.name}} + script: kubectl delete configmap {{.config.name}} --namespace={{.config.namespace}} diff --git a/fixtures/playbooks/ec2.yaml b/fixtures/playbooks/ec2.yaml index 18d83b290..d51047e2d 100644 --- a/fixtures/playbooks/ec2.yaml +++ b/fixtures/playbooks/ec2.yaml @@ -4,11 +4,11 @@ metadata: name: ec2-restart spec: description: Unconventional EC2 restart - configs: # playbooks can run on configs + configs: - type: EC2 Instance - tags: # can filter non-prod vs prod instances - a: b - actions: # 1 or more actions + tags: + telemetry: enabled + actions: - name: 'Stop EC2 instance' exec: script: aws ec2 stop-instance --instance-id {{.config.instanceId}} diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index 0f88106ba..1a8dc8e04 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -7,8 +7,7 @@ spec: configs: - type: Kubernetes::Deployment tags: - namespace: default - cluster: local-kind-cluster + environment: staging parameters: - name: replicas label: The new desired number of replicas. @@ -16,11 +15,10 @@ spec: type: any approvers: people: - # - 2611d086-926b-4d49-b616-845e61fd6fb2 - - d87243c9-3183-4ab9-9df9-c77c8278df11 # admin + - d87243c9-3183-4ab9-9df9-c77c8278df11 teams: - 018770c4-4b73-5d44-8bb5-0e849d62e461 actions: - name: 'scale deployment' exec: - script: kubectl scale --replicas={{.params.replicas}} deployment {{.config.name}} + script: kubectl scale --replicas={{.params.replicas}} --namespace={{.config.namespace}} deployment {{.config.name}} From 020158594f1189cf80c807ffe63069a18eb1a5b2 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 08:45:06 +0545 Subject: [PATCH 33/39] do not wait for start time. fetch only those runs that should have started by now. --- db/playbooks.go | 9 ++++----- playbook/queue_consumer.go | 6 +----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/db/playbooks.go b/db/playbooks.go index 0f59c5a89..5e0174d19 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "time" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" @@ -79,11 +78,11 @@ func FindPlaybooksForComponent(ctx *api.Context, configType string, tags map[str return playbooks, err } -// GetScheduledPlaybookRuns returns all the scheduled playbook runs that should be started -// before X duration from now. -func GetScheduledPlaybookRuns(ctx *api.Context, startingBefore time.Duration, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { +// GetScheduledPlaybookRuns returns all the scheduled playbook runs that are scheduled to run now +// or before now. +func GetScheduledPlaybookRuns(ctx *api.Context, limit int, exceptions ...uuid.UUID) ([]models.PlaybookRun, error) { var runs []models.PlaybookRun - if err := ctx.DB().Not(exceptions).Where("start_time <= NOW() + ?", startingBefore).Where("status = ?", models.PlaybookRunStatusScheduled).Order("start_time").Find(&runs).Error; err != nil { + if err := ctx.DB().Not(exceptions).Where("start_time <= NOW()").Where("status = ?", models.PlaybookRunStatusScheduled).Limit(limit).Order("start_time").Find(&runs).Error; err != nil { return nil, err } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index 0c9a977e1..75e0ecfe7 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -123,7 +123,7 @@ func (t *queueConsumer) onPlaybookRunNewApproval(ctx *api.Context, runID string) } func (t *queueConsumer) consumeAll(ctx *api.Context) error { - runs, err := db.GetScheduledPlaybookRuns(ctx, time.Minute*10, t.getRunIDsInRegistry()...) + runs, err := db.GetScheduledPlaybookRuns(ctx, 1, t.getRunIDsInRegistry()...) if err != nil { return fmt.Errorf("failed to get playbook runs: %w", err) } @@ -135,10 +135,6 @@ func (t *queueConsumer) consumeAll(ctx *api.Context) error { for _, r := range runs { go func(run models.PlaybookRun) { if _, loaded := t.registry.LoadOrStore(run.ID, nil); !loaded { - if !run.StartTime.After(time.Now()) { - time.Sleep(time.Until(run.StartTime)) - } - ExecuteRun(ctx, run) } From f86df806a2e3abd7088c1b1f872f91a03404681e Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 12:21:25 +0545 Subject: [PATCH 34/39] feat: In approval spec, support team name and people emails instead of ids [skip ci] --- api/global.go | 31 +++++++++++++------ api/v1/playbook_types.go | 9 +++--- auth/clerk_client.go | 3 +- auth/middleware.go | 2 +- ...ion-control.flanksource.com_playbooks.yaml | 2 ++ db/playbooks.go | 16 +++++++--- fixtures/playbooks/scale-deployment.yaml | 4 +-- playbook/approval.go | 17 ++++++---- playbook/controllers.go | 9 ++---- playbook/playbook_test.go | 4 +-- playbook/suite_test.go | 2 +- 11 files changed, 58 insertions(+), 41 deletions(-) diff --git a/api/global.go b/api/global.go index 14992b2bd..8ee529f46 100644 --- a/api/global.go +++ b/api/global.go @@ -15,9 +15,21 @@ import ( "k8s.io/client-go/kubernetes" ) -type ContextKey string +// contextKey represents an internal key for adding context fields. +type contextKey int + +// List of context keys. +// These are used to store request-scoped information. +const ( + // stores current logged in user + userContextKey contextKey = iota +) -const UserIDContextKey ContextKey = "User-ID" +// ContextUser carries basic information of the current logged in user +type ContextUser struct { + ID uuid.UUID + Email string +} const UserIDHeaderKey = "X-User-ID" @@ -63,18 +75,17 @@ func (c *Context) DB() *gorm.DB { return c.db.WithContext(c.Context) } -func (c *Context) UserID() *uuid.UUID { - id, ok := c.Context.Value(UserIDContextKey).(string) - if !ok { - return nil - } +func (c *Context) WithUser(user *ContextUser) { + c.Context = gocontext.WithValue(c.Context, userContextKey, user) +} - u, err := uuid.Parse(id) - if err != nil { +func (c *Context) User() *ContextUser { + user, ok := c.Context.Value(userContextKey).(*ContextUser) + if !ok { return nil } - return &u + return user } func (c *Context) GetEnvVarValue(input types.EnvVar) (string, error) { diff --git a/api/v1/playbook_types.go b/api/v1/playbook_types.go index 3911a1e29..c46929cd6 100644 --- a/api/v1/playbook_types.go +++ b/api/v1/playbook_types.go @@ -28,18 +28,17 @@ type PlaybookParameter struct { } type PlaybookApprovers struct { + // Emails of the approvers People []string `json:"people,omitempty" yaml:"people,omitempty"` - Teams []string `json:"teams,omitempty" yaml:"teams,omitempty"` + + // Names of the teams + Teams []string `json:"teams,omitempty" yaml:"teams,omitempty"` } func (t *PlaybookApprovers) Empty() bool { return len(t.People) == 0 && len(t.Teams) == 0 } -func (t *PlaybookApprovers) IDs() []string { - return append(t.People, t.Teams...) -} - type PlaybookApprovalType string const ( diff --git a/auth/clerk_client.go b/auth/clerk_client.go index 7836d5a2b..23baf9c4e 100644 --- a/auth/clerk_client.go +++ b/auth/clerk_client.go @@ -1,7 +1,6 @@ package auth import ( - "context" "fmt" "net/http" "strings" @@ -87,7 +86,7 @@ func (h ClerkHandler) Session(next echo.HandlerFunc) echo.HandlerFunc { c.Request().Header.Set(echo.HeaderAuthorization, fmt.Sprintf("Bearer %s", token)) c.Request().Header.Set(api.UserIDHeaderKey, user.ID.String()) - ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, user.ID.String()) + ctx.WithUser(&api.ContextUser{ID: user.ID, Email: user.Email}) return next(ctx) } } diff --git a/auth/middleware.go b/auth/middleware.go index 2a88b047d..52d2ed77d 100644 --- a/auth/middleware.go +++ b/auth/middleware.go @@ -96,7 +96,7 @@ func (k *kratosMiddleware) Session(next echo.HandlerFunc) echo.HandlerFunc { c.Request().Header.Set(api.UserIDHeaderKey, session.Identity.GetId()) ctx := c.(*api.Context) - ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, session.Identity.GetId()) + ctx.WithUser(&api.ContextUser{ID: uuid.MustParse(session.Identity.GetId()), Email: session.Identity.GetId()}) return next(ctx) } diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml index b75a75e86..76c38f3a2 100644 --- a/config/crds/mission-control.flanksource.com_playbooks.yaml +++ b/config/crds/mission-control.flanksource.com_playbooks.yaml @@ -270,10 +270,12 @@ spec: approvers: properties: people: + description: Emails of the approvers items: type: string type: array teams: + description: Names of the teams items: type: string type: array diff --git a/db/playbooks.go b/db/playbooks.go index 5e0174d19..d1f4b21e3 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -10,7 +10,6 @@ import ( "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/google/uuid" - "github.com/lib/pq" "gorm.io/gorm" ) @@ -117,9 +116,9 @@ func UpdatePlaybookRunStatusIfApproved(ctx *api.Context, playbookID string, appr return nil } - subQuery := `SELECT run_id FROM run_approvals WHERE approvers @> ?` + operator := `@>` if approval.Type == v1.PlaybookApprovalTypeAny { - subQuery = `SELECT run_id FROM run_approvals WHERE approvers && ?` + operator = `&&` } query := fmt.Sprintf(` @@ -127,13 +126,20 @@ func UpdatePlaybookRunStatusIfApproved(ctx *api.Context, playbookID string, appr SELECT run_id, ARRAY_AGG(COALESCE(person_id, team_id)) AS approvers FROM playbook_approvals GROUP BY run_id + ), + allowed_approvers AS ( + SELECT id FROM teams WHERE name IN ? + UNION + SELECT id FROM people WHERE email IN ? ) UPDATE playbook_runs SET status = ? WHERE status = ? AND playbook_id = ? - AND id IN (%s)`, subQuery) + AND id IN ( + SELECT run_id FROM run_approvals WHERE approvers %s (SELECT array_agg(id) FROM allowed_approvers) + )`, operator) - return ctx.DB().Exec(query, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID, pq.Array(approval.Approvers.IDs())).Error + return ctx.DB().Debug().Exec(query, approval.Approvers.Teams, approval.Approvers.People, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID).Error } func SavePlaybookRunApproval(ctx *api.Context, approval models.PlaybookApproval) error { diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index 1a8dc8e04..def1ab7fa 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -15,9 +15,9 @@ spec: type: any approvers: people: - - d87243c9-3183-4ab9-9df9-c77c8278df11 + - admin@local teams: - - 018770c4-4b73-5d44-8bb5-0e849d62e461 + - DevOps actions: - name: 'scale deployment' exec: diff --git a/playbook/approval.go b/playbook/approval.go index 628ff8ec6..1f04241e9 100644 --- a/playbook/approval.go +++ b/playbook/approval.go @@ -9,7 +9,12 @@ import ( "github.com/google/uuid" ) -func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error { +func ApproveRun(ctx *api.Context, playbookID, runID uuid.UUID) error { + approver := ctx.User() + if approver == nil { + return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook run") + } + playbook, err := db.FindPlaybook(ctx, playbookID) if err != nil { return api.Errorf(api.EINTERNAL, "something went wrong while finding playbook(id=%s)", playbookID).WithDebugInfo("db.FindPlaybook(id=%s): %v", playbookID, err) @@ -30,12 +35,12 @@ func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error RunID: runID, } - if collections.Contains(playbookV1.Spec.Approval.Approvers.People, approverID.String()) { - approval.PersonID = &approverID + if collections.Contains(playbookV1.Spec.Approval.Approvers.People, approver.Email) { + approval.PersonID = &approver.ID } else { - teamIDs, err := db.GetTeamIDsForUser(ctx, approverID.String()) + teamIDs, err := db.GetTeamIDsForUser(ctx, approver.ID.String()) if err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamIDsForUser(id=%s): %v", approverID, err) + return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamIDsForUser(id=%s): %v", approver.ID, err) } for _, teamID := range teamIDs { @@ -55,7 +60,7 @@ func ApproveRun(ctx *api.Context, approverID, playbookID, runID uuid.UUID) error } if err := db.SavePlaybookRunApproval(ctx, approval); err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approverID, err) + return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approver.ID, err) } return nil diff --git a/playbook/controllers.go b/playbook/controllers.go index 21d194c6b..ad1235b43 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -112,7 +112,7 @@ func HandlePlaybookRun(c echo.Context) error { PlaybookID: playbook.ID, Status: models.PlaybookRunStatusPending, Parameters: types.JSONStringMap(req.Params), - CreatedBy: ctx.UserID(), + CreatedBy: &ctx.User().ID, } if spec.Approval == nil || spec.Approval.Approvers.Empty() { @@ -187,15 +187,10 @@ func HandlePlaybookRunApproval(c echo.Context) error { ctx := c.(*api.Context) var ( - userID = ctx.UserID() playbookID = c.Param("playbook_id") runID = c.Param("run_id") ) - if userID == nil { - return c.JSON(http.StatusUnauthorized, api.HTTPError{Error: "user id is required"}) - } - playbookUUID, err := uuid.Parse(playbookID) if err != nil { return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid playbook id"}) @@ -206,7 +201,7 @@ func HandlePlaybookRunApproval(c echo.Context) error { return c.JSON(http.StatusBadRequest, api.HTTPError{Error: err.Error(), Message: "invalid run id"}) } - if err := ApproveRun(ctx, *userID, playbookUUID, runUUID); err != nil { + if err := ApproveRun(ctx, playbookUUID, runUUID); err != nil { return api.WriteError(c, err) } diff --git a/playbook/playbook_test.go b/playbook/playbook_test.go index f09fa39bd..9f18380be 100644 --- a/playbook/playbook_test.go +++ b/playbook/playbook_test.go @@ -52,8 +52,8 @@ var _ = ginkgo.Describe("Playbook runner", ginkgo.Ordered, func() { Type: v1.PlaybookApprovalTypeAny, // We have two approvers (John Doe & John Wick) and just a single approval is sufficient Approvers: v1.PlaybookApprovers{ People: []string{ - dummy.JohnDoe.ID.String(), - dummy.JohnWick.ID.String(), + dummy.JohnDoe.Email, + dummy.JohnWick.Email, }, }, }, diff --git a/playbook/suite_test.go b/playbook/suite_test.go index f2d98bac7..4bbe2205a 100644 --- a/playbook/suite_test.go +++ b/playbook/suite_test.go @@ -110,7 +110,7 @@ func mockAuthMiddleware(next echo.HandlerFunc) echo.HandlerFunc { } ctx := c.(*api.Context) - ctx.Context = context.WithValue(ctx.Context, api.UserIDContextKey, person.ID.String()) + ctx.WithUser(&api.ContextUser{ID: person.ID, Email: person.Email}) return next(c) } From c10026c187f4090fc2b57e3702e998c1b00376c1 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 14:45:25 +0545 Subject: [PATCH 35/39] fix: approval --- auth/middleware.go | 8 +++++++- db/people.go | 8 ++++---- db/playbooks.go | 31 ++++++++++++++++++++++++++++++- playbook/approval.go | 10 +++++----- playbook/queue_consumer.go | 1 + 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/auth/middleware.go b/auth/middleware.go index 52d2ed77d..e3c794959 100644 --- a/auth/middleware.go +++ b/auth/middleware.go @@ -96,7 +96,13 @@ func (k *kratosMiddleware) Session(next echo.HandlerFunc) echo.HandlerFunc { c.Request().Header.Set(api.UserIDHeaderKey, session.Identity.GetId()) ctx := c.(*api.Context) - ctx.WithUser(&api.ContextUser{ID: uuid.MustParse(session.Identity.GetId()), Email: session.Identity.GetId()}) + var email string + if traits, ok := session.Identity.GetTraits().(map[string]any); ok { + if e, ok := traits["email"].(string); ok { + email = e + } + } + ctx.WithUser(&api.ContextUser{ID: uuid.MustParse(session.Identity.GetId()), Email: email}) return next(ctx) } diff --git a/db/people.go b/db/people.go index 0f0edcb2e..815d2a7a6 100644 --- a/db/people.go +++ b/db/people.go @@ -38,10 +38,10 @@ func GetUserByID(ctx *api.Context, id string) (api.Person, error) { return user, err } -func GetTeamIDsForUser(ctx *api.Context, id string) ([]uuid.UUID, error) { - var teamIDs []uuid.UUID - err := ctx.DB().Raw("SELECT team_id FROM team_members WHERE person_id = ?", id).Scan(&teamIDs).Error - return teamIDs, err +func GetTeamsForUser(ctx *api.Context, id string) ([]models.Team, error) { + var teams []models.Team + err := ctx.DB().Raw("SELECT teams.* FROM teams LEFT JOIN team_members ON teams.id = team_members.team_id WHERE team_members.person_id = ?", id).Scan(&teams).Error + return teams, err } func GetUserByExternalID(ctx *api.Context, id string) (api.Person, error) { diff --git a/db/playbooks.go b/db/playbooks.go index d1f4b21e3..79c82a5f0 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -26,6 +26,35 @@ func FindPlaybook(ctx *api.Context, id uuid.UUID) (*models.Playbook, error) { return &p, nil } +// CanApprove returns true if the given person can approve runs of the given playbook. +func CanApprove(ctx *api.Context, personID, playbookID string) (bool, error) { + query := ` + WITH playbook_approvers AS ( + SELECT id, + ARRAY(SELECT jsonb_array_elements_text(spec->'approval'->'approvers'->'teams')) teams, + ARRAY(SELECT jsonb_array_elements_text(spec->'approval'->'approvers'->'people')) people + FROM playbooks + WHERE id = ? + ) + SELECT COUNT(*) FROM playbook_approvers WHERE + CAST(playbook_approvers.teams AS text[]) && ( -- check if the person belongs to a team that can approve + SELECT array_agg(teams.name) FROM teams LEFT JOIN team_members + ON teams.id = team_members.team_id + WHERE person_id = ? + ) + OR + CAST(playbook_approvers.people AS text[]) @> ARRAY( -- check if the person is an approver + SELECT email FROM people WHERE id = ? + )` + + var count int + if err := ctx.DB().Raw(query, playbookID, personID, personID).Scan(&count).Error; err != nil { + return false, err + } + + return count > 0, nil +} + func GetPlaybookRun(ctx *api.Context, id string) (*models.PlaybookRun, error) { var p models.PlaybookRun if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { @@ -139,7 +168,7 @@ func UpdatePlaybookRunStatusIfApproved(ctx *api.Context, playbookID string, appr SELECT run_id FROM run_approvals WHERE approvers %s (SELECT array_agg(id) FROM allowed_approvers) )`, operator) - return ctx.DB().Debug().Exec(query, approval.Approvers.Teams, approval.Approvers.People, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID).Error + return ctx.DB().Exec(query, approval.Approvers.Teams, approval.Approvers.People, models.PlaybookRunStatusScheduled, models.PlaybookRunStatusPending, playbookID).Error } func SavePlaybookRunApproval(ctx *api.Context, approval models.PlaybookApproval) error { diff --git a/playbook/approval.go b/playbook/approval.go index 1f04241e9..fc5434620 100644 --- a/playbook/approval.go +++ b/playbook/approval.go @@ -12,7 +12,7 @@ import ( func ApproveRun(ctx *api.Context, playbookID, runID uuid.UUID) error { approver := ctx.User() if approver == nil { - return api.Errorf(api.EFORBIDDEN, "you are not allowed to approve this playbook run") + return api.Errorf(api.EUNAUTHORIZED, "user not found.") } playbook, err := db.FindPlaybook(ctx, playbookID) @@ -38,14 +38,14 @@ func ApproveRun(ctx *api.Context, playbookID, runID uuid.UUID) error { if collections.Contains(playbookV1.Spec.Approval.Approvers.People, approver.Email) { approval.PersonID = &approver.ID } else { - teamIDs, err := db.GetTeamIDsForUser(ctx, approver.ID.String()) + teams, err := db.GetTeamsForUser(ctx, approver.ID.String()) if err != nil { return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamIDsForUser(id=%s): %v", approver.ID, err) } - for _, teamID := range teamIDs { - if collections.Contains(playbookV1.Spec.Approval.Approvers.Teams, teamID.String()) { - approval.TeamID = &teamID + for _, team := range teams { + if collections.Contains(playbookV1.Spec.Approval.Approvers.Teams, team.Name) { + approval.TeamID = &team.ID break } } diff --git a/playbook/queue_consumer.go b/playbook/queue_consumer.go index 75e0ecfe7..ac6ce85b2 100644 --- a/playbook/queue_consumer.go +++ b/playbook/queue_consumer.go @@ -122,6 +122,7 @@ func (t *queueConsumer) onPlaybookRunNewApproval(ctx *api.Context, runID string) return db.UpdatePlaybookRunStatusIfApproved(ctx, playbook.ID.String(), *spec.Approval) } +// TODO: Limit to X workers (5 workers, 1 batch size) func (t *queueConsumer) consumeAll(ctx *api.Context) error { runs, err := db.GetScheduledPlaybookRuns(ctx, 1, t.getRunIDsInRegistry()...) if err != nil { From eb849e5b98416e45bc2813ac85a5b5bfce82e01a Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 14:49:12 +0545 Subject: [PATCH 36/39] chore: bump duty --- go.mod | 10 ++++------ go.sum | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 53e6a4a2e..f73708ab3 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/containrrr/shoutrrr v0.7.1 github.com/fergusstrange/embedded-postgres v1.23.0 github.com/flanksource/commons v1.11.0 - github.com/flanksource/duty v1.0.159 + github.com/flanksource/duty v1.0.161 github.com/flanksource/gomplate/v3 v3.20.11 github.com/flanksource/kopper v1.0.6 github.com/google/cel-go v0.17.6 @@ -38,7 +38,7 @@ require ( ) require ( - ariga.io/atlas v0.13.1 // indirect + ariga.io/atlas v0.13.2 // indirect cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.2 // indirect @@ -101,7 +101,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect - github.com/zclconf/go-cty v1.13.2 // indirect + github.com/zclconf/go-cty v1.13.3 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect @@ -142,7 +142,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/TomOnTime/utfutil v0.0.0-20230223141146-125e65197b36 github.com/antonmedv/expr v1.14.3 // indirect - github.com/aws/aws-sdk-go v1.44.330 // indirect + github.com/aws/aws-sdk-go v1.44.331 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/cjlapao/common-go v0.0.39 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -221,5 +221,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/flanksource/duty => ../duty diff --git a/go.sum b/go.sum index f689c9678..f29495742 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ ariga.io/atlas v0.13.1 h1:oSkEYgI3qUnQZ6b6+teAEiIuizjBvkZ4YDbz0XWfCdQ= ariga.io/atlas v0.13.1/go.mod h1:+TR129FJZ5Lvzms6dvCeGWh1yR6hMvmXBhug4hrNIGk= +ariga.io/atlas v0.13.2 h1:52uuqedNjRvuTLtpHJV2KNQve1iGmBNTibAPQTwpz80= +ariga.io/atlas v0.13.2/go.mod h1:+TR129FJZ5Lvzms6dvCeGWh1yR6hMvmXBhug4hrNIGk= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -649,6 +651,7 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/antonmedv/expr v1.14.3 h1:GPrP7xKPWkFaLANPS7tPrgkNs7FMHpZdL72Dc5kFykg= @@ -669,6 +672,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.330 h1:kO41s8I4hRYtWSIuMc/O053wmEGfMTT8D4KtPSojUkA= github.com/aws/aws-sdk-go v1.44.330/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.331 h1:hEwdOTv6973uegCUY2EY8jyyq0OUg9INc0HOzcu2bjw= +github.com/aws/aws-sdk-go v1.44.331/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -766,6 +771,8 @@ github.com/flanksource/commons v1.11.0 h1:ThP3hnX4Xh4thxVl2GjQ92WvQ93jq5VqzJh46j github.com/flanksource/commons v1.11.0/go.mod h1:zYEhi6E2+diQ+loVcROUHo/Bgv+Tn61W2NYmrb5MgVI= github.com/flanksource/duty v1.0.159 h1:CkSKDZ4HYQ7cSEy8ufg1WODqcghkWsiAlk2o4I4JkqY= github.com/flanksource/duty v1.0.159/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk= +github.com/flanksource/duty v1.0.161 h1:fuEFH5A7kKAApxDP/FzFRaD2HuCrUJhwYjeuNxj/JVY= +github.com/flanksource/duty v1.0.161/go.mod h1:C3eT1PfdqTdefpGRDfUzLDVjSKuYjqZbgbIqX757FbA= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.20.11 h1:B83fXI0dqlOii2QE+qPNI0blPlcZTmfnPPU8zwVGUtA= github.com/flanksource/gomplate/v3 v3.20.11/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo= @@ -1034,6 +1041,7 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY= github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4= @@ -1396,6 +1404,8 @@ github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0= github.com/zclconf/go-cty v1.13.2/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +github.com/zclconf/go-cty v1.13.3 h1:m+b9q3YDbg6Bec5rr+KGy1MzEVzY/jC2X+YX4yqKtHI= +github.com/zclconf/go-cty v1.13.3/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= @@ -2211,14 +2221,20 @@ k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/api v0.26.4/go.mod h1:WwKEXU3R1rgCZ77AYa7DFksd9/BAIKyOmRlbVxgvjCk= k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= k8s.io/apiextensions-apiserver v0.27.4 h1:ie1yZG4nY/wvFMIR2hXBeSVq+HfNzib60FjnBYtPGSs= k8s.io/apiextensions-apiserver v0.27.4/go.mod h1:KHZaDr5H9IbGEnSskEUp/DsdXe1hMQ7uzpQcYUFt2bM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= k8s.io/component-base v0.27.4 h1:Wqc0jMKEDGjKXdae8hBXeskRP//vu1m6ypC+gwErj4c= k8s.io/component-base v0.27.4/go.mod h1:hoiEETnLc0ioLv6WPeDt8vD34DDeB35MfQnxCARq3kY= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= From ba5089c87e098a150bd528e25cbc14e783c2e4df Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 15:04:07 +0545 Subject: [PATCH 37/39] chore: move templating to exec action itself --- fixtures/playbooks/deleting-configmap.yaml | 2 +- fixtures/playbooks/scale-deployment.yaml | 2 +- playbook/actions/actions.go | 18 ++++++++++++ playbook/actions/exec.go | 9 +++++- playbook/runner.go | 34 ++++------------------ 5 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 playbook/actions/actions.go diff --git a/fixtures/playbooks/deleting-configmap.yaml b/fixtures/playbooks/deleting-configmap.yaml index 794060e4e..0d39b28ca 100644 --- a/fixtures/playbooks/deleting-configmap.yaml +++ b/fixtures/playbooks/deleting-configmap.yaml @@ -9,4 +9,4 @@ spec: actions: - name: 'Delete ConfigMap' exec: - script: kubectl delete configmap {{.config.name}} --namespace={{.config.namespace}} + script: kubectl delete configmap {{.config.name}} {{if .config.namespace}}--namespace={{.config.namespace}}{{end}} diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index def1ab7fa..d5e1e10f7 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -21,4 +21,4 @@ spec: actions: - name: 'scale deployment' exec: - script: kubectl scale --replicas={{.params.replicas}} --namespace={{.config.namespace}} deployment {{.config.name}} + script: kubectl scale --replicas={{.params.replicas}} {{if .config.namespace}}--namespace={{.config.namespace}}{{end}} deployment {{.config.name}} diff --git a/playbook/actions/actions.go b/playbook/actions/actions.go new file mode 100644 index 000000000..afbbafd21 --- /dev/null +++ b/playbook/actions/actions.go @@ -0,0 +1,18 @@ +package actions + +import "github.com/flanksource/duty/models" + +// TemplateEnv defines the config and component passed to a playbook run action. +type TemplateEnv struct { + Config *models.ConfigItem `json:"config,omitempty"` + Component *models.Component `json:"component,omitempty"` + Params map[string]string `json:"params,omitempty"` +} + +func (t *TemplateEnv) AsMap() map[string]any { + return map[string]any{ + "config": t.Config, + "component": t.Component, + "params": t.Params, + } +} diff --git a/playbook/actions/exec.go b/playbook/actions/exec.go index 158500f21..833762268 100644 --- a/playbook/actions/exec.go +++ b/playbook/actions/exec.go @@ -12,6 +12,7 @@ import ( textTemplate "text/template" "github.com/flanksource/commons/logger" + "github.com/flanksource/gomplate/v3" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" ) @@ -25,7 +26,13 @@ type ExecDetails struct { ExitCode int `json:"exitCode,omitempty"` } -func (c *ExecAction) Run(ctx *api.Context, exec v1.ExecAction) (*ExecDetails, error) { +func (c *ExecAction) Run(ctx *api.Context, exec v1.ExecAction, env TemplateEnv) (*ExecDetails, error) { + script, err := gomplate.RunTemplate(env.AsMap(), gomplate.Template{Template: exec.Script}) + if err != nil { + return nil, err + } + exec.Script = script + switch runtime.GOOS { case "windows": return execPowershell(exec, ctx) diff --git a/playbook/runner.go b/playbook/runner.go index 84296c7a4..8b5151d48 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -6,27 +6,11 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/duty/models" - "github.com/flanksource/gomplate/v3" "github.com/flanksource/incident-commander/api" v1 "github.com/flanksource/incident-commander/api/v1" "github.com/flanksource/incident-commander/playbook/actions" ) -// ActionParam defines the config and component passed to a playbook run action. -type ActionParam struct { - Config *models.ConfigItem `json:"config,omitempty"` - Component *models.Component `json:"component,omitempty"` - Params map[string]string `json:"params,omitempty"` -} - -func (t *ActionParam) AsMap() map[string]any { - return map[string]any{ - "config": t.Config, - "component": t.Component, - "params": t.Params, - } -} - func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { logger.Infof("Executing playbook run: %s", run.ID) @@ -62,15 +46,15 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return err } - actionParam := ActionParam{ + templateEnv := actions.TemplateEnv{ Params: run.Parameters, } if run.ComponentID != nil { - if err := ctx.DB().Where("id = ?", run.ComponentID).First(&actionParam.Component).Error; err != nil { + if err := ctx.DB().Where("id = ?", run.ComponentID).First(&templateEnv.Component).Error; err != nil { return err } } else if run.ConfigID != nil { - if err := ctx.DB().Where("id = ?", run.ConfigID).First(&actionParam.Config).Error; err != nil { + if err := ctx.DB().Where("id = ?", run.ConfigID).First(&templateEnv.Config).Error; err != nil { return err } } @@ -93,7 +77,7 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { "end_time": "NOW()", } - result, err := executeAction(ctx, run, action, actionParam) + result, err := executeAction(ctx, run, action, templateEnv) if err != nil { logger.Errorf("failed to execute action: %v", err) columnUpdates["status"] = models.PlaybookRunStatusFailed @@ -116,16 +100,10 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return nil } -func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookAction, actionParam ActionParam) ([]byte, error) { +func executeAction(ctx *api.Context, run models.PlaybookRun, action v1.PlaybookAction, env actions.TemplateEnv) ([]byte, error) { if action.Exec != nil { - script, err := gomplate.RunTemplate(actionParam.AsMap(), gomplate.Template{Template: action.Exec.Script}) - if err != nil { - return nil, err - } - action.Exec.Script = script - e := actions.ExecAction{} - res, err := e.Run(ctx, *action.Exec) + res, err := e.Run(ctx, *action.Exec, env) if err != nil { return nil, err } From c004db43d81c7d8d3f001376269d413b5ffe9848 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 15:07:48 +0545 Subject: [PATCH 38/39] chore: better logging --- playbook/runner.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/playbook/runner.go b/playbook/runner.go index 8b5151d48..d7a5f28a7 100644 --- a/playbook/runner.go +++ b/playbook/runner.go @@ -12,8 +12,6 @@ import ( ) func ExecuteRun(ctx *api.Context, run models.PlaybookRun) { - logger.Infof("Executing playbook run: %s", run.ID) - if err := ctx.DB().Model(&models.PlaybookRun{}).Where("id = ?", run.ID).UpdateColumn("status", models.PlaybookRunStatusRunning).Error; err != nil { logger.Errorf("failed to update playbook run status: %v", err) return @@ -46,6 +44,12 @@ func executeRun(ctx *api.Context, run models.PlaybookRun) error { return err } + logger.WithValues("playbook", playbook.Name). + WithValues("parameters", run.Parameters). + WithValues("config", run.ConfigID). + WithValues("component", run.ComponentID). + Infof("Executing playbook run: %s", run.ID) + templateEnv := actions.TemplateEnv{ Params: run.Parameters, } From 08d0927fd9a96197ae33d3afb0ccba62ed1503dc Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Fri, 25 Aug 2023 15:57:08 +0545 Subject: [PATCH 39/39] chore: update fixture. Use namespace from config tags --- fixtures/playbooks/deleting-configmap.yaml | 2 +- fixtures/playbooks/scale-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fixtures/playbooks/deleting-configmap.yaml b/fixtures/playbooks/deleting-configmap.yaml index 0d39b28ca..3974df1b8 100644 --- a/fixtures/playbooks/deleting-configmap.yaml +++ b/fixtures/playbooks/deleting-configmap.yaml @@ -9,4 +9,4 @@ spec: actions: - name: 'Delete ConfigMap' exec: - script: kubectl delete configmap {{.config.name}} {{if .config.namespace}}--namespace={{.config.namespace}}{{end}} + script: kubectl delete configmap {{.config.name}} --namespace={{.config.tags.namespace}} diff --git a/fixtures/playbooks/scale-deployment.yaml b/fixtures/playbooks/scale-deployment.yaml index d5e1e10f7..6b1240c42 100644 --- a/fixtures/playbooks/scale-deployment.yaml +++ b/fixtures/playbooks/scale-deployment.yaml @@ -21,4 +21,4 @@ spec: actions: - name: 'scale deployment' exec: - script: kubectl scale --replicas={{.params.replicas}} {{if .config.namespace}}--namespace={{.config.namespace}}{{end}} deployment {{.config.name}} + script: kubectl scale --replicas={{.params.replicas}} --namespace={{.config.tags.namespace}} deployment {{.config.name}}