Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: playbooks #499

Merged
merged 42 commits into from
Aug 25, 2023
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
d0c0b6f
init: define spec for playbooks
adityathebe Aug 15, 2023
6d31484
feat: add a new /playbook/run endpoint
adityathebe Aug 15, 2023
2a1c400
chore: register runs and return runs
adityathebe Aug 15, 2023
a1fb143
chore: add a basic playbook run queue processor
adityathebe Aug 16, 2023
7be1908
feat: add parameters validation & update playbook run on completion
adityathebe Aug 16, 2023
6c2404d
feat: implement exec action
adityathebe Aug 16, 2023
e04cd31
feat: add playbook actions
adityathebe Aug 16, 2023
0ed9909
feat: enable templating of the scripts
adityathebe Aug 16, 2023
dc757c2
chore: do not store duration & move action to separate package
adityathebe Aug 16, 2023
ec7df03
feat: add playbook run consumer and remove cron job
adityathebe Aug 16, 2023
1200a77
feat: add k8s operator
adityathebe Aug 17, 2023
2058b80
fix: playbook schema for k8 & pass params to the template renderer
adityathebe Aug 17, 2023
4bb814e
feat: only fetch scheduled runs that should start in the next 10
adityathebe Aug 17, 2023
2959f96
chore: clean up
adityathebe Aug 17, 2023
4cdc4d9
test: playbook runs
adityathebe Aug 17, 2023
a9eb8fc
improved test
adityathebe Aug 17, 2023
147be3b
feat: add user ID to the context. It's used to save the creator in a
adityathebe Aug 18, 2023
e033eb7
feat: show a list of playbooks that can be applied to a config
adityathebe Aug 21, 2023
9424835
chore: update fixtures
adityathebe Aug 21, 2023
6dacf80
chore: make use of the payload sent by pg_notify
adityathebe Aug 21, 2023
995e141
feat: implement playbook approvals
adityathebe Aug 21, 2023
429bfd6
Merge branch 'main' into feat/playbooks
adityathebe Aug 22, 2023
024b9cf
chore: fix test and context userid error
adityathebe Aug 22, 2023
6bdb6d0
feat: impl approval endpoint and better error handling
adityathebe Aug 22, 2023
cd4d84b
Merge branch 'main' into feat/playbooks
adityathebe Aug 22, 2023
3e3aea1
feat: support different approval types
adityathebe Aug 22, 2023
4085c06
feat: implement team approval
adityathebe Aug 22, 2023
50edc22
feat: add test for playbook approval
adityathebe Aug 22, 2023
9e01e4d
chore: update spec
adityathebe Aug 23, 2023
e9dc441
chore: remove Description and Templatable for exec action
adityathebe Aug 24, 2023
e9c97f2
Merge branch 'main' into feat/playbooks
adityathebe Aug 24, 2023
34c6331
chore: bump duty
adityathebe Aug 24, 2023
76508d1
chore: save exec result as an object
adityathebe Aug 24, 2023
c8331b4
fix: listing of playbooks for a component
adityathebe Aug 24, 2023
cb72e40
chore: update fixtures
adityathebe Aug 25, 2023
0201585
do not wait for start time. fetch only those runs that should have
adityathebe Aug 25, 2023
f86df80
feat: In approval spec, support team name and people emails instead o…
adityathebe Aug 25, 2023
c10026c
fix: approval
adityathebe Aug 25, 2023
eb849e5
chore: bump duty
adityathebe Aug 25, 2023
ba5089c
chore: move templating to exec action itself
adityathebe Aug 25, 2023
c004db4
chore: better logging
adityathebe Aug 25, 2023
08d0927
chore: update fixture. Use namespace from config tags
adityathebe Aug 25, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions api/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ func (c *Context) GetEnvVarValue(input types.EnvVar) (string, error) {
return duty.GetEnvValueFromCache(c.Kubernetes, input, c.Namespace)
}

func (ctx *Context) GetEnvValueFromCache(env types.EnvVar) (string, error) {
return duty.GetEnvValueFromCache(ctx.Kubernetes, env, ctx.Namespace)
}

func (c *Context) HydrateConnection(connectionName string) (*models.Connection, error) {
if connectionName == "" || !strings.HasPrefix(connectionName, "connection://") {
return nil, nil
Expand Down
173 changes: 173 additions & 0 deletions api/v1/playbook_actions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
package v1

import (
"context"
"fmt"

"github.com/flanksource/duty"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/types"
"k8s.io/client-go/kubernetes"
)

type Labels map[string]string

type Description struct {
// Description for the check
Description string `yaml:"description,omitempty" json:"description,omitempty" template:"true"`
// Name of the check
Name string `yaml:"name" json:"name" template:"true"`
// Icon for overwriting default icon on the dashboard
Icon string `yaml:"icon,omitempty" json:"icon,omitempty" template:"true"`
// Labels for the check
Labels Labels `yaml:"labels,omitempty" json:"labels,omitempty"`
// Transformed checks have a delete strategy on deletion they can either be marked healthy, unhealthy or left as is
TransformDeleteStrategy string `yaml:"transformDeleteStrategy,omitempty" json:"transformDeleteStrategy,omitempty"`
}

type Template struct {
Template string `yaml:"template,omitempty" json:"template,omitempty"`
JSONPath string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"`
Expression string `yaml:"expr,omitempty" json:"expr,omitempty"`
Javascript string `yaml:"javascript,omitempty" json:"javascript,omitempty"`
}

type Templatable struct {
Test Template `yaml:"test,omitempty" json:"test,omitempty"`
Display Template `yaml:"display,omitempty" json:"display,omitempty"`
Transform Template `yaml:"transform,omitempty" json:"transform,omitempty"`
}

type ExecAction struct {
Description `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
// Script can be a inline script or a path to a script that needs to be executed
// On windows executed via powershell and in darwin and linux executed using bash
Script string `yaml:"script" json:"script"`
Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"`
}

type ExecConnections struct {
AWS *AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"`
GCP *GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"`
Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
}

type connectionContext interface {
context.Context
HydrateConnection(connectionName string) (*models.Connection, error)
GetEnvValueFromCache(env types.EnvVar) (string, error)
}

type GCPConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint and credentials.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"`
Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"`
}

// HydrateConnection attempts to find the connection by name
// and populate the endpoint and credentials.
func (g *GCPConnection) HydrateConnection(ctx connectionContext) error {
connection, err := ctx.HydrateConnection(g.ConnectionName)
if err != nil {
return err
}

if connection != nil {
g.Credentials = &types.EnvVar{ValueStatic: connection.Certificate}
g.Endpoint = connection.URL
}

return nil
}

type AzureConnection struct {
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"`
ClientSecret *types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"`
TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"`
}

// HydrateConnection attempts to find the connection by name
// and populate the endpoint and credentials.
func (g *AzureConnection) HydrateConnection(ctx connectionContext) error {
connection, err := ctx.HydrateConnection(g.ConnectionName)
if err != nil {
return err
}

if connection != nil {
g.ClientID = &types.EnvVar{ValueStatic: connection.Username}
g.ClientSecret = &types.EnvVar{ValueStatic: connection.Password}
g.TenantID = connection.Properties["tenantID"]
}

return nil
}

type AWSConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
AccessKey types.EnvVar `yaml:"accessKey" json:"accessKey,omitempty"`
SecretKey types.EnvVar `yaml:"secretKey" json:"secretKey,omitempty"`
SessionToken types.EnvVar `yaml:"sessionToken,omitempty" json:"sessionToken,omitempty"`
Region string `yaml:"region,omitempty" json:"region,omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
// Skip TLS verify when connecting to aws
SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
// glob path to restrict matches to a subset
ObjectPath string `yaml:"objectPath,omitempty" json:"objectPath,omitempty"`
// Use path style path: http://s3.amazonaws.com/BUCKET/KEY instead of http://BUCKET.s3.amazonaws.com/KEY
UsePathStyle bool `yaml:"usePathStyle,omitempty" json:"usePathStyle,omitempty"`
}

// Populate populates an AWSConnection with credentials and other information.
// If a connection name is specified, it'll be used to populate the endpoint, accessKey and secretKey.
func (t *AWSConnection) Populate(ctx connectionContext, k8s kubernetes.Interface, namespace string) error {
if t.ConnectionName != "" {
connection, err := ctx.HydrateConnection(t.ConnectionName)
if err != nil {
return fmt.Errorf("could not parse EC2 access key: %v", err)
}

t.AccessKey.ValueStatic = connection.Username
t.SecretKey.ValueStatic = connection.Password
if t.Endpoint == "" {
t.Endpoint = connection.URL
}

t.SkipTLSVerify = connection.InsecureTLS
if t.Region == "" {
if region, ok := connection.Properties["region"]; ok {
t.Region = region
}
}
}

if accessKey, err := duty.GetEnvValueFromCache(k8s, t.AccessKey, namespace); err != nil {
return fmt.Errorf("could not parse AWS access key id: %v", err)
} else {
t.AccessKey.ValueStatic = accessKey
}

if secretKey, err := duty.GetEnvValueFromCache(k8s, t.SecretKey, namespace); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS secret access key: %v", err))
} else {
t.SecretKey.ValueStatic = secretKey
}

if sessionToken, err := duty.GetEnvValueFromCache(k8s, t.SessionToken, namespace); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS session token: %v", err))
} else {
t.SessionToken.ValueStatic = sessionToken
}

return nil
}

type PlaybookAction struct {
Name string `yaml:"name" json:"name"`
TimeoutMinutes string `yaml:"timeout-minutes,omitempty" json:"timeout-minutes,omitempty"`
Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"`
}
72 changes: 72 additions & 0 deletions api/v1/playbook_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package v1

import (
"encoding/json"

"github.com/flanksource/duty/models"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)

type Permission struct {
Role string `json:"role,omitempty" yaml:"role,omitempty"`
Team string `json:"team,omitempty" yaml:"team,omitempty"`
Ref string `json:"ref,omitempty" yaml:"ref,omitempty"`
}

// PlaybookResourceFilter defines a filter that decides whether a resource (config or a component)
// is permitted be run on the Playbook.
type PlaybookResourceFilter struct {
Type string `json:"type,omitempty" yaml:"type,omitempty"`
Tags map[string]string `json:"tags,omitempty" yaml:"tags,omitempty"`
}

// PlaybookParameter defines a parameter that a playbook needs to run.
type PlaybookParameter struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Label string `json:"label,omitempty" yaml:"label,omitempty"`
}

type PlaybookSpec struct {
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Permissions []Permission `json:"permissions,omitempty" yaml:"permissions,omitempty"`
Configs []PlaybookResourceFilter `json:"configs,omitempty" yaml:"configs,omitempty"`
Components []PlaybookResourceFilter `json:"components,omitempty" yaml:"components,omitempty"`
Parameters []PlaybookParameter `json:"parameters,omitempty" yaml:"parameters,omitempty"`
Actions []PlaybookAction `json:"actions" yaml:"actions"`
}

// PlaybookStatus defines the observed state of Playbook
type PlaybookStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
}

// +kubebuilder:object:root=true
// +kubebuilder:subresource:status

// Playbook is the schema for the Playbooks API
type Playbook struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec PlaybookSpec `json:"spec,omitempty"`
Status PlaybookStatus `json:"status,omitempty"`
}

func PlaybookFromModel(p models.Playbook) (Playbook, error) {
var spec PlaybookSpec
if err := json.Unmarshal(p.Spec, &spec); err != nil {
return Playbook{}, nil
}

out := Playbook{
ObjectMeta: metav1.ObjectMeta{
Name: p.Name,
UID: types.UID(p.ID.String()),
CreationTimestamp: metav1.Time{Time: p.CreatedAt},
},
Spec: spec,
}

return out, nil
}
Loading