Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: playbooks #499

Merged
merged 42 commits into from
Aug 25, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
d0c0b6f
init: define spec for playbooks
adityathebe Aug 15, 2023
6d31484
feat: add a new /playbook/run endpoint
adityathebe Aug 15, 2023
2a1c400
chore: register runs and return runs
adityathebe Aug 15, 2023
a1fb143
chore: add a basic playbook run queue processor
adityathebe Aug 16, 2023
7be1908
feat: add parameters validation & update playbook run on completion
adityathebe Aug 16, 2023
6c2404d
feat: implement exec action
adityathebe Aug 16, 2023
e04cd31
feat: add playbook actions
adityathebe Aug 16, 2023
0ed9909
feat: enable templating of the scripts
adityathebe Aug 16, 2023
dc757c2
chore: do not store duration & move action to separate package
adityathebe Aug 16, 2023
ec7df03
feat: add playbook run consumer and remove cron job
adityathebe Aug 16, 2023
1200a77
feat: add k8s operator
adityathebe Aug 17, 2023
2058b80
fix: playbook schema for k8 & pass params to the template renderer
adityathebe Aug 17, 2023
4bb814e
feat: only fetch scheduled runs that should start in the next 10
adityathebe Aug 17, 2023
2959f96
chore: clean up
adityathebe Aug 17, 2023
4cdc4d9
test: playbook runs
adityathebe Aug 17, 2023
a9eb8fc
improved test
adityathebe Aug 17, 2023
147be3b
feat: add user ID to the context. It's used to save the creator in a
adityathebe Aug 18, 2023
e033eb7
feat: show a list of playbooks that can be applied to a config
adityathebe Aug 21, 2023
9424835
chore: update fixtures
adityathebe Aug 21, 2023
6dacf80
chore: make use of the payload sent by pg_notify
adityathebe Aug 21, 2023
995e141
feat: implement playbook approvals
adityathebe Aug 21, 2023
429bfd6
Merge branch 'main' into feat/playbooks
adityathebe Aug 22, 2023
024b9cf
chore: fix test and context userid error
adityathebe Aug 22, 2023
6bdb6d0
feat: impl approval endpoint and better error handling
adityathebe Aug 22, 2023
cd4d84b
Merge branch 'main' into feat/playbooks
adityathebe Aug 22, 2023
3e3aea1
feat: support different approval types
adityathebe Aug 22, 2023
4085c06
feat: implement team approval
adityathebe Aug 22, 2023
50edc22
feat: add test for playbook approval
adityathebe Aug 22, 2023
9e01e4d
chore: update spec
adityathebe Aug 23, 2023
e9dc441
chore: remove Description and Templatable for exec action
adityathebe Aug 24, 2023
e9c97f2
Merge branch 'main' into feat/playbooks
adityathebe Aug 24, 2023
34c6331
chore: bump duty
adityathebe Aug 24, 2023
76508d1
chore: save exec result as an object
adityathebe Aug 24, 2023
c8331b4
fix: listing of playbooks for a component
adityathebe Aug 24, 2023
cb72e40
chore: update fixtures
adityathebe Aug 25, 2023
0201585
do not wait for start time. fetch only those runs that should have
adityathebe Aug 25, 2023
f86df80
feat: In approval spec, support team name and people emails instead o…
adityathebe Aug 25, 2023
c10026c
fix: approval
adityathebe Aug 25, 2023
eb849e5
chore: bump duty
adityathebe Aug 25, 2023
ba5089c
chore: move templating to exec action itself
adityathebe Aug 25, 2023
c004db4
chore: better logging
adityathebe Aug 25, 2023
08d0927
chore: update fixture. Use namespace from config tags
adityathebe Aug 25, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions api/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package api

import (
"errors"
"fmt"
)

// Application error codes.
//
// These are meant to be generic and they map well to HTTP error codes.
const (
ECONFLICT = "conflict"
EFORBIDDEN = "forbidden"
EINTERNAL = "internal"
EINVALID = "invalid"
ENOTFOUND = "not_found"
ENOTIMPLEMENTED = "not_implemented"
EUNAUTHORIZED = "unauthorized"
)

// Error represents an application-specific error.
type Error struct {
// Machine-readable error code.
Code string

// Human-readable error message.
Message string

// DebugInfo contains low-level internal error details that should only be logged.
// End-users should never see this.
DebugInfo string
}

// Error implements the error interface. Not used by the application otherwise.
func (e *Error) Error() string {
return fmt.Sprintf("error: code=%s message=%s", e.Code, e.Message)
}

// WithDebugInfo wraps an application error with a debug message.
func (e *Error) WithDebugInfo(msg string, args ...any) *Error {
e.DebugInfo = fmt.Sprintf(msg, args...)
return e
}

// ErrorCode unwraps an application error and returns its code.
// Non-application errors always return EINTERNAL.
func ErrorCode(err error) string {
adityathebe marked this conversation as resolved.
Show resolved Hide resolved
var e *Error
if err == nil {
return ""
} else if errors.As(err, &e) {
return e.Code
}
return EINTERNAL
}

// ErrorMessage unwraps an application error and returns its message.
// Non-application errors always return "Internal error".
func ErrorMessage(err error) string {
var e *Error
if err == nil {
return ""
} else if errors.As(err, &e) {
return e.Message
}
return "Internal error."
}

// ErrorDebugInfo unwraps an application error and returns its debug message.
func ErrorDebugInfo(err error) string {
var e *Error
if err == nil {
return ""
} else if errors.As(err, &e) {
return e.DebugInfo
}
return ""
}

// Errorf is a helper function to return an Error with a given code and formatted message.
func Errorf(code string, format string, args ...any) *Error {
return &Error{
Code: code,
Message: fmt.Sprintf(format, args...),
}
}
33 changes: 32 additions & 1 deletion api/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,24 @@ import (
"k8s.io/client-go/kubernetes"
)

// contextKey represents an internal key for adding context fields.
type contextKey int

// List of context keys.
// These are used to store request-scoped information.
const (
UserIDHeaderKey = "X-User-ID"
// stores current logged in user
userContextKey contextKey = iota
)

// ContextUser carries basic information of the current logged in user
type ContextUser struct {
ID uuid.UUID
Email string
}

const UserIDHeaderKey = "X-User-ID"

var SystemUserID *uuid.UUID
var CanaryCheckerPath string
var ApmHubPath string
Expand Down Expand Up @@ -61,10 +75,27 @@ func (c *Context) DB() *gorm.DB {
return c.db.WithContext(c.Context)
}

func (c *Context) WithUser(user *ContextUser) {
c.Context = gocontext.WithValue(c.Context, userContextKey, user)
}

func (c *Context) User() *ContextUser {
user, ok := c.Context.Value(userContextKey).(*ContextUser)
if !ok {
return nil
}

return user
}

func (c *Context) GetEnvVarValue(input types.EnvVar) (string, error) {
return duty.GetEnvValueFromCache(c.Kubernetes, input, c.Namespace)
}

func (ctx *Context) GetEnvValueFromCache(env types.EnvVar) (string, error) {
return duty.GetEnvValueFromCache(ctx.Kubernetes, env, ctx.Namespace)
}

func (c *Context) HydrateConnection(connectionName string) (*models.Connection, error) {
if connectionName == "" || !strings.HasPrefix(connectionName, "connection://") {
return nil, nil
Expand Down
39 changes: 38 additions & 1 deletion api/http.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,48 @@
package api

import (
"net/http"

"github.com/flanksource/commons/logger"
"github.com/labstack/echo/v4"
)

type HTTPError struct {
Error string `json:"error"`
Message string `json:"message"`
Message string `json:"message,omitempty"`
}

type HTTPSuccess struct {
Message string `json:"message"`
Payload any `json:"payload,omitempty"`
}

func WriteError(c echo.Context, err error) error {
code, message := ErrorCode(err), ErrorMessage(err)

if debugInfo := ErrorDebugInfo(err); debugInfo != "" {
logger.WithValues("code", code, "error", message).Errorf(debugInfo)
}

return c.JSON(ErrorStatusCode(code), &HTTPError{Error: message})
}

// lookup of application error codes to HTTP status codes.
var codes = map[string]int{
ECONFLICT: http.StatusConflict,
EINVALID: http.StatusBadRequest,
ENOTFOUND: http.StatusNotFound,
EFORBIDDEN: http.StatusForbidden,
ENOTIMPLEMENTED: http.StatusNotImplemented,
EUNAUTHORIZED: http.StatusUnauthorized,
EINTERNAL: http.StatusInternalServerError,
}

// ErrorStatusCode returns the associated HTTP status code for an application error code.
func ErrorStatusCode(code string) int {
if v, ok := codes[code]; ok {
return v
}

return http.StatusInternalServerError
}
142 changes: 142 additions & 0 deletions api/v1/playbook_actions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
package v1

import (
"context"
"fmt"

"github.com/flanksource/duty"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/types"
"k8s.io/client-go/kubernetes"
)

type ExecAction struct {
// Script can be a inline script or a path to a script that needs to be executed
// On windows executed via powershell and in darwin and linux executed using bash
Script string `yaml:"script" json:"script"`
Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"`
}

type ExecConnections struct {
AWS *AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"`
GCP *GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"`
Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
}

type connectionContext interface {
context.Context
HydrateConnection(connectionName string) (*models.Connection, error)
GetEnvValueFromCache(env types.EnvVar) (string, error)
}

type GCPConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint and credentials.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"`
Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"`
}

// HydrateConnection attempts to find the connection by name
// and populate the endpoint and credentials.
func (g *GCPConnection) HydrateConnection(ctx connectionContext) error {
connection, err := ctx.HydrateConnection(g.ConnectionName)
if err != nil {
return err
}

if connection != nil {
g.Credentials = &types.EnvVar{ValueStatic: connection.Certificate}
g.Endpoint = connection.URL
}

return nil
}

type AzureConnection struct {
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"`
ClientSecret *types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"`
TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"`
}

// HydrateConnection attempts to find the connection by name
// and populate the endpoint and credentials.
func (g *AzureConnection) HydrateConnection(ctx connectionContext) error {
connection, err := ctx.HydrateConnection(g.ConnectionName)
if err != nil {
return err
}

if connection != nil {
g.ClientID = &types.EnvVar{ValueStatic: connection.Username}
g.ClientSecret = &types.EnvVar{ValueStatic: connection.Password}
g.TenantID = connection.Properties["tenantID"]
}

return nil
}

type AWSConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
AccessKey types.EnvVar `yaml:"accessKey" json:"accessKey,omitempty"`
SecretKey types.EnvVar `yaml:"secretKey" json:"secretKey,omitempty"`
SessionToken types.EnvVar `yaml:"sessionToken,omitempty" json:"sessionToken,omitempty"`
Region string `yaml:"region,omitempty" json:"region,omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
// Skip TLS verify when connecting to aws
SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
// glob path to restrict matches to a subset
ObjectPath string `yaml:"objectPath,omitempty" json:"objectPath,omitempty"`
// Use path style path: http://s3.amazonaws.com/BUCKET/KEY instead of http://BUCKET.s3.amazonaws.com/KEY
UsePathStyle bool `yaml:"usePathStyle,omitempty" json:"usePathStyle,omitempty"`
}

// Populate populates an AWSConnection with credentials and other information.
// If a connection name is specified, it'll be used to populate the endpoint, accessKey and secretKey.
func (t *AWSConnection) Populate(ctx connectionContext, k8s kubernetes.Interface, namespace string) error {
if t.ConnectionName != "" {
connection, err := ctx.HydrateConnection(t.ConnectionName)
if err != nil {
return fmt.Errorf("could not parse EC2 access key: %v", err)
}

t.AccessKey.ValueStatic = connection.Username
t.SecretKey.ValueStatic = connection.Password
if t.Endpoint == "" {
t.Endpoint = connection.URL
}

t.SkipTLSVerify = connection.InsecureTLS
if t.Region == "" {
if region, ok := connection.Properties["region"]; ok {
t.Region = region
}
}
}

if accessKey, err := duty.GetEnvValueFromCache(k8s, t.AccessKey, namespace); err != nil {
return fmt.Errorf("could not parse AWS access key id: %v", err)
} else {
t.AccessKey.ValueStatic = accessKey
}

if secretKey, err := duty.GetEnvValueFromCache(k8s, t.SecretKey, namespace); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS secret access key: %v", err))
} else {
t.SecretKey.ValueStatic = secretKey
}

if sessionToken, err := duty.GetEnvValueFromCache(k8s, t.SessionToken, namespace); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS session token: %v", err))
} else {
t.SessionToken.ValueStatic = sessionToken
}

return nil
}

type PlaybookAction struct {
Name string `yaml:"name" json:"name"`
Exec *ExecAction `json:"exec,omitempty" yaml:"exec,omitempty"`
}
Loading
Loading