diff --git a/README.md b/README.md index b2b620af..6a6034f8 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ kube-monkey works on an opt-in model and will only schedule terminations for Kub Opt-in is done by setting the following labels on a k8s app: **`kube-monkey/enabled`**: Set to **`"enabled"`** to opt-in to kube-monkey -**`kube-monkey/mtbf`**: Mean time between failure (in days). For example, if set to **`"3"`**, the k8s app can expect to have a Pod +**`kube-monkey/mtbf`**: Mean time between failure. Examples are: 1 or 1d (1 day), 4h (4 hours) and 20m (20 minutes). A recommended value would be 2h or 3h so pods get killed at least a couple of times a day. killed approximately every third weekday. **`kube-monkey/identifier`**: A unique identifier for the k8s apps. This is used to identify the pods that belong to a k8s app as Pods inherit labels from their k8s app. So, if kube-monkey detects that app `foo` has enrolled to be a victim, kube-monkey will look for all pods that have the label `kube-monkey/identifier: foo` to determine which pods are candidates for killing. The recommendation is to set this value to be the same as the app's name. @@ -41,7 +41,7 @@ that belong to a k8s app as Pods inherit labels from their k8s app. So, if kube- * if `random-max-percent`, provide a number from `0`-`100` to specify the max `%` of pods kube-monkey can kill * if `fixed-percent`, provide a number from `0`-`100` to specify the `%` of pods to kill -#### Example of opted-in Deployment killing one pod per purge +#### Example of opted-in Deployment killing one pod once every two hours. ```yaml --- @@ -56,7 +56,7 @@ spec: labels: kube-monkey/enabled: enabled kube-monkey/identifier: monkey-victim - kube-monkey/mtbf: '2' + kube-monkey/mtbf: '2h' kube-monkey/kill-mode: "fixed" kube-monkey/kill-value: '1' [... omitted ...] @@ -74,7 +74,7 @@ metadata: labels: kube-monkey/enabled: enabled kube-monkey/identifier: monkey-victim - kube-monkey/mtbf: '2' + kube-monkey/mtbf: '2h' kube-monkey/kill-mode: "fixed" kube-monkey/kill-value: '1' spec: diff --git a/internal/pkg/calendar/calendar.go b/internal/pkg/calendar/calendar.go index 7000ae74..ab301e51 100644 --- a/internal/pkg/calendar/calendar.go +++ b/internal/pkg/calendar/calendar.go @@ -1,7 +1,11 @@ package calendar import ( + "errors" + "fmt" "math/rand" + "strconv" + "strings" "time" "github.com/golang/glog" @@ -49,19 +53,90 @@ func NextRuntime(loc *time.Location, r int) time.Time { return time.Date(year, month, day, r, 0, 0, 0, loc) } +// ParseMtbf parses an mtbf value and returns a valid time duration. +func ParseMtbf(mtbf string) (time.Duration, error) { + // time.Duration biggest valid time unit is an hour, but we want to accept + // days. Before finer grained time units this software used to accept mtbf as + // an integer interpreted as days. Hence this routine now accepts a "d" as a + // valid time unit meaning days and simply strips it, because... + if mtbf[len(mtbf) - 1] == 'd' { + mtbf = strings.TrimRight(mtbf, "d") + } + // ...below we check if a given mtbf is simply a number and backward + // compatibilty dictates us to accept a simpel number as days (see above) and + // since time.Duration does not accept hours as a valid time unit we convert + // here ourselves days into hours. + if converted_mtbf, err := strconv.Atoi(mtbf); err == nil { + mtbf = fmt.Sprintf("%dh", converted_mtbf * 24) + } + duration, err := time.ParseDuration(mtbf) + if err != nil { + return 0, err + } + one_minute, _ := time.ParseDuration("1m") + if duration < one_minute { + return 0, errors.New("smallest valid mtbf is one minute.") + } + return duration, nil +} + // RandomTimeInRange returns a random time within the range specified by startHour and endHour -func RandomTimeInRange(startHour int, endHour int, loc *time.Location) time.Time { - // calculate the number of minutes in the range - minutesInRange := (endHour - startHour) * 60 - - // calculate a random minute-offset in range [0, minutesInRange) - r := rand.New(rand.NewSource(time.Now().UnixNano())) - randMinuteOffset := r.Intn(minutesInRange) - offsetDuration := time.Duration(randMinuteOffset) * time.Minute - - // Add the minute offset to the start of the range to get a random - // time within the range - year, month, date := time.Now().Date() - rangeStart := time.Date(year, month, date, startHour, 0, 0, 0, loc) - return rangeStart.Add(offsetDuration) +func RandomTimeInRange(mtbf string, startHour int, endHour int, loc *time.Location) []time.Time { + var times []time.Time + tmptimeDuration, err := ParseMtbf(mtbf) + if err != nil { + glog.Errorf("error parsing customized mtbf %s: %v", mtbf, err) + return []time.Time{time.Now().Add(time.Duration(24*365*10) * time.Hour)} + } + + one_day, _ := time.ParseDuration("24h") + + // If the mtbf is bigger or equal to one day we will calculate one + // random time in the range. If not we will calculate several random + // times. + if tmptimeDuration >= one_day { + // calculate the number of minutes in the range + minutesInRange := (endHour - startHour) * 60 + + // calculate a random minute-offset in range [0, minutesInRange) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + randMinuteOffset := r.Intn(minutesInRange) + offsetDuration := time.Duration(randMinuteOffset) * time.Minute + + // Add the minute offset to the start of the range to get a random + // time within the range + year, month, date := time.Now().Date() + rangeStart := time.Date(year, month, date, startHour, 0, 0, 0, loc) + times = append(times, rangeStart.Add(offsetDuration)) + return times + } else { + startTime := time.Now().In(loc) + + for { + //time range should be twice of the input mean time between failure value + timeDuration := tmptimeDuration * 2 + //compute random offset time + mtbfEndTime := startTime.Add(timeDuration) + subSecond := int64(mtbfEndTime.Sub(startTime) / time.Second) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + randSecondOffset := r.Int63n(subSecond) + randCalTime := startTime.Add(time.Duration(randSecondOffset) * time.Second) + + // compute randSecondOffset between start and end hour + year, month, date := startTime.Date() + todayEndTime := time.Date(year, month, date, endHour, 0, 0, 0, loc) + todayStartTime := time.Date(year, month, date, startHour, 0, 0, 0, loc) + if startTime.Before(todayStartTime) { // now is earlier then start hour, only for test pass, normal process won't run into this condition + return []time.Time{todayStartTime} + } + if randCalTime.Before(todayEndTime) { // time offset before today's endHour + glog.V(1).Infof("RandomTimeInRange calculate time %s", randCalTime) + times = append(times, randCalTime) + // Move start time up to the calculated random time + startTime = randCalTime + } else { + return times + } + } + } } diff --git a/internal/pkg/chaos/chaosmock.go b/internal/pkg/chaos/chaosmock.go index 55c4ef3d..d53463a9 100644 --- a/internal/pkg/chaos/chaosmock.go +++ b/internal/pkg/chaos/chaosmock.go @@ -73,7 +73,7 @@ func (vm *VictimMock) IsWhitelisted() bool { } func NewVictimMock() *VictimMock { - v := victims.New(KIND, NAME, NAMESPACE, IDENTIFIER, 1) + v := victims.New(KIND, NAME, NAMESPACE, IDENTIFIER, "1h") return &VictimMock{ VictimBase: *v, } diff --git a/internal/pkg/kubemonkey/kubemonkey.go b/internal/pkg/kubemonkey/kubemonkey.go index 42375eda..05ef7722 100644 --- a/internal/pkg/kubemonkey/kubemonkey.go +++ b/internal/pkg/kubemonkey/kubemonkey.go @@ -44,10 +44,6 @@ func Run() error { } for { - // Calculate duration to sleep before next run - sleepDuration := durationToNextRun(config.RunHour(), config.Timezone()) - time.Sleep(sleepDuration) - schedule, err := schedule.New() if err != nil { glog.Fatal(err.Error()) @@ -58,6 +54,10 @@ func Run() error { } fmt.Println(schedule) ScheduleTerminations(schedule.Entries(), notificationsClient) + + // Calculate duration to sleep before next run + sleepDuration := durationToNextRun(config.RunHour(), config.Timezone()) + time.Sleep(sleepDuration) } } diff --git a/internal/pkg/schedule/schedule.go b/internal/pkg/schedule/schedule.go index 9abb812e..69d84baf 100644 --- a/internal/pkg/schedule/schedule.go +++ b/internal/pkg/schedule/schedule.go @@ -81,33 +81,43 @@ func New() (*Schedule, error) { } for _, victim := range victims { - killtime := CalculateKillTime() - - if ShouldScheduleChaos(victim.Mtbf()) { - schedule.Add(chaos.New(killtime, victim)) + mtbf := victim.Mtbf() + parsed_mtbf, err := calendar.ParseMtbf(mtbf) + if err != nil { + glog.Errorf("error parsing customized mtbf for %s/%s in namespace %s - %s: %v", victim.Kind(), victim.Name(), victim.Namespace(), mtbf, err) + continue + } + killtimes := CalculateKillTimes(mtbf) + one_day, _ := time.ParseDuration("24h") + // If the parsed mtbf value is less than one day we want to add the calculated kill times no matter + // what and otherwise we use probability to decide if we will schedule the calculated kill time. + if parsed_mtbf < one_day || ShouldScheduleChaos(float64(parsed_mtbf / one_day)) { + for _, killtime := range killtimes { + schedule.Add(chaos.New(killtime, victim)) + } } } return schedule, nil } -func CalculateKillTime() time.Time { +func CalculateKillTimes(mtbf string) []time.Time { loc := config.Timezone() if config.DebugEnabled() && config.DebugScheduleImmediateKill() { r := rand.New(rand.NewSource(time.Now().UnixNano())) // calculate a second-offset in the next minute secOffset := r.Intn(60) - return time.Now().In(loc).Add(time.Duration(secOffset) * time.Second) + return []time.Time{time.Now().In(loc).Add(time.Duration(secOffset) * time.Second)} } - return calendar.RandomTimeInRange(config.StartHour(), config.EndHour(), loc) + return calendar.RandomTimeInRange(mtbf, config.StartHour(), config.EndHour(), loc) } -func ShouldScheduleChaos(mtbf int) bool { +func ShouldScheduleChaos(mtbf float64) bool { if config.DebugEnabled() && config.DebugForceShouldKill() { return true } r := rand.New(rand.NewSource(time.Now().UnixNano())) - probability := 1 / float64(mtbf) + probability := 1 / mtbf return probability > r.Float64() } diff --git a/internal/pkg/schedule/schedule_test.go b/internal/pkg/schedule/schedule_test.go index 8d269f17..3af46394 100644 --- a/internal/pkg/schedule/schedule_test.go +++ b/internal/pkg/schedule/schedule_test.go @@ -87,16 +87,16 @@ func TestStringWithEntries(t *testing.T) { func TestCalculateKillTimeRandom(t *testing.T) { config.SetDefaults() - killtime := CalculateKillTime() + killtimes := CalculateKillTimes("1h") scheduledTime := func() (success bool) { - if killtime.Hour() >= config.StartHour() && killtime.Hour() <= config.EndHour() { + if killtimes[0].Hour() >= config.StartHour() && killtimes[0].Hour() <= config.EndHour() { success = true } return } - assert.Equal(t, killtime.Location(), config.Timezone()) + assert.Equal(t, killtimes[0].Location(), config.Timezone()) assert.Condition(t, scheduledTime) } @@ -105,10 +105,10 @@ func TestCalculateKillTimeNow(t *testing.T) { config.SetDefaults() viper.SetDefault(param.DebugEnabled, true) viper.SetDefault(param.DebugScheduleImmediateKill, true) - killtime := CalculateKillTime() + killtimes := CalculateKillTimes("1h") - assert.Equal(t, killtime.Location(), config.Timezone()) - assert.WithinDuration(t, killtime, time.Now(), time.Second*time.Duration(60)) + assert.Equal(t, killtimes[0].Location(), config.Timezone()) + assert.WithinDuration(t, killtimes[0], time.Now(), time.Second*time.Duration(60)) config.SetDefaults() } diff --git a/internal/pkg/victims/factory/daemonsets/daemonsets.go b/internal/pkg/victims/factory/daemonsets/daemonsets.go index b6fce7d3..677d716c 100644 --- a/internal/pkg/victims/factory/daemonsets/daemonsets.go +++ b/internal/pkg/victims/factory/daemonsets/daemonsets.go @@ -2,8 +2,8 @@ package daemonsets import ( "fmt" - "strconv" + "kube-monkey/internal/pkg/calendar" "kube-monkey/internal/pkg/config" "kube-monkey/internal/pkg/victims" @@ -44,20 +44,16 @@ func identifier(kubekind *appsv1.DaemonSet) (string, error) { // Read the mean-time-between-failures value defined by the DaemonSet // in the label defined by config.MtbfLabelKey -func meanTimeBetweenFailures(kubekind *appsv1.DaemonSet) (int, error) { +func meanTimeBetweenFailures(kubekind *appsv1.DaemonSet) (string, error) { mtbf, ok := kubekind.Labels[config.MtbfLabelKey] if !ok { - return -1, fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) + return "", fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) } - mtbfInt, err := strconv.Atoi(mtbf) + _, err := calendar.ParseMtbf(mtbf) if err != nil { - return -1, err + return "", fmt.Errorf("error parsing mtbf %s: %v", mtbf, err) } - if !(mtbfInt > 0) { - return -1, fmt.Errorf("Invalid value for label %s: %d", config.MtbfLabelKey, mtbfInt) - } - - return mtbfInt, nil + return mtbf, nil } diff --git a/internal/pkg/victims/factory/daemonsets/daemonsets_test.go b/internal/pkg/victims/factory/daemonsets/daemonsets_test.go index 2e262413..a46c3350 100644 --- a/internal/pkg/victims/factory/daemonsets/daemonsets_test.go +++ b/internal/pkg/victims/factory/daemonsets/daemonsets_test.go @@ -33,7 +33,7 @@ func TestNew(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) ds, err := New(&v1ds) @@ -43,14 +43,14 @@ func TestNew(t *testing.T) { assert.Equal(t, NAME, ds.Name()) assert.Equal(t, NAMESPACE, ds.Namespace()) assert.Equal(t, IDENTIFIER, ds.Identifier()) - assert.Equal(t, 1, ds.Mtbf()) + assert.Equal(t, "1h", ds.Mtbf()) } func TestInvalidIdentifier(t *testing.T) { v1ds := newDaemonSet( NAME, map[string]string{ - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) _, err := New(&v1ds) @@ -78,16 +78,5 @@ func TestInvalidMtbf(t *testing.T) { ) _, err = New(&v1ds) - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a Int type") - - v1ds = newDaemonSet( - NAME, - map[string]string{ - config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "0", - }, - ) - _, err = New(&v1ds) - - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label is lower than 1") + assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a time.Duration type") } diff --git a/internal/pkg/victims/factory/daemonsets/eligible_daemonsets_test.go b/internal/pkg/victims/factory/daemonsets/eligible_daemonsets_test.go index afd21e57..4f04e976 100644 --- a/internal/pkg/victims/factory/daemonsets/eligible_daemonsets_test.go +++ b/internal/pkg/victims/factory/daemonsets/eligible_daemonsets_test.go @@ -15,7 +15,7 @@ func TestEligibleDaemonSets(t *testing.T) { NAME, map[string]string{ "kube-monkey/identifier": "1", - "kube-monkey/mtbf": "1", + "kube-monkey/mtbf": "1h", }, ) @@ -30,7 +30,7 @@ func TestIsEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: config.EnabledLabelValue, }, ) @@ -49,7 +49,7 @@ func TestIsNotEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: "x", }, ) @@ -66,7 +66,7 @@ func TestIsNotEnrolled(t *testing.T) { func TestKillType(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killMode := "kill-mode" v1ds := newDaemonSet( @@ -104,7 +104,7 @@ func TestKillType(t *testing.T) { func TestKillValue(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killValue := "0" v1ds := newDaemonSet( diff --git a/internal/pkg/victims/factory/deployments/deployments.go b/internal/pkg/victims/factory/deployments/deployments.go index ccfb8df1..d43b57e2 100644 --- a/internal/pkg/victims/factory/deployments/deployments.go +++ b/internal/pkg/victims/factory/deployments/deployments.go @@ -2,8 +2,8 @@ package deployments import ( "fmt" - "strconv" + "kube-monkey/internal/pkg/calendar" "kube-monkey/internal/pkg/config" "kube-monkey/internal/pkg/victims" @@ -44,20 +44,16 @@ func identifier(kubekind *appsv1.Deployment) (string, error) { // Read the mean-time-between-failures value defined by the Deployment // in the label defined by config.MtbfLabelKey -func meanTimeBetweenFailures(kubekind *appsv1.Deployment) (int, error) { +func meanTimeBetweenFailures(kubekind *appsv1.Deployment) (string, error) { mtbf, ok := kubekind.Labels[config.MtbfLabelKey] if !ok { - return -1, fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) + return "", fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) } - mtbfInt, err := strconv.Atoi(mtbf) + _, err := calendar.ParseMtbf(mtbf) if err != nil { - return -1, err + return "", fmt.Errorf("error parsing mtbf %s: %v", mtbf, err) } - if !(mtbfInt > 0) { - return -1, fmt.Errorf("Invalid value for label %s: %d", config.MtbfLabelKey, mtbfInt) - } - - return mtbfInt, nil + return mtbf, nil } diff --git a/internal/pkg/victims/factory/deployments/deployments_test.go b/internal/pkg/victims/factory/deployments/deployments_test.go index 2118cb0c..016b8d8b 100644 --- a/internal/pkg/victims/factory/deployments/deployments_test.go +++ b/internal/pkg/victims/factory/deployments/deployments_test.go @@ -33,7 +33,7 @@ func TestNew(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) depl, err := New(&v1depl) @@ -43,14 +43,14 @@ func TestNew(t *testing.T) { assert.Equal(t, NAME, depl.Name()) assert.Equal(t, NAMESPACE, depl.Namespace()) assert.Equal(t, IDENTIFIER, depl.Identifier()) - assert.Equal(t, 1, depl.Mtbf()) + assert.Equal(t, "1h", depl.Mtbf()) } func TestInvalidIdentifier(t *testing.T) { v1depl := newDeployment( NAME, map[string]string{ - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) _, err := New(&v1depl) @@ -78,16 +78,5 @@ func TestInvalidMtbf(t *testing.T) { ) _, err = New(&v1depl) - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a Int type") - - v1depl = newDeployment( - NAME, - map[string]string{ - config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "0", - }, - ) - _, err = New(&v1depl) - - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label is lower than 1") + assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a time.Duration type") } diff --git a/internal/pkg/victims/factory/deployments/eligible_deployments_test.go b/internal/pkg/victims/factory/deployments/eligible_deployments_test.go index bea63ca1..d0758622 100644 --- a/internal/pkg/victims/factory/deployments/eligible_deployments_test.go +++ b/internal/pkg/victims/factory/deployments/eligible_deployments_test.go @@ -15,7 +15,7 @@ func TestEligibleDeployments(t *testing.T) { NAME, map[string]string{ "kube-monkey/identifier": "1", - "kube-monkey/mtbf": "1", + "kube-monkey/mtbf": "1h", }, ) @@ -30,7 +30,7 @@ func TestIsEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: config.EnabledLabelValue, }, ) @@ -49,7 +49,7 @@ func TestIsNotEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: "x", }, ) @@ -66,7 +66,7 @@ func TestIsNotEnrolled(t *testing.T) { func TestKillType(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killMode := "kill-mode" v1depl := newDeployment( @@ -104,7 +104,7 @@ func TestKillType(t *testing.T) { func TestKillValue(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killValue := "0" v1depl := newDeployment( diff --git a/internal/pkg/victims/factory/statefulsets/eligible_statefulsets_test.go b/internal/pkg/victims/factory/statefulsets/eligible_statefulsets_test.go index d88f1842..ee6022e4 100644 --- a/internal/pkg/victims/factory/statefulsets/eligible_statefulsets_test.go +++ b/internal/pkg/victims/factory/statefulsets/eligible_statefulsets_test.go @@ -15,7 +15,7 @@ func TestEligibleStatefulSets(t *testing.T) { NAME, map[string]string{ "kube-monkey/identifier": "1", - "kube-monkey/mtbf": "1", + "kube-monkey/mtbf": "1h", }, ) @@ -30,7 +30,7 @@ func TestIsEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: config.EnabledLabelValue, }, ) @@ -49,7 +49,7 @@ func TestIsNotEnrolled(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: "1", - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", config.EnabledLabelKey: "x", }, ) @@ -66,7 +66,7 @@ func TestIsNotEnrolled(t *testing.T) { func TestKillType(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killMode := "kill-mode" v1stfs := newStatefulSet( @@ -104,7 +104,7 @@ func TestKillType(t *testing.T) { func TestKillValue(t *testing.T) { ident := "1" - mtbf := "1" + mtbf := "1h" killValue := "0" v1stfs := newStatefulSet( diff --git a/internal/pkg/victims/factory/statefulsets/statefulset_test.go b/internal/pkg/victims/factory/statefulsets/statefulset_test.go index f00ffe41..77d2c533 100644 --- a/internal/pkg/victims/factory/statefulsets/statefulset_test.go +++ b/internal/pkg/victims/factory/statefulsets/statefulset_test.go @@ -33,7 +33,7 @@ func TestNew(t *testing.T) { NAME, map[string]string{ config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) stfs, err := New(&v1stfs) @@ -43,14 +43,14 @@ func TestNew(t *testing.T) { assert.Equal(t, NAME, stfs.Name()) assert.Equal(t, NAMESPACE, stfs.Namespace()) assert.Equal(t, IDENTIFIER, stfs.Identifier()) - assert.Equal(t, 1, stfs.Mtbf()) + assert.Equal(t, "1h", stfs.Mtbf()) } func TestInvalidIdentifier(t *testing.T) { v1stfs := newStatefulSet( NAME, map[string]string{ - config.MtbfLabelKey: "1", + config.MtbfLabelKey: "1h", }, ) _, err := New(&v1stfs) @@ -78,16 +78,5 @@ func TestInvalidMtbf(t *testing.T) { ) _, err = New(&v1stfs) - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a Int type") - - v1stfs = newStatefulSet( - NAME, - map[string]string{ - config.IdentLabelKey: IDENTIFIER, - config.MtbfLabelKey: "0", - }, - ) - _, err = New(&v1stfs) - - assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label is lower than 1") + assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a time.Duration type") } diff --git a/internal/pkg/victims/factory/statefulsets/statefulsets.go b/internal/pkg/victims/factory/statefulsets/statefulsets.go index 75bdac83..19fba6ad 100644 --- a/internal/pkg/victims/factory/statefulsets/statefulsets.go +++ b/internal/pkg/victims/factory/statefulsets/statefulsets.go @@ -2,8 +2,8 @@ package statefulsets import ( "fmt" - "strconv" + "kube-monkey/internal/pkg/calendar" "kube-monkey/internal/pkg/config" "kube-monkey/internal/pkg/victims" @@ -44,20 +44,16 @@ func identifier(kubekind *corev1.StatefulSet) (string, error) { // Read the mean-time-between-failures value defined by the StatefulSet // in the label defined by config.MtbfLabelKey -func meanTimeBetweenFailures(kubekind *corev1.StatefulSet) (int, error) { +func meanTimeBetweenFailures(kubekind *corev1.StatefulSet) (string, error) { mtbf, ok := kubekind.Labels[config.MtbfLabelKey] if !ok { - return -1, fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) + return "", fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey) } - mtbfInt, err := strconv.Atoi(mtbf) + _, err := calendar.ParseMtbf(mtbf) if err != nil { - return -1, err + return "", fmt.Errorf("error parsing mtbf %s: %v", mtbf, err) } - if !(mtbfInt > 0) { - return -1, fmt.Errorf("Invalid value for label %s: %d", config.MtbfLabelKey, mtbfInt) - } - - return mtbfInt, nil + return mtbf, nil } diff --git a/internal/pkg/victims/victims.go b/internal/pkg/victims/victims.go index ef2ae406..43345de9 100644 --- a/internal/pkg/victims/victims.go +++ b/internal/pkg/victims/victims.go @@ -33,7 +33,7 @@ type VictimBaseTemplate interface { Name() string Namespace() string Identifier() string - Mtbf() int + Mtbf() string VictimAPICalls } @@ -67,12 +67,12 @@ type VictimBase struct { name string namespace string identifier string - mtbf int + mtbf string VictimBaseTemplate } -func New(kind, name, namespace, identifier string, mtbf int) *VictimBase { +func New(kind, name, namespace, identifier string, mtbf string) *VictimBase { return &VictimBase{kind: kind, name: name, namespace: namespace, identifier: identifier, mtbf: mtbf} } @@ -92,7 +92,7 @@ func (v *VictimBase) Identifier() string { return v.identifier } -func (v *VictimBase) Mtbf() int { +func (v *VictimBase) Mtbf() string { return v.mtbf } diff --git a/internal/pkg/victims/victims_test.go b/internal/pkg/victims/victims_test.go index f54ecafa..e56a8f89 100644 --- a/internal/pkg/victims/victims_test.go +++ b/internal/pkg/victims/victims_test.go @@ -59,7 +59,7 @@ func generateNRunningPods(namePrefix string, n int) []runtime.Object { } func newVictimBase() *VictimBase { - return New(KIND, NAME, NAMESPACE, IDENTIFIER, 1) + return New(KIND, NAME, NAMESPACE, IDENTIFIER, "1h") } func getPodList(client kube.Interface) *corev1.PodList { @@ -75,7 +75,7 @@ func TestVictimBaseTemplateGetters(t *testing.T) { assert.Equal(t, "name", v.Name()) assert.Equal(t, NAMESPACE, v.Namespace()) assert.Equal(t, IDENTIFIER, v.Identifier()) - assert.Equal(t, 1, v.Mtbf()) + assert.Equal(t, "1h", v.Mtbf()) } func TestRunningPods(t *testing.T) { @@ -304,7 +304,7 @@ func TestIsBlacklisted(t *testing.T) { b := v.IsBlacklisted() assert.False(t, b, "%s namespace should not be blacklisted", NAMESPACE) - v = New("Pod", "name", metav1.NamespaceSystem, IDENTIFIER, 1) + v = New("Pod", "name", metav1.NamespaceSystem, IDENTIFIER, "1h") b = v.IsBlacklisted() assert.True(t, b, "%s namespace should be blacklisted", metav1.NamespaceSystem)