From fb6469bd3d2a9b26b5e034f348db926780b59ebf Mon Sep 17 00:00:00 2001 From: Wan Date: Tue, 28 Sep 2021 15:39:12 -0700 Subject: [PATCH 1/4] #28 Reset retryCount and error description for Ready case Signed-off-by: Wan --- api/v1alpha1/wavefrontalert_types.go | 2 +- .../alertmanager.keikoproj.io_alertsconfigs.yaml | 1 + .../alertmanager.keikoproj.io_wavefrontalerts.yaml | 1 + controllers/alertsconfig_controller.go | 12 ++++++++++-- go.mod | 2 +- go.sum | 2 ++ 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/wavefrontalert_types.go b/api/v1alpha1/wavefrontalert_types.go index 1962471..38d92b3 100644 --- a/api/v1alpha1/wavefrontalert_types.go +++ b/api/v1alpha1/wavefrontalert_types.go @@ -135,7 +135,7 @@ type AlertStatus struct { LastChangeChecksum string `json:"lastChangeChecksum,omitempty"` AssociatedAlert AssociatedAlert `json:"associatedAlert,omitempty"` AssociatedAlertsConfig AssociatedAlertsConfig `json:"associatedAlertsConfig,omitempty"` - ErrorDescription string `json:"errorDescription,omitempty"` + ErrorDescription string `json:"errorDescription"` } type AssociatedAlertsConfig struct { diff --git a/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml b/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml index 378d3e8..97cc6b6 100644 --- a/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml +++ b/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml @@ -144,6 +144,7 @@ spec: type: string required: - alertName + - errorDescription - id type: object description: AlertsStatus details includes individual alert details diff --git a/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml b/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml index e405f1e..6f1804f 100644 --- a/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml +++ b/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml @@ -159,6 +159,7 @@ spec: type: string required: - alertName + - errorDescription - id type: object description: AlertsStatus details includes individual alert details diff --git a/controllers/alertsconfig_controller.go b/controllers/alertsconfig_controller.go index 5b4a501..8635e0d 100644 --- a/controllers/alertsconfig_controller.go +++ b/controllers/alertsconfig_controller.go @@ -161,6 +161,7 @@ func (r *AlertsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request LastChangeChecksum: reqChecksum, Link: fmt.Sprintf("https://%s/alerts/%s", internalconfig.Props.WavefrontAPIUrl(), *alert.ID), State: alertmanagerv1alpha1.Ready, + ErrorDescription: "", AssociatedAlert: alertmanagerv1alpha1.AssociatedAlert{ CR: alertName, }, @@ -193,8 +194,9 @@ func (r *AlertsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request alertStatus := alertHashMap[alertName] alertStatus.LastChangeChecksum = reqChecksum - // Update the individual alert status state to be ready + // Update the individual alert status state to be ready and cleanup the error message alertStatus.State = alertmanagerv1alpha1.Ready + alertStatus.ErrorDescription = "" if err := r.CommonClient.PatchWfAlertAndAlertsConfigStatus(ctx, alertmanagerv1alpha1.Ready, &wfAlert, &alertsConfig, alertStatus); err != nil { log.Error(err, "unable to patch wfalert and alertsconfig status objects") return r.PatchIndividualAlertsConfigError(ctx, &alertsConfig, alertName, alertmanagerv1alpha1.Error, err) @@ -221,8 +223,9 @@ func (r *AlertsConfigReconciler) HandleIndividalAlertConfigRemoval(ctx context.C tempStatusConfig := updatedAlertsConfig.Status.AlertsStatus tempState := updatedAlertsConfig.Status.State var toBeDeleted []string + areAlertsReady := true - for key, status := range updatedAlertsConfig.Status.AlertsStatus { + for key, status := range tempStatusConfig { // This is for sure delete use case if _, ok := updatedAlertsConfig.Spec.Alerts[key]; !ok { //This means we didn't find this in spec anymore @@ -235,6 +238,7 @@ func (r *AlertsConfigReconciler) HandleIndividalAlertConfigRemoval(ctx context.C } else { if status.State == alertmanagerv1alpha1.Error { tempState = alertmanagerv1alpha1.Error + areAlertsReady = false } } } @@ -245,6 +249,10 @@ func (r *AlertsConfigReconciler) HandleIndividalAlertConfigRemoval(ctx context.C // update the count updatedAlertsConfig.Status.AlertsCount = len(updatedAlertsConfig.Spec.Alerts) updatedAlertsConfig.Status.AlertsStatus = tempStatusConfig + // reset the retry count if all the alerts are Ready + if areAlertsReady { + updatedAlertsConfig.Status.RetryCount = 0 + } // update the status return r.CommonClient.UpdateStatus(ctx, &updatedAlertsConfig, tempState, errRequeueTime) } diff --git a/go.mod b/go.mod index 2b78c5a..b9ac631 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.16.0 golang.org/x/mod v0.5.1 // indirect - golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect + golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect golang.org/x/tools v0.1.6 // indirect k8s.io/api v0.19.2 k8s.io/apimachinery v0.19.2 diff --git a/go.sum b/go.sum index 8287e48..05f68ab 100644 --- a/go.sum +++ b/go.sum @@ -603,6 +603,8 @@ golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3 h1:3Ad41xy2WCESpufXwgs7NpDSu golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 h1:c20P3CcPbopVp2f7099WLOqSNKURf30Z0uq66HpijZY= golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 254e3962c45dcaf94b22ad568e0624028101bd43 Mon Sep 17 00:00:00 2001 From: Wan Date: Tue, 28 Sep 2021 15:40:07 -0700 Subject: [PATCH 2/4] Removed testing code Signed-off-by: Wan --- controllers/alertsconfig_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/alertsconfig_controller.go b/controllers/alertsconfig_controller.go index 8635e0d..a1daef3 100644 --- a/controllers/alertsconfig_controller.go +++ b/controllers/alertsconfig_controller.go @@ -225,7 +225,7 @@ func (r *AlertsConfigReconciler) HandleIndividalAlertConfigRemoval(ctx context.C var toBeDeleted []string areAlertsReady := true - for key, status := range tempStatusConfig { + for key, status := range updatedAlertsConfig.Status.AlertsStatus { // This is for sure delete use case if _, ok := updatedAlertsConfig.Spec.Alerts[key]; !ok { //This means we didn't find this in spec anymore From da9ee2db35cc35c380a11b2cba13a98a540ab0b3 Mon Sep 17 00:00:00 2001 From: Wan Date: Wed, 29 Sep 2021 10:05:58 -0700 Subject: [PATCH 3/4] Remove the cleanup error description code --- api/v1alpha1/wavefrontalert_types.go | 2 +- controllers/alertsconfig_controller.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/api/v1alpha1/wavefrontalert_types.go b/api/v1alpha1/wavefrontalert_types.go index 38d92b3..1962471 100644 --- a/api/v1alpha1/wavefrontalert_types.go +++ b/api/v1alpha1/wavefrontalert_types.go @@ -135,7 +135,7 @@ type AlertStatus struct { LastChangeChecksum string `json:"lastChangeChecksum,omitempty"` AssociatedAlert AssociatedAlert `json:"associatedAlert,omitempty"` AssociatedAlertsConfig AssociatedAlertsConfig `json:"associatedAlertsConfig,omitempty"` - ErrorDescription string `json:"errorDescription"` + ErrorDescription string `json:"errorDescription,omitempty"` } type AssociatedAlertsConfig struct { diff --git a/controllers/alertsconfig_controller.go b/controllers/alertsconfig_controller.go index a1daef3..27cdda1 100644 --- a/controllers/alertsconfig_controller.go +++ b/controllers/alertsconfig_controller.go @@ -161,7 +161,6 @@ func (r *AlertsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request LastChangeChecksum: reqChecksum, Link: fmt.Sprintf("https://%s/alerts/%s", internalconfig.Props.WavefrontAPIUrl(), *alert.ID), State: alertmanagerv1alpha1.Ready, - ErrorDescription: "", AssociatedAlert: alertmanagerv1alpha1.AssociatedAlert{ CR: alertName, }, @@ -196,7 +195,6 @@ func (r *AlertsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request alertStatus.LastChangeChecksum = reqChecksum // Update the individual alert status state to be ready and cleanup the error message alertStatus.State = alertmanagerv1alpha1.Ready - alertStatus.ErrorDescription = "" if err := r.CommonClient.PatchWfAlertAndAlertsConfigStatus(ctx, alertmanagerv1alpha1.Ready, &wfAlert, &alertsConfig, alertStatus); err != nil { log.Error(err, "unable to patch wfalert and alertsconfig status objects") return r.PatchIndividualAlertsConfigError(ctx, &alertsConfig, alertName, alertmanagerv1alpha1.Error, err) From bb7bf30c92d15f8986ae226b86943df57bb905f8 Mon Sep 17 00:00:00 2001 From: Wan Date: Wed, 29 Sep 2021 10:26:44 -0700 Subject: [PATCH 4/4] Remove the cleanup error description code --- config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml | 1 - config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml b/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml index 97cc6b6..378d3e8 100644 --- a/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml +++ b/config/crd/bases/alertmanager.keikoproj.io_alertsconfigs.yaml @@ -144,7 +144,6 @@ spec: type: string required: - alertName - - errorDescription - id type: object description: AlertsStatus details includes individual alert details diff --git a/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml b/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml index 6f1804f..e405f1e 100644 --- a/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml +++ b/config/crd/bases/alertmanager.keikoproj.io_wavefrontalerts.yaml @@ -159,7 +159,6 @@ spec: type: string required: - alertName - - errorDescription - id type: object description: AlertsStatus details includes individual alert details