-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #767 from cloudflare/alerts/external_labels
Added alerts/external_labels check
- Loading branch information
Showing
16 changed files
with
1,043 additions
and
459 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| --- | ||
| layout: default | ||
| parent: Checks | ||
| grand_parent: Documentation | ||
| --- | ||
|
|
||
| # alerts/external_labels | ||
|
|
||
| Alerting rules can be templated to render the value of external labels | ||
| configured for the Prometheus server these rules are being evaluated | ||
| using `$externalLabels` variable. | ||
| [See docs](https://prometheus.io/docs/prometheus/latest/configuration/template_reference/#alert-field-templates). | ||
|
|
||
| This check will look for alerting rules referencing external labels that are | ||
| not present on given Prometheus server. | ||
|
|
||
| If we define `cluster` label in `global:external_labels`, example: | ||
|
|
||
| ```yaml | ||
| global: | ||
| external_labels: | ||
| cluster: mycluster | ||
| ``` | ||
| Then we can access it in alert rules deployed to that Prometheus server | ||
| by using `$externalLabels.cluster` variable: | ||
|
|
||
| ```yaml | ||
| - alert: Abc Is Down | ||
| expr: up{job="abc"} == 0 | ||
| annotations: | ||
| summary: "{{ $labels.job }} is down in {{ $externalLabels.cluster }} cluster" | ||
| ``` | ||
|
|
||
| But if we try to do that without `cluster` in `global:external_labels` configuration | ||
| then `$externalLabels.cluster` will be empty, and this is what this check would | ||
| report. | ||
|
|
||
| ## Configuration | ||
|
|
||
| This check doesn't have any configuration options. | ||
|
|
||
| ## How to enable it | ||
|
|
||
| This check is enabled by default for all configured Prometheus servers. | ||
|
|
||
| Example: | ||
|
|
||
| ```js | ||
| prometheus "prod" { | ||
| uri = "https://prometheus-prod.example.com" | ||
| timeout = "60s" | ||
| include = [ | ||
| "rules/prod/.*", | ||
| "rules/common/.*", | ||
| ] | ||
| } | ||
| prometheus "dev" { | ||
| uri = "https://prometheus-dev.example.com" | ||
| timeout = "30s" | ||
| include = [ | ||
| "rules/dev/.*", | ||
| "rules/common/.*", | ||
| ] | ||
| } | ||
| ``` | ||
|
|
||
| ## How to disable it | ||
|
|
||
| You can disable this check globally by adding this config block: | ||
|
|
||
| ```js | ||
| checks { | ||
| disabled = ["alerts/external_labels"] | ||
| } | ||
| ``` | ||
|
|
||
| You can also disable it for all rules inside given file by adding | ||
| a comment anywhere in that file. Example: | ||
|
|
||
| ```yaml | ||
| # pint file/disable alerts/external_labels | ||
| ``` | ||
|
|
||
| Or you can disable it per rule by adding a comment to it. Example: | ||
|
|
||
| ```yaml | ||
| # pint disable alerts/external_labels | ||
| ``` | ||
|
|
||
| If you want to disable only individual instances of this check | ||
| you can add a more specific comment. | ||
|
|
||
| ```yaml | ||
| # pint disable alerts/external_labels($prometheus) | ||
| ``` | ||
|
|
||
| Where `$prometheus` is the name of Prometheus server to disable. | ||
|
|
||
| Example: | ||
|
|
||
| ```yaml | ||
| # pint disable alerts/external_labels(prod) | ||
| ``` | ||
|
|
||
| ## How to snooze it | ||
|
|
||
| You can disable this check until given time by adding a comment to it. Example: | ||
|
|
||
| ```yaml | ||
| # pint snooze $TIMESTAMP alerts/external_labels | ||
| ``` | ||
|
|
||
| Where `$TIMESTAMP` is either use [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) | ||
| formatted or `YYYY-MM-DD`. | ||
| Adding this comment will disable `alerts/external_labels` _until_ `$TIMESTAMP`, after that | ||
| check will be re-enabled. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| package checks | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/cloudflare/pint/internal/discovery" | ||
| "github.com/cloudflare/pint/internal/parser" | ||
| "github.com/cloudflare/pint/internal/promapi" | ||
| ) | ||
|
|
||
| const ( | ||
| AlertsExternalLabelsCheckName = "alerts/external_labels" | ||
| ) | ||
|
|
||
| func NewAlertsExternalLabelsCheck(prom *promapi.FailoverGroup) AlertsExternalLabelsCheck { | ||
| return AlertsExternalLabelsCheck{ | ||
| prom: prom, | ||
| } | ||
| } | ||
|
|
||
| type AlertsExternalLabelsCheck struct { | ||
| prom *promapi.FailoverGroup | ||
| } | ||
|
|
||
| func (c AlertsExternalLabelsCheck) Meta() CheckMeta { | ||
| return CheckMeta{IsOnline: true} | ||
| } | ||
|
|
||
| func (c AlertsExternalLabelsCheck) String() string { | ||
| return fmt.Sprintf("%s(%s)", AlertsExternalLabelsCheckName, c.prom.Name()) | ||
| } | ||
|
|
||
| func (c AlertsExternalLabelsCheck) Reporter() string { | ||
| return AlertsExternalLabelsCheckName | ||
| } | ||
|
|
||
| func (c AlertsExternalLabelsCheck) Check(ctx context.Context, _ string, rule parser.Rule, _ []discovery.Entry) (problems []Problem) { | ||
| if rule.AlertingRule == nil { | ||
| return problems | ||
| } | ||
|
|
||
| if rule.AlertingRule.Expr.SyntaxError != nil { | ||
| return problems | ||
| } | ||
|
|
||
| cfg, err := c.prom.Config(ctx) | ||
| if err != nil { | ||
| text, severity := textAndSeverityFromError(err, c.Reporter(), c.prom.Name(), Bug) | ||
| problems = append(problems, Problem{ | ||
| Fragment: fmt.Sprintf("%s: %s", rule.AlertingRule.Alert.Key.Value, rule.AlertingRule.Alert.Value.Value), | ||
| Lines: rule.AlertingRule.Lines(), | ||
| Reporter: c.Reporter(), | ||
| Text: text, | ||
| Severity: severity, | ||
| }) | ||
| return problems | ||
| } | ||
|
|
||
| if rule.AlertingRule.Labels != nil { | ||
| for _, label := range rule.AlertingRule.Labels.Items { | ||
| for _, name := range checkExternalLabels(label.Key.Value, label.Key.Value, cfg.Config.Global.ExternalLabels) { | ||
| problems = append(problems, Problem{ | ||
| Fragment: fmt.Sprintf("%s: %s", label.Key.Value, label.Value.Value), | ||
| Lines: label.Lines(), | ||
| Reporter: c.Reporter(), | ||
| Text: fmt.Sprintf("template is using %q external label but %s doesn't have this label configured in global:external_labels", name, promText(c.prom.Name(), cfg.URI)), | ||
| Severity: Bug, | ||
| }) | ||
| } | ||
| for _, name := range checkExternalLabels(label.Key.Value, label.Value.Value, cfg.Config.Global.ExternalLabels) { | ||
| problems = append(problems, Problem{ | ||
| Fragment: fmt.Sprintf("%s: %s", label.Key.Value, label.Value.Value), | ||
| Lines: label.Lines(), | ||
| Reporter: c.Reporter(), | ||
| Text: fmt.Sprintf("template is using %q external label but %s doesn't have this label configured in global:external_labels", name, promText(c.prom.Name(), cfg.URI)), Severity: Bug, | ||
| }) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if rule.AlertingRule.Annotations != nil { | ||
| for _, annotation := range rule.AlertingRule.Annotations.Items { | ||
| for _, name := range checkExternalLabels(annotation.Key.Value, annotation.Key.Value, cfg.Config.Global.ExternalLabels) { | ||
| problems = append(problems, Problem{ | ||
| Fragment: fmt.Sprintf("%s: %s", annotation.Key.Value, annotation.Value.Value), | ||
| Lines: annotation.Lines(), | ||
| Reporter: c.Reporter(), | ||
| Text: fmt.Sprintf("template is using %q external label but %s doesn't have this label configured in global:external_labels", name, promText(c.prom.Name(), cfg.URI)), | ||
| Severity: Bug, | ||
| }) | ||
| } | ||
| for _, name := range checkExternalLabels(annotation.Key.Value, annotation.Value.Value, cfg.Config.Global.ExternalLabels) { | ||
| problems = append(problems, Problem{ | ||
| Fragment: fmt.Sprintf("%s: %s", annotation.Key.Value, annotation.Value.Value), | ||
| Lines: annotation.Lines(), | ||
| Reporter: c.Reporter(), | ||
| Text: fmt.Sprintf("template is using %q external label but %s doesn't have this label configured in global:external_labels", name, promText(c.prom.Name(), cfg.URI)), | ||
| Severity: Bug, | ||
| }) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return problems | ||
| } | ||
|
|
||
| func checkExternalLabels(name, text string, externalLabels map[string]string) (labels []string) { | ||
| vars, aliases, ok := findTemplateVariables(name, text) | ||
| if !ok { | ||
| return nil | ||
| } | ||
|
|
||
| done := map[string]struct{}{} | ||
| externalLabelsAliases := aliases.varAliases(".ExternalLabels") | ||
| for _, v := range vars { | ||
| for _, a := range externalLabelsAliases { | ||
| if len(v) > 1 && v[0] == a { | ||
| name := v[1] | ||
| if _, ok = done[name]; ok { | ||
| continue | ||
| } | ||
| if _, ok := externalLabels[name]; !ok { | ||
| labels = append(labels, name) | ||
| } | ||
| done[name] = struct{}{} | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return labels | ||
| } |
Oops, something went wrong.