From 1348867a4076e7f31543aa6ea8f8ea216a022bc4 Mon Sep 17 00:00:00 2001 From: Lukasz Mierzwa Date: Fri, 26 Jul 2024 13:42:10 +0100 Subject: [PATCH] Added alerts/absent check --- cmd/pint/tests/0023_enabled_checks.txt | 8 +- cmd/pint/tests/0025_config.txt | 1 + .../tests/0054_watch_metrics_prometheus.txt | 2 + .../0057_watch_metrics_prometheus_ignore.txt | 2 + cmd/pint/tests/0113_config_env_expand.txt | 1 + cmd/pint/tests/0115_file_disable_tag.txt | 2 +- docs/changelog.md | 8 +- docs/checks/alerts/absent.md | 94 +++++ go.mod | 8 +- go.sum | 20 +- internal/checks/alerts_absent.go | 99 +++++ internal/checks/alerts_absent_test.go | 240 ++++++++++++ internal/checks/base.go | 2 + .../config/__snapshots__/config_test.snap | 367 ++++++++++-------- internal/config/config.go | 5 + internal/config/config_test.go | 51 ++- 16 files changed, 716 insertions(+), 194 deletions(-) create mode 100644 docs/checks/alerts/absent.md create mode 100644 internal/checks/alerts_absent.go create mode 100644 internal/checks/alerts_absent_test.go diff --git a/cmd/pint/tests/0023_enabled_checks.txt b/cmd/pint/tests/0023_enabled_checks.txt index f42839f5..af50ce7c 100644 --- a/cmd/pint/tests/0023_enabled_checks.txt +++ b/cmd/pint/tests/0023_enabled_checks.txt @@ -1,9 +1,9 @@ pint.error -l debug --no-color lint rules ! stdout . -stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)"] path=rules/1.yaml rule=one' -stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)"] path=rules/1.yaml rule=two' -stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)"] path=rules/2.yaml rule=one' -stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)"] path=rules/2.yaml rule=two' +stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/1.yaml rule=one' +stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/1.yaml rule=two' +stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/2.yaml rule=one' +stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/2.yaml rule=two' -- rules/1.yaml -- - record: one diff --git a/cmd/pint/tests/0025_config.txt b/cmd/pint/tests/0025_config.txt index 4dd154cd..745ce5a2 100644 --- a/cmd/pint/tests/0025_config.txt +++ b/cmd/pint/tests/0025_config.txt @@ -14,6 +14,7 @@ level=INFO msg="Loading configuration file" path=.pint.hcl "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", diff --git a/cmd/pint/tests/0054_watch_metrics_prometheus.txt b/cmd/pint/tests/0054_watch_metrics_prometheus.txt index edbdba4b..362cad17 100644 --- a/cmd/pint/tests/0054_watch_metrics_prometheus.txt +++ b/cmd/pint/tests/0054_watch_metrics_prometheus.txt @@ -42,6 +42,8 @@ parser { -- metrics.txt -- # HELP pint_check_duration_seconds How long did a check took to complete # TYPE pint_check_duration_seconds summary +pint_check_duration_seconds_sum{check="alerts/absent"} +pint_check_duration_seconds_count{check="alerts/absent"} pint_check_duration_seconds_sum{check="alerts/comparison"} pint_check_duration_seconds_count{check="alerts/comparison"} pint_check_duration_seconds_sum{check="alerts/external_labels"} diff --git a/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt b/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt index 201879e1..30726bbb 100644 --- a/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt +++ b/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt @@ -40,6 +40,8 @@ parser { -- metrics.txt -- # HELP pint_check_duration_seconds How long did a check took to complete # TYPE pint_check_duration_seconds summary +pint_check_duration_seconds_sum{check="alerts/absent"} +pint_check_duration_seconds_count{check="alerts/absent"} pint_check_duration_seconds_sum{check="alerts/comparison"} pint_check_duration_seconds_count{check="alerts/comparison"} pint_check_duration_seconds_sum{check="alerts/external_labels"} diff --git a/cmd/pint/tests/0113_config_env_expand.txt b/cmd/pint/tests/0113_config_env_expand.txt index 70e94036..493364ae 100644 --- a/cmd/pint/tests/0113_config_env_expand.txt +++ b/cmd/pint/tests/0113_config_env_expand.txt @@ -19,6 +19,7 @@ level=INFO msg="Loading configuration file" path=.pint.hcl }, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", diff --git a/cmd/pint/tests/0115_file_disable_tag.txt b/cmd/pint/tests/0115_file_disable_tag.txt index 73683e26..502d0c8a 100644 --- a/cmd/pint/tests/0115_file_disable_tag.txt +++ b/cmd/pint/tests/0115_file_disable_tag.txt @@ -11,7 +11,7 @@ level=INFO msg="Configured new Prometheus server" name=prom uris=1 uptime=up tag level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1:7103 workers=16 level=DEBUG msg="Generated all Prometheus servers" count=1 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:test1 lines=6-8 -level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","alerts/external_labels(prom)","promql/counter(prom)"] path=rules/0001.yml rule=colo:test1 +level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","alerts/external_labels(prom)","promql/counter(prom)","alerts/absent(prom)"] path=rules/0001.yml rule=colo:test1 level=DEBUG msg="Scheduling Prometheus metrics metadata query" uri=http://127.0.0.1:7103 metric=foo level=DEBUG msg="Getting prometheus metrics metadata" uri=http://127.0.0.1:7103 metric=foo level=ERROR msg="Query returned an error" err="failed to query Prometheus metrics metadata: Get \"http://127.0.0.1:7103/api/v1/metadata?metric=foo\": dial tcp 127.0.0.1:7103: connect: connection refused" uri=http://127.0.0.1:7103 query=foo diff --git a/docs/changelog.md b/docs/changelog.md index afda65fa..5f3c9693 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,15 +2,19 @@ ## v0.63.0 -### Fixed +### Added -- Fixed false positive warnings from [alerts/comparison](checks/alerts/comparison.md) when using `absent_over_time()`. +- Added [alerts/absent](checks/alerts/absent.md) check. ### Changed - [promql/vector_matching](checks/promql/vector_matching.md) will now report more details, including which Prometheus server reports problems and which part of the query is the issue. - GitHub report code was refactored, it should behave as before. +### Fixed + +- Fixed false positive warnings from [alerts/comparison](checks/alerts/comparison.md) when using `absent_over_time()`. + ## v0.62.2 ### Fixed diff --git a/docs/checks/alerts/absent.md b/docs/checks/alerts/absent.md new file mode 100644 index 00000000..3b4e5113 --- /dev/null +++ b/docs/checks/alerts/absent.md @@ -0,0 +1,94 @@ +--- +layout: default +parent: Checks +grand_parent: Documentation +--- + +# alerts/absent + +This check will warn you about alerting rules that are using `absent()` calls without having `for` option set +to at least 2x scrape interval. +Using `absent()` without `for` can cause false positive alerts when Prometheus is restarted and the rule +is evaluated before the metrics tested using `absent()` are scraped. Adding a `for` option with at least +2x scrape interval is usually enough to prevent this from happening. + +## Configuration + +This check doesn't have any configuration options. + +## How to enable it + +This check is enabled by default for all configured Prometheus servers. + +Example: + +```js +prometheus "prod" { + uri = "https://prometheus-prod.example.com" + timeout = "60s" + include = [ + "rules/prod/.*", + "rules/common/.*", + ] +} + +prometheus "dev" { + uri = "https://prometheus-dev.example.com" + timeout = "30s" + include = [ + "rules/dev/.*", + "rules/common/.*", + ] +} +``` + +## How to disable it + +You can disable this check globally by adding this config block: + +```js +checks { + disabled = ["alerts/absent"] +} +``` + +You can also disable it for all rules inside given file by adding +a comment anywhere in that file. Example: + +```yaml +# pint file/disable alerts/absent +``` + +Or you can disable it per rule by adding a comment to it. Example: + +```yaml +# pint disable alerts/absent +``` + +If you want to disable only individual instances of this check +you can add a more specific comment. + +```yaml +# pint disable alerts/absent($prometheus) +``` + +Where `$prometheus` is the name of Prometheus server to disable. + +Example: + +```yaml +# pint disable alerts/absent(prod) +``` + +## How to snooze it + +You can disable this check until given time by adding a comment to it. Example: + +```yaml +# pint snooze $TIMESTAMP alerts/absent +``` + +Where `$TIMESTAMP` is either use [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) +formatted or `YYYY-MM-DD`. +Adding this comment will disable `alerts/absent` *until* `$TIMESTAMP`, after that +check will be re-enabled. diff --git a/go.mod b/go.mod index 5544aa9f..a86ecd8a 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.5.3 go.uber.org/ratelimit v0.3.1 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/oauth2 v0.21.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -72,11 +72,11 @@ require ( go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect - golang.org/x/mod v0.18.0 // indirect + golang.org/x/mod v0.19.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.22.0 // indirect + golang.org/x/tools v0.23.0 // indirect google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index 8f0e6356..58f56de6 100644 --- a/go.sum +++ b/go.sum @@ -190,18 +190,18 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,8 +214,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= @@ -226,8 +226,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/checks/alerts_absent.go b/internal/checks/alerts_absent.go new file mode 100644 index 00000000..c40870b3 --- /dev/null +++ b/internal/checks/alerts_absent.go @@ -0,0 +1,99 @@ +package checks + +import ( + "context" + "fmt" + "time" + + "github.com/cloudflare/pint/internal/discovery" + "github.com/cloudflare/pint/internal/output" + "github.com/cloudflare/pint/internal/parser" + "github.com/cloudflare/pint/internal/promapi" + + "github.com/prometheus/common/model" + promParser "github.com/prometheus/prometheus/promql/parser" +) + +const ( + AlertsAbsentCheckName = "alerts/absent" + AlertsAbsentCheckDetails = "When Prometheus restart this alert rule might be evaluated before your service is scraped, which can cause false positives from absent() call.\nAdding `for` option that is at least 2x scrape interval will prevent this from happening." +) + +func NewAlertsAbsentCheck(prom *promapi.FailoverGroup) AlertsAbsentCheck { + return AlertsAbsentCheck{ + prom: prom, + } +} + +type AlertsAbsentCheck struct { + prom *promapi.FailoverGroup +} + +func (c AlertsAbsentCheck) Meta() CheckMeta { + return CheckMeta{ + States: []discovery.ChangeType{ + discovery.Noop, + discovery.Added, + discovery.Modified, + discovery.Moved, + }, + IsOnline: true, + } +} + +func (c AlertsAbsentCheck) String() string { + return fmt.Sprintf("%s(%s)", AlertsAbsentCheckName, c.prom.Name()) +} + +func (c AlertsAbsentCheck) Reporter() string { + return AlertsAbsentCheckName +} + +func (c AlertsAbsentCheck) Check(ctx context.Context, _ discovery.Path, rule parser.Rule, _ []discovery.Entry) (problems []Problem) { + if rule.AlertingRule == nil { + return problems + } + + if rule.AlertingRule.Expr.SyntaxError != nil { + return problems + } + + if n, ok := rule.AlertingRule.Expr.Query.Expr.(*promParser.Call); !ok || n.Func.Name != "absent" { + return problems + } + + cfg, err := c.prom.Config(ctx, 0) + if err != nil { + text, severity := textAndSeverityFromError(err, c.Reporter(), c.prom.Name(), Warning) + problems = append(problems, Problem{ + Lines: rule.AlertingRule.Expr.Value.Lines, + Reporter: c.Reporter(), + Text: text, + Severity: severity, + }) + return problems + } + + if rule.AlertingRule.For != nil { + forDur, err := model.ParseDuration(rule.AlertingRule.For.Value) + if err != nil { + return problems + } + if time.Duration(forDur) >= cfg.Config.Global.ScrapeInterval*2 { + return problems + } + } + + problems = append(problems, Problem{ + Lines: rule.AlertingRule.Expr.Value.Lines, + Reporter: c.Reporter(), + Text: fmt.Sprintf("Alert query is using absent() which might cause false positives when %s restarts, please add `for: %s` to avoid this.", + promText(c.prom.Name(), cfg.URI), + output.HumanizeDuration((cfg.Config.Global.ScrapeInterval * 2).Round(time.Minute)), + ), + Details: AlertsAbsentCheckDetails, + Severity: Warning, + }) + + return problems +} diff --git a/internal/checks/alerts_absent_test.go b/internal/checks/alerts_absent_test.go new file mode 100644 index 00000000..761e5c4b --- /dev/null +++ b/internal/checks/alerts_absent_test.go @@ -0,0 +1,240 @@ +package checks_test + +import ( + "fmt" + "testing" + "time" + + "github.com/cloudflare/pint/internal/checks" + "github.com/cloudflare/pint/internal/parser" + "github.com/cloudflare/pint/internal/promapi" +) + +func newAlertsAbsentCheck(prom *promapi.FailoverGroup) checks.RuleChecker { + return checks.NewAlertsAbsentCheck(prom) +} + +func absentForNeeded(prom, uri, d string) string { + return fmt.Sprintf("Alert query is using absent() which might cause false positives when `%s` Prometheus server at %s restarts, please add `for: %s` to avoid this.", prom, uri, d) +} + +func TestAlertsAbsentCheck(t *testing.T) { + testCases := []checkTest{ + { + description: "ignores recording rules", + content: "- record: foo\n expr: sum(foo)\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: noProblems, + }, + { + description: "ignores rules with syntax errors", + content: "- alert: foo\n expr: sum(foo) without(\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: noProblems, + }, + { + description: "ignores rules with invalid duration", + content: "- alert: foo\n expr: absent(foo)\n for: abc\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: noProblems, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:\n scrape_interval: 1m\n"}, + }, + }, + }, + { + description: "absent() without for", + content: "- alert: foo\n expr: absent(foo)\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: absentForNeeded("prom", uri, "2m"), + Details: checks.AlertsAbsentCheckDetails, + Severity: checks.Warning, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:\n scrape_interval: 1m\n"}, + }, + }, + }, + { + description: "absent() < 2x scrape_interval", + content: "- alert: foo\n expr: absent(foo)\n for: 1m\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: absentForNeeded("prom", uri, "2m"), + Details: checks.AlertsAbsentCheckDetails, + Severity: checks.Warning, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:\n scrape_interval: 1m\n"}, + }, + }, + }, + { + description: "absent() < 2x scrape_interval, 53s", + content: "- alert: foo\n expr: absent(foo)\n for: 1m\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: absentForNeeded("prom", uri, "2m"), + Details: checks.AlertsAbsentCheckDetails, + Severity: checks.Warning, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:\n scrape_interval: 53s\n"}, + }, + }, + }, + { + description: "absent() < 2x scrape_interval, no config", + content: "- alert: foo\n expr: absent(foo)\n for: 30s\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: absentForNeeded("prom", uri, "2m"), + Details: checks.AlertsAbsentCheckDetails, + Severity: checks.Warning, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "{}"}, + }, + }, + }, + { + description: "absent() == 2x scrape_interval", + content: "- alert: foo\n expr: absent(foo)\n for: 2m\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: noProblems, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:\n scrape_interval: 1m\n"}, + }, + }, + }, + { + description: "invalid status", + content: "- alert: foo\n expr: absent(foo)\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: checkErrorBadData("prom", uri, "bad_data: bad input data"), + Severity: checks.Warning, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: respondWithBadData(), + }, + }, + }, + { + description: "invalid YAML", + content: "- alert: foo\n expr: absent(foo)\n", + checker: newAlertsAbsentCheck, + prometheus: newSimpleProm, + problems: func(uri string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: checkErrorUnableToRun(checks.AlertsAbsentCheckName, "prom", uri, fmt.Sprintf("failed to decode config data in %s response: yaml: line 2: could not find expected ':'", uri)), + Severity: checks.Bug, + }, + } + }, + mocks: []*prometheusMock{ + { + conds: []requestCondition{requireConfigPath}, + resp: configResponse{yaml: "global:::\nglobal:{}{}{}\n"}, + }, + }, + }, + { + description: "connection refused", + content: "- alert: foo\n expr: absent(foo)\n", + checker: newAlertsAbsentCheck, + prometheus: func(_ string) *promapi.FailoverGroup { + return simpleProm("prom", "http://127.0.0.1:1111", time.Second, true) + }, + problems: func(_ string) []checks.Problem { + return []checks.Problem{ + { + Lines: parser.LineRange{ + First: 2, + Last: 2, + }, + Reporter: checks.AlertsAbsentCheckName, + Text: checkErrorUnableToRun(checks.AlertsAbsentCheckName, "prom", "http://127.0.0.1:1111", "connection refused"), + Severity: checks.Bug, + }, + } + }, + }, + } + runTests(t, testCases) +} diff --git a/internal/checks/base.go b/internal/checks/base.go index b86c96c1..13300bdb 100644 --- a/internal/checks/base.go +++ b/internal/checks/base.go @@ -12,6 +12,7 @@ import ( var ( CheckNames = []string{ + AlertsAbsentCheckName, AnnotationCheckName, AlertsCheckName, AlertsExternalLabelsCheckName, @@ -37,6 +38,7 @@ var ( RejectCheckName, } OnlineChecks = []string{ + AlertsAbsentCheckName, AlertsCheckName, AlertsExternalLabelsCheckName, LabelsConflictCheckName, diff --git a/internal/config/__snapshots__/config_test.snap b/internal/config/__snapshots__/config_test.snap index 0006e376..e225ca46 100755 --- a/internal/config/__snapshots__/config_test.snap +++ b/internal/config/__snapshots__/config_test.snap @@ -138,6 +138,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -176,6 +177,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -228,6 +230,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -283,6 +286,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -335,6 +339,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -376,6 +381,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -435,6 +441,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -495,6 +502,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -542,6 +550,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -638,6 +647,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -698,6 +708,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -757,6 +768,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -816,6 +828,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -875,6 +888,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -934,6 +948,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -993,6 +1008,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1046,6 +1062,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1100,6 +1117,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1153,6 +1171,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1206,6 +1225,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1259,6 +1279,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1312,6 +1333,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1365,6 +1387,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1419,6 +1442,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1472,6 +1496,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1526,6 +1551,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1585,6 +1611,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1637,6 +1664,132 @@ } --- +[TestGetChecksForRule/two_prometheus_servers_/_disable_all_checks_via_comment - 1] +{ + "ci": { + "baseBranch": "master", + "maxCommits": 20 + }, + "parser": {}, + "checks": { + "enabled": [ + "alerts/absent", + "alerts/annotation", + "alerts/count", + "alerts/external_labels", + "alerts/for", + "alerts/template", + "labels/conflict", + "promql/aggregate", + "alerts/comparison", + "promql/fragile", + "promql/range_query", + "promql/rate", + "promql/regexp", + "promql/syntax", + "promql/vector_matching", + "query/cost", + "promql/counter", + "promql/series", + "rule/dependency", + "rule/duplicate", + "rule/for", + "rule/label", + "rule/link", + "rule/reject" + ], + "disabled": [ + "alerts/template", + "alerts/external_labels" + ] + }, + "owners": {}, + "prometheus": [ + { + "name": "prom1", + "uri": "http://localhost/1", + "timeout": "1s", + "uptime": "up", + "concurrency": 16, + "rateLimit": 100, + "required": false + }, + { + "name": "prom2", + "uri": "http://localhost/2", + "timeout": "1s", + "uptime": "up", + "concurrency": 16, + "rateLimit": 100, + "required": false + } + ] +} +--- + +[TestGetChecksForRule/two_prometheus_servers_/_disable_checks_via_file/disable_comment - 1] +{ + "ci": { + "baseBranch": "master", + "maxCommits": 20 + }, + "parser": {}, + "checks": { + "enabled": [ + "alerts/absent", + "alerts/annotation", + "alerts/count", + "alerts/external_labels", + "alerts/for", + "alerts/template", + "labels/conflict", + "promql/aggregate", + "alerts/comparison", + "promql/fragile", + "promql/range_query", + "promql/rate", + "promql/regexp", + "promql/syntax", + "promql/vector_matching", + "query/cost", + "promql/counter", + "promql/series", + "rule/dependency", + "rule/duplicate", + "rule/for", + "rule/label", + "rule/link", + "rule/reject" + ], + "disabled": [ + "alerts/template", + "alerts/external_labels" + ] + }, + "owners": {}, + "prometheus": [ + { + "name": "prom1", + "uri": "http://localhost/1", + "timeout": "1s", + "uptime": "up", + "concurrency": 16, + "rateLimit": 100, + "required": false + }, + { + "name": "prom2", + "uri": "http://localhost/2", + "timeout": "1s", + "uptime": "up", + "concurrency": 16, + "rateLimit": 100, + "required": false + } + ] +} +--- + [TestGetChecksForRule/single_prometheus_server - 1] { "ci": { @@ -1646,6 +1799,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1695,6 +1849,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1748,6 +1903,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1800,6 +1956,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1855,7 +2012,7 @@ } --- -[TestGetChecksForRule/prometheus_check_with_prometheus_servers_and_disable_comment - 1] +[TestGetChecksForRule/tag_snoozes_all_prometheus_checks - 1] { "ci": { "baseBranch": "master", @@ -1864,6 +2021,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1893,11 +2051,13 @@ "prometheus": [ { "name": "prom1", - "uri": "http://localhost", - "timeout": "1s", + "uri": "http://localhost/1", + "timeout": "2m0s", "uptime": "up", - "include": [ - "rules.yml" + "tags": [ + "foo", + "disable", + "bar" ], "concurrency": 16, "rateLimit": 100, @@ -1905,26 +2065,30 @@ }, { "name": "prom2", - "uri": "http://localhost", - "timeout": "1s", + "uri": "http://localhost/2", + "timeout": "2m0s", "uptime": "up", - "include": [ - "rules.yml" - ], "concurrency": 16, "rateLimit": 100, "required": false - } - ], - "rules": [ + }, { - "cost": {} + "name": "prom3", + "uri": "http://localhost/3", + "timeout": "2m0s", + "uptime": "up", + "tags": [ + "foo" + ], + "concurrency": 16, + "rateLimit": 100, + "required": false } ] } --- -[TestGetChecksForRule/two_prometheus_servers_/_disable_all_checks_via_comment - 1] +[TestGetChecksForRule/prometheus_check_with_prometheus_servers_and_disable_comment - 1] { "ci": { "baseBranch": "master", @@ -1933,6 +2097,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -1956,32 +2121,39 @@ "rule/label", "rule/link", "rule/reject" - ], - "disabled": [ - "alerts/template", - "alerts/external_labels" ] }, "owners": {}, "prometheus": [ { "name": "prom1", - "uri": "http://localhost/1", + "uri": "http://localhost", "timeout": "1s", "uptime": "up", + "include": [ + "rules.yml" + ], "concurrency": 16, "rateLimit": 100, "required": false }, { "name": "prom2", - "uri": "http://localhost/2", + "uri": "http://localhost", "timeout": "1s", "uptime": "up", + "include": [ + "rules.yml" + ], "concurrency": 16, "rateLimit": 100, "required": false } + ], + "rules": [ + { + "cost": {} + } ] } --- @@ -1995,6 +2167,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2079,6 +2252,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2109,7 +2283,8 @@ "promql/vector_matching", "promql/range_query", "rule/duplicate", - "labels/conflict" + "labels/conflict", + "alerts/absent" ] }, "owners": {}, @@ -2174,7 +2349,7 @@ } --- -[TestGetChecksForRule/two_prometheus_servers_/_disable_checks_via_file/disable_comment - 1] +[TestGetChecksForRule/two_prometheus_servers_/_snoozed_checks_via_comment - 1] { "ci": { "baseBranch": "master", @@ -2183,6 +2358,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2209,7 +2385,7 @@ ], "disabled": [ "alerts/template", - "alerts/external_labels" + "promql/regexp" ] }, "owners": {}, @@ -2245,6 +2421,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2298,7 +2475,7 @@ } --- -[TestGetChecksForRule/tag_snoozes_all_prometheus_checks - 1] +[TestGetChecksForRule/tag_disables_all_prometheus_checks - 1] { "ci": { "baseBranch": "master", @@ -2307,6 +2484,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2382,6 +2560,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", @@ -2431,143 +2610,6 @@ } --- -[TestGetChecksForRule/tag_disables_all_prometheus_checks - 1] -{ - "ci": { - "baseBranch": "master", - "maxCommits": 20 - }, - "parser": {}, - "checks": { - "enabled": [ - "alerts/annotation", - "alerts/count", - "alerts/external_labels", - "alerts/for", - "alerts/template", - "labels/conflict", - "promql/aggregate", - "alerts/comparison", - "promql/fragile", - "promql/range_query", - "promql/rate", - "promql/regexp", - "promql/syntax", - "promql/vector_matching", - "query/cost", - "promql/counter", - "promql/series", - "rule/dependency", - "rule/duplicate", - "rule/for", - "rule/label", - "rule/link", - "rule/reject" - ] - }, - "owners": {}, - "prometheus": [ - { - "name": "prom1", - "uri": "http://localhost/1", - "timeout": "2m0s", - "uptime": "up", - "tags": [ - "foo", - "disable", - "bar" - ], - "concurrency": 16, - "rateLimit": 100, - "required": false - }, - { - "name": "prom2", - "uri": "http://localhost/2", - "timeout": "2m0s", - "uptime": "up", - "concurrency": 16, - "rateLimit": 100, - "required": false - }, - { - "name": "prom3", - "uri": "http://localhost/3", - "timeout": "2m0s", - "uptime": "up", - "tags": [ - "foo" - ], - "concurrency": 16, - "rateLimit": 100, - "required": false - } - ] -} ---- - -[TestGetChecksForRule/two_prometheus_servers_/_snoozed_checks_via_comment - 1] -{ - "ci": { - "baseBranch": "master", - "maxCommits": 20 - }, - "parser": {}, - "checks": { - "enabled": [ - "alerts/annotation", - "alerts/count", - "alerts/external_labels", - "alerts/for", - "alerts/template", - "labels/conflict", - "promql/aggregate", - "alerts/comparison", - "promql/fragile", - "promql/range_query", - "promql/rate", - "promql/regexp", - "promql/syntax", - "promql/vector_matching", - "query/cost", - "promql/counter", - "promql/series", - "rule/dependency", - "rule/duplicate", - "rule/for", - "rule/label", - "rule/link", - "rule/reject" - ], - "disabled": [ - "alerts/template", - "promql/regexp" - ] - }, - "owners": {}, - "prometheus": [ - { - "name": "prom1", - "uri": "http://localhost/1", - "timeout": "1s", - "uptime": "up", - "concurrency": 16, - "rateLimit": 100, - "required": false - }, - { - "name": "prom2", - "uri": "http://localhost/2", - "timeout": "1s", - "uptime": "up", - "concurrency": 16, - "rateLimit": 100, - "required": false - } - ] -} ---- - [TestGetChecksForRule/alerts/count_full - 1] { "ci": { @@ -2577,6 +2619,7 @@ "parser": {}, "checks": { "enabled": [ + "alerts/absent", "alerts/annotation", "alerts/count", "alerts/external_labels", diff --git a/internal/config/config.go b/internal/config/config.go index af2b8365..eb389f53 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -157,6 +157,11 @@ func (cfg *Config) GetChecksForRule(ctx context.Context, gen *PrometheusGenerato check: checks.NewCounterCheck(p), tags: p.Tags(), }) + allChecks = append(allChecks, checkMeta{ + name: checks.CounterCheckName, + check: checks.NewAlertsAbsentCheck(p), + tags: p.Tags(), + }) } for _, rule := range cfg.Rules { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 08c46b55..bb120005 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -194,7 +194,8 @@ prometheus "prom" { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RateCheckName + "(prom)", + checks.RegexpCheckName, + checks.RateCheckName + "(prom)", checks.SeriesCheckName + "(prom)", checks.VectorMatchingCheckName + "(prom)", checks.RangeQueryCheckName + "(prom)", @@ -202,6 +203,7 @@ prometheus "prom" { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", }, }, { @@ -227,7 +229,8 @@ prometheus "prom" { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RateCheckName + "(prom)", + checks.RegexpCheckName, + checks.RateCheckName + "(prom)", checks.SeriesCheckName + "(prom)", checks.VectorMatchingCheckName + "(prom)", checks.RangeQueryCheckName + "(prom)", @@ -235,6 +238,7 @@ prometheus "prom" { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", }, }, { @@ -380,7 +384,8 @@ prometheus "prom" { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RateCheckName + "(prom)", + checks.RegexpCheckName, + checks.RateCheckName + "(prom)", checks.SeriesCheckName + "(prom)", checks.VectorMatchingCheckName + "(prom)", checks.RangeQueryCheckName + "(prom)", @@ -388,6 +393,7 @@ prometheus "prom" { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", }, }, { @@ -418,7 +424,8 @@ prometheus "ignore" { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RateCheckName + "(prom)", + checks.RegexpCheckName, + checks.RateCheckName + "(prom)", checks.SeriesCheckName + "(prom)", checks.VectorMatchingCheckName + "(prom)", checks.RangeQueryCheckName + "(prom)", @@ -426,6 +433,7 @@ prometheus "ignore" { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", }, }, { @@ -475,7 +483,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.AggregationCheckName + "(job:true)", + checks.RegexpCheckName, + checks.AggregationCheckName + "(job:true)", checks.AggregationCheckName + "(instance:false)", checks.AggregationCheckName + "(rack:false)", }, @@ -512,7 +521,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.AggregationCheckName + "(job:true)", + checks.RegexpCheckName, + checks.AggregationCheckName + "(job:true)", checks.AggregationCheckName + "(rack:false)", }, }, @@ -591,11 +601,13 @@ prometheus "prom2" { checks.RangeQueryCheckName + "(prom1)", checks.LabelsConflictCheckName + "(prom1)", checks.AlertsExternalLabelsCheckName + "(prom1)", + checks.AlertsAbsentCheckName + "(prom1)", checks.SeriesCheckName + "(prom2)", checks.VectorMatchingCheckName + "(prom2)", checks.RangeQueryCheckName + "(prom2)", checks.RuleDuplicateCheckName + "(prom2)", checks.CounterCheckName + "(prom2)", + checks.AlertsAbsentCheckName + "(prom2)", checks.CostCheckName + "(prom1)", }, }, @@ -656,7 +668,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.LabelCheckName + "(team:true)", + checks.RegexpCheckName, + checks.LabelCheckName + "(team:true)", checks.AnnotationCheckName + "(summary:true)", checks.LabelCheckName + "(team:false)", checks.AnnotationCheckName + "(summary=~^foo.+$:true)", @@ -763,7 +776,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RejectCheckName + "(key=~'^http://.+$')", + checks.RegexpCheckName, + checks.RejectCheckName + "(key=~'^http://.+$')", checks.RejectCheckName + "(val=~'^http://.+$')", checks.RejectCheckName + "(key=~'^.* +.*$')", checks.RejectCheckName + "(val=~'^$')", @@ -902,7 +916,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.LabelCheckName + "(priority=~^(1|2|3|4|5)$:true)", + checks.RegexpCheckName, + checks.LabelCheckName + "(priority=~^(1|2|3|4|5)$:true)", }, }, { @@ -1006,7 +1021,8 @@ rule { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.LabelCheckName + "(priority=~^(1|2|3|4|5)$:true)", + checks.RegexpCheckName, + checks.LabelCheckName + "(priority=~^(1|2|3|4|5)$:true)", }, }, { @@ -1027,6 +1043,7 @@ checks { "promql/range_query", "rule/duplicate", "labels/conflict", + "alerts/absent", ] } prometheus "prom1" { @@ -1103,6 +1120,7 @@ prometheus "prom1" { checks.LabelsConflictCheckName + "(prom1)", checks.AlertsExternalLabelsCheckName + "(prom1)", checks.CounterCheckName + "(prom1)", + checks.AlertsAbsentCheckName + "(prom1)", checks.AlertsCheckName + "(prom1)", }, }, @@ -1655,6 +1673,7 @@ checks { checks.LabelsConflictCheckName + "(prom1)", checks.AlertsExternalLabelsCheckName + "(prom1)", checks.CounterCheckName + "(prom1)", + checks.AlertsAbsentCheckName + "(prom1)", checks.SeriesCheckName + "(prom2)", checks.VectorMatchingCheckName + "(prom2)", checks.RangeQueryCheckName + "(prom2)", @@ -1662,6 +1681,7 @@ checks { checks.LabelsConflictCheckName + "(prom2)", checks.AlertsExternalLabelsCheckName + "(prom2)", checks.CounterCheckName + "(prom2)", + checks.AlertsAbsentCheckName + "(prom2)", }, disabledChecks: []string{"promql/rate"}, }, @@ -1698,6 +1718,7 @@ prometheus "prom3" { # pint disable promql/rate(+disable) # pint disable promql/vector_matching(+disable) # pint disable rule/duplicate(+disable) +# pint disable alerts/absent(+disable) - record: foo expr: sum(foo) `), @@ -1716,6 +1737,7 @@ prometheus "prom3" { checks.LabelsConflictCheckName + "(prom2)", checks.AlertsExternalLabelsCheckName + "(prom2)", checks.CounterCheckName + "(prom2)", + checks.AlertsAbsentCheckName + "(prom2)", checks.RateCheckName + "(prom3)", checks.SeriesCheckName + "(prom3)", checks.VectorMatchingCheckName + "(prom3)", @@ -1724,6 +1746,7 @@ prometheus "prom3" { checks.LabelsConflictCheckName + "(prom3)", checks.AlertsExternalLabelsCheckName + "(prom3)", checks.CounterCheckName + "(prom3)", + checks.AlertsAbsentCheckName + "(prom3)", }, }, { @@ -1759,6 +1782,7 @@ prometheus "prom3" { # pint snooze 2099-11-28 promql/rate(+disable) # pint snooze 2099-11-28 promql/vector_matching(+disable) # pint snooze 2099-11-28 rule/duplicate(+disable) +# pint snooze 2099-11-28 alers/absent(+disable) - record: foo expr: sum(foo) `), @@ -1769,7 +1793,8 @@ prometheus "prom3" { checks.ComparisonCheckName, checks.TemplateCheckName, checks.FragileCheckName, - checks.RegexpCheckName, checks.RateCheckName + "(prom2)", + checks.RegexpCheckName, + checks.RateCheckName + "(prom2)", checks.SeriesCheckName + "(prom2)", checks.VectorMatchingCheckName + "(prom2)", checks.RangeQueryCheckName + "(prom2)", @@ -1777,6 +1802,7 @@ prometheus "prom3" { checks.LabelsConflictCheckName + "(prom2)", checks.AlertsExternalLabelsCheckName + "(prom2)", checks.CounterCheckName + "(prom2)", + checks.AlertsAbsentCheckName + "(prom2)", checks.RateCheckName + "(prom3)", checks.SeriesCheckName + "(prom3)", checks.VectorMatchingCheckName + "(prom3)", @@ -1785,6 +1811,7 @@ prometheus "prom3" { checks.LabelsConflictCheckName + "(prom3)", checks.AlertsExternalLabelsCheckName + "(prom3)", checks.CounterCheckName + "(prom3)", + checks.AlertsAbsentCheckName + "(prom3)", }, }, { @@ -1825,6 +1852,7 @@ rule { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", checks.AlertsCheckName + "(prom)", }, }, @@ -1869,6 +1897,7 @@ rule { checks.LabelsConflictCheckName + "(prom)", checks.AlertsExternalLabelsCheckName + "(prom)", checks.CounterCheckName + "(prom)", + checks.AlertsAbsentCheckName + "(prom)", checks.AlertsCheckName + "(prom)", }, },