Skip to content

Commit

Permalink
refact cscli metric processing (crowdsecurity#2816)
Browse files Browse the repository at this point in the history 
* typos
* refact cscli metric processing
* lint
  • Loading branch information
@mmetc
mmetc authored Feb 7, 2024
1 parent 3208a40 commit af1df06
Show file tree
Hide file tree
Showing 3 changed files with 240 additions and 164 deletions.
2 changes: 1 addition & 1 deletion .golangci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ linters-settings:

gocognit:
# lower this after refactoring
min-complexity: 150
min-complexity: 145

gocyclo:
# lower this after refactoring
Expand Down
185 changes: 48 additions & 137 deletions cmd/crowdsec-cli/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -42,8 +43,14 @@ type (
}
)

var (
ErrMissingConfig = errors.New("prometheus section missing, can't show metrics")
ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")

)

type metricSection interface {
Table(io.Writer, bool, bool)
Table(out io.Writer, noUnit bool, showEmpty bool)
Description() (string, string)
}

Expand Down Expand Up @@ -154,6 +161,9 @@ func (ms metricStore) Fetch(url string) error {
origin := metric.Labels["origin"]
action := metric.Labels["action"]

appsecEngine := metric.Labels["appsec_engine"]
appsecRule := metric.Labels["rule_name"]

mtype := metric.Labels["type"]

fval, err := strconv.ParseFloat(value, 32)
Expand All @@ -162,178 +172,78 @@ func (ms metricStore) Fetch(url string) error {
}

ival := int(fval)

switch fam.Name {
//
// buckets
//
case "cs_bucket_created_total":
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
mBucket[name]["instantiation"] += ival
mBucket.Process(name, "instantiation", ival)
case "cs_buckets":
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
mBucket[name]["curr_count"] += ival
mBucket.Process(name, "curr_count", ival)
case "cs_bucket_overflowed_total":
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
mBucket[name]["overflow"] += ival
mBucket.Process(name, "overflow", ival)
case "cs_bucket_poured_total":
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mBucket[name]["pour"] += ival
mAcquis[source]["pour"] += ival
mBucket.Process(name, "pour", ival)
mAcquis.Process(source, "pour", ival)
case "cs_bucket_underflowed_total":
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)
}
mBucket[name]["underflow"] += ival
mBucket.Process(name, "underflow", ival)
//
// parsers
//
case "cs_parser_hits_total":
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mAcquis[source]["reads"] += ival
mAcquis.Process(source, "reads", ival)
case "cs_parser_hits_ok_total":
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mAcquis[source]["parsed"] += ival
mAcquis.Process(source, "parsed", ival)
case "cs_parser_hits_ko_total":
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mAcquis[source]["unparsed"] += ival
mAcquis.Process(source, "unparsed", ival)
case "cs_node_hits_total":
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
}
mParser[name]["hits"] += ival
mParser.Process(name, "hits", ival)
case "cs_node_hits_ok_total":
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
}
mParser[name]["parsed"] += ival
mParser.Process(name, "parsed", ival)
case "cs_node_hits_ko_total":
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)
}
mParser[name]["unparsed"] += ival
mParser.Process(name, "unparsed", ival)
//
// whitelists
//
case "cs_node_wl_hits_total":
if _, ok := mWhitelist[name]; !ok {
mWhitelist[name] = make(map[string]map[string]int)
}
if _, ok := mWhitelist[name][reason]; !ok {
mWhitelist[name][reason] = make(map[string]int)
}
mWhitelist[name][reason]["hits"] += ival
mWhitelist.Process(name, reason, "hits", ival)
case "cs_node_wl_hits_ok_total":
if _, ok := mWhitelist[name]; !ok {
mWhitelist[name] = make(map[string]map[string]int)
}
if _, ok := mWhitelist[name][reason]; !ok {
mWhitelist[name][reason] = make(map[string]int)
}
mWhitelist[name][reason]["whitelisted"] += ival
mWhitelist.Process(name, reason, "whitelisted", ival)
// track as well whitelisted lines at acquis level
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)
}
mAcquis[source]["whitelisted"] += ival
mAcquis.Process(source, "whitelisted", ival)
//
// lapi
//
case "cs_lapi_route_requests_total":
if _, ok := mLapi[route]; !ok {
mLapi[route] = make(map[string]int)
}
mLapi[route][method] += ival
mLapi.Process(route, method, ival)
case "cs_lapi_machine_requests_total":
if _, ok := mLapiMachine[machine]; !ok {
mLapiMachine[machine] = make(map[string]map[string]int)
}
if _, ok := mLapiMachine[machine][route]; !ok {
mLapiMachine[machine][route] = make(map[string]int)
}
mLapiMachine[machine][route][method] += ival
mLapiMachine.Process(machine, route, method, ival)
case "cs_lapi_bouncer_requests_total":
if _, ok := mLapiBouncer[bouncer]; !ok {
mLapiBouncer[bouncer] = make(map[string]map[string]int)
}
if _, ok := mLapiBouncer[bouncer][route]; !ok {
mLapiBouncer[bouncer][route] = make(map[string]int)
}
mLapiBouncer[bouncer][route][method] += ival
mLapiBouncer.Process(bouncer, route, method, ival)
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
if _, ok := mLapiDecision[bouncer]; !ok {
mLapiDecision[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := mLapiDecision[bouncer]
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
mLapiDecision[bouncer] = x
mLapiDecision.Process(bouncer, fam.Name, ival)
//
// decisions
//
case "cs_active_decisions":
if _, ok := mDecision[reason]; !ok {
mDecision[reason] = make(map[string]map[string]int)
}
if _, ok := mDecision[reason][origin]; !ok {
mDecision[reason][origin] = make(map[string]int)
}
mDecision[reason][origin][action] += ival
mDecision.Process(reason, origin, action, ival)
case "cs_alerts":
mAlert[reason] += ival
mAlert.Process(reason, ival)
//
// stash
//
case "cs_cache_size":
mStash[name] = struct {
Type string
Count int
}{Type: mtype, Count: ival}
mStash.Process(name, mtype, ival)
//
// appsec
//
case "cs_appsec_reqs_total":
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
mAppsecEngine[metric.Labels["appsec_engine"]]["processed"] = ival
mAppsecEngine.Process(appsecEngine, "processed", ival)
case "cs_appsec_block_total":
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
mAppsecEngine[metric.Labels["appsec_engine"]]["blocked"] = ival
mAppsecEngine.Process(appsecEngine, "blocked", ival)
case "cs_appsec_rule_hits":
appsecEngine := metric.Labels["appsec_engine"]
ruleID := metric.Labels["rule_name"]
if _, ok := mAppsecRule[appsecEngine]; !ok {
mAppsecRule[appsecEngine] = make(map[string]map[string]int, 0)
}
if _, ok := mAppsecRule[appsecEngine][ruleID]; !ok {
mAppsecRule[appsecEngine][ruleID] = make(map[string]int, 0)
}
mAppsecRule[appsecEngine][ruleID]["triggered"] = ival
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
default:
log.Debugf("unknown: %+v", fam.Name)
continue
Expand Down Expand Up @@ -380,13 +290,13 @@ func (ms metricStore) Format(out io.Writer, sections []string, formatType string
case "json":
x, err := json.MarshalIndent(want, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
case "raw":
x, err := yaml.Marshal(want)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
default:
Expand All @@ -404,11 +314,11 @@ func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
}

if cfg.Prometheus == nil {
return fmt.Errorf("prometheus section missing, can't show metrics")
return ErrMissingConfig
}

if !cfg.Prometheus.Enabled {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
return ErrMetricsDisabled
}

ms := NewMetricStore()
Expand All @@ -427,6 +337,7 @@ func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -468,6 +379,7 @@ cscli metrics list`,
// expandAlias returns a list of sections. The input can be a list of sections or alias.
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
ret := []string{}

for _, section := range args {
switch section {
case "engine":
Expand Down Expand Up @@ -522,8 +434,8 @@ cscli metrics show acquisition parsers buckets stash -o json`,

func (cli *cliMetrics) list() error {
type metricType struct {
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
}

Expand Down Expand Up @@ -553,13 +465,13 @@ func (cli *cliMetrics) list() error {
case "json":
x, err := json.MarshalIndent(allMetrics, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics: %w", err)
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
case "raw":
x, err := yaml.Marshal(allMetrics)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics: %w", err)
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
}
Expand All @@ -575,8 +487,7 @@ func (cli *cliMetrics) newListCmd() *cobra.Command {
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cli.list()
return nil
return cli.list()
},
}

Expand Down
Loading

0 comments on commit af1df06

Please sign in to comment.