From 61c9efa0292ae384f1467969934e1085316d85ec Mon Sep 17 00:00:00 2001 From: Bhavina Date: Mon, 15 Mar 2021 18:41:13 +0000 Subject: [PATCH] Adding metric scraping configuration via a series of env variables --- README.md | 16 ++ exporter.py | 13 +- helpers/prometheus.py | 393 ++++++++++++++++++++++-------------------- libs/sentry.py | 10 +- 4 files changed, 233 insertions(+), 199 deletions(-) diff --git a/README.md b/README.md index 430c04e..dfed55a 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,20 @@ docker-compose up -d - `sentry_issues`: Gauge Histogram of open issues split into 3 buckets: 1h, 24h, and 14d - `sentry_events`: Total events counts per project +### Metric Configuration +By default all metrics are scraped, however, issue or event-related metrics can be disabled by setting the relevant variable to False: +```sh +export SENTRY_SCRAPE_ISSUE_METRICS=False +export SENTRY_SCRAPE_EVENT_METRICS=False +``` +By default, if `SENTRY_SCRAPE_ISSUE_METRICS=True or is unset` issue metrics are scraped for `1hour`, `24hours` and `14days`. Any of these can be disabled by setting the relevant variable to False: +```sh +export SENTRY_ISSUES_1H=False +export SENTRY_ISSUES_24H=False +export SENTRY_ISSUES_14D=False +``` +As with `SENTRY_AUTH_TOKEN`, all of these variables can be passed in through the `docker run -e VAR_NAME=<>` command or via the `.env` file if using Docker Compose. + ## Samples **Grafana Dashboard** @@ -98,6 +112,8 @@ scrape_configs: - Use a high `scrape_timeout` for the exporter job > General recomendation is to set `scrape_interval - 1` (i.e.: `4m`) +- If the scraping of particular metrics are disabled the values above can be reduced depending on your setup. + ## 📒 Documentation [Sentry Prometheus Exporter documentation](https://italux.github.io/sentry-prometheus-exporter/) diff --git a/exporter.py b/exporter.py index b95bda4..87e22cb 100644 --- a/exporter.py +++ b/exporter.py @@ -29,6 +29,17 @@ app = Flask(__name__) +def get_metric_config(): + """Get metric scraping options.""" + scrape_issue_metrics = getenv("SENTRY_SCRAPE_ISSUE_METRICS") or "True" + scrape_events_metrics = getenv("SENTRY_SCRAPE_EVENT_METRICS") or "True" + default_for_time_metrics = "True" if scrape_issue_metrics == "True" else "False" + get_1h_metrics = getenv("SENTRY_ISSUES_1H") or default_for_time_metrics + get_24h_metrics = getenv("SENTRY_ISSUES_24H") or default_for_time_metrics + get_14d_metrics = getenv("SENTRY_ISSUES_14D") or default_for_time_metrics + return [scrape_issue_metrics, scrape_events_metrics, get_1h_metrics, get_24h_metrics, get_14d_metrics] + + @app.route("/") def hello_world(): return "

Sentry Issues & Events Exporter

\ @@ -41,7 +52,7 @@ def sentry_exporter(): sentry = SentryAPI(BASE_URL, AUTH_TOKEN) log.info("exporter: cleaning registry collectors...") clean_registry() - REGISTRY.register(SentryCollector(sentry, ORG_SLUG, PROJECTS_SLUG)) + REGISTRY.register(SentryCollector(sentry, ORG_SLUG, get_metric_config(), PROJECTS_SLUG)) exporter = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()}) return exporter diff --git a/helpers/prometheus.py b/helpers/prometheus.py index 6b6f66a..c0aea6f 100644 --- a/helpers/prometheus.py +++ b/helpers/prometheus.py @@ -42,12 +42,17 @@ class SentryCollector(object): >>> REGISTRY.register(SentryCollector(sentry, org_slug, projects_slug)) """ - def __init__(self, sentry_api, sentry_org_slug, sentry_projects_slug=None): + def __init__(self, sentry_api, sentry_org_slug, metric_scraping_config, sentry_projects_slug=None): """Inits SentryCollector with a SentryAPI object""" super(SentryCollector, self).__init__() self.__sentry_api = sentry_api self.sentry_org_slug = sentry_org_slug self.sentry_projects_slug = sentry_projects_slug + self.issue_metrics = metric_scraping_config[0] + self.events_metrics = metric_scraping_config[1] + self.get_1h_metrics = metric_scraping_config[2] + self.get_24h_metrics = metric_scraping_config[3] + self.get_14d_metrics = metric_scraping_config[4] def __build_sentry_data_from_api(self): """Build a local data structure from sentry API calls. @@ -132,51 +137,55 @@ def __build_sentry_data_from_api(self): "projects_envs": projects_envs, } } - - __metadata = data.get("metadata") - - projects_issue_data = {} - - for project in __metadata.get("projects"): - projects_issue_data[project.get("slug")] = {} - envs = __metadata.get("projects_envs").get(project.get("slug")) - for env in envs: - log.debug( - "metadata: getting issues from api - project: {proj} env: {env} age: 1h".format( - proj=project.get("slug"), env=env - ) - ) - project_issues_1h = self.__sentry_api.issues( - self.org.get("slug"), project, env, age="1h" - ) - log.debug( - "metadata: getting issues from api - project: {proj} env: {env} age: 24h".format( - proj=project.get("slug"), env=env - ) - ) - project_issues_24h = self.__sentry_api.issues( - self.org.get("slug"), project, env, age="24h" - ) - log.debug( - "metadata: getting issues from api - project: {proj} env: {env} age: 14d".format( - proj=project.get("slug"), env=env - ) - ) - project_issues_14d = self.__sentry_api.issues( - self.org.get("slug"), project, env, age="14d" - ) - - log.debug("data structure: building projects issues data") - for k, v in project_issues_1h.items(): - projects_issue_data[project.get("slug")][k] = {"1h": v} - - for k, v in project_issues_24h.items(): - projects_issue_data[project.get("slug")][k].update({"24h": v}) - - for k, v in project_issues_14d.items(): - projects_issue_data[project.get("slug")][k].update({"14d": v}) - - data["projects_data"] = projects_issue_data + if self.issue_metrics == "True": + __metadata = data.get("metadata") + + projects_issue_data = {} + + for project in __metadata.get("projects"): + projects_issue_data[project.get("slug")] = {} + envs = __metadata.get("projects_envs").get(project.get("slug")) + for env in envs: + project_issues_1h = project_issues_24h = project_issues_14d = {} + if self.get_1h_metrics == "True": + log.debug( + "metadata: getting issues from api - project: {proj} env: {env} age: 1h".format( + proj=project.get("slug"), env=env + ) + ) + project_issues_1h = self.__sentry_api.issues( + self.org.get("slug"), project, env, age="1h" + ) + if self.get_24h_metrics == "True": + log.debug( + "metadata: getting issues from api - project: {proj} env: {env} age: 24h".format( + proj=project.get("slug"), env=env + ) + ) + project_issues_24h = self.__sentry_api.issues( + self.org.get("slug"), project, env, age="24h" + ) + if self.get_14d_metrics == "True": + log.debug( + "metadata: getting issues from api - project: {proj} env: {env} age: 14d".format( + proj=project.get("slug"), env=env + ) + ) + project_issues_14d = self.__sentry_api.issues( + self.org.get("slug"), project, env, age="14d" + ) + + log.debug("data structure: building projects issues data"): + for k, v in project_issues_1h.items(): + projects_issue_data[project.get("slug")][k] = {"1h": v} + + for k, v in project_issues_24h.items(): + projects_issue_data[project.get("slug")][k].update({"24h": v}) + + for k, v in project_issues_14d.items(): + projects_issue_data[project.get("slug")][k].update({"14d": v}) + + data["projects_data"] = projects_issue_data write_cache(JSON_CACHE_FILE, data, DEFAULT_CACHE_EXPIRE_TIMESTAMP) log.debug("cache: writing data structure to file: {cache}".format(cache=JSON_CACHE_FILE)) @@ -205,157 +214,161 @@ def collect(self): self.org = __metadata.get("org") self.projects_data = {} - issues_histogram_metrics = GaugeHistogramMetricFamily( - "sentry_issues", - "Number of open issues (aka is:unresolved) per project", - buckets=None, - gsum_value=None, - labels=[ - "project_slug", - "environment", - ], - unit="", - ) - - log.info("collector: loading projects issues") - for project in __metadata.get("projects"): - envs = __metadata.get("projects_envs").get(project.get("slug")) - project_issues = __projects_data.get(project.get("slug")) - for env in envs: - log.debug( - "collector: loading issues - project: {proj} env: {env}".format( - proj=project.get("slug"), env=env + if self.issue_metrics == "True": + issues_histogram_metrics = GaugeHistogramMetricFamily( + "sentry_issues", + "Number of open issues (aka is:unresolved) per project", + buckets=None, + gsum_value=None, + labels=[ + "project_slug", + "environment", + ], + unit="", + ) + + log.info("collector: loading projects issues") + for project in __metadata.get("projects"): + envs = __metadata.get("projects_envs").get(project.get("slug")) + project_issues = __projects_data.get(project.get("slug")) + for env in envs: + log.debug( + "collector: loading issues - project: {proj} env: {env}".format( + proj=project.get("slug"), env=env + ) ) - ) - project_issues_1h = project_issues.get(env).get("1h") - project_issues_24h = project_issues.get(env).get("24h") - project_issues_14d = project_issues.get(env).get("14d") + project_issues_1h = project_issues.get(env).get("1h") + project_issues_24h = project_issues.get(env).get("24h") + project_issues_14d = project_issues.get(env).get("14d") + + events_1h = 0 + events_24h = 0 + events_14d = 0 + + if project_issues_1h: + for issue in project_issues_1h: + events_1h += int(issue.get("count") or 0) + + if project_issues_24h: + for issue in project_issues_24h: + events_24h += int(issue.get("count") or 0) + + if project_issues_14d: + for issue in project_issues_14d: + events_14d += int(issue.get("count") or 0) + + sum_events = events_1h + events_24h + events_14d + histo_buckets = [] + if self.get_1h_metrics == "True": + histo_buckets.append(("1h", float(events_1h))) + if self.get_24h_metrics == "True": + histo_buckets.append(("24h", float(events_24h))) + if self.get_14d_metrics == "True": + histo_buckets.append(("+Inf", float(events_14d))) + issues_histogram_metrics.add_metric( + labels=[ + str(project.get("slug")), + str(env), + ], + buckets=histo_buckets, + gsum_value=int(sum_events), + ) - events_1h = 0 - events_24h = 0 - events_14d = 0 + yield issues_histogram_metrics + + issues_metrics = GaugeMetricFamily( + "sentry_open_issue_events", + "Number of open issues (aka is:unresolved) per project", + labels=[ + "issue_id", + "logger", + "level", + "status", + "platform", + "project_slug", + "environment", + "release", + "isUnhandled", + "firstSeen", + "lastSeen", + ], + ) - if project_issues_1h: + for project in __metadata.get("projects"): + envs = __metadata.get("projects_envs").get(project.get("slug")) + project_issues = __projects_data.get(project.get("slug")) + for env in envs: + project_issues_1h = project_issues.get(env).get("1h") for issue in project_issues_1h: - events_1h += int(issue.get("count") or 0) - - if project_issues_24h: - for issue in project_issues_24h: - events_24h += int(issue.get("count") or 0) - - if project_issues_14d: - for issue in project_issues_14d: - events_14d += int(issue.get("count") or 0) - - sum_events = events_1h + events_24h + events_14d - - issues_histogram_metrics.add_metric( - labels=[ - str(project.get("slug")), - str(env), - ], - buckets=[ - ("1h", float(events_1h)), - ("24h", float(events_24h)), - ("+Inf", float(events_14d)), - ], - gsum_value=int(sum_events), - ) - - yield issues_histogram_metrics - - issues_metrics = GaugeMetricFamily( - "sentry_open_issue_events", - "Number of open issues (aka is:unresolved) per project", - labels=[ - "issue_id", - "logger", - "level", - "status", - "platform", - "project_slug", - "environment", - "release", - "isUnhandled", - "firstSeen", - "lastSeen", - ], - ) - - for project in __metadata.get("projects"): - envs = __metadata.get("projects_envs").get(project.get("slug")) - project_issues = __projects_data.get(project.get("slug")) - for env in envs: - project_issues_1h = project_issues.get(env).get("1h") - for issue in project_issues_1h: - release = self.__sentry_api.issue_release(issue.get("id"), env) - issues_metrics.add_metric( - [ - str(issue.get("id")), - str(issue.get("logger")) or "None", - str(issue.get("level")), - str(issue.get("status")), - str(issue.get("platform")), - str(issue.get("project").get("slug")), - str(env), - str(release), - str(issue.get("isUnhandled")), - str( - datetime.strftime( - datetime.strptime( - str( - issue.get("firstSeen") - # if the issue age is recent, firstSeen returns None - # and we'll return datetime.now() as default - or datetime.strftime( - datetime.now(), "%Y-%m-%dT%H:%M:%SZ" - ) + release = self.__sentry_api.issue_release(issue.get("id"), env) + issues_metrics.add_metric( + [ + str(issue.get("id")), + str(issue.get("logger")) or "None", + str(issue.get("level")), + str(issue.get("status")), + str(issue.get("platform")), + str(issue.get("project").get("slug")), + str(env), + str(release), + str(issue.get("isUnhandled")), + str( + datetime.strftime( + datetime.strptime( + str( + issue.get("firstSeen") + # if the issue age is recent, firstSeen returns None + # and we'll return datetime.now() as default + or datetime.strftime( + datetime.now(), "%Y-%m-%dT%H:%M:%SZ" + ) + ), + "%Y-%m-%dT%H:%M:%SZ", ), - "%Y-%m-%dT%H:%M:%SZ", - ), - "%Y-%m-%d", - ) - ), - str( - datetime.strftime( - datetime.strptime( - str( - issue.get("lastSeen") - # if the issue age is recent, lastSeen returns None - # and we'll return datetime.now() as default - or datetime.strftime( - datetime.now(), "%Y-%m-%dT%H:%M:%SZ" - ) + "%Y-%m-%d", + ) + ), + str( + datetime.strftime( + datetime.strptime( + str( + issue.get("lastSeen") + # if the issue age is recent, lastSeen returns None + # and we'll return datetime.now() as default + or datetime.strftime( + datetime.now(), "%Y-%m-%dT%H:%M:%SZ" + ) + ), + "%Y-%m-%dT%H:%M:%SZ", ), - "%Y-%m-%dT%H:%M:%SZ", - ), - "%Y-%m-%d", - ) - ), + "%Y-%m-%d", + ) + ), + ], + int(issue.get("count")), + ) + yield issues_metrics + + if self.events_metrics == "True": + project_events_metrics = CounterMetricFamily( + "sentry_events", + "Total events counts per project", + labels=[ + "project_slug", + "stat", + ], + ) + + for project in __metadata.get("projects"): + events = self.__sentry_api.project_stats(self.org.get("slug"), project.get("slug")) + for stat, value in events.items(): + project_events_metrics.add_metric( + [ + str(project.get("slug")), + str(stat), ], - int(issue.get("count")), + int(value), ) - yield issues_metrics - - project_events_metrics = CounterMetricFamily( - "sentry_events", - "Total events counts per project", - labels=[ - "project_slug", - "stat", - ], - ) - - for project in __metadata.get("projects"): - events = self.__sentry_api.project_stats(self.org.get("slug"), project.get("slug")) - for stat, value in events.items(): - project_events_metrics.add_metric( - [ - str(project.get("slug")), - str(stat), - ], - int(value), - ) - yield project_events_metrics + yield project_events_metrics diff --git a/libs/sentry.py b/libs/sentry.py index 4aa0fa2..bba642b 100644 --- a/libs/sentry.py +++ b/libs/sentry.py @@ -170,8 +170,7 @@ def project_stats(self, org_slug, project_slug): for stat_name, values in stats.items(): for stat in values: - if type(stat) != str: - events += stat[1] + events += stat[1] project_events[stat_name] = events return project_events @@ -198,8 +197,6 @@ def environments(self, org_slug, project): org=org_slug, proj_slug=project.get("slug") ), ) - if resp.status_code == 404: - return [] environments = [env.get("name") for env in resp.json()] return environments @@ -235,10 +232,7 @@ def issues(self, org_slug, project, environment=None, age="24h"): issues = {} issues_url = issues_url + "&environment={env}".format(env=environment) resp = self.__get(issues_url) - if resp.status_code == 404: - issues[environment] = [] - else: - issues[environment] = resp.json() + issues[environment] = resp.json() return issues else: resp = self.__get(issues_url)