diff --git a/.dockerignore b/.dockerignore index ad62951..d7036c0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ -node_modules -npm-debug.log .env -.idea .git -tests +.idea +node_modules +npm-debug.log diff --git a/.env.default b/.env.default index 3e9c8a0..e7e0940 100644 --- a/.env.default +++ b/.env.default @@ -1,5 +1,14 @@ APP_PORT=3000 APP_ALERTMANAGER_SECRET= + +# Optional URLs used for alerts with links back to your alertman instance and dashboards +APP_ALERTMANAGER_URL=https://alertman.domain/#/alerts +APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL=https://grafana.domain/dashboard/path?orgId=2&var-job=nodes&var-node=All&var-hostname= +APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME=true + +# DEBUG | VERBOSE | INFO | WARNING | ERROR +LOG_LEVEL=VERBOSE + MATRIX_HOMESERVER_URL=https://homeserver.tld # The rooms to send alerts to, separated by a | # Each entry contains the receiver name (from alertmanager) and the diff --git a/Dockerfile b/Dockerfile index c33c9b8..1262108 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY package*.json ./ RUN npm install --only=production -COPY . . +COPY src /app/src EXPOSE 3000 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c49f3c2 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +test: + docker run -it --rm \ + --workdir /app \ + -v $(PWD):/app \ + -e LOG_LEVEL=INFO \ + --userns host \ + node:18-alpine \ + sh -c "npm install && npm run test" + +build: + docker build -t matrix-alertmanager . diff --git a/README.md b/README.md index 5b9816f..11dc5c6 100644 --- a/README.md +++ b/README.md @@ -19,23 +19,29 @@ Main features: ### Configuration -Whether running manually or via the Docker image, the configuration is set +Whether running manually or via the Docker image, the configuration is set via environment variables. When running manually, copy `.env.default` -into `.env`, set the values and they will be loaded automatically. -When using the Docker image, set the environment variables when running +into `.env`, set the values and they will be loaded automatically. +When using the Docker image, set the environment variables when running the container. ### Docker The [Docker image](https://cloud.docker.com/repository/docker/jaywink/matrix-alertmanager) `jaywink/matrix-alertmanager:latest` is the easiest way to get the service running. Ensure you set the required environment variables listed in `.env.default` in this repository. +#### Building locally + +You can build the container locally: + + make build + ### Alertmanager You will need to configure a webhook receiver in Alertmanager. It should looks something like this: ```yaml receivers: -- name: 'myreceiver' +- name: 'receiver1' webhook_configs: - url: 'https://my-matrix-alertmanager.tld/alerts?secret=veryverysecretkeyhere' ``` @@ -46,7 +52,7 @@ Alternatively, put the secret in a separate file and use basic auth with usernam ```yaml receivers: -- name: 'myreceiver' +- name: 'receiver2' webhook_configs: - url: 'https://my-matrix-alertmanager.tld/alerts' http_config: @@ -55,6 +61,15 @@ receivers: password_file: /path/to/password.secret ``` +Note that the receiver `name` must match a configured receiver in the +`MATRIX_ROOMS` env variable. + +Optional convenience links can be configured back to your alertmanager +instance and/or a default dashboard. + +If an alert is received with a `url` in the `annotations` that URL will be used +rather than the default dashboard URL. + ### Prometheus rules Add some styling to your prometheus rules @@ -64,6 +79,7 @@ rules: - alert: High Memory Usage of Container annotations: description: Container named {{\$labels.container_name}} in {{\$labels.pod_name}} in {{\$labels.namespace}} is using more than 75% of Memory Limit + url: https://grafana.domain/path/do/dashboard?orgId=12&var-host={\$labels.container_name} expr: | ((( sum(container_memory_usage_bytes{image!=\"\",container_name!=\"POD\", namespace!=\"kube-system\"}) by (namespace,container_name,pod_name, instance) / sum(container_spec_memory_limit_bytes{image!=\"\",container_name!=\"POD\",namespace!=\"kube-system\"}) by (namespace,container_name,pod_name, instance) ) * 100 ) < +Inf ) > 75 for: 5m @@ -73,6 +89,16 @@ rules: NOTE! Currently, the bot cannot talk HTTPS, so you need to have a reverse proxy in place to terminate SSL, or use unsecure unencrypted connections. +## Testing + +The tests can either be run locally with: + + npm test + +..or from within a docker container: + + make test + ## TODO * Registering an account instead of having to use an existing account diff --git a/src/app.js b/src/app.js index ac7e469..8f136e3 100644 --- a/src/app.js +++ b/src/app.js @@ -1,6 +1,7 @@ const express = require('express') const client = require('./client') const routes = require('./routes') +const log = require('./log') // Config require('dotenv').config() @@ -13,14 +14,11 @@ app.get('/', routes.getRoot) app.post('/alerts', routes.postAlerts) // Initialize Matrix client client.init().then(() => { - // eslint-disable-next-line no-console - console.log('matrix-alertmanager initialized and ready') + log.info('matrix-alertmanager initialized and ready') app.listen(process.env.APP_PORT, () => {}) }).catch(e => { - // eslint-disable-next-line no-console - console.error('initialization failed') - // eslint-disable-next-line no-console - console.error(e) + log.error('initialization failed') + log.error(e) }) module.exports = app diff --git a/src/client.js b/src/client.js index f389bf0..7fbb15f 100644 --- a/src/client.js +++ b/src/client.js @@ -1,5 +1,6 @@ const matrix = require('matrix-js-sdk') const striptags = require('striptags') +const log = require('./log') let joinedRoomsCache = [] @@ -12,7 +13,7 @@ const client = { joinedRoomsCache.push(room.roomId) } } catch (ex) { - console.warn(`Could not join room ${roomId} - ${ex}`) + log.warn(`Could not join room ${roomId} - ${ex}`) } } }, diff --git a/src/log.js b/src/log.js new file mode 100644 index 0000000..d09aeb3 --- /dev/null +++ b/src/log.js @@ -0,0 +1,30 @@ +// DEBUG | VERBOSE | INFO | WARNING | ERROR +const logLevel = process.env.LOG_LEVEL || 'INFO' + +const log = { + error: message => { + console.error(message) + }, + warn: message => { + if ( ['DEBUG', 'VERBOSE', 'INFO', 'WARNING'].indexOf(logLevel) > -1 ) { + console.warning(message) + } + }, + info: message => { + if ( ['DEBUG', 'VERBOSE', 'INFO'].indexOf(logLevel) > -1 ) { + console.info(message) + } + }, + verbose: message => { + if ( ['DEBUG', 'VERBOSE'].indexOf(logLevel) > -1 ) { + console.log(message) + } + }, + debug: message => { + if ( ['DEBUG'].indexOf(logLevel) > -1 ) { + console.debug(message) + } + } +} + +module.exports = log diff --git a/src/routes.js b/src/routes.js index 83935ca..bf94781 100644 --- a/src/routes.js +++ b/src/routes.js @@ -1,5 +1,6 @@ const client = require('./client') const utils = require('./utils') +const log = require('./log') const routes = { getRoot: (req, res) => { @@ -11,16 +12,23 @@ const routes = { res.status(403).end() return } + if ( process.env.LOG_LEVEL === 'DEBUG' ) { + log.verbose(JSON.stringify(req.body)) + } const alerts = utils.parseAlerts(req.body) if (!alerts) { - res.json({'result': 'no alerts found in payload'}) + const msg = 'no alerts found in payload' + log.info(msg) + res.json({'result': msg}) return } const roomId = utils.getRoomForReceiver(req.body.receiver) if (!roomId) { - res.json({'result': 'no rooms configured for this receiver'}) + const msg = 'no rooms configured for this receiver' + log.info(msg) + res.json({'result': msg}) return } @@ -29,10 +37,9 @@ const routes = { await Promise.all(promises) res.json({'result': 'ok'}) } catch (e) { - // eslint-disable-next-line no-console - console.error(e) + log.error(e) res.status(500) - res.json({'result': 'error'}) + res.json({'result': 'error sending alert'}) } }, } diff --git a/src/utils.js b/src/utils.js index 4b438ab..0d3224c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -28,7 +28,6 @@ const utils = { Format a single alert into a message string. */ let parts = [] - //console.log(data) if (data.status === 'firing') { if (process.env.MENTION_ROOM === "1") { @@ -44,7 +43,11 @@ const utils = { return '#dc3545'; // red } })(data.labels.severity); - parts.push('FIRING:') + if ( data.labels.severity !== undefined ) { + parts.push('' + data.labels.severity.toUpperCase() + ':') + } else { + parts.push('FIRING:') + } } else if (data.status === 'resolved') { parts.push('RESOLVED:') } else { @@ -53,17 +56,34 @@ const utils = { // name and location of occurrence if (data.labels.alertname !== undefined) { - parts.push('', data.labels.alertname, '') - if (data.labels.host !== undefined || data.labels.instance !== undefined) { - parts.push(' at ') + if ( process.env.APP_ALERTMANAGER_URL !== undefined ) { + parts.push('', data.labels.alertname, '') + } else { + parts.push('', data.labels.alertname, '') + } + if (data.labels.host !== undefined || data.labels.hostname !== undefined || data.labels.instance !== undefined) { + parts.push(' on ') } } - if (data.labels.host !== undefined) { - parts.push(data.labels.host) - } else if (data.labels.instance !== undefined) { - parts.push(data.labels.instance) + + let host = data.labels.host || data.labels.hostname || data.labels.instance; + + if (data.labels.parent !== undefined && data.labels.parent != host) { + parts.push(data.labels.parent + '/' + host) + } else { + parts.push(host) } + parts.push(' (') + let labels = new Set() + for (const [label, value] of Object.entries(data.labels)) { + if (['alertname', 'host', 'hostname', 'instance', 'parent', 'severity'].indexOf(label) == -1) { + labels.add(value) + } + } + parts.push(...labels) + parts.push(')') + // additional descriptive content if (data.annotations.message !== undefined) { parts.push('
', data.annotations.message.replace("\n", "
")) @@ -71,7 +91,23 @@ const utils = { if (data.annotations.description !== undefined) { parts.push('
', data.annotations.description) } - parts.push('
Alert link') + parts.push('
Alert link') + + if ( data.annotations.url !== undefined) { + parts.push(' | Other') + } else if ( + data.labels.hostname !== undefined + && process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL !== undefined + ) { + let dashboard_url = process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL + if ( + process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME !== undefined + && process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME === 'true' + ) { + dashboard_url += data.labels.hostname + } + parts.push(' | Dashboard') + } return parts.join(' ') },