Richer alerts; improved docs; easier testing

.dockerignore

@@ -1,6 +1,5 @@
-node_modules
-npm-debug.log
 .env
-.idea
 .git
-tests
+.idea
+node_modules
+npm-debug.log

.env.default

@@ -1,5 +1,14 @@
 APP_PORT=3000
 APP_ALERTMANAGER_SECRET=<secret key for the webhook events>
+
+# Optional URLs used for alerts with links back to your alertman instance and dashboards
+APP_ALERTMANAGER_URL=https://alertman.domain/#/alerts
+APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL=https://grafana.domain/dashboard/path?orgId=2&var-job=nodes&var-node=All&var-hostname=
+APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME=true
+
+# DEBUG | VERBOSE | INFO | WARNING | ERROR
+LOG_LEVEL=VERBOSE
+
 MATRIX_HOMESERVER_URL=https://homeserver.tld
 # The rooms to send alerts to, separated by a |
 # Each entry contains the receiver name (from alertmanager) and the

Dockerfile

@@ -8,7 +8,7 @@ COPY package*.json ./
 
 RUN npm install --only=production
 
-COPY . .
+COPY src /app/src
 
 EXPOSE 3000
 

Makefile

@@ -0,0 +1,11 @@
+test:
+	docker run -it --rm \
+		--workdir /app \
+		-v $(PWD):/app \
+		-e LOG_LEVEL=INFO \
+		--userns host \
+		node:18-alpine \
+		sh -c "npm install && npm run test"
+
+build:
+	docker build -t matrix-alertmanager .

README.md

@@ -19,23 +19,29 @@ Main features:
 
 ### Configuration
 
-Whether running manually or via the Docker image, the configuration is set 
+Whether running manually or via the Docker image, the configuration is set
 via environment variables. When running manually, copy `.env.default`
-into `.env`, set the values and they will be loaded automatically. 
-When using the Docker image, set the environment variables when running 
+into `.env`, set the values and they will be loaded automatically.
+When using the Docker image, set the environment variables when running
 the container.
 
 ### Docker
 
 The [Docker image](https://cloud.docker.com/repository/docker/jaywink/matrix-alertmanager) `jaywink/matrix-alertmanager:latest` is the easiest way to get the service running. Ensure you set the required environment variables listed in `.env.default` in this repository.
 
+#### Building locally
+
+You can build the container locally:
+
+    make build
+
 ### Alertmanager
 
 You will need to configure a webhook receiver in Alertmanager. It should looks something like this:
 
 ```yaml
 receivers:
-- name: 'myreceiver'
+- name: 'receiver1'
   webhook_configs:
   - url: 'https://my-matrix-alertmanager.tld/alerts?secret=veryverysecretkeyhere'
 ```
@@ -46,7 +52,7 @@ Alternatively, put the secret in a separate file and use basic auth with usernam
 
 ```yaml
 receivers:
-- name: 'myreceiver'
+- name: 'receiver2'
   webhook_configs:
   - url: 'https://my-matrix-alertmanager.tld/alerts'
     http_config:
@@ -55,6 +61,15 @@ receivers:
         password_file: /path/to/password.secret
 ```
 
+Note that the receiver `name` must match a configured recevier in the
+`MATRIX_ROOMS` env variable.
+
+Optional convenience links can be configured back to your alertmanager
+instance and/or a default dashboard.
+
+If an alert is received with a `url` in the `annotations` that URL will be used
+rather than the default dashboard URL.
+
 ### Prometheus rules
 
 Add some styling to your prometheus rules
@@ -64,6 +79,7 @@ rules:
 - alert: High Memory Usage of Container
   annotations:
     description: Container named <strong>{{\$labels.container_name}}</strong> in <strong>{{\$labels.pod_name}}</strong> in <strong>{{\$labels.namespace}}</strong> is using more than 75% of Memory Limit
+    url: https://grafana.domain/path/do/dashboard?orgId=12&var-host={\$labels.container_name}
   expr: |
     ((( sum(container_memory_usage_bytes{image!=\"\",container_name!=\"POD\", namespace!=\"kube-system\"}) by (namespace,container_name,pod_name, instance)  / sum(container_spec_memory_limit_bytes{image!=\"\",container_name!=\"POD\",namespace!=\"kube-system\"}) by (namespace,container_name,pod_name, instance) ) * 100 ) < +Inf ) > 75
   for: 5m
@@ -73,6 +89,16 @@ rules:
 
 NOTE! Currently, the bot cannot talk HTTPS, so you need to have a reverse proxy in place to terminate SSL, or use unsecure unencrypted connections.
 
+## Testing
+
+The tests can either be run locally with:
+
+    npm test
+
+..or from within a docker container:
+
+    make test
+
 ## TODO
 
 * Registering an account instead of having to use an existing account

src/app.js

@@ -1,6 +1,7 @@
 const express = require('express')
 const client = require('./client')
 const routes = require('./routes')
+const log = require('./log')
 
 // Config
 require('dotenv').config()
@@ -13,14 +14,11 @@ app.get('/', routes.getRoot)
 app.post('/alerts', routes.postAlerts)
 // Initialize Matrix client
 client.init().then(() => {
-    // eslint-disable-next-line no-console
-    console.log('matrix-alertmanager initialized and ready')
+    log.info('matrix-alertmanager initialized and ready')
     app.listen(process.env.APP_PORT, () => {})
 }).catch(e => {
-    // eslint-disable-next-line no-console
-    console.error('initialization failed')
-    // eslint-disable-next-line no-console
-    console.error(e)
+    log.error('initialization failed')
+    log.error(e)
 })
 
 module.exports = app

src/client.js

@@ -1,5 +1,6 @@
 const matrix = require('matrix-js-sdk')
 const striptags = require('striptags')
+const log = require('./log')
 
 let joinedRoomsCache = []
 
@@ -12,7 +13,7 @@ const client = {
                     joinedRoomsCache.push(room.roomId)
                 }
             } catch (ex) {
-                console.warn(`Could not join room ${roomId} - ${ex}`)
+                log.warn(`Could not join room ${roomId} - ${ex}`)
             }
         }
     },

src/log.js

@@ -0,0 +1,30 @@
+// DEBUG | VERBOSE | INFO | WARNING | ERROR
+const logLevel = process.env.LOG_LEVEL || 'INFO'
+
+const log = {
+    error: message => {
+        console.error(message)
+    },
+    warn: message => {
+        if ( ['DEBUG', 'VERBOSE', 'INFO', 'WARNING'].indexOf(logLevel) > -1 ) {
+            console.warning(message)
+        }
+    },
+    info: message => {
+        if ( ['DEBUG', 'VERBOSE', 'INFO'].indexOf(logLevel) > -1 ) {
+            console.info(message)
+        }
+    },
+    verbose: message => {
+        if ( ['DEBUG', 'VERBOSE'].indexOf(logLevel) > -1 ) {
+            console.log(message)
+        }
+    },
+    debug: message => {
+        if ( ['DEBUG'].indexOf(logLevel) > -1 ) {
+            console.debug(message)
+        }
+    }
+}
+
+module.exports = log

src/routes.js

@@ -1,5 +1,6 @@
 const client = require('./client')
 const utils = require('./utils')
+const log = require('./log')
 
 const routes = {
     getRoot: (req, res) => {
@@ -11,16 +12,23 @@ const routes = {
             res.status(403).end()
             return
         }
+        if ( process.env.LOG_LEVEL === 'DEBUG' ) {
+            log.verbose(JSON.stringify(req.body))
+        }
         const alerts = utils.parseAlerts(req.body)
 
         if (!alerts) {
-            res.json({'result': 'no alerts found in payload'})
+            const msg = 'no alerts found in payload'
+            log.info(msg)
+            res.json({'result': msg})
             return
         }
 
         const roomId = utils.getRoomForReceiver(req.body.receiver)
         if (!roomId) {
-            res.json({'result': 'no rooms configured for this receiver'})
+            const msg = 'no rooms configured for this receiver'
+            log.info(msg)
+            res.json({'result': msg})
             return
         }
 
@@ -29,10 +37,9 @@ const routes = {
             await Promise.all(promises)
             res.json({'result': 'ok'})
         } catch (e) {
-            // eslint-disable-next-line no-console
-            console.error(e)
+            log.error(e)
             res.status(500)
-            res.json({'result': 'error'})
+            res.json({'result': 'error sending alert'})
         }
     },
 }

src/utils.js

@@ -28,7 +28,6 @@ const utils = {
         Format a single alert into a message string.
          */
         let parts = []
-        //console.log(data)
 
         if (data.status === 'firing') {
             if (process.env.MENTION_ROOM === "1") {
@@ -44,7 +43,11 @@ const utils = {
                     return '#dc3545'; // red
                 }
               })(data.labels.severity);
-            parts.push('<strong><font color=\"' + color + '\">FIRING:</font></strong>')
+            if ( data.labels.severity !== undefined ) {
+              parts.push('<strong><font color=\"' + color + '\">' + data.labels.severity.toUpperCase() + ':</font></strong>')
+            } else {
+              parts.push('<strong><font color=\"' + color + '\">FIRING:</font></strong>')
+            }
         } else if (data.status === 'resolved') {
             parts.push('<strong><font color=\"#33cc33\">RESOLVED:</font></strong>')
         } else {
@@ -53,25 +56,58 @@ const utils = {
 
         // name and location of occurrence
         if (data.labels.alertname !== undefined) {
-            parts.push('<i>', data.labels.alertname, '</i>')
-            if (data.labels.host !== undefined || data.labels.instance !== undefined) {
-                parts.push(' at ')
+            if ( process.env.APP_ALERTMANAGER_URL !== undefined ) {
+                parts.push('<a href=\"'+process.env.APP_ALERTMANAGER_URL+'\"><i>', data.labels.alertname, '</i></a>')
+            } else {
+                parts.push('<i>', data.labels.alertname, '</i>')
+            }
+            if (data.labels.host !== undefined || data.labels.hostname !== undefined || data.labels.instance !== undefined) {
+                parts.push(' on ')
             }
         }
-        if (data.labels.host !== undefined) {
-            parts.push(data.labels.host)
-        } else if (data.labels.instance !== undefined) {
-            parts.push(data.labels.instance)
+
+        let host = data.labels.host || data.labels.hostname || data.labels.instance;
+
+        if (data.labels.parent !== undefined && data.labels.parent != host) {
+            parts.push(data.labels.parent + '/' + host)
+        } else {
+            parts.push(host)
         }
 
+        parts.push(' (')
+        let labels = new Set()
+        for (const [label, value] of Object.entries(data.labels)) {
+            if (['alertname', 'host', 'hostname', 'instance', 'parent', 'severity'].indexOf(label) == -1) {
+                labels.add(value)
+            }
+        }
+        parts.push(...labels)
+        parts.push(')')
+
         // additional descriptive content
         if (data.annotations.message !== undefined) {
             parts.push('<br>', data.annotations.message.replace("\n", "<br>"))
         }
         if (data.annotations.description !== undefined) {
             parts.push('<br>', data.annotations.description)
         }
-        parts.push('<br><a href="', data.generatorURL,'">Alert link</a>')
+        parts.push('<br><a href="'+ data.generatorURL +'">Alert link</a>')
+
+        if ( data.annotations.url !== undefined) {
+            parts.push(' | <a href="' + data.annotations.url + '">Other</a>')
+        } else if (
+                data.labels.hostname !== undefined
+                && process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL !== undefined
+                ) {
+            let dashboard_url = process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL
+            if (
+                    process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME !== undefined
+                    && process.env.APP_ALERTMANAGER_DEFAULT_DASHBOARD_URL_APPEND_HOSTNAME === 'true'
+                ) {
+                dashboard_url += data.labels.hostname
+            }
+            parts.push(' | <a href="' + dashboard_url + '">Dashboard</a>')
+        }
 
         return parts.join(' ')
     },