From c8b2d94afee6c11c56cf56397d6988f84587c879 Mon Sep 17 00:00:00 2001 From: GilbN <24592972+GilbN@users.noreply.github.com> Date: Wed, 11 Sep 2024 20:47:53 +0200 Subject: [PATCH] Add new geoip2influx files from upstream (#955) * Add new geoip2influx files from upstream --- Dockerfile | 9 ++- README.md | 67 +++++++++++++++---- .../run | 35 ++++++++-- .../s6-rc.d/svc-mod-swag-geoip2influx/run | 2 +- 4 files changed, 92 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5bd925b..ba54f9ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,14 @@ FROM scratch as buildstage -ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx.py /root-layer/geoip2influx/geoip2influx.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/run.py /root-layer/geoip2influx/run.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/__init__.py /root-layer/geoip2influx/geoip2influx/__init__.py +ADD https://raw.githubusercontent.com/GilbN/geoip2influx/master/geoip2influx/constants.py /root-layer/geoip2influx/geoip2influx/constants.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx_base.py /root-layer/geoip2influx/geoip2influx/influx_base.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx.py /root-layer/geoip2influx/geoip2influx/influx.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influxv2.py /root-layer/geoip2influx/geoip2influx/influxv2.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logger.py /root-layer/geoip2influx/geoip2influx/logger.py +ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logparser.py /root-layer/geoip2influx/geoip2influx/logparser.py COPY root/ /root-layer/ # runtime stage diff --git a/README.md b/README.md index f479531a..7d79b59a 100644 --- a/README.md +++ b/README.md @@ -17,31 +17,74 @@ Add the ones that differ on your system. | Environment Variable | Example Value | Description | | -------------------- | ------------- | ----------- | -| NGINX_LOG_PATH | /config/log/nginx/access.log | Optional, defaults to the example. | +| NGINX_LOG_PATH | /config/log/nginx/access.log | Container path for Nginx logfile , defaults to the example. | +| GEO_MEASUREMENT | geoip2influx | InfluxDB measurement name for geohashes. Optional, defaults to the example. | +| LOG_MEASUREMENT | nginx_access_logs | InfluxDB measurement name for nginx logs. Optional, defaults to the example. | +| SEND_NGINX_LOGS | true | Set to `false` to disable nginx logs. Optional, defaults to `true`. | +| GEOIP2INFLUX_LOG_LEVEL | info | Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. | +| GEOIP2INFLUX_LOG_PATH | /config/log/geoip2influx/geoip2influx.log | Optional. Defaults to example. | +| GEOIP_DB_PATH | /config/geoip2db/GeoLite2-City.mmdb | Optional. Defaults to example. | +| MAXMINDDB_LICENSE_KEY | xxxxxxx | Add your Maxmind licence key | +| MAXMINDDB_USER_ID | xxxxxxx| Add your Maxmind account id | + +**InfluxDB v1.8.x values** + +| Environment Variable | Example Value | Description | +| -------------------- | ------------- | ----------- | | INFLUX_HOST | localhost | Host running InfluxDB. | | INFLUX_HOST_PORT | 8086 | Optional, defaults to 8086. | | INFLUX_DATABASE | geoip2influx | Optional, defaults to geoip2influx. | | INFLUX_USER | root | Optional, defaults to root. | | INFLUX_PASS | root | Optional, defaults to root. | -| GEO_MEASUREMENT | geoip2influx | Optional, InfluxDB measurement name for geohashes. Defaults to the example. | -| LOG_MEASUREMENT | nginx_access_logs | Optional, InfluxDB measurement name for nginx logs. Defaults to the example. | -| SEND_NGINX_LOGS | true | Optional, set to `false` to disable nginx logs. Defaults to `true`. | -| GEOIP2INFLUX_LOG_LEVEL | info | Optional. Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. | -| INFLUX_RETENTION | 7d | Optional. Sets the retention for the database. Defaults to example.| -| INFLUX_SHARD | 1d | Optional. Set the shard for the database. Defaults to example. | +| INFLUX_RETENTION | 7d | Sets the retention for the database. Optional, defaults to example.| +| INFLUX_SHARD | 1d | Set the shard for the database. Optional, defaults to example. | -*** +**InfluxDB v2.x values** + +| Environment Variable | Example Value | Description | +| -------------------- | ------------- | ----------- | +| USE_INFLUXDB_V2 | true | Required if using InfluxDB2. Defaults to false | +| INFLUXDB_V2_TOKEN | secret-token | Required | +| INFLUXDB_V2_URL | http://localhost:8086 | Optional, defaults to http://localhost:8086 | +| INFLUXDB_V2_ORG | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. | +| INFLUXDB_V2_BUCKET | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. | +| INFLUXDB_V2_RETENTION | 604800 | Optional, defaults to 604800. 7 days in seconds | +| INFLUXDB_V2_DEBUG | false | Optional, defaults to false. Enables the debug mode for the influxdb-client package. | +| INFLUXDB_V2_BATCHING | true | Optional, defaults to false. Enables batch writing of data. | +| INFLUXDB_V2_BATCH_SIZE | 100 | Optional, defaults to 10. | +| INFLUXDB_V2_FLUSH_INTERVAL | 30000 | Optional, defaults to 15000. How often in milliseconds to write a batch | + +### INFLUXDB_V2_TOKEN + +If the organization or bucket does not exist, it will try and create them with the token. -### InfluxDB +> [!NOTE] +> The minimim level of rights needed is write access to the bucket. -#### InfluxDB v2.x is not supported. Use v1.8.x. +### MaxMind Geolite2 -The InfluxDB database will be created automatically with the name you choose. +Get your licence key here: https://www.maxmind.com/en/geolite2/signup + +## InfluxDB + +### InfluxDB v2.x and v1.8x is supported. + +#### Note: The Grafana dashboard currently only supports InfluxDB v1.8.x + +The InfluxDB database/bucket and retention rules will be created automatically with the name you choose. + +``` +-e INFLUX_DATABASE=geoip2influx or -e INFLUXDB_V2_BUCKET=geoip2influx +``` *** ## Grafana dashboard: -### [Grafana Dashboard Link](https://grafana.com/grafana/dashboards/12268/) + +Use [https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json](https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json) + +> [!NOTE] +> Dashboard currently only supports InfluxDB 1.8.x. *** diff --git a/root/etc/s6-overlay/s6-rc.d/init-mod-swag-geoip2influx-add-package/run b/root/etc/s6-overlay/s6-rc.d/init-mod-swag-geoip2influx-add-package/run index 484bb0c6..d99e1978 100755 --- a/root/etc/s6-overlay/s6-rc.d/init-mod-swag-geoip2influx-add-package/run +++ b/root/etc/s6-overlay/s6-rc.d/init-mod-swag-geoip2influx-add-package/run @@ -1,14 +1,19 @@ #!/usr/bin/with-contenv bash +echo -e "[init-geoip2influx-setup] Starting" + + if ! pip list 2>&1 | grep -q "influxdb"; then echo '-------------------------------------------------------------' echo '| Mod by Gilbn |' echo '| Adding required modules for Geoip2Influx to install list |' echo '-------------------------------------------------------------' echo "\ - geoip2==3.0.0 \ + geoip2==4.8.0 \ geohash2==1.1 \ - influxdb==5.3.0 \ + influxdb==5.3.2 \ + python-dotenv \ + influxdb-client==1.44.0 \ IPy==1.01" >> /mod-pip-packages-to-install.list fi @@ -25,19 +30,35 @@ lsiown -R abc:abc \ /geoip2influx \ /config/log/geoip2influx -chmod +x /geoip2influx/geoip2influx.py +chmod +x /geoip2influx/run.py # Display variables for troubleshooting -echo -e "Variables set:\\n\ -NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\ +echo -e "[init-geoip2influx-setup] Variables set:\\n\\n INFLUX_HOST=${INFLUX_HOST}\\n\ INFLUX_HOST_PORT=${INFLUX_HOST_PORT}\\n\ INFLUX_DATABASE=${INFLUX_DATABASE}\\n\ INFLUX_USER=${INFLUX_USER}\\n\ INFLUX_PASS=${INFLUX_PASS}\\n\ INFLUX_RETENTION=${INFLUX_RETENTION}\\n\ -INFLUX_SHARD=${INFLUX_SHARD}\\n\ +INFLUX_SHARD=${INFLUX_SHARD}\\n\\n +INFLUXDB_V2_TOKEN=${INFLUXDB_V2_TOKEN}\\n\ +INFLUXDB_V2_URL=${INFLUXDB_V2_URL}\\n\ +INFLUXDB_V2_ORG=${INFLUXDB_V2_ORG}\\n\ +INFLUXDB_V2_BUCKET=${INFLUXDB_V2_BUCKET}\\n\ +INFLUXDB_V2_RETENTION=${INFLUXDB_V2_RETENTION}\\n\ +INFLUXDB_V2_DEBUG=${INFLUXDB_V2_DEBUG}\\n\ +INFLUXDB_V2_BATCHING=${INFLUXDB_V2_BATCHING}\\n\ +INFLUXDB_V2_BATCH_SIZE=${INFLUXDB_V2_BATCH_SIZE}\\n\ +INFLUXDB_V2_FLUSH_INTERVAL=${INFLUXDB_V2_FLUSH_INTERVAL}\\n\\n GEO_MEASUREMENT=${GEO_MEASUREMENT}\\n\ LOG_MEASUREMENT=${LOG_MEASUREMENT}\\n\ +NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\ SEND_NGINX_LOGS=${SEND_NGINX_LOGS}\\n\ -GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n" +GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n\ +GEOIP2INFLUX_LOG_PATH=${GEOIP2INFLUX_LOG_PATH}\\n\ +GEOIP_DB_PATH=${GEOIP_DB_PATH}\\n\ +USE_INFLUXDB_V2=${USE_INFLUXDB_V2}\\n\ +MAXMINDDB_USER_ID=${MAXMINDDB_USER_ID}\\n\ +MAXMINDDB_LICENSE_KEY=${MAXMINDDB_LICENSE_KEY}\\n" + +echo -e "[init-geoip2influx-setup] Finished" diff --git a/root/etc/s6-overlay/s6-rc.d/svc-mod-swag-geoip2influx/run b/root/etc/s6-overlay/s6-rc.d/svc-mod-swag-geoip2influx/run index 9dc871a5..07a4979b 100755 --- a/root/etc/s6-overlay/s6-rc.d/svc-mod-swag-geoip2influx/run +++ b/root/etc/s6-overlay/s6-rc.d/svc-mod-swag-geoip2influx/run @@ -1,4 +1,4 @@ #!/usr/bin/with-contenv bash exec \ - python3 /geoip2influx/geoip2influx.py \ No newline at end of file + python3 /geoip2influx/run.py \ No newline at end of file