add SKIP_IP_RANGES/improve crowdsec docs/dep updates

Signed-off-by: Zoey <zoey@z0ey.de>

Dockerfile

@@ -31,8 +31,9 @@ RUN apk add --no-cache ca-certificates nodejs-current yarn && \
 
 
 FROM --platform="$BUILDPLATFORM" alpine:3.19.1 as crowdsec
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
 
-ARG CSNB_VER=v1.0.6-rc5
+ARG CSNB_VER=v1.0.7
 
 WORKDIR /src
 RUN apk add --no-cache ca-certificates git build-base && \
@@ -46,15 +47,17 @@ RUN apk add --no-cache ca-certificates git build-base && \
     sed -i "s|ENABLED=.*|ENABLED=false|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
     sed -i "s|API_URL=.*|API_URL=http://127.0.0.1:8080|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
     sed -i "s|BAN_TEMPLATE_PATH=.*|BAN_TEMPLATE_PATH=/data/etc/crowdsec/ban.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
-    sed -i "s|CAPTCHA_TEMPLATE_PATH=.*|CAPTCHA_TEMPLATE_PATH=/data/etc/crowdsec/captcha.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf
+    sed -i "s|CAPTCHA_TEMPLATE_PATH=.*|CAPTCHA_TEMPLATE_PATH=/data/etc/crowdsec/captcha.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
+    echo "APPSEC_URL=http://127.0.0.1:7422" | tee -a /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
+    echo "APPSEC_FAILURE_ACTION=deny" | tee -a /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf
 
-FROM zoeyvid/nginx-quic:252
+FROM zoeyvid/nginx-quic:256
 SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
 
 ARG CRS_VER=v4.0/dev
 
 COPY rootfs /
-COPY --from=zoeyvid/certbot-docker:21  /usr/local          /usr/local
+COPY --from=zoeyvid/certbot-docker:24  /usr/local          /usr/local
 COPY --from=zoeyvid/curl-quic:370      /usr/local/bin/curl /usr/local/bin/curl
 
 RUN apk add --no-cache ca-certificates tzdata tini \
@@ -116,6 +119,7 @@ ENV PUID=0 \
     NGINX_LOG_NOT_FOUND=false \
     CLEAN=true \
     FULLCLEAN=false \
+    SKIP_IP_RANGES=false \
     LOGROTATE=false \
     LOGROTATIONS=3 \
     GOA=false \

README.md

@@ -18,9 +18,9 @@ running at home or otherwise, including free TLS, without having to know too muc
 **Note: If you don't use network mode host, which I don't recommend, don't forget to enable IPv6 in Docker, see [here](https://github.com/nextcloud/all-in-one/blob/main/docker-ipv6-support.md), you only need to edit the daemon.json and restart docker, if you use the bridge network, otherwise please enable IPv6 in your custom docker network!** <br>
 **Note: Don't forget to open Port 80 (tcp) and 443 (tcp AND udp, http3/quic needs udp) in your firewall (because of network mode host, you also need to open this ports in ufw, if you use ufw).** <br>
 **Note: ModSecurity overblocking (403 Error)? Please see `/opt/npm/etc/modsecurity`, if you also use CRS please see [here](https://coreruleset.org/docs/concepts/false_positives_tuning).** <br>
-**Note: Internal/LAN Instance? Please disable `must-staple` in `/opt/npm/tls/certbot/config.ini`.** <br>
+**Note: Internal/LAN Instance? Please disable `must-staple` in `/opt/npm/tls/certbot/config.ini` before creating your certificates.** <br>
 **Note: Other Databases like MariaDB may work, but are unsupported.** <br>
-**Note: access.log, logrotate and goaccess are NOT enabled by default bceuase of GDPR.** <br>
+**Note: access.log/stream.log, logrotate and goaccess are NOT enabled by default bceuase of GDPR, you can enable them in the compose.yaml.** <br>
 
 
 ## Project Goal
@@ -104,12 +104,39 @@ so that the barrier for entry here is low.
 
 # Crowdsec
 1. Install crowdsec using this compose file: https://github.com/ZoeyVid/NPMplus/blob/develop/compose.crowdsec.yaml
-2. make sure to use `network_mode: host` in your compose file
-3. run `docker exec crowdsec cscli bouncers add npmplus -o raw` and save the output
-4. open `/opt/npm/etc/crowdsec/crowdsec.conf`
-5. set `ENABLED` to `true`
-6. use the output of step 3 as `API_KEY`
-7. make sure `API_URL` is set to `http://127.0.0.1:8080`
+2. open `/opt/crowdsec/conf/acquis.d/appsec.yaml` and fill it with:
+```yaml
+listen_addr: 127.0.0.1:7422
+appsec_config: crowdsecurity/virtual-patching
+name: myAppSecComponent
+source: appsec
+labels:
+  type: appsec
+```
+3. open `/opt/crowdsec/conf/acquis.d/npmplus.yaml` and fill it with:
+```yaml
+filenames:
+  - /opt/npm/nginx/access.log
+labels:
+  type: npmplus
+---
+source: docker
+container_name:
+ - npmplus
+labels:
+  type: npmplus
+---
+source: docker
+container_name:
+ - npmplus
+labels:
+  type: modsecurity
+```
+4. make sure to use `network_mode: host` in your compose file
+5. run `docker exec crowdsec cscli bouncers add npmplus -o raw` and save the output
+6. open `/opt/npm/etc/crowdsec/crowdsec.conf`
+7. set `ENABLED` to `true`
+8. use the output of step 5 as `API_KEY`
 9. save the file
 10. restart the npm
 

backend/internal/ip_ranges.js

@@ -20,8 +20,10 @@ const internalIpRanges = {
 	iteration_count:     0,
 
 	initTimer: () => {
-		logger.info('IP Ranges Renewal Timer initialized');
-		internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
+		if (process.env.SKIP_IP_RANGES === false) {
+			logger.info('IP Ranges Renewal Timer initialized');
+			internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
+		}
 	},
 
 	fetchUrl: (url) => {
@@ -47,7 +49,7 @@ const internalIpRanges = {
 	 * Triggered at startup and then later by a timer, this will fetch the ip ranges from services and apply them to nginx.
 	 */
 	fetch: () => {
-		if (!internalIpRanges.interval_processing) {
+		if (!internalIpRanges.interval_processing && process.env.SKIP_IP_RANGES === false) {
 			internalIpRanges.interval_processing = true;
 			logger.info('Fetching IP Ranges from online services...');
 

compose.crowdsec.yaml

@@ -5,9 +5,13 @@ services:
     restart: always
     network_mode: bridge
     ports:
+      - "127.0.0.1:7422:7422"
       - "127.0.0.1:8080:8080"
     environment:
       - "TZ=Europe/Berlin"
+      - "COLLECTIONS=ZoeyVid/npmplus crowdsecurity/appsec-virtual-patching"
     volumes:
       - "/opt/crowdsec/conf:/etc/crowdsec"
       - "/opt/crowdsec/data:/var/lib/crowdsec/data"
+      - "/opt/npm/nginx:/opt/npm/nginx:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"

compose.yaml

@@ -33,7 +33,8 @@ services:
 #      - "NGINX_LOG_NOT_FOUND=true" # Allow logging of 404 errors, default false
 #      - "CLEAN=false" # Clean folders, default true
 #      - "FULLCLEAN=true" # Clean unused config folders, default false
-#      - "LOGROTATE=true" # Enables writing http access logs to /opt/npm/nginx/access.log and daily logrotation, default false
+#      - "SKIP_IP_RANGES=true" # Skip feteching/whitelisting ip ranges from aws and cloudflare, default false
+#      - "LOGROTATE=true" # Enables writing http access logs to /opt/npm/nginx/access.log, stream access logs to /opt/npm/nginx/stream.log and enables daily logrotation, default false
 #      - "LOGROTATIONS=7" # Set how often the access.log should be rotated until it is deleted, default 3
 #      - "GOA=true" # Enables goaccess, overrides LOGROTATE, default false --- if you download the GeoLite2-Country.mmdb, GeoLite2-City.mmdb AND GeoLite2-ASN.mmdb file from MaxMind and place them in /opt/npm/etc/goaccess/geoip it will automatically enable GeoIP in goaccess after restarting NPMplus (no need to change GOACLA below), you may also use the compose.geoip.yaml
 #      - "GOACLA=--agent-list --real-os --double-decode --anonymize-ip --anonymize-level=2 --keep-last=7 --with-output-resolver --no-query-string" # Arguments that should be passed to goaccess, default: https://github.com/ZoeyVid/NPMplus/blob/develop/rootfs/usr/local/bin/launch.sh#L50 and: --agent-list --real-os --double-decode --anonymize-ip --anonymize-level=1 --keep-last=30 --with-output-resolver --no-query-string

frontend/package.json

@@ -8,7 +8,7 @@
     "babel-core": "6.26.3",
     "babel-loader": "8.3.0",
     "babel-preset-env": "1.7.0",
-    "backbone": "1.5.0",
+    "backbone": "1.6.0",
     "backbone.marionette": "4.1.3",
     "copy-webpack-plugin": "5.1.2",
     "css-loader": "5.2.7",

rootfs/etc/logrotate

@@ -1,4 +1,4 @@
-/data/nginx/access.log {
+/data/nginx/*.log {
         daily
         rotate 3
         missingok

rootfs/usr/local/bin/start.sh

@@ -146,6 +146,11 @@ if ! echo "$FULLCLEAN" | grep -q "^true$\|^false$"; then
     sleep inf
 fi
 
+if ! echo "$SKIP_IP_RANGES" | grep -q "^true$\|^false$"; then
+    echo "SKIP_IP_RANGES needs to be true or false."
+    sleep inf
+fi
+
 if ! echo "$LOGROTATE" | grep -q "^true$\|^false$"; then
     echo "LOGROTATE needs to be true or false."
     sleep inf
@@ -356,7 +361,10 @@ if [ "$LOGROTATE" = "true" ]; then
     sed -i "s|rotate [0-9]\+|rotate $LOGROTATIONS|g" /etc/logrotate
 elif [ "$FULLCLEAN" = "true" ]; then
     rm -vrf /data/etc/logrotate.status \
-            /data/nginx/access.log.*
+            /data/nginx/access.log \
+            /data/nginx/access.log.* \
+            /data/nginx/stream.log \
+            /data/nginx/stream.log.*
 fi
 
 mkdir -p /tmp/acme-challenge \
@@ -662,12 +670,6 @@ sed -i "s|#\?ssl_certificate_key .*|ssl_certificate_key $DEFAULT_KEY;|g" /usr/lo
 if [ -n "$DEFAULT_CHAIN" ]; then sed -i "s|#\?ssl_trusted_certificate .*|ssl_trusted_certificate $DEFAULT_CHAIN;|g" /usr/local/nginx/conf/conf.d/include/goaccess-no-server-name.conf; fi
 
 
-if [ "$DISABLE_IPV6" = "true" ]; then
-    sed -i "s|#\?resolver .*|resolver local=on valid=10s ipv6=off;|g" /usr/local/nginx/conf/nginx.conf
-else
-    sed -i "s|#\?resolver .*|resolver local=on valid=10s;|g" /usr/local/nginx/conf/nginx.conf
-fi
-
 sed -i "s|48693|$NIBEP|g" /app/index.js
 sed -i "s|48693|$NIBEP|g" /usr/local/nginx/conf/conf.d/npm.conf
 
@@ -732,9 +734,11 @@ else
 fi
 
 if [ "$LOGROTATE" = "true" ]; then
-    sed -i "s|access_log.*|access_log /data/nginx/access.log log;|g" /usr/local/nginx/conf/nginx.conf
+    sed -i "s|access_log off; # http|access_log /data/nginx/access.log log;|g" /usr/local/nginx/conf/nginx.conf
+    sed -i "s|access_log off; # stream|access_log /data/nginx/stream.log proxy;|g" /usr/local/nginx/conf/nginx.conf
 else
-    sed -i "s|access_log.*|access_log off;|g" /usr/local/nginx/conf/nginx.conf
+    sed -i "s|access_log /data/nginx/access.log log;|access_log off; # http|g" /usr/local/nginx/conf/nginx.conf
+    sed -i "s|access_log /data/nginx/stream.log proxy;|access_log off; # stream|g" /usr/local/nginx/conf/nginx.conf
 fi
 
 if [ ! -s /data/nginx/default.conf ]; then

rootfs/usr/local/nginx/conf/nginx.conf

@@ -16,7 +16,7 @@ events {
 
 http {
     log_format log '[$time_local] $host $remote_addr $request_time "$request" $status $body_bytes_sent $bytes_sent $http_referer $http_user_agent';
-    access_log off;
+    access_log off; # http
     log_not_found off;
 
     include mime.types;
@@ -49,7 +49,7 @@ http {
     quic_retry on;
     ssl_dyn_rec_enable on;
 
-    #resolver ;
+    resolver local=on valid=10s ipv6=off;
     fastcgi_index index.php;
     index index.php index.html;
 
@@ -139,6 +139,9 @@ http {
 }
 
 stream {
+    log_format proxy '$remote_addr [$time_local] $protocol $status $bytes_sent $bytes_received $session_time "$upstream_addr" "$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
+    access_log off; # stream
+
     # Custom
     include /data/nginx/custom/stream_top.conf;