Speed up restore, allow IPv6 chains to have different name

- When writing restore script from save, group 16/line for restore speed - Save now only includes the selected address family (-4 or -6) - Add raw format to save (e.g. to feed to BlockCountries for permanence) - Don't install rule if action is flush, list, remove, or save (NOP) - Optimize list, slightly - Yet another stab at detecting dirty working directory - Install .conf as .conf.new if one exists (instead of not installing)
tlhackque · Jan 18, 2023 · 9ebcaa1 · 9ebcaa1
1 parent 81d91e5
commit 9ebcaa1
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 62 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 .git*           export-ignore
+Makefile        export-subst ident
 ipblock         export-subst ident
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@
 *.sig
 ipblock-*
 *.tar *.tar.gz *.tar.xz *tar.lzop *.tar.lzma *.tar.Z *.tar.zst *.tar.bz *.tar.bz2
+.tagged
diff --git a/Makefile b/Makefile
@@ -1,5 +1,7 @@
 # Copyright (C) 2023 Timothe Litt litt at acm ddot org
 
+# $Id$
+
 # Install targets - can override on command line
 
 # Note that DESTDIR is supported for staging environments
@@ -74,6 +76,14 @@ ipblock$(man1ext) : README.md ipblock Makefile
 	$(SED) -e's,^`ipblock -h` for complete help$$,./ipblock -h,e' $< | \
 	    $(LOWDOWN) -s -t man --parse-codeindent -M "title=ipblock" -M "date=$$(date -r ipblock +%d-%b-%Y)" -Msection=8 -o $@ -
 
+.PHONY : viewreadme viewman
+
+viewreadme : README.md
+	@LANG=C $(LOWDOWN) -s -t term --parse-codeindent $< | less
+
+viewman : ipblock$(man1ext)
+	@LANG=C less $<
+
 # Make tarball kits - various compressions
 
 .PHONY : dist unsigned-dist signed-dist
@@ -112,7 +122,7 @@ install_dirs := $(DESTDIR)$(bindir) $(DESTDIR)$(man1dir) $(DESTDIR)$(confdir)
 install : ipblock ipblock$(man1ext) config/ipblock.conf installdirs
 	$(INSTALL_PROGRAM) ipblock $(DESTDIR)$(bindir)/ipblock
 	$(INSTALL_DATA) ipblock$(man1ext) $(DESTDIR)$(man1dir)/ipblock$(man1ext)
-	-if [ -f "$(confdir)/ipblock.conf" ]; then true ; else  $(INSTALL_DATA) config/ipblock.conf $(DESTDIR)$(confdir)/ipblock.conf; fi
+	-if [ -f "$(confdir)/ipblock.conf" ]; then $(INSTALL_DATA) config/ipblock.conf $(DESTDIR)$(confdir)/ipblock.conf.new ; else  $(INSTALL_DATA) config/ipblock.conf $(DESTDIR)$(confdir)/ipblock.conf; fi
 	@echo ""
 	@echo "Please read 'man 1 ipblock' before using the command'"
 
@@ -166,7 +176,7 @@ tag : .tagged
 
 .tagged : $(shell git ls-tree --full-tree --name-only -r HEAD) unsigned-dist
 	@if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*' >/dev/null 2>/dev/null || \
-	    [ -n "$$(git diff --stat)" ]; then \
+	    [ -n "$$(git diff --name-only)$$(git diff --name-only --staged)" ]; then \
 	    echo " *** Not tagging V$(kitversion) because working directory is dirty"; echo ""; false ;\
 	 elif [ "$(strip $(gittag))" == "V$(kitversion)" ]; then                 \
 	    echo " *** Not tagging because V$(kitversion) already exists";       \

diff --git a/README.md b/README.md
@@ -33,13 +33,16 @@ Options, including the desired chain, should be specified in the configuration f
 `ipblock -h` for complete help
 
 ## Installation
-Download the latest `Vn.m-Release` tarball using the `tar.gz` link at [GitHub](https://github.com/tlhackque/ipblock/releases).
+Download the latest `ipblock-n.m.o-Release` tarball and signature using
+the `tar.gz` or `tar.xz` and `.sig` links at
+[GitHub](https://github.com/tlhackque/ipblock/releases).
 
-- Do **not** select the `.zip` file, as it does not preserve file permissions.
-- Do **NOT** use the **Clone or download** link on the main `ipblock` page, as it provides a `.zip` file.
+- Do **NOT** use the **Clone or download** link on the main `ipblock` page.
+- Building or installing from source requires `lowdown`
 
-Unpack the `tar.gz`:  
-    tar -xzf ipblock&lt;n&gt;.&lt;m&gt;-Release.tar.gz
+Verify and npack the `tar.gz`:  
+    gpg --verify ipblock&lt;n&gt;.&lt;m&gt;-Release.tar.gz.sig && \\  
+    tar -xzf     ipblock&lt;n&gt;.&lt;m&gt;-Release.tar.gz
 
 This will create a subdirectory named ipblock-&lt;version&gt;.
 

diff --git a/config/ipblock.conf b/config/ipblock.conf
@@ -21,6 +21,10 @@
 # If you leave -C at the default, "INPUT", the ipblock rule will
 # supersede any protection that you hav established, so don't.
 #
+# The name of the IPv6 chain is defaulted to be the same as the IPv4 chain's.
+# If they differ, specify the IPv6 chain name with -c (lowercase).
+# Otherwise, -C will be used for both.
+#
 #  See ipblock -h for more information.
 
 #OPTIONS="-C mychain"

diff --git a/ipblock b/ipblock
@@ -8,15 +8,15 @@
 VERSION='$Id$'
 
 # shellcheck disable=SC2034 # used by Makefile
-RELEASE='1.6.0'
+RELEASE='1.7.0'
 
 SELF="$(basename "$0")"
 
 function displayVersion() {
     if [[ "$VERSION" =~ ^'$''Id: '[[:xdigit:]]{24}([[:xdigit:]]{4})([[:xdigit:]]{4})([[:xdigit:]]{4})([[:xdigit:]]{4})' $'$ ]]; then
         printf "$SELF version %s-%s-%s-%s\n%*s release %s\n" \
                "${BASH_REMATCH[-4]}" "${BASH_REMATCH[-3]}" "${BASH_REMATCH[-2]}" "${BASH_REMATCH[-1]}" "${#SELF}" "" "$RELEASE"
-        exit 0
+        return 0
     fi
     echo "$SELF version '$VERSION' - format error"
     exit 1
@@ -39,6 +39,7 @@ fi
 
 BlackList="BlackList"
 Chain="INPUT"
+Chain6=
 DateFormat="+%a %d-%b-%Y %T %Z"
 
 # NOTE: The help text below is automagically inserted into README.md
@@ -68,25 +69,33 @@ EOF
     -L        List currently blocked IP addresses and last seen time (default)
     -S file   Save current addresses as a script.  (use - for stdout)
     -a        With -S, add to existing output file
+    -r        With -S, save in raw format (just IP address list, e.g. for BlockCountries)
     -T tps    Use tps as the jiffys/sec (only if /boot/config-<kernel> is not available.
     -t        Estimate -T
-    -X        Disable ipblock and remove its rule.  Does not fluah list.
+    -X        Disable ipblock and remove its rule.  May not fluah list.
     -V        Display version
 
     -D fmt    Date format (strftime) (Default is '%a %d-%b-%Y %T %Z'), null for ctime
 
     -C:chain  Specify chain to hook (default is INPUT)
+    -c:chain  Specify chain for IPv6 hook (default is same name as IPv4)
     -N:table  Recent table name that maintains list. (Default is BlackList)
               IPv6 adds '6' to the specified name (Thus, IPv6 default is BlackList6)
 ```
 EOF
     cat <<EOF
 
+Most options should be placed in ${SELF}.conf, making use very simple.
+
+The most common usage is
+
+    $SELF address
+
 If neither -4 nor -6 is specified:
     If a numeric address is specified, the address family is used.
     Otherwise, the default is -4
 
-Options -L and -S ignore -4 & -6; they access all tables present.
+Option -L ignores -4 & -6; it accesses all tables present.
 
 Option -T is only required when the kernel configuration file is not mounted on /boot, as
 happens with some VPS providers.  The value is the jiffies (ticks) per second of
@@ -139,6 +148,7 @@ V=
 APPEND=
 TICKS=
 NOHOST=
+RAW=
 
 if [ -f "/sys/module/ipt_recent/parameters/ip_list_tot" ]; then
     MAXENT="$(cat /sys/module/ipt_recent/parameters/ip_list_tot)"
@@ -148,8 +158,7 @@ else
     MAXENT=100
 fi
 
-DD=
-while getopts "46AC:dD:hFLN:nRaS:tT:vXV-:" opt; do
+while getopts "46AC:c:dD:hFLN:nrRaS:tT:vXV-:" opt; do
     case $opt in
         4)
             IPV="4"
@@ -160,6 +169,9 @@ while getopts "46AC:dD:hFLN:nRaS:tT:vXV-:" opt; do
         C)
             Chain="$OPTARG"
             ;;
+        c)
+            Chain6="$OPTARG"
+            ;;
         A)
             ACTION="add"
             ;;
@@ -206,6 +218,9 @@ while getopts "46AC:dD:hFLN:nRaS:tT:vXV-:" opt; do
         a)
             APPEND="y"
             ;;
+        r)
+            RAW="y"
+            ;;
         v)
             V="y"
             ;;
@@ -214,10 +229,12 @@ while getopts "46AC:dD:hFLN:nRaS:tT:vXV-:" opt; do
             ;;
         V)
             displayVersion
+            exit 0
             ;;
         -)
             if [ "$OPTARG" == "version" ]; then
                 displayVersion
+                exit 0
             elif [ "$OPTARG" == 'help' ]; then
                  displayHelp
             else
@@ -254,17 +271,16 @@ if [ -z "$IPV" ]; then
         fi
     fi
 fi
-if [ "$IPV" == '4' ]; then
-    ProcMatch="/proc/net/ip_tables_matches"
-else
-    ProcMatch="/proc/net/ip6_tables_matches"
-fi
 
 IPT="iptables"
 BlackBase="$BlackList"
 if [ "$IPV" == "6" ]; then
     IPT="ip6tables"
     BlackList="${BlackList}6"
+    ProcMatch="/proc/net/ip6_tables_matches"
+    [ -n "$Chain6" ] && Chain="$Chain6"
+else
+    ProcMatch="/proc/net/ip_tables_matches"
 fi
 
 if ! command -v "$IPT" >/dev/null 2>&1 ; then
@@ -288,7 +304,10 @@ fi
 
 # shellcheck disable=SC2086 # RULE needs word splitting
 if ! $IPT -C $RULE >/dev/null 2>&1 ; then
-    if $IPT -I $RULE ; then
+    if [[ "$ACTION" =~ ^(flush|list|remove|save)$ ]]; then
+        [ -n "$V" ]     && echo "Not installed for IPv$IPV"
+        exit 0
+    elif $IPT -I $RULE ; then
         [ -n "$V" ]     && echo "Installed IPv$IPV input rule"
         [ -n "$DEBUG" ] && echo "  $IPT -I $RULE"
     else
@@ -351,61 +370,90 @@ fi
 if [ "$ACTION" == "save" ]; then
     export LC_ALL="C"
 
-    function save () {
-        cat <<EOF
+    function savelines() {
+        local v="$1" nohdr="$2" n=0
+
+        if [ -n "$RAW" ]; then
+            while IFS= read -r IP; do
+                echo "$IP"
+                ((++n))
+            done
+            [ -z "$nohdr" ] && echo "# Count = $n"
+            return 0;
+        fi
+
+        while IFS= read -r IP; do
+            if [[ $((n++ % 16)) == 0 ]]; then
+                printf "\n%s %s %s" "$PROG" "$v" "$IP"
+            else
+                printf " %s" "$IP"
+            fi
+        done
+        [[ $((n % 16)) != 0 ]] && printf "\n"
+        [ -z "$nohdr" ] && echo "# Count = $n"
+        return 0
+    }
+    function saveHdr () {
+        if [ -z "$RAW" ]; then
+            cat <<EOF
 #!/bin/bash
 
-# Auto-generated by $SELF V$VERSION
+EOF
+        fi
+        cat <<EOF
+# Auto-generated by $SELF $(displayVersion | head -n1) on $(date '+%d-%b-%Y %T')
 
 EOF
+    }
+    function save () {
+        local nohdr="$1"
+        [ -z "$nohdr" ] && saveHdr
         PROG="$(readlink -en "$0")"
-        if [ -f "$ProcRecent/$BlackBase" ]; then
+        if [ "$IPV" == "4" ] && [ -f "$ProcRecent/$BlackBase" ]; then
             # shellcheck disable=SC2002 # Redirecting input doesn't work here
             ( cat "$ProcRecent/$BlackBase" | while IFS= read -r LINE; do
                     sed -e's/^src=\([^ ]*\) .*$/\1/' <<<"$LINE"
                 done ) |
-            sort -u -t . -k1,1n -k2,2n -k3,3n -k4,4n |
-            while IFS= read -r SRT; do
-                echo "$PROG -4 $SRT"
-            done
+            sort -u -t . -k1,1n -k2,2n -k3,3n -k4,4n | savelines -4 "$nohdr"
         fi
 
-        if [ -f "$ProcRecent/${BlackBase}6" ]; then
+        if [ "$IPV" == "6" ] && [ -f "$ProcRecent/${BlackBase}6" ]; then
             # shellcheck disable=SC2002 # Redirecting input doesn't work here
             ( cat "$ProcRecent/${BlackBase}6" | while IFS= read -r LINE; do
                     sed -e's/^src=\([^ ]*\) .*$/\1/' <<<"$LINE"
                 done ) |
-            sort -u -t : -f -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 -k6,6 -k7,7 -k8,8 |
-            while IFS= read -r SRT; do
-                echo "$PROG -6 $SRT"
-            done
+            sort -u -t : -f -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 -k6,6 -k7,7 -k8,8 | savelines -6 "$nohdr"
         fi
-        echo "# EOF"
+        [ -z "$nohdr" ] && echo "# EOF"
     }
-    if [ "$SCRIPT" = '-' ]; then
-        save
+    if [ "${SCRIPT}" = '-' ]; then
+        save | grep -vP '^# Count = ([0-9]+).*$'
     else
-        if [ -n "$APPEND" ] && [ -f "$SCRIPT" ]; then
-            rm -f "$SCRIPT".tmp
-            mv "$SCRIPT" "$SCRIPT".tmp
-            cat >"$SCRIPT".tmph <<EOF
-#!/bin/bash
-
-# Auto-generated by $SELF V$VERSION
-
-EOF
-            save >>"$SCRIPT".tmp
-            grep -vh '^#' "$SCRIPT".tmp | sort -u | cat "$SCRIPT".tmph - >"$SCRIPT"
-            echo "# EOF" >>"$SCRIPT"
-            rm -f "$SCRIPT".tmp "$SCRIPT".tmph
+        if [ -n "$APPEND" ] && [ -f "${SCRIPT}" ]; then
+            rm -f "${SCRIPT}.tmp"
+            sed "${SCRIPT}" -Ee"/^#/d;/^\$/d;s,^[^ ]*/?${SELF}( +-[46])? *,,;;s/ /\n/g;" >"${SCRIPT}.tmp"
+            saveHdr >"${SCRIPT}"
+            RAW="y" save 'nohdr' >>"${SCRIPT}.tmp"
+            if [ "$IPV" == "4" ]; then
+                sort -u -t . -k1,1n -k2,2n -k3,3n -k4,4n "${SCRIPT}.tmp" | savelines -4 >>"${SCRIPT}"
+            else
+                sort -u -t : -f -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 -k6,6 -k7,7 -k8,8 "${SCRIPT}.tmp" | savelines -6 >>"${SCRIPT}"
+            fi
+            echo "# EOF" >>"${SCRIPT}"
+            rm -f "${SCRIPT}.tmp"
+        else
+            save >"${SCRIPT}"
+        fi
+        if [ -z "$RAW" ]; then
+            chmod +x "${SCRIPT}"
         else
-            save >"$SCRIPT"
+            chmod -x "${SCRIPT}"
         fi
-        chmod +x "$SCRIPT"
-        [ -n "$V" ] && echo "Wrote $(readlink -en "$SCRIPT")"
-        CNT="$( grep -c -- ' -4 \| -6 ' "$SCRIPT")"
+        [ -n "$V" ] && echo "Wrote $(readlink -en "${SCRIPT}")"
+        CNT="$(sed "${SCRIPT}" -nEe's/^# Count = ([0-9]+).*$/\1/p')"
+        sed -i "${SCRIPT}" -Ee'/^# Count = ([0-9]+).*$/d'
         if [ "$CNT" -gt "$MAXENT" ]; then
-            echo "$(readlink -en "$SCRIPT") lists $CNT addresses, but only $MAXENT will be retained.  See -h for more information."
+            echo "$(readlink -en "${SCRIPT}") lists $CNT addresses, but only $MAXENT will be retained.  See -h for more information."
             exit 2
         fi
     fi
@@ -490,7 +538,7 @@ if [ "$ACTION" == "list" ]; then
             fi
             [ -n "$DEBUG" ] && echo "IP: $IP NOW: $NOW UP: $UPT TICKS: $TICKS SEEN: $SEEN SSE: $SSE DATE: $DATE$HN"
 
-            echo "$IP ${Dots:0:$(( ${#Dots} - ${#IP} ))} $DATE$HN"
+            echo "$IP ${Dots:${#IP}} $DATE$HN"
         done
 
         if [ -n "$V" ]; then
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ @@
     *.sig
     ipblock-*
     *.tar *.tar.gz *.tar.xz *tar.lzop *.tar.lzma *.tar.Z *.tar.zst *.tar.bz *.tar.bz2
+    .tagged