v1.0.1

larsonzh · Apr 9, 2023 · 381c8b2 · 381c8b2
1 parent 93d1a59
commit 381c8b2
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 173 deletions.
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@ Multi process parallel acquisition tool for IP address data of ISP network opera
 
 *哈哈，牛逼玩意儿 ！！！*
 
-**v1.0.0**
+**v1.0.1**
 
 工具采用 Shell 脚本编写，参考并借鉴 clangcn（ https://github.com/clangcn/everyday-update-cn-isp-ip.git ）项目代码和思路，通过多进程并行处理技术，对信息检索和数据写入过程进行优化，极大提高 ISP 运营商分项地址数据生成效率，减少运行时间。在提供 IPv4 数据获取的同时，增加 IPv6 数据获取功能，以及基于 CIDR 网段聚合算法的 IPv4/6 CIDR 地址数据的生成功能。
 
@@ -65,11 +65,11 @@ Multi process parallel acquisition tool for IP address data of ISP network opera
 
 二、安装项目脚本
 
-<ul>1.下载本工具的软件压缩包 lzsipcn-[version ID].tgz（例如：lzispro-v1.0.0.tgz）。</ul>
+<ul>1.下载本工具的软件压缩包 lzsipcn-[version ID].tgz（例如：lzispro-v1.0.1.tgz）。</ul>
 
 <ul>2.将压缩包复制到设备的任意有读写权限的目录。</ul>
 
-<ul>3.在 Shell 终端中使用解压缩命令在当前目录中解压缩，生成 lzispro-[version ID] 目录（例如：lzispro-v1.0.0），其中包含一个 lzispro 目录，是脚本所在目录。</ul>
+<ul>3.在 Shell 终端中使用解压缩命令在当前目录中解压缩，生成 lzispro-[version ID] 目录（例如：lzispro-v1.0.1），其中包含一个 lzispro 目录，是脚本所在目录。</ul>
 <ul>
 
 ```markdown

diff --git a/release/lzispro/func/lzispdata.sh b/release/lzispro/func/lzispdata.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-# lzispdata.sh v1.0.0
+# lzispdata.sh v1.0.1
 # By LZ 妙妙呜 (larsonzhang@gmail.com)
 
 # Obtain ISP attribution data script
@@ -19,118 +19,34 @@ IPV_TYPE="${1}"
 # Source Data File index number
 SRC_INDEX="$( printf "%u\n" "${2}" )"
 
-# Source Data File Path
-TRAN_PATH_SRC=""
-
-# Destination data file path
-TRAN_PATH_DST=""
-
-# China ISP IPv4 Raw Data Target File Name
-TRAN_ISP_DATA_0="lz_all_cn.txt"
-TRAN_ISP_DATA_1="lz_chinatelecom.txt"
-TRAN_ISP_DATA_2="lz_unicom_cnc.txt"
-TRAN_ISP_DATA_3="lz_cmcc.txt"
-TRAN_ISP_DATA_4="lz_crtc.txt"
-TRAN_ISP_DATA_5="lz_cernet.txt"
-TRAN_ISP_DATA_6="lz_gwbn.txt"
-TRAN_ISP_DATA_7="lz_othernet.txt"
-TRAN_ISP_DATA_8="lz_hk.txt"
-TRAN_ISP_DATA_9="lz_mo.txt"
-TRAN_ISP_DATA_10="lz_tw.txt"
-
-# China ISP IPv6 Raw Data Target File Name
-TRAN_ISP_IPV6_DATA_0="lz_all_cn_ipv6.txt"
-TRAN_ISP_IPV6_DATA_1="lz_chinatelecom_ipv6.txt"
-TRAN_ISP_IPV6_DATA_2="lz_unicom_cnc_ipv6.txt"
-TRAN_ISP_IPV6_DATA_3="lz_cmcc_ipv6.txt"
-TRAN_ISP_IPV6_DATA_4="lz_crtc_ipv6.txt"
-TRAN_ISP_IPV6_DATA_5="lz_cernet_ipv6.txt"
-TRAN_ISP_IPV6_DATA_6="lz_gwbn_ipv6.txt"
-TRAN_ISP_IPV6_DATA_7="lz_othernet_ipv6.txt"
-TRAN_ISP_IPV6_DATA_8="lz_hk_ipv6.txt"
-TRAN_ISP_IPV6_DATA_9="lz_mo_ipv6.txt"
-TRAN_ISP_IPV6_DATA_10="lz_tw_ipv6.txt"
-
-# IP Address Information Details Query Host
-TRAN_WHOIS_HOST="whois.apnic.net"
-
-# Maximum Number Of Retries After IP Address Query Failure
-# 0--Unlimited, 5--Default
-TRAN_RETRY_NUM="5"
-
-# Synchronization Lock File Path & Name
-TRAN_PATH_LOCK="/var/lock"
-TRAN_LOCK_FILE="lzispro.lock"
-
 # ------------------ Function -------------------
 
 init_param() {
-    LOCK_ENABLE="1"
     [ "${1}" != "2" ] && return "1"
     [ "${IPV_TYPE}" != "ipv4" ] && [ "${IPV_TYPE}" != "ipv6" ] && return "1"
     SRC_INDEX="$( printf "%u\n" "${SRC_INDEX}" )"
     ! echo "${SRC_INDEX}" | grep -qE '^[0-9][0-9]*$' && return "1"
     # Source Data File Path
-    PATH_SRC="${TRAN_PATH_SRC}"
+    PATH_SRC="${PATH_TMP}"
     [ ! -d "${PATH_SRC}" ] && return "1"
-    local index="0" ipv4_data="" ipv6_data=""
-    until [ "${index}" -gt "10" ]
-    do
-        # China ISP IPv4 Raw Data Target File Name
-        eval "ISP_DATA_${index}=\${TRAN_ISP_DATA_${index}}"
-        eval [ -z "\${ISP_DATA_${index}}" ] && return "1"
-        eval "unset TRAN_ISP_DATA_${index}"
-        # China ISP IPv6 Raw Data Target File Name
-        eval "ISP_IPV6_DATA_${index}=\${TRAN_ISP_IPV6_DATA_${index}}"
-        eval [ -z "\${ISP_IPV6_DATA_${index}}" ] && return "1"
-        eval "unset TRAN_ISP_IPV6_DATA_${index}"
-        index="$(( index + 1 ))"
-    done
     # Source Data File name
     SRC_FILENAME="${ISP_DATA_0%.*}.dat_${SRC_INDEX}"
     [ "${IPV_TYPE}" != "ipv4" ] && SRC_FILENAME="${ISP_IPV6_DATA_0%.*}.dat_${SRC_INDEX}"
     [ ! -f "${PATH_SRC}/${SRC_FILENAME}" ] && return "1"
     # Destination data file path
-    PATH_DST="${TRAN_PATH_DST}"
+    PATH_DST="${PATH_TMP}"
     [ ! -d "${PATH_DST}" ] && return "1"
     # IP Address Information Details Query Host
-    WHOIS_HOST="${TRAN_WHOIS_HOST}"
     [ -z "${WHOIS_HOST}" ] && return "1"
     # Maximum Number Of Retries After IP Address Query Failure
-    RETRY_NUM="$( printf "%u\n" "${TRAN_RETRY_NUM}" )"
+    RETRY_NUM="$( printf "%u\n" "${RETRY_NUM}" )"
     ! echo "${RETRY_NUM}" | grep -qE '^[0-9][0-9]*$' && return "1"
     # Synchronization Lock File Path & Name
-    PATH_LOCK="${TRAN_PATH_LOCK}"
-    LOCK_FILE="${TRAN_LOCK_FILE}"
     [ ! -d "${PATH_LOCK}" ] && return "1"
     [ ! -f "${PATH_LOCK}/${LOCK_FILE}" ] && return "1"
-    LOCK_FILE="${LOCK_FILE%.*}_${SRC_INDEX}.lock"
-    LOCK_ENABLE="0"
-    unset TRAN_PATH_SRC TRAN_PATH_DST TRAN_WHOIS_HOST TRAN_RETRY_NUM TRAN_PATH_LOCK TRAN_LOCK_FILE
     return "0"
 }
 
-set_lock() {
-    [ ! -d "${PATH_LOCK}" ] && {
-        LOCK_ENABLE="1"
-        return "1"
-    }
-    [ -f "${PATH_LOCK}/${LOCK_FILE}" ] && {
-        LOCK_ENABLE="1"
-        return "1"
-    }
-    touch "${PATH_LOCK}/${LOCK_FILE}"
-    [ ! -f "${PATH_LOCK}/${LOCK_FILE}" ] && {
-        LOCK_ENABLE="1"
-        return "1"
-    }
-    return "0"
-}
-
-unset_lock() {
-    [ "${LOCK_ENABLE}" = "0" ] && [ -f "${PATH_LOCK}/${LOCK_FILE}" ] && rm -f "${PATH_LOCK}/${LOCK_FILE}" 2> /dev/null
-}
-
 init_isp_data_buf() {
     local index="1"
     until [ "${index}" -gt "7" ]
@@ -258,11 +174,9 @@ get_isp_data() {
 while true
 do
     init_param "${#}" || break
-    set_lock || break
     get_isp_data
     break
 done
-unset_lock
 
 exit "0"
 

diff --git a/release/lzispro/lzispro.sh b/release/lzispro/lzispro.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-# lzispro.sh v1.0.0
+# lzispro.sh v1.0.1
 # By LZ 妙妙呜 (larsonzhang@gmail.com)
 
 # Multi process parallel acquisition tool for IP address data of ISP network operators in China
@@ -148,7 +148,7 @@ LOCK_FILE_ID="323"
 # Forced Unlocking Command Word
 UNLOCK_CMD="unlock"
 
-LZ_VERSION="v1.0.0"
+LZ_VERSION="v1.0.1"
 
 # ------------------ Function -------------------
 
@@ -217,7 +217,6 @@ kill_child_processes() {
     ps | awk '$0 ~ "'"${ISP_DATA_SCRIPT}"'" && !/awk/ {system("kill -9 "$1" > /dev/null 2>&1")}'
     remove_div_data "ipv4"
     remove_div_data "ipv6"
-    rm -f "${PATH_LOCK}/${LOCK_FILE%.*}_"*
 }
 
 forced_unlock() {
@@ -417,76 +416,37 @@ init_project_dir() {
     return "0"
 }
 
+export_env_var() {
+    local index="0"
+    until [ "${index}" -gt "10" ]
+    do
+        export ISP_DATA_"${index}"
+        export ISP_IPV6_DATA_"${index}"
+        index="$(( index + 1 ))"
+    done
+    export PATH_TMP
+    export WHOIS_HOST
+    export RETRY_NUM
+    export PATH_LOCK
+    export LOCK_FILE
+}
+
 init_isp_data_script() {
     if ! grep -qEm 1 '^[ ]*#![\/]bin[\/]sh[ ]|^[ ]*#![\/]bin[\/]sh$' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
         lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
         lz_echo "Game Over !!!"
         return "1"
     fi
-    if ! grep -qE '^[ ]*TRAN_PATH_SRC=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    if ! grep -qE '^[ ]*TRAN_PATH_DST=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    if ! grep -qE '^[ ]*TRAN_WHOIS_HOST=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    if ! grep -qE '^[ ]*TRAN_RETRY_NUM=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    if ! grep -qE '^[ ]*TRAN_PATH_LOCK=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    if ! grep -qE '^[ ]*TRAN_LOCK_FILE=' "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-        lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-        lz_echo "Game Over !!!"
-        return "1"
-    fi
-    sed -i -e "s:^[ ]*TRAN_PATH_SRC=.*$:TRAN_PATH_SRC=\"${PATH_TMP}\":g" \
-        -e "s:^[ ]*TRAN_PATH_DST=.*$:TRAN_PATH_DST=\"${PATH_TMP}\":g" \
-        -e "s:^[ ]*TRAN_WHOIS_HOST=.*$:TRAN_WHOIS_HOST=\"${WHOIS_HOST}\":g" \
-        -e "s:^[ ]*TRAN_RETRY_NUM=.*$:TRAN_RETRY_NUM=\"${RETRY_NUM}\":g" \
-        -e "s:^[ ]*TRAN_PATH_LOCK=.*$:TRAN_PATH_LOCK=\"${PATH_LOCK}\":g" \
-        -e "s:^[ ]*TRAN_LOCK_FILE=.*$:TRAN_LOCK_FILE=\"${LOCK_FILE}\":g" "${PATH_FUNC}/${ISP_DATA_SCRIPT}"
-    local index="0" ipv4_data="" ipv6_data=""
-    until [ "${index}" -gt "10" ]
-    do
-        if ! grep -qE "^[ ]*TRAN_ISP_DATA_${index}=" "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-            lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-            lz_echo "Game Over !!!"
-            return "1"
-        fi
-        if ! grep -qE "^[ ]*TRAN_ISP_IPV6_DATA_${index}=" "${PATH_FUNC}/${ISP_DATA_SCRIPT}"; then
-            lz_echo "${PATH_FUNC}/${ISP_DATA_SCRIPT} file is damaged."
-            lz_echo "Game Over !!!"
-            return "1"
-        fi
-        eval ipv4_data="\${ISP_DATA_${index}}"
-        eval ipv6_data="\${ISP_IPV6_DATA_${index}}"
-        sed -i -e "s/^[ ]*TRAN_ISP_DATA_${index}=.*$/TRAN_ISP_DATA_${index}=\"${ipv4_data}\"/g" \
-            -e "s/^[ ]*TRAN_ISP_IPV6_DATA_${index}=.*$/TRAN_ISP_IPV6_DATA_${index}=\"${ipv6_data}\"/g" "${PATH_FUNC}/${ISP_DATA_SCRIPT}"
-        index="$(( index + 1 ))"
-    done
     chmod +x "${PATH_FUNC}/${ISP_DATA_SCRIPT}"
+    export_env_var
     return "0"
 }
 
 get_apnic_info() {
     [ -z "${DOWNLOAD_URL}" ] && DOWNLOAD_URL="http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest"
     local progress="--progress=bar:force"
     [ "${PROGRESS_BAR:="0"}" != "0" ] && progress="-q"
-    lz_echo "Download......"
+    lz_echo "Exciting fetch......"
     eval wget -c "${progress}" --prefer-family=IPv4 --no-check-certificate "${DOWNLOAD_URL}" -O "${PATH_TMP}/${APNIC_IP_INFO%.*}.dat"
     if [ ! -f "${PATH_TMP}/${APNIC_IP_INFO%.*}.dat" ]; then
         lz_echo "${APNIC_IP_INFO} Failed. Game Over !!!"
@@ -625,43 +585,38 @@ check_isp_data() {
 
 get_shell_cmd() {
     local sh_str=""
-    if ps | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qw 'bash'; then
+    if ps | awk '$1 == "'"$$"'" && !/awk/' | grep -qw 'bash'; then
         sh_str="bash"
-        ps | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qEw '[\/]bin[\/]bash' && sh_str="/bin/bash"
-    elif ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qw 'bash'; then
+        ps | awk '$1 == "'"$$"'" && !/awk/' | grep -qEw '[\/]bin[\/]bash' && sh_str="/bin/bash"
+    elif ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/' | grep -qw 'bash'; then
         sh_str="bash"
-        ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qEw '[\/]bin[\/]bash' && sh_str="/bin/bash"
-    elif ps | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qw 'sh'; then
+        ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/' | grep -qEw '[\/]bin[\/]bash' && sh_str="/bin/bash"
+    elif ps | awk '$1 == "'"$$"'" && !/awk/' | grep -qw 'sh'; then
         sh_str="sh"
-        ps | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qEw '[\/]bin[\/]sh' && sh_str="/bin/sh"
-    elif ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qw 'sh'; then
+        ps | awk '$1 == "'"$$"'" && !/awk/' | grep -qEw '[\/]bin[\/]sh' && sh_str="/bin/sh"
+    elif ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/' | grep -qw 'sh'; then
         sh_str="sh"
-        ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/ && !/grep/' | grep -qEw '[\/]bin[\/]sh' && sh_str="/bin/sh"
+        ps a 2> /dev/null | awk '$1 == "'"$$"'" && !/awk/' | grep -qEw '[\/]bin[\/]sh' && sh_str="/bin/sh"
     fi
     echo "${sh_str}"
 }
 
-fetch_isp_data() {
-    local fname="${ISP_DATA_0}" sh_str="$( get_shell_cmd )" findex="0" pflag="0" index="0"
+start_isp_data_proc() {
+    local fname="${ISP_DATA_0}" prefix_str="$( get_shell_cmd )" findex="0"
     [ "${1}" != "ipv4" ] && fname="${ISP_IPV6_DATA_0}"
-    which nohup > /dev/null 2>&1 && sh_str="nohup ${sh_str}"
     until [ "${findex}" -ge "${PARA_QUERY_PROC_NUM}" ]
     do
         [ -f "${PATH_TMP}/${fname%.*}.dat_${findex}" ] && \
-            eval "${sh_str}" "${PATH_FUNC}/${ISP_DATA_SCRIPT}" "${1}" "${findex}" > /dev/null 2>&1 &
+            eval "${prefix_str}" "${PATH_FUNC}/${ISP_DATA_SCRIPT}" "${1}" "${findex}" > /dev/null 2>&1 &
         findex="$(( findex + 1 ))"
     done
     sleep 5s
+    prefix_str=""
+    ps a 2> /dev/null | awk -v count="0" '$1 == "'"$$"'" && !/awk/ {count++} END{if (count == "0") exit(1)}' && prefix_str="a"
     while true
     do
         [ "${PROGRESS_BAR}" = "0" ] && echo -n "."
-        index="0" pflag="0"
-        until [ "${index}" -ge "${PARA_QUERY_PROC_NUM}" ]
-        do
-            [ -f "${PATH_LOCK}/${LOCK_FILE%.*}_${index}.lock" ] && { pflag="1"; break; }
-            index="$(( index + 1 ))"
-        done
-        [ "${pflag}" = "0" ] && break
+        ! ps "${prefix_str}" | awk -v count="0" '$0 ~ "'"${ISP_DATA_SCRIPT}"'" && !/awk/ {count++} END{if (count == "0") exit(1)}' && break
         sleep 5s
     done
     [ "${PROGRESS_BAR}" = "0" ] && echo ""
@@ -684,7 +639,7 @@ get_isp_data() {
     lz_echo "Use ${PARA_QUERY_PROC_NUM} process${suffix_str} for parallel query processing."
     lz_echo "Don't interrupt & Please wait......"
     [ "${PROGRESS_BAR}" = "0" ] && echo -n "."
-    fetch_isp_data "${1}" || return "1"
+    start_isp_data_proc "${1}" || return "1"
     check_isp_data "${1}" || return "1"
     return "0"
 }
@@ -1017,7 +972,7 @@ get_file_time_stamp() {
 show_header() {
     BEGIN_TIME="$( date +%s -d "$( date +"%F %T" )" )"
     lz_echo
-    lz_echo "LZ ISPRO ${LZ_VERSION:="v1.0.0"} script commands start......"
+    lz_echo "LZ ISPRO ${LZ_VERSION:="v1.0.1"} script commands start......"
     lz_echo "By LZ (larsonzhang@gmail.com)"
     lz_echo "---------------------------------------------"
     lz_echo "Command (in the ${PATH_CURRENT})"