Build Database Update #1890
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Database Update | |
| on: | |
| push: | |
| paths: | |
| - 'input/**' | |
| - '.github/workflows/build-list.yml' | |
| schedule: | |
| - cron: '45 8 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| build_and_commit: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| list: | |
| - name: vpn | |
| validate_low: 1500000 | |
| validate_high: 3000000 | |
| - name: datacenter | |
| validate_low: 200000000 | |
| validate_high: 300000000 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Dependencies | |
| run: | | |
| sudo apt-get install ipcalc | |
| - name: Downloading ASN database | |
| run: | | |
| curl https://iptoasn.com/data/ip2asn-v4.tsv.gz | gzip -d > /tmp/asndb.tsv | |
| - name: ASN -> IP Range | |
| run: | | |
| cat "$GITHUB_WORKSPACE/input/${{matrix.list.name}}/ASN.txt" | grep -v '^#' | awk '{print $1}' | grep '^AS' | while read asn; do | |
| echo "Processing $asn" | |
| awk '{if($3 == '${asn:2}') print "ipcalc -rn "$1"-"$2" | tail -n+2"}' /tmp/asndb.tsv | bash >> /tmp/asn-processed.txt | |
| done | |
| if [[ "${{matrix.list.name}}" == "datacenter" ]]; then | |
| cat "$GITHUB_WORKSPACE/input/vpn/ASN.txt" | grep -v '^#' | awk '{print $1}' | grep '^AS' | while read asn; do | |
| echo "Processing $asn" | |
| awk '{if($3 == '${asn:2}') print "ipcalc -rn "$1"-"$2" | tail -n+2"}' /tmp/asndb.tsv | bash >> /tmp/asn-processed.txt | |
| done | |
| fi | |
| - name: Merge Lists | |
| run: | | |
| # Remove too small allocations from ASN list (anything less than a /24 can't be announced) | |
| # TODO: This might remove some SWIP'ed ranges (needs evaluation) | |
| perl ./helpers/cleanup.pl /tmp/asn-processed.txt | grep -E '/(2[0-4]|1[0-9]|[0-9])$' > /tmp/asn-cleaned.txt | |
| # Combine lists | |
| cat $GITHUB_WORKSPACE/input/${{matrix.list.name}}/ips/*.txt | grep -v '^#' | awk '{print $1}' | sed '/^$/d' > /tmp/manual-processed.txt | |
| cat /tmp/asn-cleaned.txt /tmp/manual-processed.txt | sort -n > /tmp/ipv4.txt | |
| # Final cleanup | |
| perl ./helpers/cleanup.pl /tmp/ipv4.txt > ipv4.txt | |
| - name: Test generated Lists | |
| run: | | |
| if [[ $(grep ^0. ipv4.txt | wc -l) != "0" ]]; then | |
| echo "Lists can not contain 0.0.0.0/8" | |
| exit 1 | |
| fi | |
| if [[ $(grep ^127. ipv4.txt | wc -l) != "0" ]]; then | |
| echo "Lists can not contain 127.0.0.1/8" | |
| exit 1 | |
| fi | |
| ips_covered=$(awk -F/ 'BEGIN{a=0}{if($2==""){a+=1}else{a+=2^(32-$2)}}END{print a}' ipv4.txt) | |
| if [[ "$ips_covered" -lt ${{matrix.list.validate_low}} ]]; then | |
| echo "Too few addresses covered ($ips_covered)" | |
| exit 1 | |
| fi | |
| if [[ "$ips_covered" -gt ${{matrix.list.validate_high}} ]]; then | |
| echo "Too many addresses covered ($ips_covered)" | |
| exit 1 | |
| fi | |
| - name: Push Output of ipv4.txt | |
| uses: X4BNet/copy_file_to_another_repo_action@main | |
| env: | |
| API_TOKEN_GITHUB: ${{ secrets.MY_GITHUB_TOKEN }} | |
| with: | |
| source_file: 'ipv4.txt' | |
| destination_repo: '${{ github.repository }}' | |
| destination_folder: '/output/${{matrix.list.name}}/' | |
| user_email: 'automated@x4b.net' | |
| user_name: 'listbuilder' | |
| destination_branch: "main" | |
| - name: Push Output of ipv4.txt | |
| uses: X4BNet/copy_file_to_another_repo_action@main | |
| env: | |
| API_TOKEN_GITHUB: ${{ secrets.MY_GITHUB_TOKEN }} | |
| if: ${{ matrix.list.name == 'datacenter' }} | |
| with: | |
| source_file: 'ipv4.txt' | |
| destination_repo: '${{ github.repository }}' | |
| destination_folder: '/' | |
| user_email: 'automated@x4b.net' | |
| user_name: 'listbuilder' | |
| destination_branch: "main" | |