From 0cb3b40b9fd752caa766babd3ebbc0536e7adb7d Mon Sep 17 00:00:00 2001 From: Rath Rene Date: Fri, 17 May 2024 16:15:58 +0200 Subject: [PATCH] build fix --- .gitignore | 3 ++- README.md | 18 ++++++++++-------- build.py | 36 +++++++++++++++++++++--------------- scripts/build.sh | 19 +++++++++++++++++++ scripts/lint.sh | 20 ++++++++++++++++++++ 5 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 scripts/build.sh create mode 100644 scripts/lint.sh diff --git a/.gitignore b/.gitignore index 43a4441..ff489ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ peeringdb.sqlite3 *.tsv *.mmdb -*.csv \ No newline at end of file +*.csv +*.zip \ No newline at end of file diff --git a/README.md b/README.md index ba0d412..0bd869d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Open IP to ASN/Internet Provider Database -If you have on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions) +If you have an idea on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions) ## Data sources @@ -18,19 +18,20 @@ Data is updated daily. ### Download -* [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip) (*recommended*) -* [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip) (*recommended*) - -* [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip) +* [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip) (*recommended*) * [IPv6 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv6_small.mmdb.zip) +* [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip) +* [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip) + + Limit: 1 Download per day @@ -54,6 +55,7 @@ Limits: * 10 Requests per min * 50 Requests per 10 min * 100 Requests per hour +* 200 Requests per day ---- diff --git a/build.py b/build.py index 0cbffa6..07e30e9 100755 --- a/build.py +++ b/build.py @@ -33,6 +33,7 @@ DATA_FILE_IP6 = 'data-raw-table_ip6.tsv' DESCRIPTION = 'OXL ASN Database (BSD 3-Clause License)' BGP_SOURCE = 'https://thyme.apnic.net/.combined' +PDB_FILE = Path('peeringdb.sqlite3') mmdb_ip4_full = MMDBWriter(ip_version=4, description=DESCRIPTION) mmdb_ip6_full = MMDBWriter(ip_version=6, description=DESCRIPTION) mmdb_ip4_small = MMDBWriter(ip_version=4, description=DESCRIPTION) @@ -190,29 +191,31 @@ def contacts_small(self) -> dict: def cleanup(): try: - remove_file(DATA_FILE_IP4) - remove_file(DATA_FILE_IP6) + if IPv4: + remove_file(DATA_FILE_IP4) + + if IPv6: + remove_file(DATA_FILE_IP6) except FileNotFoundError: pass -if not REFRESH_DATA and ( - Path('peeringdb.sqlite3').stat().st_size < 10_000_000 or - (IPv4 and not Path(DATA_FILE_IP4).is_file()) or - (IPv6 and not Path(DATA_FILE_IP6).is_file()) -): - REFRESH_DATA = True - -if REFRESH_DATA: - print('\n### DOWNLOADING DATA ###') - - cleanup() +if not PDB_FILE.is_file() or PDB_FILE.stat().st_size < 10_000_000: + print('\n### SYNCING PEERINGDB ###') + # running 'peeringdb sync' programmatically Sync().handle( check_load_config(getcwd()), verbose=False, quiet=False, init=False, since=-1, fetch_private=False, ) + +if REFRESH_DATA or \ + (IPv4 and not Path(DATA_FILE_IP4).is_file()) or \ + (IPv6 and not Path(DATA_FILE_IP6).is_file()): + print('\n### DOWNLOADING BGP-DATA ###') + cleanup() + if IPv4: urlretrieve(f'{BGP_SOURCE}/data-raw-table', DATA_FILE_IP4) @@ -229,7 +232,8 @@ def cleanup(): if c % STATUS_INTERVAL == 0: print('PARSING', c) - net, asn = line.split('\t') + net, asn = line.strip().split('\t') + asn = int(asn) if asn not in asn_nets: asn_nets[asn] = ASN(asn) @@ -246,13 +250,15 @@ def cleanup(): print('PARSING', c) # not tab separated :( - net, asn = line.split(' ', 1) + net, asn = line.strip().split(' ', 1) asn = asn.strip() if not asn.isdigit(): # some bad data continue + asn = int(asn) + if asn not in asn_nets: asn_nets[asn] = ASN(asn) diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..4713748 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -eu + +cd "$(dirname "$0")/.." + +python3 build.py + +rm -f *.zip + +echo '' +echo '### PACKING ###' + +zip -9 'asn_ipv4_full.mmdb.zip' 'asn_ipv4_full.mmdb' LICENSE_db.txt +zip -9 'asn_ipv4_small.mmdb.zip' 'asn_ipv4_small.mmdb' LICENSE_db.txt +zip -9 'asn_ipv6_full.mmdb.zip' 'asn_ipv6_full.mmdb' LICENSE_db.txt +zip -9 'asn_ipv6_small.mmdb.zip' 'asn_ipv6_small.mmdb' LICENSE_db.txt + +echo '### DONE ###' diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100644 index 0000000..be8ebb7 --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +cd "$(dirname "$0")/.." + +export AW_INIT=1 + +echo '' +echo 'LINTING Python' +echo '' + +pylint --recursive=y . + +echo '' +echo 'LINTING YAML' +echo '' + +yamllint . +