build fix

.gitignore

@@ -1,4 +1,5 @@
 peeringdb.sqlite3
 *.tsv
 *.mmdb
-*.csv
+*.csv
+*.zip

README.md

@@ -1,6 +1,6 @@
 # Open IP to ASN/Internet Provider Database
 
-If you have on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions)
+If you have an idea on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions)
 
 ## Data sources
 
@@ -18,19 +18,20 @@ Data is updated daily.
 
 ### Download
 
-* [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip) (*recommended*)
-* [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip) (*recommended*)
-<!--
-* [IPv4 Full in CVS format](https://geoip.oxl.at/file/asn_ipv4_full.csv.zip)
-* [IPv6 Full in CVS format](https://geoip.oxl.at/file/asn_ipv6_full.csv.zip)
--->
-* [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip)
+* [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip) (*recommended*)
 * [IPv6 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv6_small.mmdb.zip)
 <!--
 * [IPv4 Stripped in CSV format](https://geoip.oxl.at/file/asn_ipv4_small.csv.zip)
 * [IPv6 Stripped in CSV format](https://geoip.oxl.at/file/asn_ipv4_small.csv.zip)
 -->
 
+* [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip)
+* [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip)
+<!--
+* [IPv4 Full in CVS format](https://geoip.oxl.at/file/asn_ipv4_full.csv.zip)
+* [IPv6 Full in CVS format](https://geoip.oxl.at/file/asn_ipv6_full.csv.zip)
+-->
+
 Limit: 1 Download per day
 
 
@@ -54,6 +55,7 @@ Limits:
 * 10 Requests per min
 * 50 Requests per 10 min
 * 100 Requests per hour
+* 200 Requests per day
 
 ----
 

build.py

@@ -33,6 +33,7 @@
 DATA_FILE_IP6 = 'data-raw-table_ip6.tsv'
 DESCRIPTION = 'OXL ASN Database (BSD 3-Clause License)'
 BGP_SOURCE = 'https://thyme.apnic.net/.combined'
+PDB_FILE = Path('peeringdb.sqlite3')
 mmdb_ip4_full = MMDBWriter(ip_version=4, description=DESCRIPTION)
 mmdb_ip6_full = MMDBWriter(ip_version=6, description=DESCRIPTION)
 mmdb_ip4_small = MMDBWriter(ip_version=4, description=DESCRIPTION)
@@ -190,29 +191,31 @@ def contacts_small(self) -> dict:
 
 def cleanup():
  try:
- remove_file(DATA_FILE_IP4)
- remove_file(DATA_FILE_IP6)
+ if IPv4:
+ remove_file(DATA_FILE_IP4)
+
+ if IPv6:
+ remove_file(DATA_FILE_IP6)
 
  except FileNotFoundError:
  pass
 
 
-if not REFRESH_DATA and (
- Path('peeringdb.sqlite3').stat().st_size < 10_000_000 or
- (IPv4 and not Path(DATA_FILE_IP4).is_file()) or
- (IPv6 and not Path(DATA_FILE_IP6).is_file())
-):
- REFRESH_DATA = True
-
-if REFRESH_DATA:
- print('\n### DOWNLOADING DATA ###')
-
- cleanup()
+if not PDB_FILE.is_file() or PDB_FILE.stat().st_size < 10_000_000:
+ print('\n### SYNCING PEERINGDB ###')
+ # running 'peeringdb sync' programmatically
  Sync().handle(
  check_load_config(getcwd()),
  verbose=False, quiet=False, init=False, since=-1,
  fetch_private=False,
  )
+
+if REFRESH_DATA or \
+ (IPv4 and not Path(DATA_FILE_IP4).is_file()) or \
+ (IPv6 and not Path(DATA_FILE_IP6).is_file()):
+ print('\n### DOWNLOADING BGP-DATA ###')
+ cleanup()
+
  if IPv4:
  urlretrieve(f'{BGP_SOURCE}/data-raw-table', DATA_FILE_IP4)
 
@@ -229,7 +232,8 @@ def cleanup():
  if c % STATUS_INTERVAL == 0:
  print('PARSING', c)
 
- net, asn = line.split('\t')
+ net, asn = line.strip().split('\t')
+ asn = int(asn)
  if asn not in asn_nets:
  asn_nets[asn] = ASN(asn)
 
@@ -246,13 +250,15 @@ def cleanup():
  print('PARSING', c)
 
  # not tab separated :(
- net, asn = line.split(' ', 1)
+ net, asn = line.strip().split(' ', 1)
  asn = asn.strip()
 
  if not asn.isdigit():
  # some bad data
  continue
 
+ asn = int(asn)
+
  if asn not in asn_nets:
  asn_nets[asn] = ASN(asn)
 

scripts/build.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -eu
+
+cd "$(dirname "$0")/.."
+
+python3 build.py
+
+rm -f *.zip
+
+echo ''
+echo '### PACKING ###'
+
+zip -9 'asn_ipv4_full.mmdb.zip' 'asn_ipv4_full.mmdb' LICENSE_db.txt
+zip -9 'asn_ipv4_small.mmdb.zip' 'asn_ipv4_small.mmdb' LICENSE_db.txt
+zip -9 'asn_ipv6_full.mmdb.zip' 'asn_ipv6_full.mmdb' LICENSE_db.txt
+zip -9 'asn_ipv6_small.mmdb.zip' 'asn_ipv6_small.mmdb' LICENSE_db.txt
+
+echo '### DONE ###'

scripts/lint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+export AW_INIT=1
+
+echo ''
+echo 'LINTING Python'
+echo ''
+
+pylint --recursive=y .
+
+echo ''
+echo 'LINTING YAML'
+echo ''
+
+yamllint .
+