Skip to content

Commit

Permalink
added JSON format
Browse files Browse the repository at this point in the history
  • Loading branch information
@superstes
superstes committed May 21, 2024
1 parent 80024cb commit 248589c
Show file tree
Hide file tree
Showing 6 changed files with 407 additions and 23 deletions.
26 changes: 15 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

If you have an idea on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions)

Thanks go to the author of [hackitu.de](https://www.hackitu.de/geoip/) for sharing his knowledge about this topic.

## Data sources

IP to ASN:
Expand All @@ -20,25 +22,27 @@ Data is updated daily.

* [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip) (*recommended*)
* [IPv6 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv6_small.mmdb.zip)
<!--
* [IPv4 Stripped in CSV format](https://geoip.oxl.at/file/asn_ipv4_small.csv.zip)
* [IPv6 Stripped in CSV format](https://geoip.oxl.at/file/asn_ipv4_small.csv.zip)
-->
* [IPv4 Stripped in JSON format](https://geoip.oxl.at/file/asn_ipv4_small.json.zip)
* [IPv6 Stripped in JSON format](https://geoip.oxl.at/file/asn_ipv6_small.json.zip)

* [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip)
* [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip)
<!--
* [IPv4 Full in CVS format](https://geoip.oxl.at/file/asn_ipv4_full.csv.zip)
* [IPv6 Full in CVS format](https://geoip.oxl.at/file/asn_ipv6_full.csv.zip)
-->
* [IPv4 Full in JSON format](https://geoip.oxl.at/file/asn_ipv4_full.json.zip)
* [IPv6 Full in JSON format](https://geoip.oxl.at/file/asn_ipv6_full.json.zip)

* [IPv4+IPv6 Stripped in JSON format](https://geoip.oxl.at/file/asn_small.json.zip)
* [IPv4+IPv6 Full in JSON format](https://geoip.oxl.at/file/asn_full.json.zip)

Limit: 1 Download per day
Limit: 2 Downloads per day

Note: Databases in MMDB format might be faster and cheaper to query.

#### Schema Examples

* [Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/full.json)
* [Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/small.json)
* [MMDB Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/mmdb_full.json)
* [MMDB Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/mmdb_small.json)
* [JSON Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/json_full.json)
* [JSON Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/json_small.json)

----

Expand Down
109 changes: 97 additions & 12 deletions build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@
# ASN infos
# https://www.peeringdb.com/

# todo: implement CSV db format
# NOTE: for automated runs you might want to export the env-var 'PYTHONUNBUFFERED=1' beforehand

import logging
from pathlib import Path
from urllib.request import urlretrieve
from os import remove as remove_file
from os import getcwd
from json import dumps as json_dumps

from netaddr import IPSet
from mmdb_writer import MMDBWriter
Expand All @@ -29,6 +30,8 @@
IPv6 = True
PROGRESS = False # enable for debugging
REFRESH_DATA = True
MMDB = True
JSON = True

# you may not need to modify those
STATUS_INTERVAL = 10_000
Expand All @@ -38,11 +41,27 @@
DATA_FILE_IP4 = 'data-raw-table_ip4.tsv'
DATA_FILE_IP6 = 'data-raw-table_ip6.tsv'
PDB_FILE = Path('peeringdb.sqlite3')
DUMP_FILE_IP4_MMDB_FULL = 'asn_ipv4_full.mmdb'
DUMP_FILE_IP4_MMDB_SMALL = 'asn_ipv4_small.mmdb'
DUMP_FILE_IP6_MMDB_FULL = 'asn_ipv6_full.mmdb'
DUMP_FILE_IP6_MMDB_SMALL = 'asn_ipv6_small.mmdb'
DUMP_FILE_ALL_JSON_FULL = 'asn_full.json'
DUMP_FILE_ALL_JSON_SMALL = 'asn_small.json'
DUMP_FILE_IP4_JSON_FULL = 'asn_ipv4_full.json'
DUMP_FILE_IP4_JSON_SMALL = 'asn_ipv4_small.json'
DUMP_FILE_IP6_JSON_FULL = 'asn_ipv6_full.json'
DUMP_FILE_IP6_JSON_SMALL = 'asn_ipv6_small.json'

mmdb_ip4_full = MMDBWriter(ip_version=4, description=DESCRIPTION)
mmdb_ip6_full = MMDBWriter(ip_version=6, description=DESCRIPTION)
mmdb_ip4_small = MMDBWriter(ip_version=4, description=DESCRIPTION)
mmdb_ip6_small = MMDBWriter(ip_version=6, description=DESCRIPTION)
json_all_full = {}
json_all_small = {}
json_ip4_full = {}
json_ip4_small = {}
json_ip6_full = {}
json_ip6_small = {}


def _empty(v: any) -> any:
Expand All @@ -52,6 +71,11 @@ def _empty(v: any) -> any:
return v


def serialize_ipset(ipset: IPSet) -> list:
# pylint: disable=W0212
return [str(_) for _ in sorted(ipset._cidrs)]


class ASN:
def __init__(self, asn_id: int):
# pylint: disable=E1101
Expand Down Expand Up @@ -279,30 +303,91 @@ def cleanup():
print('ADDING', c)

# add additional ASN info below
dump_full = {'asn': asn, 'organization': data.organization, 'info': data.info, 'contacts': data.contacts}
dump_small = {
'asn': asn,
dump_full_data = {'organization': data.organization, 'info': data.info, 'contacts': data.contacts}
dump_small_data = {
'organization': data.organization_small,
'info': data.info_small,
'contacts': data.contacts_small,
}
dump_mmdb_full = {'asn': asn, **dump_full_data}
dump_mmdb_small = {'asn': asn, **dump_small_data}
if IPv4:
mmdb_ip4_full.insert_network(data.ip4, dump_full)
mmdb_ip4_small.insert_network(data.ip4, dump_small)
if MMDB:
mmdb_ip4_full.insert_network(data.ip4, dump_mmdb_full)
mmdb_ip4_small.insert_network(data.ip4, dump_mmdb_small)

if JSON:
json_ip4_full[asn] = {'ipv4': serialize_ipset(data.ip4), **dump_full_data}
json_ip4_small[asn] = {'ipv4': serialize_ipset(data.ip4), **dump_small_data}

if IPv6:
mmdb_ip6_full.insert_network(data.ip6, dump_full)
mmdb_ip6_small.insert_network(data.ip6, dump_small)
if MMDB:
mmdb_ip6_full.insert_network(data.ip6, dump_mmdb_full)
mmdb_ip6_small.insert_network(data.ip6, dump_mmdb_small)

if JSON:
json_ip6_full[asn] = {'ipv6': serialize_ipset(data.ip6), **dump_full_data}
json_ip6_small[asn] = {'ipv6': serialize_ipset(data.ip6), **dump_small_data}

if IPv4 and IPv6 and JSON:
json_all_full[asn] = {
'ipv4': json_ip4_small[asn]['ipv4'],
'ipv6': json_ip6_small[asn]['ipv6'],
**dump_full_data,
}
json_all_small[asn] = {
'ipv4': json_ip4_small[asn]['ipv4'],
'ipv6': json_ip6_small[asn]['ipv6'],
**dump_small_data,
}

c += 1

print('\n### WRITING ###')

if IPv4 and IPv6 and JSON:
with open(DUMP_FILE_ALL_JSON_FULL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_all_full))

with open(DUMP_FILE_ALL_JSON_SMALL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_all_small))

if IPv4:
mmdb_ip4_full.to_db_file('asn_ipv4_full.mmdb')
mmdb_ip4_small.to_db_file('asn_ipv4_small.mmdb')
if MMDB:
remove_file(DUMP_FILE_IP4_MMDB_FULL)
mmdb_ip4_full.to_db_file(DUMP_FILE_IP4_MMDB_FULL)
del mmdb_ip4_full

remove_file(DUMP_FILE_IP4_MMDB_SMALL)
mmdb_ip4_small.to_db_file(DUMP_FILE_IP4_MMDB_SMALL)
del mmdb_ip4_small

if JSON:
with open(DUMP_FILE_IP4_JSON_FULL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_ip4_full))
del json_ip4_full

with open(DUMP_FILE_IP4_JSON_SMALL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_ip4_small))
del json_ip4_small

if IPv6:
mmdb_ip6_full.to_db_file('asn_ipv6_full.mmdb')
mmdb_ip6_small.to_db_file('asn_ipv6_small.mmdb')
if MMDB:
remove_file(DUMP_FILE_IP6_MMDB_FULL)
mmdb_ip6_full.to_db_file(DUMP_FILE_IP6_MMDB_FULL)
del mmdb_ip6_full

remove_file(DUMP_FILE_IP6_MMDB_SMALL)
mmdb_ip6_small.to_db_file(DUMP_FILE_IP6_MMDB_SMALL)
del mmdb_ip6_small

if JSON:
with open(DUMP_FILE_IP6_JSON_FULL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_ip6_full))
del json_ip6_full

with open(DUMP_FILE_IP6_JSON_SMALL, 'w', encoding='utf-8') as f:
f.write(json_dumps(json_ip6_small))
del json_ip6_small

print('\n### DONE ###')
176 changes: 176 additions & 0 deletions example/json_full.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
{
"212384": {
"ipv4": [
"2.59.59.0/24",
"5.105.12.0/24",
"23.230.250.0/23",
"37.140.244.0/24",
"45.8.218.0/24",
"45.9.124.0/22",
"45.12.104.0/22",
"45.14.144.0/22",
"45.81.59.0/24",
"45.87.187.0/24",
"45.134.13.0/24",
"45.140.39.0/24",
"45.142.238.0/23",
"45.146.83.0/24",
"45.147.225.0/24",
"45.149.63.0/24",
"45.149.166.0/24",
"45.150.59.0/24",
"45.157.180.0/22",
"46.37.103.0/24",
"50.114.17.0/24",
"50.114.72.0/24",
"62.112.30.0/24",
"65.110.46.0/24",
"67.203.21.0/24",
"69.166.229.0/24",
"81.21.3.0/24",
"81.199.29.0/24",
"83.147.24.0/24",
"84.32.82.0/24",
"84.32.86.0/24",
"85.209.179.0/24",
"85.209.221.0/24",
"86.38.238.0/24",
"87.254.31.0/24",
"88.209.231.0/24",
"88.216.37.0/24",
"88.216.196.0/24",
"89.34.171.0/24",
"91.132.227.0/24",
"91.199.47.0/24",
"91.199.168.0/24",
"91.213.173.0/24",
"91.226.97.0/24",
"94.103.177.0/24",
"102.129.208.0/24",
"103.212.82.0/24",
"104.234.197.0/24",
"104.254.195.0/24",
"140.99.220.0/24",
"140.228.22.0/24",
"141.11.121.0/24",
"146.19.85.0/24",
"146.247.106.0/24",
"152.89.248.0/24",
"154.16.56.0/24",
"154.16.88.0/24",
"154.127.63.0/24",
"157.254.220.0/24",
"173.211.93.0/24",
"174.140.207.0/24",
"176.57.48.0/23",
"176.57.58.0/24",
"176.105.224.0/23",
"176.105.229.0/24",
"176.223.181.0/24",
"178.253.215.0/24",
"179.61.130.0/24",
"181.214.63.0/24",
"181.214.169.0/24",
"181.214.241.0/24",
"181.215.37.0/24",
"185.11.140.0/23",
"185.106.194.0/23",
"185.108.206.0/23",
"185.139.3.0/24",
"185.139.239.0/24",
"185.147.55.0/24",
"185.158.105.0/24",
"185.170.42.0/24",
"185.189.33.0/24",
"185.232.207.0/24",
"188.209.138.0/24",
"188.211.249.0/24",
"188.244.113.0/24",
"190.93.76.0/24",
"191.96.34.0/24",
"191.96.109.0/24",
"191.96.211.0/24",
"191.101.93.0/24",
"191.101.122.0/24",
"193.30.102.0/24",
"193.109.196.0/24",
"193.178.52.0/24",
"194.4.54.0/24",
"194.5.66.0/24",
"194.5.149.0/24",
"194.113.227.0/24",
"199.48.176.0/24",
"202.43.7.0/24",
"203.17.123.0/24",
"206.53.60.0/24",
"206.197.211.0/24",
"209.145.43.0/24",
"212.46.35.0/24",
"212.46.42.0/24",
"213.139.76.0/24",
"216.107.130.0/24",
"216.247.98.0/24",
"217.28.128.0/24"
],
"organization": {
"status": "ok",
"address1": "Lygumų g. 80-21",
"address2": "87142 Telšiai",
"city": "Telšiai",
"state": "Telšių r. sav.",
"zipcode": "87142",
"country": "LT",
"suite": "",
"floor": "",
"latitude": 55.976777,
"longitude": 22.269666,
"name": "Simoresta UAB",
"aka": "",
"name_long": "",
"website": "https://simoresta.lt",
"social_media": [
{
"service": "website",
"identifier": "https://simoresta.lt"
}
],
"notes": ""
},
"info": {
"status": "ok",
"name": "Simoresta GL",
"aka": "",
"name_long": "",
"irr_as_set": "",
"website": "https://simoresta.lt",
"social_media": [
{
"service": "website",
"identifier": "https://simoresta.lt"
}
],
"looking_glass": "",
"route_server": "",
"notes": "",
"info_traffic": "10-20Gbps",
"info_ratio": "Balanced",
"info_scope": "Global",
"info_types": [],
"info_prefixes4": 0,
"info_prefixes6": 0,
"info_unicast": false,
"info_multicast": false,
"info_ipv6": false,
"info_never_via_route_servers": false,
"policy_url": "",
"policy_general": "Open",
"policy_locations": "Not Required",
"policy_ratio": false,
"policy_contracts": "Not Required",
"status_dashboard": "",
"rir_status": "ok",
"rir_status_updated": "2022-07-27 05:29:57"
},
"contacts": {}
}
}
Loading

0 comments on commit 248589c

Please sign in to comment.