From 248589cd21aa5fa634136f103896b3107765cc94 Mon Sep 17 00:00:00 2001 From: Rath Rene Date: Tue, 21 May 2024 20:03:54 +0200 Subject: [PATCH] added JSON format --- README.md | 26 ++-- build.py | 109 +++++++++++++-- example/json_full.json | 176 ++++++++++++++++++++++++ example/json_small.json | 119 ++++++++++++++++ example/{full.json => mmdb_full.json} | 0 example/{small.json => mmdb_small.json} | 0 6 files changed, 407 insertions(+), 23 deletions(-) create mode 100644 example/json_full.json create mode 100644 example/json_small.json rename example/{full.json => mmdb_full.json} (100%) rename example/{small.json => mmdb_small.json} (100%) diff --git a/README.md b/README.md index 85b3dc3..1347a1b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ If you have an idea on how to improve this project: [feel free to start a discussion](https://github.com/O-X-L/geoip-asn/discussions) +Thanks go to the author of [hackitu.de](https://www.hackitu.de/geoip/) for sharing his knowledge about this topic. + ## Data sources IP to ASN: @@ -20,25 +22,27 @@ Data is updated daily. * [IPv4 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv4_small.mmdb.zip) (*recommended*) * [IPv6 Stripped in MMDB format](https://geoip.oxl.at/file/asn_ipv6_small.mmdb.zip) - +* [IPv4 Stripped in JSON format](https://geoip.oxl.at/file/asn_ipv4_small.json.zip) +* [IPv6 Stripped in JSON format](https://geoip.oxl.at/file/asn_ipv6_small.json.zip) * [IPv4 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv4_full.mmdb.zip) * [IPv6 Full in MMDB format](https://geoip.oxl.at/file/asn_ipv6_full.mmdb.zip) - +* [IPv4 Full in JSON format](https://geoip.oxl.at/file/asn_ipv4_full.json.zip) +* [IPv6 Full in JSON format](https://geoip.oxl.at/file/asn_ipv6_full.json.zip) + +* [IPv4+IPv6 Stripped in JSON format](https://geoip.oxl.at/file/asn_small.json.zip) +* [IPv4+IPv6 Full in JSON format](https://geoip.oxl.at/file/asn_full.json.zip) -Limit: 1 Download per day +Limit: 2 Downloads per day +Note: Databases in MMDB format might be faster and cheaper to query. #### Schema Examples -* [Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/full.json) -* [Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/small.json) +* [MMDB Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/mmdb_full.json) +* [MMDB Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/mmdb_small.json) +* [JSON Full](https://github.com/O-X-L/geoip-asn/blob/latest/example/json_full.json) +* [JSON Small](https://github.com/O-X-L/geoip-asn/blob/latest/example/json_small.json) ---- diff --git a/build.py b/build.py index d46469f..f2881db 100755 --- a/build.py +++ b/build.py @@ -9,13 +9,14 @@ # ASN infos # https://www.peeringdb.com/ -# todo: implement CSV db format +# NOTE: for automated runs you might want to export the env-var 'PYTHONUNBUFFERED=1' beforehand import logging from pathlib import Path from urllib.request import urlretrieve from os import remove as remove_file from os import getcwd +from json import dumps as json_dumps from netaddr import IPSet from mmdb_writer import MMDBWriter @@ -29,6 +30,8 @@ IPv6 = True PROGRESS = False # enable for debugging REFRESH_DATA = True +MMDB = True +JSON = True # you may not need to modify those STATUS_INTERVAL = 10_000 @@ -38,11 +41,27 @@ DATA_FILE_IP4 = 'data-raw-table_ip4.tsv' DATA_FILE_IP6 = 'data-raw-table_ip6.tsv' PDB_FILE = Path('peeringdb.sqlite3') +DUMP_FILE_IP4_MMDB_FULL = 'asn_ipv4_full.mmdb' +DUMP_FILE_IP4_MMDB_SMALL = 'asn_ipv4_small.mmdb' +DUMP_FILE_IP6_MMDB_FULL = 'asn_ipv6_full.mmdb' +DUMP_FILE_IP6_MMDB_SMALL = 'asn_ipv6_small.mmdb' +DUMP_FILE_ALL_JSON_FULL = 'asn_full.json' +DUMP_FILE_ALL_JSON_SMALL = 'asn_small.json' +DUMP_FILE_IP4_JSON_FULL = 'asn_ipv4_full.json' +DUMP_FILE_IP4_JSON_SMALL = 'asn_ipv4_small.json' +DUMP_FILE_IP6_JSON_FULL = 'asn_ipv6_full.json' +DUMP_FILE_IP6_JSON_SMALL = 'asn_ipv6_small.json' mmdb_ip4_full = MMDBWriter(ip_version=4, description=DESCRIPTION) mmdb_ip6_full = MMDBWriter(ip_version=6, description=DESCRIPTION) mmdb_ip4_small = MMDBWriter(ip_version=4, description=DESCRIPTION) mmdb_ip6_small = MMDBWriter(ip_version=6, description=DESCRIPTION) +json_all_full = {} +json_all_small = {} +json_ip4_full = {} +json_ip4_small = {} +json_ip6_full = {} +json_ip6_small = {} def _empty(v: any) -> any: @@ -52,6 +71,11 @@ def _empty(v: any) -> any: return v +def serialize_ipset(ipset: IPSet) -> list: + # pylint: disable=W0212 + return [str(_) for _ in sorted(ipset._cidrs)] + + class ASN: def __init__(self, asn_id: int): # pylint: disable=E1101 @@ -279,30 +303,91 @@ def cleanup(): print('ADDING', c) # add additional ASN info below - dump_full = {'asn': asn, 'organization': data.organization, 'info': data.info, 'contacts': data.contacts} - dump_small = { - 'asn': asn, + dump_full_data = {'organization': data.organization, 'info': data.info, 'contacts': data.contacts} + dump_small_data = { 'organization': data.organization_small, 'info': data.info_small, 'contacts': data.contacts_small, } + dump_mmdb_full = {'asn': asn, **dump_full_data} + dump_mmdb_small = {'asn': asn, **dump_small_data} if IPv4: - mmdb_ip4_full.insert_network(data.ip4, dump_full) - mmdb_ip4_small.insert_network(data.ip4, dump_small) + if MMDB: + mmdb_ip4_full.insert_network(data.ip4, dump_mmdb_full) + mmdb_ip4_small.insert_network(data.ip4, dump_mmdb_small) + + if JSON: + json_ip4_full[asn] = {'ipv4': serialize_ipset(data.ip4), **dump_full_data} + json_ip4_small[asn] = {'ipv4': serialize_ipset(data.ip4), **dump_small_data} if IPv6: - mmdb_ip6_full.insert_network(data.ip6, dump_full) - mmdb_ip6_small.insert_network(data.ip6, dump_small) + if MMDB: + mmdb_ip6_full.insert_network(data.ip6, dump_mmdb_full) + mmdb_ip6_small.insert_network(data.ip6, dump_mmdb_small) + + if JSON: + json_ip6_full[asn] = {'ipv6': serialize_ipset(data.ip6), **dump_full_data} + json_ip6_small[asn] = {'ipv6': serialize_ipset(data.ip6), **dump_small_data} + + if IPv4 and IPv6 and JSON: + json_all_full[asn] = { + 'ipv4': json_ip4_small[asn]['ipv4'], + 'ipv6': json_ip6_small[asn]['ipv6'], + **dump_full_data, + } + json_all_small[asn] = { + 'ipv4': json_ip4_small[asn]['ipv4'], + 'ipv6': json_ip6_small[asn]['ipv6'], + **dump_small_data, + } c += 1 print('\n### WRITING ###') + +if IPv4 and IPv6 and JSON: + with open(DUMP_FILE_ALL_JSON_FULL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_all_full)) + + with open(DUMP_FILE_ALL_JSON_SMALL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_all_small)) + if IPv4: - mmdb_ip4_full.to_db_file('asn_ipv4_full.mmdb') - mmdb_ip4_small.to_db_file('asn_ipv4_small.mmdb') + if MMDB: + remove_file(DUMP_FILE_IP4_MMDB_FULL) + mmdb_ip4_full.to_db_file(DUMP_FILE_IP4_MMDB_FULL) + del mmdb_ip4_full + + remove_file(DUMP_FILE_IP4_MMDB_SMALL) + mmdb_ip4_small.to_db_file(DUMP_FILE_IP4_MMDB_SMALL) + del mmdb_ip4_small + + if JSON: + with open(DUMP_FILE_IP4_JSON_FULL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_ip4_full)) + del json_ip4_full + + with open(DUMP_FILE_IP4_JSON_SMALL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_ip4_small)) + del json_ip4_small if IPv6: - mmdb_ip6_full.to_db_file('asn_ipv6_full.mmdb') - mmdb_ip6_small.to_db_file('asn_ipv6_small.mmdb') + if MMDB: + remove_file(DUMP_FILE_IP6_MMDB_FULL) + mmdb_ip6_full.to_db_file(DUMP_FILE_IP6_MMDB_FULL) + del mmdb_ip6_full + + remove_file(DUMP_FILE_IP6_MMDB_SMALL) + mmdb_ip6_small.to_db_file(DUMP_FILE_IP6_MMDB_SMALL) + del mmdb_ip6_small + + if JSON: + with open(DUMP_FILE_IP6_JSON_FULL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_ip6_full)) + del json_ip6_full + + with open(DUMP_FILE_IP6_JSON_SMALL, 'w', encoding='utf-8') as f: + f.write(json_dumps(json_ip6_small)) + del json_ip6_small print('\n### DONE ###') diff --git a/example/json_full.json b/example/json_full.json new file mode 100644 index 0000000..123bd5f --- /dev/null +++ b/example/json_full.json @@ -0,0 +1,176 @@ +{ + "212384": { + "ipv4": [ + "2.59.59.0/24", + "5.105.12.0/24", + "23.230.250.0/23", + "37.140.244.0/24", + "45.8.218.0/24", + "45.9.124.0/22", + "45.12.104.0/22", + "45.14.144.0/22", + "45.81.59.0/24", + "45.87.187.0/24", + "45.134.13.0/24", + "45.140.39.0/24", + "45.142.238.0/23", + "45.146.83.0/24", + "45.147.225.0/24", + "45.149.63.0/24", + "45.149.166.0/24", + "45.150.59.0/24", + "45.157.180.0/22", + "46.37.103.0/24", + "50.114.17.0/24", + "50.114.72.0/24", + "62.112.30.0/24", + "65.110.46.0/24", + "67.203.21.0/24", + "69.166.229.0/24", + "81.21.3.0/24", + "81.199.29.0/24", + "83.147.24.0/24", + "84.32.82.0/24", + "84.32.86.0/24", + "85.209.179.0/24", + "85.209.221.0/24", + "86.38.238.0/24", + "87.254.31.0/24", + "88.209.231.0/24", + "88.216.37.0/24", + "88.216.196.0/24", + "89.34.171.0/24", + "91.132.227.0/24", + "91.199.47.0/24", + "91.199.168.0/24", + "91.213.173.0/24", + "91.226.97.0/24", + "94.103.177.0/24", + "102.129.208.0/24", + "103.212.82.0/24", + "104.234.197.0/24", + "104.254.195.0/24", + "140.99.220.0/24", + "140.228.22.0/24", + "141.11.121.0/24", + "146.19.85.0/24", + "146.247.106.0/24", + "152.89.248.0/24", + "154.16.56.0/24", + "154.16.88.0/24", + "154.127.63.0/24", + "157.254.220.0/24", + "173.211.93.0/24", + "174.140.207.0/24", + "176.57.48.0/23", + "176.57.58.0/24", + "176.105.224.0/23", + "176.105.229.0/24", + "176.223.181.0/24", + "178.253.215.0/24", + "179.61.130.0/24", + "181.214.63.0/24", + "181.214.169.0/24", + "181.214.241.0/24", + "181.215.37.0/24", + "185.11.140.0/23", + "185.106.194.0/23", + "185.108.206.0/23", + "185.139.3.0/24", + "185.139.239.0/24", + "185.147.55.0/24", + "185.158.105.0/24", + "185.170.42.0/24", + "185.189.33.0/24", + "185.232.207.0/24", + "188.209.138.0/24", + "188.211.249.0/24", + "188.244.113.0/24", + "190.93.76.0/24", + "191.96.34.0/24", + "191.96.109.0/24", + "191.96.211.0/24", + "191.101.93.0/24", + "191.101.122.0/24", + "193.30.102.0/24", + "193.109.196.0/24", + "193.178.52.0/24", + "194.4.54.0/24", + "194.5.66.0/24", + "194.5.149.0/24", + "194.113.227.0/24", + "199.48.176.0/24", + "202.43.7.0/24", + "203.17.123.0/24", + "206.53.60.0/24", + "206.197.211.0/24", + "209.145.43.0/24", + "212.46.35.0/24", + "212.46.42.0/24", + "213.139.76.0/24", + "216.107.130.0/24", + "216.247.98.0/24", + "217.28.128.0/24" + ], + "organization": { + "status": "ok", + "address1": "Lygumų g. 80-21", + "address2": "87142 Telšiai", + "city": "Telšiai", + "state": "Telšių r. sav.", + "zipcode": "87142", + "country": "LT", + "suite": "", + "floor": "", + "latitude": 55.976777, + "longitude": 22.269666, + "name": "Simoresta UAB", + "aka": "", + "name_long": "", + "website": "https://simoresta.lt", + "social_media": [ + { + "service": "website", + "identifier": "https://simoresta.lt" + } + ], + "notes": "" + }, + "info": { + "status": "ok", + "name": "Simoresta GL", + "aka": "", + "name_long": "", + "irr_as_set": "", + "website": "https://simoresta.lt", + "social_media": [ + { + "service": "website", + "identifier": "https://simoresta.lt" + } + ], + "looking_glass": "", + "route_server": "", + "notes": "", + "info_traffic": "10-20Gbps", + "info_ratio": "Balanced", + "info_scope": "Global", + "info_types": [], + "info_prefixes4": 0, + "info_prefixes6": 0, + "info_unicast": false, + "info_multicast": false, + "info_ipv6": false, + "info_never_via_route_servers": false, + "policy_url": "", + "policy_general": "Open", + "policy_locations": "Not Required", + "policy_ratio": false, + "policy_contracts": "Not Required", + "status_dashboard": "", + "rir_status": "ok", + "rir_status_updated": "2022-07-27 05:29:57" + }, + "contacts": {} + } +} diff --git a/example/json_small.json b/example/json_small.json new file mode 100644 index 0000000..3cae9d5 --- /dev/null +++ b/example/json_small.json @@ -0,0 +1,119 @@ +{ + "20773": { + "ipv4": [ + "5.35.224.0/19", + "5.175.0.0/19", + "37.61.192.0/19", + "62.138.0.0/18", + "62.138.128.0/18", + "79.170.40.0/24", + "79.170.42.0/24", + "79.170.44.0/24", + "80.237.128.0/23", + "80.237.130.0/24", + "80.237.135.0/24", + "80.237.136.0/24", + "80.237.142.0/24", + "80.237.194.0/23", + "80.237.212.0/24", + "80.237.214.0/23", + "80.237.248.0/23", + "80.237.251.0/24", + "80.237.252.0/22", + "80.246.48.0/20", + "83.169.32.0/20", + "85.119.152.0/21", + "85.195.75.0/24", + "85.195.76.0/24", + "85.195.82.0/23", + "85.195.85.0/24", + "85.195.86.0/24", + "85.195.88.0/22", + "85.195.92.0/23", + "85.195.95.0/24", + "85.195.96.0/24", + "85.195.98.0/23", + "85.195.100.0/24", + "85.195.102.0/23", + "85.195.107.0/24", + "85.195.109.0/24", + "85.195.110.0/24", + "85.195.114.0/23", + "85.195.116.0/24", + "85.195.119.0/24", + "85.195.120.0/24", + "85.195.124.0/24", + "85.195.127.0/24", + "87.230.64.0/24", + "87.230.110.0/23", + "87.230.112.0/21", + "88.80.192.0/19", + "91.109.2.0/23", + "91.250.64.0/21", + "92.42.111.0/24", + "92.51.170.0/24", + "92.51.180.0/22", + "92.204.162.0/24", + "92.204.171.0/24", + "92.204.186.0/24", + "92.204.189.0/24", + "92.204.191.0/24", + "92.204.194.0/23", + "92.204.240.0/24", + "93.92.144.0/21", + "94.136.40.0/24", + "94.199.240.0/21", + "134.119.182.0/24", + "134.119.207.0/24", + "146.0.226.0/23", + "146.0.228.0/24", + "146.0.230.0/24", + "146.0.232.0/24", + "146.0.234.0/23", + "146.0.236.0/23", + "146.0.245.0/24", + "146.0.247.0/24", + "146.0.249.0/24", + "146.0.250.0/23", + "146.0.255.0/24", + "151.106.31.0/24", + "151.106.57.0/24", + "160.153.12.0/22", + "160.153.24.0/22", + "160.153.128.0/17", + "176.28.0.0/22", + "176.28.4.0/24", + "178.77.104.0/21", + "178.77.112.0/21", + "185.19.216.0/22", + "185.34.212.0/22", + "185.85.50.0/23", + "185.136.158.0/24", + "185.136.160.0/24", + "188.121.32.0/19", + "195.225.104.0/22", + "217.115.128.0/21", + "217.115.143.0/24", + "217.115.145.0/24", + "217.115.146.0/23", + "217.115.152.0/21", + "217.172.162.0/24" + ], + "organization": { + "name": "Host Europe GmbH", + "name_long": "", + "country": "DE", + "state": "", + "city": "Cologne", + "latitude": 50.940693, + "longitude": 6.939616, + "website": "http://www.heg.com" + }, + "info": { + "name": "Host Europe GmbH", + "name_long": "", + "website": "http://www.hosteurope.de/" + }, + "contacts": {} + } +} \ No newline at end of file diff --git a/example/full.json b/example/mmdb_full.json similarity index 100% rename from example/full.json rename to example/mmdb_full.json diff --git a/example/small.json b/example/mmdb_small.json similarity index 100% rename from example/small.json rename to example/mmdb_small.json