From 5d12fc464f821f5bedaf612c1cc81e2ad4d0237e Mon Sep 17 00:00:00 2001 From: David Tran Date: Tue, 5 Dec 2023 12:16:10 -0800 Subject: [PATCH 1/4] Fix ipv4 to asn recipe to use donwloaded pfx2as file --- .../recipe/map_ipv4_address_to_asn/README.md | 44 ++++++++++++++----- .../ip_asn_pyipmeta.py | 15 ++++--- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/sources/recipe/map_ipv4_address_to_asn/README.md b/sources/recipe/map_ipv4_address_to_asn/README.md index bb6813b8..64998dff 100644 --- a/sources/recipe/map_ipv4_address_to_asn/README.md +++ b/sources/recipe/map_ipv4_address_to_asn/README.md @@ -45,38 +45,53 @@ Detailed installation and usage instructions [here]( https://github.com/CAIDA/py The following script returns a dictionary `ip2asn` that maps ips to origin asns. -### Map between ips and origin asns using PyIPMeta +### Map between IPs and origin ASNs using PyIPMeta -For this solution, clone **PyIPMeta** from [here]( https://github.com/CAIDA/pyipmeta) -More data can be found at http://data.caida.org/datasets/routing/routeviews-prefix2as/ +For this solution, clone **PyIPMeta** from [here]( https://github.com/CAIDA/pyipmeta). -Sample ips.txt found [here]( http://data.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz) +Download a prefix2asn file by following the directions [here](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-f` flag. -**Usage** : `$ python3 ip_asn.py -i ips.txt` +Sample lists of IPs found [here]( http://data.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz). + +**Usage** : `$ python3 ip_asn_pyipmeta.py -f -i ` ~~~python import _pyipmeta import argparse +import datetime +import os +import psutil -parser = argparse.ArgumentParser() -parser.add_argument('-i', dest = 'ips_file', default = '', help = 'Please enter the file name of the ips file') -args = parser.parse_args() +def returnTime(): + return datetime.datetime.now() +def returnMemUsage(): + process = psutil.Process(os.getpid()) + return process.memory_info()[0] + ipm = _pyipmeta.IpMeta() +# print(ipm) + +parser = argparse.ArgumentParser() +parser.add_argument('-p', dest = 'prefix2asn_file', default = '', help = 'Please enter the prefix2asn file name') +parser.add_argument('-i', dest = 'ips_file', default = '', help = 'Please enter the file name of the ips file') +args = parser.parse_args() # print("Getting/enabling pfx2as provider (using included test data)") prov = ipm.get_provider_by_name("pfx2as") -print(ipm.enable_provider(prov, "-f http://data.caida.org/datasets/routing/routeviews-prefix2as/2017/03/routeviews-rv2-20170329-0200.pfx2as.gz")) +print(ipm.enable_provider(prov, f"-f {args.prefix2asn_file}")) print() -# Create list of ips from test file +# Create list of ips from test file ips = [] with open(args.ips_file) as f: for line in f: line = line.rstrip().split("\t")[1] ips.append(line) +begin_time = returnTime() +begin_mem = returnMemUsage() # Map between ipv4 addresses and origin asns ip2asn = {} @@ -84,10 +99,15 @@ for ip in ips: if ipm.lookup(ip): (res,) = ipm.lookup(ip) if res.get('asns'): - ip2asn[ip] = res.get('asns')[-1] - + ip2asn[ip] = res.get('asns') # print(ip2asn) +end_time = returnTime() +end_mem = returnMemUsage() + +# hour:minute:second:microsecond +print("Delta time:" , end_time - begin_time) +print("Delta memory:", end_mem - begin_mem) ~~~ ### Background diff --git a/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py b/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py index 16210f7a..84500e35 100644 --- a/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py +++ b/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py @@ -45,6 +45,7 @@ #!/usr/bin/env python import _pyipmeta +import argparse import datetime import os import psutil @@ -60,15 +61,19 @@ def returnMemUsage(): ipm = _pyipmeta.IpMeta() # print(ipm) +parser = argparse.ArgumentParser() +parser.add_argument('-p', dest = 'prefix2asn_file', default = '', help = 'Please enter the prefix2asn file name') +parser.add_argument('-i', dest = 'ips_file', default = '', help = 'Please enter the file name of the ips file') +args = parser.parse_args() + # print("Getting/enabling pfx2as provider (using included test data)") prov = ipm.get_provider_by_name("pfx2as") -# print(prov) -print(ipm.enable_provider(prov, "-f /test/pfx2as/routeviews-rv2-20170329-0200.pfx2as.gz")) +print(ipm.enable_provider(prov, f"-f {args.prefix2asn_file}")) print() - +# Create list of ips from test file ips = [] -with open('ips.txt') as f: +with open(args.ips_file) as f: for line in f: line = line.rstrip().split("\t")[1] ips.append(line) @@ -76,6 +81,7 @@ def returnMemUsage(): begin_time = returnTime() begin_mem = returnMemUsage() +# Map between ipv4 addresses and origin asns ip2asn = {} for ip in ips: if ipm.lookup(ip): @@ -83,7 +89,6 @@ def returnMemUsage(): if res.get('asns'): ip2asn[ip] = res.get('asns') - # print(ip2asn) end_time = returnTime() end_mem = returnMemUsage() From 724baf1e139d0434b3afaf0e04fff8ccb9560ad4 Mon Sep 17 00:00:00 2001 From: richmass1 <103288353+richmass1@users.noreply.github.com> Date: Fri, 15 Dec 2023 21:59:47 -0800 Subject: [PATCH 2/4] Correction to README and make script print mapping --- sources/recipe/map_ipv4_address_to_asn/README.md | 6 +++--- sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sources/recipe/map_ipv4_address_to_asn/README.md b/sources/recipe/map_ipv4_address_to_asn/README.md index 64998dff..09fa8b9d 100644 --- a/sources/recipe/map_ipv4_address_to_asn/README.md +++ b/sources/recipe/map_ipv4_address_to_asn/README.md @@ -49,11 +49,11 @@ The following script returns a dictionary `ip2asn` that maps ips to origin asns. For this solution, clone **PyIPMeta** from [here]( https://github.com/CAIDA/pyipmeta). -Download a prefix2asn file by following the directions [here](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-f` flag. +Download a prefix2asn file by following the directions [here](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-p` flag. Sample lists of IPs found [here]( http://data.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz). -**Usage** : `$ python3 ip_asn_pyipmeta.py -f -i ` +**Usage** : `$ python3 ip_asn_pyipmeta.py -p -i ` ~~~python import _pyipmeta @@ -101,7 +101,7 @@ for ip in ips: if res.get('asns'): ip2asn[ip] = res.get('asns') -# print(ip2asn) +print(ip2asn) end_time = returnTime() end_mem = returnMemUsage() diff --git a/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py b/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py index 84500e35..9d106aa0 100644 --- a/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py +++ b/sources/recipe/map_ipv4_address_to_asn/ip_asn_pyipmeta.py @@ -89,7 +89,7 @@ def returnMemUsage(): if res.get('asns'): ip2asn[ip] = res.get('asns') -# print(ip2asn) +print(ip2asn) end_time = returnTime() end_mem = returnMemUsage() From 03510f0cd38adc36a64076bc4799f5f12c17fbca Mon Sep 17 00:00:00 2001 From: richmass1 <103288353+richmass1@users.noreply.github.com> Date: Sat, 16 Dec 2023 09:51:22 -0800 Subject: [PATCH 3/4] Change sample IP list link in README --- sources/recipe/map_ipv4_address_to_asn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/recipe/map_ipv4_address_to_asn/README.md b/sources/recipe/map_ipv4_address_to_asn/README.md index 09fa8b9d..81704a41 100644 --- a/sources/recipe/map_ipv4_address_to_asn/README.md +++ b/sources/recipe/map_ipv4_address_to_asn/README.md @@ -51,7 +51,7 @@ For this solution, clone **PyIPMeta** from [here]( https://github.com/CAIDA/pyip Download a prefix2asn file by following the directions [here](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-p` flag. -Sample lists of IPs found [here]( http://data.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz). +Sample lists of IPs found [here]( http://publicdata.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz). **Usage** : `$ python3 ip_asn_pyipmeta.py -p -i ` From a5a2f40c919e53fd709ac399e6a19a208139af74 Mon Sep 17 00:00:00 2001 From: richmass1 <103288353+richmass1@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:33:55 -0800 Subject: [PATCH 4/4] Changing "here" links Changing the text of links that previously said "here", as requested by a comment on #726. I also removed some links that had sparse information or were no longer active. --- .../recipe/map_ipv4_address_to_asn/README.md | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/sources/recipe/map_ipv4_address_to_asn/README.md b/sources/recipe/map_ipv4_address_to_asn/README.md index 81704a41..ae4d1ec0 100644 --- a/sources/recipe/map_ipv4_address_to_asn/README.md +++ b/sources/recipe/map_ipv4_address_to_asn/README.md @@ -39,7 +39,7 @@ Before installing PyIPMeta, you will need: - Python setuptools (`sudo apt install python-setuptools` on Ubuntu) - Python development headers (`sudo apt install python-dev` on Ubuntu) -Detailed installation and usage instructions [here]( https://github.com/CAIDA/pyipmeta ). +Detailed installation and usage instructions on the [PyIPMeta repo]( https://github.com/CAIDA/pyipmeta ). # solution # @@ -47,11 +47,11 @@ The following script returns a dictionary `ip2asn` that maps ips to origin asns. ### Map between IPs and origin ASNs using PyIPMeta -For this solution, clone **PyIPMeta** from [here]( https://github.com/CAIDA/pyipmeta). +For this solution, clone **PyIPMeta** from the [PyIPMeta repo]( https://github.com/CAIDA/pyipmeta). -Download a prefix2asn file by following the directions [here](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-p` flag. +Download a prefix2asn file by following the directions on the [Routeviews Prefix-to-AS Mappings dataset page](https://catalog.caida.org/dataset/routeviews_prefix2as). When using the script, pass in the file name after the `-p` flag. -Sample lists of IPs found [here]( http://publicdata.caida.org/datasets/topology/ark/ipv4/dns-names/2019/05/dns-names.l7.20190501.txt.gz). +This script takes a list of IP addresses in the format used by CAIDA's [DNS Names dataset](https://catalog.caida.org/dataset/ark_ipv4_dns_names). **Usage** : `$ python3 ip_asn_pyipmeta.py -p -i ` @@ -118,8 +118,7 @@ print("Delta memory:", end_mem - begin_mem) - An *IPv4 address prefix* is the prefix of an IPv4 address. - e.g. Consider the IPV4 address : 182.24.0.0/18 - In this case, 18 is the length of the prefix. -- The prefix is the first 18 bits of the IP address. -- More information on IPv4 addresses can be found [here]( https://docs.oracle.com/cd/E19455-01/806-0916/6ja85399u/index.html#:~:text=The%20IPv4%20address%20is%20a,bit%20fields%20separated%20by%20periods ) +- The prefix is the first 18 bits of the IP address. **What is forwarding/How does forwarding work?** - Fowarding means sending incoming information packets to the appropriate destination interface. This is done by routers with the help of a forwarding table. @@ -137,15 +136,14 @@ It finds the prefix matching the given IP address and returns the corresponding | 192.168.0.0/16 | B | - For example, for the given incoming IP address: 192.168.20.19 -- **Node A** is selected as the destination router node as it contains the *longer matching prefix* i.e. 192.168.20.16 -- Source: [link]( https://www.lewuathe.com/longest-prefix-match-with-trie-tree.html ) -- More information can be found [here]( https://www.geeksforgeeks.org/longest-prefix-matching-in-routers/ ) +- **Node A** is selected as the destination router node as it contains the *longer matching prefix* i.e. 192.168.20.16 +- More information, from [GeeksforGeeks](https://www.geeksforgeeks.org/longest-prefix-matching-in-routers/ ) **What is an AS?** - AS stands for Autonomous system. - It can be broadly be thought of as a single organization, or a collection of routers that route groups of IP addresses under a common administration, typically a large organization or an ISP (Internet Service Provider). - It is a connected group of one or more IP addresses (known as IP prefixes) that provide a common way to route internet traffic to systems outside the AS. -- More information on AS can be found [here]( https://www.cs.rutgers.edu/~pxk/352/notes/autonomous_systems.html) and [here](https://catalog.caida.org/details/media/2016_as_intro_topology_windas_intro_topology_wind.pdf) +- More information, from [Rutgers University](https://www.cs.rutgers.edu/~pxk/352/notes/autonomous_systems.html) ### Caveats - **Multi-origin AS** : Some prefixes originate from multiple AS's (which could be siblings or distinct organizations). @@ -158,7 +156,7 @@ This makes it more challenging to interpret the appearance of a matching destina - `pyasn` can also be used for mapping between ipv4 addresses and origin asns. - The `pyasn` object is be initialized using an IPASN datafile. - It also provides extremely fast lookups for IP addresses, as it returns the origin asns and the BGP prefixes it matches. -- Detailed installation instructions and more information on Usage and IPASN data files [found here]( https://github.com/hadiasghari/pyasn ). +- Detailed installation instructions, more information on usage, and IPASN data files can be found on the [PyASN GitHub repo](https://github.com/hadiasghari/pyasn ). - Note that the current `pyipmeta` **does not support** `ipv6`, whereas `pyasn` does. However, `pyipmeyta` provides **greater flexbility** as it provides the geographical information as well.