how to shrink consecutive B ip classes in bigger one - ip

Question

After reading List of IP Space used by Facebook:
"Real" list is the last answer, but I wonder how Igy (with the answer marked as solution) managed to shrink the list a lot by adding consecutive classes into a bigger one (by decreasing accordingly from network mask for each new consecutive network), is there a tool, or only manually ?
This is a HUGE improvement for firewall, where number of rules counts (the shorter the better).

Answer 1

A simple solution is to use netaddr:
import netaddr
ips = netaddr.IPSet()
for addr in all_addrs:
ips.add(addr)
ips.compact()
for cidr in ips.iter_cidrs():
print(str(cidr))

Answer 2

Following Python 3 script can do what you want:
#!/usr/bin/python3
# -*- coding: utf-8 -*-
from functools import reduce
import sys
def add_mark(regions, mark, k):
r = regions[:]
i = 0
j = len(r)
while i < j:
m = (i + j) // 2
if mark < r[m][0]:
j = m
elif r[m][0] < mark:
i = m + 1
else:
r[m][1] += k
if r[m][1] == 0:
del r[m]
return r
r.insert(i, [mark, k])
return r
def add_region(regions, start, end):
return add_mark(add_mark(regions, start, 1), end, -1)
def parse_network(n):
pos = n.find('/')
return ip_to_int(n[:pos]), 2**(32 - int(n[pos+1:]))
def ip_to_int(ip):
return reduce(lambda a, b: 256*a + b, map(int, ip.split('.')))
def print_summary(r):
if len(r) == 0:
return
start = None
level = 0
for item in r:
level += item[1]
if start is None:
start = item[0]
elif level == 0:
summarize_networks(start, item[0])
start = None
def summarize_networks(start, end):
while start < end:
mask = 32
amount = 1
while start % amount == 0 and start + amount - 1< end:
mask -= 1
amount *= 2
mask += 1
amount //= 2
print('{0}/{1}'.format(int_to_ip(start), mask))
start += amount
def int_to_ip(n):
n, o4 = divmod(n, 256)
n, o3 = divmod(n, 256)
o1, o2 = divmod(n, 256)
return '.'.join(map(str, [o1, o2, o3, o4]))
def main():
regions = []
while True:
line = sys.stdin.readline()
if len(line) == 0:
break
for item in line.strip().split():
start, amount = parse_network(item)
regions = add_region(regions, start, start + amount)
print_summary(regions)
if __name__ == "__main__":
main()
Example:
./unique_networks.py <<EOF
192.168.0.0/24
192.168.0.128/25
192.168.0.248/30
192.168.1.0/24
192.168.100.0/24
EOF
192.168.0.0/23
192.168.100.0/24
For a facebook's list of
204.15.20.0/22
69.63.176.0/20
66.220.144.0/20
66.220.144.0/21
69.63.184.0/21
69.63.176.0/21
74.119.76.0/22
69.171.255.0/24
173.252.64.0/18
69.171.224.0/19
69.171.224.0/20
103.4.96.0/22
69.63.176.0/24
173.252.64.0/19
173.252.70.0/24
31.13.64.0/18
31.13.24.0/21
66.220.152.0/21
66.220.159.0/24
69.171.239.0/24
69.171.240.0/20
31.13.64.0/19
31.13.64.0/24
31.13.65.0/24
31.13.67.0/24
31.13.68.0/24
31.13.69.0/24
31.13.70.0/24
31.13.71.0/24
31.13.72.0/24
31.13.73.0/24
31.13.74.0/24
31.13.75.0/24
31.13.76.0/24
31.13.77.0/24
31.13.96.0/19
31.13.66.0/24
173.252.96.0/19
69.63.178.0/24
31.13.78.0/24
31.13.79.0/24
31.13.80.0/24
31.13.82.0/24
31.13.83.0/24
31.13.84.0/24
31.13.85.0/24
31.13.86.0/24
31.13.87.0/24
31.13.88.0/24
31.13.89.0/24
31.13.90.0/24
31.13.91.0/24
31.13.92.0/24
31.13.93.0/24
31.13.94.0/24
31.13.95.0/24
69.171.253.0/24
69.63.186.0/24
31.13.81.0/24
179.60.192.0/22
179.60.192.0/24
179.60.193.0/24
179.60.194.0/24
179.60.195.0/24
185.60.216.0/22
45.64.40.0/22
185.60.216.0/24
185.60.217.0/24
185.60.218.0/24
185.60.219.0/24
129.134.0.0/16
157.240.0.0/16
204.15.20.0/22
69.63.176.0/20
69.63.176.0/21
69.63.184.0/21
66.220.144.0/20
69.63.176.0/20
networks this script summarizes them to:
31.13.24.0/21
31.13.64.0/18
45.64.40.0/22
66.220.144.0/20
69.63.176.0/20
69.171.224.0/19
74.119.76.0/22
103.4.96.0/22
129.134.0.0/16
157.240.0.0/16
173.252.64.0/18
179.60.192.0/22
185.60.216.0/22
204.15.20.0/22

Answer 3

With the help of sds' answer (netaddr is beautiful, it even sorts the output) I came up with the following to convert facebook IP ranges to ipset:
ipset create facebook4 hash:net comment
whois -h whois.radb.net -- '-i origin AS32934' | awk '/^route:/ {print $2}' | ./netaddr-compact.py | sed 's/^/ipset add facebook4 /' | sh -x
ipset create facebook6 hash:net family inet6 comment
whois -h whois.radb.net -- '-i origin AS32934' | awk '/^route6:/ {print $2}' | ./netaddr-compact.py | sed 's/^/ipset add facebook6 /' | sh -x
ipset create facebook list:set comment
ipset add facebook facebook4
ipset add facebook facebook6
The netaddr-compact.py file is simple:
#!/usr/bin/env python3
import netaddr # on ubuntu: apt install python3-netaddr
import fileinput
ips = netaddr.IPSet()
for addr in fileinput.input():
ips.add(addr)
ips.compact()
for cidr in ips.iter_cidrs():
print(str(cidr))