fast network membership check

The rendering code checks whether endpoint IPs are part of
cluster-local networks. Due to the prevalence of endpoints - medium
sized reports can contain many thousands of endpoints - this is
performance critical. Alas the existing code performs the check via a
linear scan of a list of networks. That is slow when there are more
than a few. Unfortunately in some common k8s network setups, e.g. on
AWS, a cluster can contain hundreds of networks, due to /32 networks
derived from interfaces with multiple IPs.

Here we change representation of the set of networks to a prefix
tree (aka trie), which is well-suited for IP network membership checks
since networks are in fact a bitstring prefixes.

The specific representation is a crit-bit tree, but that choice was
purely based on implementation convenience - the chosen library is the
only one I could find that directly supports IP networks.

render/container_test.go

@@ -2,7 +2,6 @@ package render_test
 
 import (
 	"fmt"
-	"net"
 	"testing"
 
 	"github.com/weaveworks/common/test"
@@ -46,11 +45,10 @@ type testcase struct {
 }
 
 func testMap(t *testing.T, f render.MapFunc, input testcase) {
-	_, ipNet, err := net.ParseCIDR("1.2.3.0/16")
-	if err != nil {
+	localNetworks := report.NewNetworks()
+	if err := localNetworks.AddCIDR("1.2.3.0/16"); err != nil {
 		t.Fatalf(err.Error())
 	}
-	localNetworks := report.Networks([]*net.IPNet{ipNet})
 	if have := f(input.n, localNetworks); input.ok != (len(have) > 0) {
 		name := input.name
 		if name == "" {

render/theinternet.go

@@ -1,7 +1,6 @@
 package render
 
 import (
-	"net"
 	"regexp"
 	"strings"
 
@@ -68,26 +67,15 @@ func isKnownService(hostname string) bool {
 // used to determine which nodes in the report are "remote", i.e. outside of
 // our infrastructure.
 func LocalNetworks(r report.Report) report.Networks {
-	var (
-		result   = report.Networks{}
-		networks = map[string]struct{}{}
-	)
+	networks := report.NewNetworks()
 
 	for _, topology := range []report.Topology{r.Host, r.Overlay} {
 		for _, md := range topology.Nodes {
 			nets, _ := md.Sets.Lookup(host.LocalNetworks)
 			for _, s := range nets {
-				_, ipNet, err := net.ParseCIDR(s)
-				if err != nil {
-					continue
-				}
-				_, ok := networks[ipNet.String()]
-				if !ok {
-					result = append(result, ipNet)
-					networks[ipNet.String()] = struct{}{}
-				}
+				networks.AddCIDR(s)
 			}
 		}
 	}
-	return result
+	return networks
 }

render/theinternet_test.go

@@ -1,7 +1,6 @@
 package render_test
 
 import (
-	"net"
 	"reflect"
 	"testing"
 
@@ -30,21 +29,14 @@ func TestReportLocalNetworks(t *testing.T) {
 			},
 		},
 	})
-	want := report.Networks([]*net.IPNet{
-		mustParseCIDR("10.0.0.1/8"),
-		mustParseCIDR("192.168.1.1/24"),
-		mustParseCIDR("10.32.0.1/12"),
-	})
+	want := report.NewNetworks()
+	for _, cidr := range []string{"10.0.0.1/8", "192.168.1.1/24", "10.32.0.1/12"} {
+		if err := want.AddCIDR(cidr); err != nil {
+			panic(err)
+		}
+	}
 	have := render.LocalNetworks(r)
 	if !reflect.DeepEqual(want, have) {
 		t.Errorf("%s", test.Diff(want, have))
 	}
 }
-
-func mustParseCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
-	if err != nil {
-		panic(err)
-	}
-	return ipNet
-}