Merge pull request #2625 from weaveworks/fast-network-membership-check

fast network membership check
This commit is contained in:
Matthias Radestock
2017-06-21 12:45:12 +01:00
committed by GitHub
10 changed files with 747 additions and 55 deletions

View File

@@ -2,7 +2,6 @@ package render_test
import (
"fmt"
"net"
"testing"
"github.com/weaveworks/common/test"
@@ -46,11 +45,10 @@ type testcase struct {
}
func testMap(t *testing.T, f render.MapFunc, input testcase) {
_, ipNet, err := net.ParseCIDR("1.2.3.0/16")
if err != nil {
localNetworks := report.NewNetworks()
if err := localNetworks.AddCIDR("1.2.3.0/16"); err != nil {
t.Fatalf(err.Error())
}
localNetworks := report.Networks([]*net.IPNet{ipNet})
if have := f(input.n, localNetworks); input.ok != (len(have) > 0) {
name := input.name
if name == "" {

View File

@@ -1,7 +1,6 @@
package render
import (
"net"
"regexp"
"strings"
@@ -68,26 +67,15 @@ func isKnownService(hostname string) bool {
// used to determine which nodes in the report are "remote", i.e. outside of
// our infrastructure.
func LocalNetworks(r report.Report) report.Networks {
var (
result = report.Networks{}
networks = map[string]struct{}{}
)
networks := report.NewNetworks()
for _, topology := range []report.Topology{r.Host, r.Overlay} {
for _, md := range topology.Nodes {
nets, _ := md.Sets.Lookup(host.LocalNetworks)
for _, s := range nets {
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
continue
}
_, ok := networks[ipNet.String()]
if !ok {
result = append(result, ipNet)
networks[ipNet.String()] = struct{}{}
}
networks.AddCIDR(s)
}
}
}
return result
return networks
}

View File

@@ -1,7 +1,6 @@
package render_test
import (
"net"
"reflect"
"testing"
@@ -30,21 +29,14 @@ func TestReportLocalNetworks(t *testing.T) {
},
},
})
want := report.Networks([]*net.IPNet{
mustParseCIDR("10.0.0.1/8"),
mustParseCIDR("192.168.1.1/24"),
mustParseCIDR("10.32.0.1/12"),
})
want := report.NewNetworks()
for _, cidr := range []string{"10.0.0.1/8", "192.168.1.1/24", "10.32.0.1/12"} {
if err := want.AddCIDR(cidr); err != nil {
panic(err)
}
}
have := render.LocalNetworks(r)
if !reflect.DeepEqual(want, have) {
t.Errorf("%s", test.Diff(want, have))
}
}
func mustParseCIDR(s string) *net.IPNet {
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
panic(err)
}
return ipNet
}

View File

@@ -3,25 +3,38 @@ package report
import (
"net"
"strings"
"github.com/k-sone/critbitgo"
)
// Networks represent a set of subnets
type Networks []*net.IPNet
type Networks struct{ *critbitgo.Net }
// LocalNetworks helps in determining which addresses a probe reports
// as being host-scoped.
//
// TODO this design is broken, make it consistent with probe networks.
var LocalNetworks = Networks{}
var LocalNetworks = NewNetworks()
// NewNetworks creates a datastructure representing a set of networks.
func NewNetworks() Networks {
return Networks{critbitgo.NewNet()}
}
// Add adds a network.
func (n Networks) Add(ipnet *net.IPNet) error {
return n.Net.Add(ipnet, struct{}{})
}
// AddCIDR adds a network, represented as CIDR.
func (n Networks) AddCIDR(cidr string) error {
return n.Net.AddCIDR(cidr, struct{}{})
}
// Contains returns true if IP is in Networks.
func (n Networks) Contains(ip net.IP) bool {
for _, net := range n {
if net.Contains(ip) {
return true
}
}
return false
network, _, _ := n.MatchIP(ip)
return network != nil
}
// LocalAddresses returns a list of the local IP addresses.
@@ -67,7 +80,9 @@ func AddLocalBridge(name string) error {
return err
}
LocalNetworks = ipv4Nets(addrs)
for _, ipnet := range ipv4Nets(addrs) {
LocalNetworks.Add(ipnet)
}
return nil
}
@@ -82,7 +97,7 @@ func GetLocalNetworks() ([]*net.IPNet, error) {
}
func ipv4Nets(addrs []net.Addr) []*net.IPNet {
nets := Networks{}
nets := []*net.IPNet{}
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.To4() != nil {
nets = append(nets, ipnet)

View File

@@ -8,10 +8,12 @@ import (
)
func TestContains(t *testing.T) {
networks := report.Networks([]*net.IPNet{
mustParseCIDR("10.0.0.1/8"),
mustParseCIDR("192.168.1.1/24"),
})
networks := report.NewNetworks()
for _, cidr := range []string{"10.0.0.1/8", "192.168.1.1/24"} {
if err := networks.AddCIDR(cidr); err != nil {
panic(err)
}
}
if networks.Contains(net.ParseIP("52.52.52.52")) {
t.Errorf("52.52.52.52 not in %v", networks)
@@ -21,11 +23,3 @@ func TestContains(t *testing.T) {
t.Errorf("10.0.0.1 in %v", networks)
}
}
func mustParseCIDR(s string) *net.IPNet {
_, ipNet, err := net.ParseCIDR(s)
if err != nil {
panic(err)
}
return ipNet
}

22
vendor/github.com/k-sone/critbitgo/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Keita Sone
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

390
vendor/github.com/k-sone/critbitgo/critbit.go generated vendored Normal file
View File

@@ -0,0 +1,390 @@
package critbitgo
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"os"
"strconv"
)
// The matrix of most significant bit
var msbMatrix [256]byte
func buildMsbMatrix() {
for i := 0; i < len(msbMatrix); i++ {
b := byte(i)
b |= b >> 1
b |= b >> 2
b |= b >> 4
msbMatrix[i] = b &^ (b >> 1)
}
}
type node struct {
internal *internal
external *external
}
type internal struct {
child [2]node
offset int
bit byte
cont bool // if true, key of child[1] contains key of child[0]
}
type external struct {
key []byte
value interface{}
}
// finding the critical bit.
func (n *external) criticalBit(key []byte) (offset int, bit byte, cont bool) {
nlen := len(n.key)
klen := len(key)
mlen := nlen
if nlen > klen {
mlen = klen
}
// find first differing byte and bit
for offset = 0; offset < mlen; offset++ {
if a, b := key[offset], n.key[offset]; a != b {
bit = msbMatrix[a^b]
return
}
}
if nlen < klen {
bit = msbMatrix[key[offset]]
} else if nlen > klen {
bit = msbMatrix[n.key[offset]]
} else {
// two keys are equal
offset = -1
}
return offset, bit, true
}
// calculate direction.
func (n *internal) direction(key []byte) int {
if n.offset < len(key) && (key[n.offset]&n.bit != 0 || n.cont) {
return 1
}
return 0
}
// Crit-bit Tree
type Trie struct {
root node
size int
}
// searching the tree.
func (t *Trie) search(key []byte) *node {
n := &t.root
for n.internal != nil {
n = &n.internal.child[n.internal.direction(key)]
}
return n
}
// membership testing.
func (t *Trie) Contains(key []byte) bool {
if n := t.search(key); n.external != nil && bytes.Equal(n.external.key, key) {
return true
}
return false
}
// get member.
// if `key` is in Trie, `ok` is true.
func (t *Trie) Get(key []byte) (value interface{}, ok bool) {
if n := t.search(key); n.external != nil && bytes.Equal(n.external.key, key) {
return n.external.value, true
}
return
}
// insert into the tree (replaceable).
func (t *Trie) insert(key []byte, value interface{}, replace bool) bool {
// an empty tree
if t.size == 0 {
t.root.external = &external{
key: key,
value: value,
}
t.size = 1
return true
}
n := t.search(key)
newOffset, newBit, newCont := n.external.criticalBit(key)
// already exists in the tree
if newOffset == -1 {
if replace {
n.external.value = value
return true
}
return false
}
// allocate new node
newNode := &internal{
offset: newOffset,
bit: newBit,
cont: newCont,
}
direction := newNode.direction(key)
newNode.child[direction].external = &external{
key: key,
value: value,
}
// insert new node
wherep := &t.root
for in := wherep.internal; in != nil; in = wherep.internal {
if in.offset > newOffset || (in.offset == newOffset && in.bit < newBit) {
break
}
wherep = &in.child[in.direction(key)]
}
if wherep.internal != nil {
newNode.child[1-direction].internal = wherep.internal
} else {
newNode.child[1-direction].external = wherep.external
wherep.external = nil
}
wherep.internal = newNode
t.size += 1
return true
}
// insert into the tree.
// if `key` is alredy in Trie, return false.
func (t *Trie) Insert(key []byte, value interface{}) bool {
return t.insert(key, value, false)
}
// set into the tree.
func (t *Trie) Set(key []byte, value interface{}) {
t.insert(key, value, true)
}
// deleting elements.
// if `key` is in Trie, `ok` is true.
func (t *Trie) Delete(key []byte) (value interface{}, ok bool) {
// an empty tree
if t.size == 0 {
return
}
var direction int
var whereq *node // pointer to the grandparent
var wherep *node = &t.root
// finding the best candidate to delete
for in := wherep.internal; in != nil; in = wherep.internal {
direction = in.direction(key)
whereq = wherep
wherep = &in.child[direction]
}
// checking that we have the right element
if !bytes.Equal(wherep.external.key, key) {
return
}
value = wherep.external.value
ok = true
// removing the node
if whereq == nil {
wherep.external = nil
} else {
othern := whereq.internal.child[1-direction]
whereq.internal = othern.internal
whereq.external = othern.external
}
t.size -= 1
return
}
// clearing a tree.
func (t *Trie) Clear() {
t.root.internal = nil
t.root.external = nil
t.size = 0
}
// return the number of key in a tree.
func (t *Trie) Size() int {
return t.size
}
// fetching elements with a given prefix.
// handle is called with arguments key and value (if handle returns `false`, the iteration is aborted)
func (t *Trie) Allprefixed(prefix []byte, handle func(key []byte, value interface{}) bool) bool {
// an empty tree
if t.size == 0 {
return true
}
// walk tree, maintaining top pointer
p := &t.root
top := p
if len(prefix) > 0 {
for q := p.internal; q != nil; q = p.internal {
p = &q.child[q.direction(prefix)]
if q.offset < len(prefix) {
top = p
}
}
// check prefix
if !bytes.Contains(p.external.key, prefix) {
return true
}
}
return allprefixed(top, handle)
}
func allprefixed(n *node, handle func([]byte, interface{}) bool) bool {
if n.internal != nil {
// dealing with an internal node while recursing
for i := 0; i < 2; i++ {
if !allprefixed(&n.internal.child[i], handle) {
return false
}
}
} else {
// dealing with an external node while recursing
return handle(n.external.key, n.external.value)
}
return true
}
// Search for the longest matching key from the beginning of the given key.
// if `key` is in Trie, `ok` is true.
func (t *Trie) LongestPrefix(given []byte) (key []byte, value interface{}, ok bool) {
// an empty tree
if t.size == 0 {
return
}
return longestPrefix(&t.root, given)
}
func longestPrefix(n *node, key []byte) ([]byte, interface{}, bool) {
if n.internal != nil {
direction := n.internal.direction(key)
if k, v, ok := longestPrefix(&n.internal.child[direction], key); ok {
return k, v, ok
}
if direction == 1 {
return longestPrefix(&n.internal.child[0], key)
}
} else {
if bytes.HasPrefix(key, n.external.key) {
return n.external.key, n.external.value, true
}
}
return nil, nil, false
}
// Iterating elements from a given start key.
// handle is called with arguments key and value (if handle returns `false`, the iteration is aborted)
func (t *Trie) Walk(start []byte, handle func(key []byte, value interface{}) bool) bool {
var seek bool
if start != nil {
seek = true
}
return walk(&t.root, start, &seek, handle)
}
func walk(n *node, key []byte, seek *bool, handle func([]byte, interface{}) bool) bool {
if n.internal != nil {
var direction int
if *seek {
direction = n.internal.direction(key)
}
if !walk(&n.internal.child[direction], key, seek, handle) {
return false
}
if !(*seek) && direction == 0 {
// iteration another side
return walk(&n.internal.child[1], key, seek, handle)
}
return true
} else {
if *seek {
if bytes.Equal(n.external.key, key) {
// seek completed
*seek = false
} else {
// key is not in Trie
return false
}
}
return handle(n.external.key, n.external.value)
}
}
// dump tree. (for debugging)
func (t *Trie) Dump(w io.Writer) {
if t.root.internal == nil && t.root.external == nil {
return
}
if w == nil {
w = os.Stdout
}
dump(w, &t.root, true, "")
}
func dump(w io.Writer, n *node, right bool, prefix string) {
var ownprefix string
if right {
ownprefix = prefix
} else {
ownprefix = prefix[:len(prefix)-1] + "`"
}
if in := n.internal; in != nil {
fmt.Fprintf(w, "%s-- off=%d, bit=%08b(%02x), cont=%v\n", ownprefix, in.offset, in.bit, in.bit, in.cont)
for i := 0; i < 2; i++ {
var nextprefix string
switch i {
case 0:
nextprefix = prefix + " |"
right = true
case 1:
nextprefix = prefix + " "
right = false
}
dump(w, &in.child[i], right, nextprefix)
}
} else {
fmt.Fprintf(w, "%s-- key=%d (%s)\n", ownprefix, n.external.key, key2str(n.external.key))
}
return
}
func key2str(key []byte) string {
for _, c := range key {
if !strconv.IsPrint(rune(c)) {
return hex.EncodeToString(key)
}
}
return string(key)
}
// create a tree.
func NewTrie() *Trie {
return &Trie{}
}
func init() {
buildMsbMatrix()
}

57
vendor/github.com/k-sone/critbitgo/map.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
package critbitgo
import (
"unsafe"
)
// The map is sorted according to the natural ordering of its keys
type SortedMap struct {
trie *Trie
}
func (m *SortedMap) Contains(key string) bool {
return m.trie.Contains(*(*[]byte)(unsafe.Pointer(&key)))
}
func (m *SortedMap) Get(key string) (value interface{}, ok bool) {
return m.trie.Get(*(*[]byte)(unsafe.Pointer(&key)))
}
func (m *SortedMap) Set(key string, value interface{}) {
m.trie.Set([]byte(key), value)
}
func (m *SortedMap) Delete(key string) (value interface{}, ok bool) {
return m.trie.Delete(*(*[]byte)(unsafe.Pointer(&key)))
}
func (m *SortedMap) Clear() {
m.trie.Clear()
}
func (m *SortedMap) Size() int {
return m.trie.Size()
}
// Returns a slice of sorted keys
func (m *SortedMap) Keys() []string {
keys := make([]string, 0, m.Size())
m.trie.Allprefixed([]byte{}, func(k []byte, v interface{}) bool {
keys = append(keys, string(k))
return true
})
return keys
}
// Executes a provided function for each element that has a given prefix.
// if handle returns `false`, the iteration is aborted.
func (m *SortedMap) Each(prefix string, handle func(key string, value interface{}) bool) bool {
return m.trie.Allprefixed([]byte(prefix), func(k []byte, v interface{}) bool {
return handle(string(k), v)
})
}
// Create a SortedMap
func NewSortedMap() *SortedMap {
return &SortedMap{NewTrie()}
}

228
vendor/github.com/k-sone/critbitgo/net.go generated vendored Normal file
View File

@@ -0,0 +1,228 @@
package critbitgo
import (
"net"
)
var (
mask32 = net.IPMask{0xff, 0xff, 0xff, 0xff}
mask128 = net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
)
// IP routing table.
type Net struct {
trie *Trie
}
// Add a route.
// If `r` is not IPv4/IPv6 network, returns an error.
func (n *Net) Add(r *net.IPNet, value interface{}) (err error) {
var ip net.IP
if ip, _, err = netValidateIPNet(r); err == nil {
n.trie.Set(netIPNetToKey(ip, r.Mask), value)
}
return
}
// Add a route.
// If `s` is not CIDR notation, returns an error.
func (n *Net) AddCIDR(s string, value interface{}) (err error) {
var r *net.IPNet
if _, r, err = net.ParseCIDR(s); err == nil {
n.Add(r, value)
}
return
}
// Delete a specific route.
// If `r` is not IP4/IPv6 network or a route is not found, `ok` is false.
func (n *Net) Delete(r *net.IPNet) (value interface{}, ok bool, err error) {
var ip net.IP
if ip, _, err = netValidateIPNet(r); err == nil {
value, ok = n.trie.Delete(netIPNetToKey(ip, r.Mask))
}
return
}
// Delete a specific route.
// If `s` is not CIDR notation or a route is not found, `ok` is false.
func (n *Net) DeleteCIDR(s string) (value interface{}, ok bool, err error) {
var r *net.IPNet
if _, r, err = net.ParseCIDR(s); err == nil {
value, ok, err = n.Delete(r)
}
return
}
// Get a specific route.
// If `r` is not IPv4/IPv6 network or a route is not found, `ok` is false.
func (n *Net) Get(r *net.IPNet) (value interface{}, ok bool, err error) {
var ip net.IP
if ip, _, err = netValidateIPNet(r); err == nil {
value, ok = n.trie.Get(netIPNetToKey(ip, r.Mask))
}
return
}
// Get a specific route.
// If `s` is not CIDR notation or a route is not found, `ok` is false.
func (n *Net) GetCIDR(s string) (value interface{}, ok bool, err error) {
var r *net.IPNet
if _, r, err = net.ParseCIDR(s); err == nil {
value, ok, err = n.Get(r)
}
return
}
// Return a specific route by using the longest prefix matching.
// If `r` is not IPv4/IPv6 network or a route is not found, `route` is nil.
func (n *Net) Match(r *net.IPNet) (route *net.IPNet, value interface{}, err error) {
var ip net.IP
if ip, _, err = netValidateIP(r.IP); err == nil {
if k, v := n.match(netIPNetToKey(ip, r.Mask)); k != nil {
route = netKeyToIPNet(k)
value = v
}
}
return
}
// Return a specific route by using the longest prefix matching.
// If `s` is not CIDR notation, or a route is not found, `route` is nil.
func (n *Net) MatchCIDR(s string) (route *net.IPNet, value interface{}, err error) {
var r *net.IPNet
if _, r, err = net.ParseCIDR(s); err == nil {
route, value, err = n.Match(r)
}
return
}
// Return a specific route by using the longest prefix matching.
// If `ip` is invalid IP, or a route is not found, `route` is nil.
func (n *Net) MatchIP(ip net.IP) (route *net.IPNet, value interface{}, err error) {
var isV4 bool
ip, isV4, err = netValidateIP(ip)
if err != nil {
return
}
var mask net.IPMask
if isV4 {
mask = mask32
} else {
mask = mask128
}
if k, v := n.match(netIPNetToKey(ip, mask)); k != nil {
route = netKeyToIPNet(k)
value = v
}
return
}
func (n *Net) match(key []byte) ([]byte, interface{}) {
if n.trie.size > 0 {
if node := lookup(&n.trie.root, key, false); node != nil {
return node.external.key, node.external.value
}
}
return nil, nil
}
func lookup(p *node, key []byte, backtracking bool) *node {
if p.internal != nil {
var direction int
if p.internal.offset == len(key)-1 {
// selecting the larger side when comparing the mask
direction = 1
} else if backtracking {
direction = 0
} else {
direction = p.internal.direction(key)
}
if c := lookup(&p.internal.child[direction], key, backtracking); c != nil {
return c
}
if direction == 1 {
// search other node
return lookup(&p.internal.child[0], key, true)
}
return nil
} else {
nlen := len(p.external.key)
if nlen != len(key) {
return nil
}
// check mask
mask := p.external.key[nlen-1]
if mask > key[nlen-1] {
return nil
}
// compare both keys with mask
div := int(mask >> 3)
for i := 0; i < div; i++ {
if p.external.key[i] != key[i] {
return nil
}
}
if mod := uint(mask & 0x07); mod > 0 {
bit := 8 - mod
if p.external.key[div] != key[div]&(0xff>>bit<<bit) {
return nil
}
}
return p
}
}
// Deletes all routes.
func (n *Net) Clear() {
n.trie.Clear()
}
// Returns number of routes.
func (n *Net) Size() int {
return n.trie.Size()
}
// Create IP routing table
func NewNet() *Net {
return &Net{NewTrie()}
}
func netValidateIP(ip net.IP) (nIP net.IP, isV4 bool, err error) {
if v4 := ip.To4(); v4 != nil {
nIP = v4
isV4 = true
} else if ip.To16() != nil {
nIP = ip
} else {
err = &net.AddrError{Err: "Invalid IP address", Addr: ip.String()}
}
return
}
func netValidateIPNet(r *net.IPNet) (nIP net.IP, isV4 bool, err error) {
if r == nil {
err = &net.AddrError{Err: "IP network is nil"}
return
}
return netValidateIP(r.IP)
}
func netIPNetToKey(ip net.IP, mask net.IPMask) []byte {
// +--------------+------+
// | ip address.. | mask |
// +--------------+------+
ones, _ := mask.Size()
return append(ip, byte(ones))
}
func netKeyToIPNet(k []byte) *net.IPNet {
iplen := len(k) - 1
return &net.IPNet{
IP: net.IP(k[:iplen]),
Mask: net.CIDRMask(int(k[iplen]), iplen*8),
}
}

8
vendor/manifest vendored
View File

@@ -1048,6 +1048,14 @@
"revision": "77ed1c8a01217656d2080ad51981f6e99adaa177",
"branch": "master"
},
{
"importpath": "github.com/k-sone/critbitgo",
"repository": "https://github.com/k-sone/critbitgo",
"vcs": "git",
"revision": "327359a051d71948cb756142d112c7df283fac4d",
"branch": "master",
"notests": true
},
{
"importpath": "github.com/kr/pretty",
"repository": "https://github.com/kr/pretty",