From 98f036359b1e6581c9462b44a0ae527ec784cdcf Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 20 Jun 2017 19:31:11 +0100 Subject: [PATCH] fast network membership check The rendering code checks whether endpoint IPs are part of cluster-local networks. Due to the prevalence of endpoints - medium sized reports can contain many thousands of endpoints - this is performance critical. Alas the existing code performs the check via a linear scan of a list of networks. That is slow when there are more than a few. Unfortunately in some common k8s network setups, e.g. on AWS, a cluster can contain hundreds of networks, due to /32 networks derived from interfaces with multiple IPs. Here we change representation of the set of networks to a prefix tree (aka trie), which is well-suited for IP network membership checks since networks are in fact a bitstring prefixes. The specific representation is a crit-bit tree, but that choice was purely based on implementation convenience - the chosen library is the only one I could find that directly supports IP networks. --- render/container_test.go | 6 +- render/theinternet.go | 18 +- render/theinternet_test.go | 20 +- report/networks.go | 35 +- report/networks_test.go | 18 +- vendor/github.com/k-sone/critbitgo/LICENSE | 22 + vendor/github.com/k-sone/critbitgo/critbit.go | 390 ++++++++++++++++++ vendor/github.com/k-sone/critbitgo/map.go | 57 +++ vendor/github.com/k-sone/critbitgo/net.go | 228 ++++++++++ vendor/manifest | 8 + 10 files changed, 747 insertions(+), 55 deletions(-) create mode 100644 vendor/github.com/k-sone/critbitgo/LICENSE create mode 100644 vendor/github.com/k-sone/critbitgo/critbit.go create mode 100644 vendor/github.com/k-sone/critbitgo/map.go create mode 100644 vendor/github.com/k-sone/critbitgo/net.go diff --git a/render/container_test.go b/render/container_test.go index d873c601b..f376f22ba 100644 --- a/render/container_test.go +++ b/render/container_test.go @@ -2,7 +2,6 @@ package render_test import ( "fmt" - "net" "testing" "github.com/weaveworks/common/test" @@ -46,11 +45,10 @@ type testcase struct { } func testMap(t *testing.T, f render.MapFunc, input testcase) { - _, ipNet, err := net.ParseCIDR("1.2.3.0/16") - if err != nil { + localNetworks := report.NewNetworks() + if err := localNetworks.AddCIDR("1.2.3.0/16"); err != nil { t.Fatalf(err.Error()) } - localNetworks := report.Networks([]*net.IPNet{ipNet}) if have := f(input.n, localNetworks); input.ok != (len(have) > 0) { name := input.name if name == "" { diff --git a/render/theinternet.go b/render/theinternet.go index aa675d22e..06996e49f 100644 --- a/render/theinternet.go +++ b/render/theinternet.go @@ -1,7 +1,6 @@ package render import ( - "net" "regexp" "strings" @@ -68,26 +67,15 @@ func isKnownService(hostname string) bool { // used to determine which nodes in the report are "remote", i.e. outside of // our infrastructure. func LocalNetworks(r report.Report) report.Networks { - var ( - result = report.Networks{} - networks = map[string]struct{}{} - ) + networks := report.NewNetworks() for _, topology := range []report.Topology{r.Host, r.Overlay} { for _, md := range topology.Nodes { nets, _ := md.Sets.Lookup(host.LocalNetworks) for _, s := range nets { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - continue - } - _, ok := networks[ipNet.String()] - if !ok { - result = append(result, ipNet) - networks[ipNet.String()] = struct{}{} - } + networks.AddCIDR(s) } } } - return result + return networks } diff --git a/render/theinternet_test.go b/render/theinternet_test.go index fabf2335f..1cb540cd1 100644 --- a/render/theinternet_test.go +++ b/render/theinternet_test.go @@ -1,7 +1,6 @@ package render_test import ( - "net" "reflect" "testing" @@ -30,21 +29,14 @@ func TestReportLocalNetworks(t *testing.T) { }, }, }) - want := report.Networks([]*net.IPNet{ - mustParseCIDR("10.0.0.1/8"), - mustParseCIDR("192.168.1.1/24"), - mustParseCIDR("10.32.0.1/12"), - }) + want := report.NewNetworks() + for _, cidr := range []string{"10.0.0.1/8", "192.168.1.1/24", "10.32.0.1/12"} { + if err := want.AddCIDR(cidr); err != nil { + panic(err) + } + } have := render.LocalNetworks(r) if !reflect.DeepEqual(want, have) { t.Errorf("%s", test.Diff(want, have)) } } - -func mustParseCIDR(s string) *net.IPNet { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - panic(err) - } - return ipNet -} diff --git a/report/networks.go b/report/networks.go index c2797760d..4cd1d8a70 100644 --- a/report/networks.go +++ b/report/networks.go @@ -3,25 +3,38 @@ package report import ( "net" "strings" + + "github.com/k-sone/critbitgo" ) // Networks represent a set of subnets -type Networks []*net.IPNet +type Networks struct{ *critbitgo.Net } // LocalNetworks helps in determining which addresses a probe reports // as being host-scoped. // // TODO this design is broken, make it consistent with probe networks. -var LocalNetworks = Networks{} +var LocalNetworks = NewNetworks() + +// Create datastructure for set of subnets. +func NewNetworks() Networks { + return Networks{critbitgo.NewNet()} +} + +// Add a network. +func (n Networks) Add(ipnet *net.IPNet) error { + return n.Net.Add(ipnet, struct{}{}) +} + +// Add a network, represented as CIDR. +func (n Networks) AddCIDR(cidr string) error { + return n.Net.AddCIDR(cidr, struct{}{}) +} // Contains returns true if IP is in Networks. func (n Networks) Contains(ip net.IP) bool { - for _, net := range n { - if net.Contains(ip) { - return true - } - } - return false + network, _, _ := n.MatchIP(ip) + return network != nil } // LocalAddresses returns a list of the local IP addresses. @@ -67,7 +80,9 @@ func AddLocalBridge(name string) error { return err } - LocalNetworks = ipv4Nets(addrs) + for _, ipnet := range ipv4Nets(addrs) { + LocalNetworks.Add(ipnet) + } return nil } @@ -82,7 +97,7 @@ func GetLocalNetworks() ([]*net.IPNet, error) { } func ipv4Nets(addrs []net.Addr) []*net.IPNet { - nets := Networks{} + nets := []*net.IPNet{} for _, addr := range addrs { if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.To4() != nil { nets = append(nets, ipnet) diff --git a/report/networks_test.go b/report/networks_test.go index 47ca616ac..dbc49de21 100644 --- a/report/networks_test.go +++ b/report/networks_test.go @@ -8,10 +8,12 @@ import ( ) func TestContains(t *testing.T) { - networks := report.Networks([]*net.IPNet{ - mustParseCIDR("10.0.0.1/8"), - mustParseCIDR("192.168.1.1/24"), - }) + networks := report.NewNetworks() + for _, cidr := range []string{"10.0.0.1/8", "192.168.1.1/24"} { + if err := networks.AddCIDR(cidr); err != nil { + panic(err) + } + } if networks.Contains(net.ParseIP("52.52.52.52")) { t.Errorf("52.52.52.52 not in %v", networks) @@ -21,11 +23,3 @@ func TestContains(t *testing.T) { t.Errorf("10.0.0.1 in %v", networks) } } - -func mustParseCIDR(s string) *net.IPNet { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - panic(err) - } - return ipNet -} diff --git a/vendor/github.com/k-sone/critbitgo/LICENSE b/vendor/github.com/k-sone/critbitgo/LICENSE new file mode 100644 index 000000000..f36eed92a --- /dev/null +++ b/vendor/github.com/k-sone/critbitgo/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Keita Sone + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/k-sone/critbitgo/critbit.go b/vendor/github.com/k-sone/critbitgo/critbit.go new file mode 100644 index 000000000..477f4b0a0 --- /dev/null +++ b/vendor/github.com/k-sone/critbitgo/critbit.go @@ -0,0 +1,390 @@ +package critbitgo + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "os" + "strconv" +) + +// The matrix of most significant bit +var msbMatrix [256]byte + +func buildMsbMatrix() { + for i := 0; i < len(msbMatrix); i++ { + b := byte(i) + b |= b >> 1 + b |= b >> 2 + b |= b >> 4 + msbMatrix[i] = b &^ (b >> 1) + } +} + +type node struct { + internal *internal + external *external +} + +type internal struct { + child [2]node + offset int + bit byte + cont bool // if true, key of child[1] contains key of child[0] +} + +type external struct { + key []byte + value interface{} +} + +// finding the critical bit. +func (n *external) criticalBit(key []byte) (offset int, bit byte, cont bool) { + nlen := len(n.key) + klen := len(key) + mlen := nlen + if nlen > klen { + mlen = klen + } + + // find first differing byte and bit + for offset = 0; offset < mlen; offset++ { + if a, b := key[offset], n.key[offset]; a != b { + bit = msbMatrix[a^b] + return + } + } + + if nlen < klen { + bit = msbMatrix[key[offset]] + } else if nlen > klen { + bit = msbMatrix[n.key[offset]] + } else { + // two keys are equal + offset = -1 + } + return offset, bit, true +} + +// calculate direction. +func (n *internal) direction(key []byte) int { + if n.offset < len(key) && (key[n.offset]&n.bit != 0 || n.cont) { + return 1 + } + return 0 +} + +// Crit-bit Tree +type Trie struct { + root node + size int +} + +// searching the tree. +func (t *Trie) search(key []byte) *node { + n := &t.root + for n.internal != nil { + n = &n.internal.child[n.internal.direction(key)] + } + return n +} + +// membership testing. +func (t *Trie) Contains(key []byte) bool { + if n := t.search(key); n.external != nil && bytes.Equal(n.external.key, key) { + return true + } + return false +} + +// get member. +// if `key` is in Trie, `ok` is true. +func (t *Trie) Get(key []byte) (value interface{}, ok bool) { + if n := t.search(key); n.external != nil && bytes.Equal(n.external.key, key) { + return n.external.value, true + } + return +} + +// insert into the tree (replaceable). +func (t *Trie) insert(key []byte, value interface{}, replace bool) bool { + // an empty tree + if t.size == 0 { + t.root.external = &external{ + key: key, + value: value, + } + t.size = 1 + return true + } + + n := t.search(key) + newOffset, newBit, newCont := n.external.criticalBit(key) + + // already exists in the tree + if newOffset == -1 { + if replace { + n.external.value = value + return true + } + return false + } + + // allocate new node + newNode := &internal{ + offset: newOffset, + bit: newBit, + cont: newCont, + } + direction := newNode.direction(key) + newNode.child[direction].external = &external{ + key: key, + value: value, + } + + // insert new node + wherep := &t.root + for in := wherep.internal; in != nil; in = wherep.internal { + if in.offset > newOffset || (in.offset == newOffset && in.bit < newBit) { + break + } + wherep = &in.child[in.direction(key)] + } + + if wherep.internal != nil { + newNode.child[1-direction].internal = wherep.internal + } else { + newNode.child[1-direction].external = wherep.external + wherep.external = nil + } + wherep.internal = newNode + t.size += 1 + return true +} + +// insert into the tree. +// if `key` is alredy in Trie, return false. +func (t *Trie) Insert(key []byte, value interface{}) bool { + return t.insert(key, value, false) +} + +// set into the tree. +func (t *Trie) Set(key []byte, value interface{}) { + t.insert(key, value, true) +} + +// deleting elements. +// if `key` is in Trie, `ok` is true. +func (t *Trie) Delete(key []byte) (value interface{}, ok bool) { + // an empty tree + if t.size == 0 { + return + } + + var direction int + var whereq *node // pointer to the grandparent + var wherep *node = &t.root + + // finding the best candidate to delete + for in := wherep.internal; in != nil; in = wherep.internal { + direction = in.direction(key) + whereq = wherep + wherep = &in.child[direction] + } + + // checking that we have the right element + if !bytes.Equal(wherep.external.key, key) { + return + } + value = wherep.external.value + ok = true + + // removing the node + if whereq == nil { + wherep.external = nil + } else { + othern := whereq.internal.child[1-direction] + whereq.internal = othern.internal + whereq.external = othern.external + } + t.size -= 1 + return +} + +// clearing a tree. +func (t *Trie) Clear() { + t.root.internal = nil + t.root.external = nil + t.size = 0 +} + +// return the number of key in a tree. +func (t *Trie) Size() int { + return t.size +} + +// fetching elements with a given prefix. +// handle is called with arguments key and value (if handle returns `false`, the iteration is aborted) +func (t *Trie) Allprefixed(prefix []byte, handle func(key []byte, value interface{}) bool) bool { + // an empty tree + if t.size == 0 { + return true + } + + // walk tree, maintaining top pointer + p := &t.root + top := p + if len(prefix) > 0 { + for q := p.internal; q != nil; q = p.internal { + p = &q.child[q.direction(prefix)] + if q.offset < len(prefix) { + top = p + } + } + + // check prefix + if !bytes.Contains(p.external.key, prefix) { + return true + } + } + + return allprefixed(top, handle) +} + +func allprefixed(n *node, handle func([]byte, interface{}) bool) bool { + if n.internal != nil { + // dealing with an internal node while recursing + for i := 0; i < 2; i++ { + if !allprefixed(&n.internal.child[i], handle) { + return false + } + } + } else { + // dealing with an external node while recursing + return handle(n.external.key, n.external.value) + } + return true +} + +// Search for the longest matching key from the beginning of the given key. +// if `key` is in Trie, `ok` is true. +func (t *Trie) LongestPrefix(given []byte) (key []byte, value interface{}, ok bool) { + // an empty tree + if t.size == 0 { + return + } + return longestPrefix(&t.root, given) +} + +func longestPrefix(n *node, key []byte) ([]byte, interface{}, bool) { + if n.internal != nil { + direction := n.internal.direction(key) + if k, v, ok := longestPrefix(&n.internal.child[direction], key); ok { + return k, v, ok + } + if direction == 1 { + return longestPrefix(&n.internal.child[0], key) + } + } else { + if bytes.HasPrefix(key, n.external.key) { + return n.external.key, n.external.value, true + } + } + return nil, nil, false +} + +// Iterating elements from a given start key. +// handle is called with arguments key and value (if handle returns `false`, the iteration is aborted) +func (t *Trie) Walk(start []byte, handle func(key []byte, value interface{}) bool) bool { + var seek bool + if start != nil { + seek = true + } + return walk(&t.root, start, &seek, handle) +} + +func walk(n *node, key []byte, seek *bool, handle func([]byte, interface{}) bool) bool { + if n.internal != nil { + var direction int + if *seek { + direction = n.internal.direction(key) + } + if !walk(&n.internal.child[direction], key, seek, handle) { + return false + } + if !(*seek) && direction == 0 { + // iteration another side + return walk(&n.internal.child[1], key, seek, handle) + } + return true + } else { + if *seek { + if bytes.Equal(n.external.key, key) { + // seek completed + *seek = false + } else { + // key is not in Trie + return false + } + } + return handle(n.external.key, n.external.value) + } +} + +// dump tree. (for debugging) +func (t *Trie) Dump(w io.Writer) { + if t.root.internal == nil && t.root.external == nil { + return + } + if w == nil { + w = os.Stdout + } + dump(w, &t.root, true, "") +} + +func dump(w io.Writer, n *node, right bool, prefix string) { + var ownprefix string + if right { + ownprefix = prefix + } else { + ownprefix = prefix[:len(prefix)-1] + "`" + } + + if in := n.internal; in != nil { + fmt.Fprintf(w, "%s-- off=%d, bit=%08b(%02x), cont=%v\n", ownprefix, in.offset, in.bit, in.bit, in.cont) + for i := 0; i < 2; i++ { + var nextprefix string + switch i { + case 0: + nextprefix = prefix + " |" + right = true + case 1: + nextprefix = prefix + " " + right = false + } + dump(w, &in.child[i], right, nextprefix) + } + } else { + fmt.Fprintf(w, "%s-- key=%d (%s)\n", ownprefix, n.external.key, key2str(n.external.key)) + } + return +} + +func key2str(key []byte) string { + for _, c := range key { + if !strconv.IsPrint(rune(c)) { + return hex.EncodeToString(key) + } + } + return string(key) +} + +// create a tree. +func NewTrie() *Trie { + return &Trie{} +} + +func init() { + buildMsbMatrix() +} diff --git a/vendor/github.com/k-sone/critbitgo/map.go b/vendor/github.com/k-sone/critbitgo/map.go new file mode 100644 index 000000000..17bcb0b88 --- /dev/null +++ b/vendor/github.com/k-sone/critbitgo/map.go @@ -0,0 +1,57 @@ +package critbitgo + +import ( + "unsafe" +) + +// The map is sorted according to the natural ordering of its keys +type SortedMap struct { + trie *Trie +} + +func (m *SortedMap) Contains(key string) bool { + return m.trie.Contains(*(*[]byte)(unsafe.Pointer(&key))) +} + +func (m *SortedMap) Get(key string) (value interface{}, ok bool) { + return m.trie.Get(*(*[]byte)(unsafe.Pointer(&key))) +} + +func (m *SortedMap) Set(key string, value interface{}) { + m.trie.Set([]byte(key), value) +} + +func (m *SortedMap) Delete(key string) (value interface{}, ok bool) { + return m.trie.Delete(*(*[]byte)(unsafe.Pointer(&key))) +} + +func (m *SortedMap) Clear() { + m.trie.Clear() +} + +func (m *SortedMap) Size() int { + return m.trie.Size() +} + +// Returns a slice of sorted keys +func (m *SortedMap) Keys() []string { + keys := make([]string, 0, m.Size()) + m.trie.Allprefixed([]byte{}, func(k []byte, v interface{}) bool { + keys = append(keys, string(k)) + return true + }) + return keys +} + +// Executes a provided function for each element that has a given prefix. +// if handle returns `false`, the iteration is aborted. +func (m *SortedMap) Each(prefix string, handle func(key string, value interface{}) bool) bool { + return m.trie.Allprefixed([]byte(prefix), func(k []byte, v interface{}) bool { + return handle(string(k), v) + }) +} + +// Create a SortedMap +func NewSortedMap() *SortedMap { + return &SortedMap{NewTrie()} +} diff --git a/vendor/github.com/k-sone/critbitgo/net.go b/vendor/github.com/k-sone/critbitgo/net.go new file mode 100644 index 000000000..7e155b245 --- /dev/null +++ b/vendor/github.com/k-sone/critbitgo/net.go @@ -0,0 +1,228 @@ +package critbitgo + +import ( + "net" +) + +var ( + mask32 = net.IPMask{0xff, 0xff, 0xff, 0xff} + mask128 = net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +) + +// IP routing table. +type Net struct { + trie *Trie +} + +// Add a route. +// If `r` is not IPv4/IPv6 network, returns an error. +func (n *Net) Add(r *net.IPNet, value interface{}) (err error) { + var ip net.IP + if ip, _, err = netValidateIPNet(r); err == nil { + n.trie.Set(netIPNetToKey(ip, r.Mask), value) + } + return +} + +// Add a route. +// If `s` is not CIDR notation, returns an error. +func (n *Net) AddCIDR(s string, value interface{}) (err error) { + var r *net.IPNet + if _, r, err = net.ParseCIDR(s); err == nil { + n.Add(r, value) + } + return +} + +// Delete a specific route. +// If `r` is not IP4/IPv6 network or a route is not found, `ok` is false. +func (n *Net) Delete(r *net.IPNet) (value interface{}, ok bool, err error) { + var ip net.IP + if ip, _, err = netValidateIPNet(r); err == nil { + value, ok = n.trie.Delete(netIPNetToKey(ip, r.Mask)) + } + return +} + +// Delete a specific route. +// If `s` is not CIDR notation or a route is not found, `ok` is false. +func (n *Net) DeleteCIDR(s string) (value interface{}, ok bool, err error) { + var r *net.IPNet + if _, r, err = net.ParseCIDR(s); err == nil { + value, ok, err = n.Delete(r) + } + return +} + +// Get a specific route. +// If `r` is not IPv4/IPv6 network or a route is not found, `ok` is false. +func (n *Net) Get(r *net.IPNet) (value interface{}, ok bool, err error) { + var ip net.IP + if ip, _, err = netValidateIPNet(r); err == nil { + value, ok = n.trie.Get(netIPNetToKey(ip, r.Mask)) + } + return +} + +// Get a specific route. +// If `s` is not CIDR notation or a route is not found, `ok` is false. +func (n *Net) GetCIDR(s string) (value interface{}, ok bool, err error) { + var r *net.IPNet + if _, r, err = net.ParseCIDR(s); err == nil { + value, ok, err = n.Get(r) + } + return +} + +// Return a specific route by using the longest prefix matching. +// If `r` is not IPv4/IPv6 network or a route is not found, `route` is nil. +func (n *Net) Match(r *net.IPNet) (route *net.IPNet, value interface{}, err error) { + var ip net.IP + if ip, _, err = netValidateIP(r.IP); err == nil { + if k, v := n.match(netIPNetToKey(ip, r.Mask)); k != nil { + route = netKeyToIPNet(k) + value = v + } + } + return +} + +// Return a specific route by using the longest prefix matching. +// If `s` is not CIDR notation, or a route is not found, `route` is nil. +func (n *Net) MatchCIDR(s string) (route *net.IPNet, value interface{}, err error) { + var r *net.IPNet + if _, r, err = net.ParseCIDR(s); err == nil { + route, value, err = n.Match(r) + } + return +} + +// Return a specific route by using the longest prefix matching. +// If `ip` is invalid IP, or a route is not found, `route` is nil. +func (n *Net) MatchIP(ip net.IP) (route *net.IPNet, value interface{}, err error) { + var isV4 bool + ip, isV4, err = netValidateIP(ip) + if err != nil { + return + } + var mask net.IPMask + if isV4 { + mask = mask32 + } else { + mask = mask128 + } + if k, v := n.match(netIPNetToKey(ip, mask)); k != nil { + route = netKeyToIPNet(k) + value = v + } + return +} + +func (n *Net) match(key []byte) ([]byte, interface{}) { + if n.trie.size > 0 { + if node := lookup(&n.trie.root, key, false); node != nil { + return node.external.key, node.external.value + } + } + return nil, nil +} + +func lookup(p *node, key []byte, backtracking bool) *node { + if p.internal != nil { + var direction int + if p.internal.offset == len(key)-1 { + // selecting the larger side when comparing the mask + direction = 1 + } else if backtracking { + direction = 0 + } else { + direction = p.internal.direction(key) + } + + if c := lookup(&p.internal.child[direction], key, backtracking); c != nil { + return c + } + if direction == 1 { + // search other node + return lookup(&p.internal.child[0], key, true) + } + return nil + } else { + nlen := len(p.external.key) + if nlen != len(key) { + return nil + } + + // check mask + mask := p.external.key[nlen-1] + if mask > key[nlen-1] { + return nil + } + + // compare both keys with mask + div := int(mask >> 3) + for i := 0; i < div; i++ { + if p.external.key[i] != key[i] { + return nil + } + } + if mod := uint(mask & 0x07); mod > 0 { + bit := 8 - mod + if p.external.key[div] != key[div]&(0xff>>bit<