Files
troubleshoot/pkg/analyze/weave.go
Andrew Reed 91eb94baaa Weave report analyzers
The IPAM pool analyzer checks that utilization of the pod IP subnet is
less than 85%. For example, if using 10.32.0.0/12, this analyzer will
warn if 3,482 IPs are currently allocated to pods.

The pending allocation analyzer checks that the IPAM status in the
report has no items for the PendingAllocates field. This indicates the
IPAM service is not ready according to the code in the weave status
template
e3712152d2/prog/weaver/http.go (L186).

The weave connections analyzer checks that all connections to remote
peers are in the established state. The state will be "pending" if UDP
is blocked between nodes and will be "failed" if the weave pod on the
remote node is in a crash loop. To force a pending state for testing,
run the commands `iptables -A INPUT -p udp --dport 6784 -j REJECT` and
`iptables -A INPUT -p udp --dport 6783 -j REJECT` on a peer.

The weave connections analyzer also checks that all connections are
using the fastdp protocol. A commopn issue seen in the field on
CentOS/RHEL 7 is that some sides of a connection are using fastdp and
other sides have fallen back to sleeve. Set the WEAVE_NO_FASTDP env var
on the weave daemonset to "true" to test this analyzer.
2021-09-08 21:29:38 +00:00

174 lines
4.2 KiB
Go

package analyzer
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/pkg/errors"
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
)
// relevant fields from https://github.com/weaveworks/weave/blob/e3712152d2a0fe3bc998964c948e45bdf8ff6144/prog/weaver/http.go#L295
type WeaveReport struct {
Router WeaveRouter
IPAM WeaveIPAM
}
type WeaveRouter struct {
NickName string // this is the hostname
Connections []WeaveConnection
}
type WeaveIPAM struct {
RangeNumIPs int
ActiveIPs int
PendingAllocates []string
}
type WeaveConnection struct {
State string
Info string
Attrs WeaveAttributes
}
type WeaveAttributes struct {
Encrypted bool `json:"encrypted"`
MTU int `json:"mtu"`
Name string `json:"name"`
}
func analyzeWeaveReport(analyzer *troubleshootv1beta2.WeaveReportAnalyze, findFiles func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
files, err := findFiles(analyzer.ReportFileGlob)
if err != nil {
return nil, errors.Wrapf(err, "failed to find weave report files in %q", analyzer.ReportFileGlob)
}
if len(files) == 0 {
return nil, nil
}
reports := map[string]WeaveReport{}
for name, file := range files {
report := WeaveReport{}
if err := json.Unmarshal(file, &report); err != nil {
return nil, errors.Wrapf(err, "failed to unmarshal weave report json from %s", name)
}
reports[report.Router.NickName] = report
}
results := []*AnalyzeResult{}
if result := analyzeWeaveIPAMPools(reports); result != nil {
results = append(results, result)
}
if result := analyzeWeavePendingAllocation(reports); result != nil {
results = append(results, result)
}
if result := analyzeWeaveConnections(reports); result != nil {
results = append(results, result)
}
if len(results) == 0 {
results = append(results, &AnalyzeResult{
Title: "Weave Report",
IsPass: true,
Message: "No issues detected in weave report",
})
}
return results, nil
}
func analyzeWeaveIPAMPools(reports map[string]WeaveReport) *AnalyzeResult {
for _, report := range reports {
if result := analyzeWeaveIPAMPool(&report); result != nil {
return result
}
}
return nil
}
func analyzeWeaveIPAMPool(report *WeaveReport) *AnalyzeResult {
if report.IPAM.RangeNumIPs == 0 {
return nil
}
ipsUsed := float64(report.IPAM.ActiveIPs) / float64(report.IPAM.RangeNumIPs)
if ipsUsed < 0.85 {
return nil
}
return &AnalyzeResult{
Title: "Available Pod IPs",
IsWarn: true,
Message: fmt.Sprintf("%d of %d total available IPs have been assigned", report.IPAM.ActiveIPs, report.IPAM.RangeNumIPs),
}
}
func analyzeWeavePendingAllocation(reports map[string]WeaveReport) *AnalyzeResult {
for _, report := range reports {
if len(report.IPAM.PendingAllocates) > 0 {
return &AnalyzeResult{
Title: "Pending IP Allocation",
IsWarn: true,
Message: "Waiting for IPs to become available",
}
}
}
return nil
}
// Get the peer hostname for logging purposes from a string like: "encrypted fastdp 1a:5b:a9:53:2b:11(areed-aka-kkz0)"
var weaveConnectionInfoPeerRegex = regexp.MustCompile(`\(([^)]+)\)$`)
func parseWeaveConnectionInfoHostname(info string) string {
matches := weaveConnectionInfoPeerRegex.FindStringSubmatch(info)
if len(matches) == 2 {
return matches[1]
}
return ""
}
func analyzeWeaveConnections(reports map[string]WeaveReport) *AnalyzeResult {
for host, report := range reports {
for _, connection := range report.Router.Connections {
// older versions of weave show connection to self as failed
if strings.HasPrefix(connection.Info, "cannot connect to ourself") {
continue
}
peer := parseWeaveConnectionInfoHostname(connection.Info)
if peer == "" {
peer = "peer"
}
if connection.State != "established" {
return &AnalyzeResult{
Title: "Weave Inter-Node Connections",
IsWarn: true,
Message: fmt.Sprintf("Connection from %s to %s is %s", host, peer, connection.State),
}
}
if connection.Attrs.Name != "" && connection.Attrs.Name != "fastdp" {
return &AnalyzeResult{
Title: "Weave Inter-Node Connections",
IsWarn: true,
Message: fmt.Sprintf("Connection from %s to %s protocol is %q, not fastdp", host, peer, connection.Attrs.Name),
}
}
}
}
return nil
}