From 450d7570ebfdba0864c551222f5934ed536393c6 Mon Sep 17 00:00:00 2001 From: Andrew Reed Date: Fri, 12 Feb 2021 21:22:27 +0000 Subject: [PATCH] Analyze HTTP load balancer --- .../preflight/host-http-load-balancer.yaml | 33 ++++ pkg/analyze/analyzer.go | 7 + pkg/analyze/host_httploadbalancer.go | 90 ++++++++++ .../v1beta2/hostanalyzer_shared.go | 9 +- .../v1beta2/zz_generated.deepcopy.go | 32 ++++ pkg/collect/host_collector.go | 2 + pkg/collect/host_httploadbalancer.go | 164 ++++++++++++++++++ pkg/collect/host_network.go | 11 +- pkg/collect/host_tcploadbalancer.go | 10 -- 9 files changed, 342 insertions(+), 16 deletions(-) create mode 100644 examples/preflight/host-http-load-balancer.yaml create mode 100644 pkg/analyze/host_httploadbalancer.go create mode 100644 pkg/collect/host_httploadbalancer.go diff --git a/examples/preflight/host-http-load-balancer.yaml b/examples/preflight/host-http-load-balancer.yaml new file mode 100644 index 00000000..26b3e98a --- /dev/null +++ b/examples/preflight/host-http-load-balancer.yaml @@ -0,0 +1,33 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: HostPreflight +metadata: + name: httploadbalancer +spec: + collectors: + - httpLoadBalancer: + collectorName: httploadbalancer + port: 80 + address: http://app.corporate.internal + timeout: 10s + analyzers: + - httpLoadBalancer: + collectorName: httploadbalancer + outcomes: + - fail: + when: "connection-refused" + message: Connection to port 80 via load balancer was refused. + - fail: + when: "address-in-use" + message: Another process was already listening on port 80. + - fail: + when: "connection-timeout" + message: Timed out connecting to port 80 via load balancer. Check your firewall. + - fail: + when: "bind-permission-denied" + message: Bind permission denied. Try running as root. + - fail: + when: "error" + message: Failed to connect to port 80 via load balancer. + - pass: + when: "connected" + message: Successfully connected to port 80 via load balancer. diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go index 60d907da..01a6a17d 100644 --- a/pkg/analyze/analyzer.go +++ b/pkg/analyze/analyzer.go @@ -55,6 +55,13 @@ func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getColle } return []*AnalyzeResult{result}, nil } + if hostAnalyzer.HTTPLoadBalancer != nil { + result, err := analyzeHostHTTPLoadBalancer(hostAnalyzer.HTTPLoadBalancer, getFile) + if err != nil { + return nil, err + } + return []*AnalyzeResult{result}, nil + } if hostAnalyzer.DiskUsage != nil { result, err := analyzeHostDiskUsage(hostAnalyzer.DiskUsage, getFile) if err != nil { diff --git a/pkg/analyze/host_httploadbalancer.go b/pkg/analyze/host_httploadbalancer.go new file mode 100644 index 00000000..ad2ba771 --- /dev/null +++ b/pkg/analyze/host_httploadbalancer.go @@ -0,0 +1,90 @@ +package analyzer + +import ( + "encoding/json" + "fmt" + "path" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/collect" +) + +func analyzeHostHTTPLoadBalancer(hostAnalyzer *troubleshootv1beta2.HTTPLoadBalancerAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) { + collectorName := hostAnalyzer.CollectorName + if collectorName == "" { + collectorName = "httpLoadBalancer" + } + fullPath := path.Join("httpLoadBalancer", fmt.Sprintf("%s.json", collectorName)) + + collected, err := getCollectedFileContents(fullPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read collected file name: %s", fullPath) + } + actual := collect.NetworkStatusResult{} + if err := json.Unmarshal(collected, &actual); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal collected") + } + + result := AnalyzeResult{} + + title := hostAnalyzer.CheckName + if title == "" { + title = "HTTP Load Balancer" + } + result.Title = title + + for _, outcome := range hostAnalyzer.Outcomes { + if outcome.Fail != nil { + if outcome.Fail.When == "" { + result.IsFail = true + result.Message = outcome.Fail.Message + result.URI = outcome.Fail.URI + + return &result, nil + } + + if string(actual.Status) == outcome.Fail.When { + result.IsFail = true + result.Message = outcome.Fail.Message + result.URI = outcome.Fail.URI + + return &result, nil + } + } else if outcome.Warn != nil { + if outcome.Warn.When == "" { + result.IsWarn = true + result.Message = outcome.Warn.Message + result.URI = outcome.Warn.URI + + return &result, nil + } + + if string(actual.Status) == outcome.Warn.When { + result.IsWarn = true + result.Message = outcome.Warn.Message + result.URI = outcome.Warn.URI + + return &result, nil + } + } else if outcome.Pass != nil { + if outcome.Pass.When == "" { + result.IsPass = true + result.Message = outcome.Pass.Message + result.URI = outcome.Pass.URI + + return &result, nil + } + + if string(actual.Status) == outcome.Pass.When { + result.IsPass = true + result.Message = outcome.Pass.Message + result.URI = outcome.Pass.URI + + return &result, nil + } + } + } + + return &result, nil +} diff --git a/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go index 915a1d8d..2da58727 100644 --- a/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go @@ -16,6 +16,12 @@ type TCPLoadBalancerAnalyze struct { Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` } +type HTTPLoadBalancerAnalyze struct { + AnalyzeMeta `json:",inline" yaml:",inline"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` +} + type TCPPortStatusAnalyze struct { AnalyzeMeta `json:",inline" yaml:",inline"` CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` @@ -47,7 +53,8 @@ type BlockDevicesAnalyze struct { type HostAnalyze struct { CPU *CPUAnalyze `json:"cpu,omitempty" yaml:"cpu,omitempty"` // - TCPLoadBalancer *TCPLoadBalancerAnalyze `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"` + TCPLoadBalancer *TCPLoadBalancerAnalyze `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"` + HTTPLoadBalancer *HTTPLoadBalancerAnalyze `json:"httpLoadBalancer,omitempty" yaml:"httpLoadBalancer,omitempty"` DiskUsage *DiskUsageAnalyze `json:"diskUsage,omitempty" yaml:"diskUsage,omitempty"` diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index 50a68dc6..8e487ead 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -1097,6 +1097,33 @@ func (in *HTTPLoadBalancer) DeepCopy() *HTTPLoadBalancer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPLoadBalancerAnalyze) DeepCopyInto(out *HTTPLoadBalancerAnalyze) { + *out = *in + out.AnalyzeMeta = in.AnalyzeMeta + if in.Outcomes != nil { + in, out := &in.Outcomes, &out.Outcomes + *out = make([]*Outcome, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Outcome) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPLoadBalancerAnalyze. +func (in *HTTPLoadBalancerAnalyze) DeepCopy() *HTTPLoadBalancerAnalyze { + if in == nil { + return nil + } + out := new(HTTPLoadBalancerAnalyze) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostAnalyze) DeepCopyInto(out *HostAnalyze) { *out = *in @@ -1110,6 +1137,11 @@ func (in *HostAnalyze) DeepCopyInto(out *HostAnalyze) { *out = new(TCPLoadBalancerAnalyze) (*in).DeepCopyInto(*out) } + if in.HTTPLoadBalancer != nil { + in, out := &in.HTTPLoadBalancer, &out.HTTPLoadBalancer + *out = new(HTTPLoadBalancerAnalyze) + (*in).DeepCopyInto(*out) + } if in.DiskUsage != nil { in, out := &in.DiskUsage, &out.DiskUsage *out = new(DiskUsageAnalyze) diff --git a/pkg/collect/host_collector.go b/pkg/collect/host_collector.go index 37dbd25a..389ade51 100644 --- a/pkg/collect/host_collector.go +++ b/pkg/collect/host_collector.go @@ -24,6 +24,8 @@ func (c *HostCollector) RunCollectorSync() (result map[string][]byte, err error) result, err = HostMemory(c) } else if c.Collect.TCPLoadBalancer != nil { result, err = HostTCPLoadBalancer(c) + } else if c.Collect.HTTPLoadBalancer != nil { + result, err = HostHTTPLoadBalancer(c) } else if c.Collect.DiskUsage != nil { result, err = HostDiskUsage(c) } else if c.Collect.TCPPortStatus != nil { diff --git a/pkg/collect/host_httploadbalancer.go b/pkg/collect/host_httploadbalancer.go new file mode 100644 index 00000000..26b5b285 --- /dev/null +++ b/pkg/collect/host_httploadbalancer.go @@ -0,0 +1,164 @@ +package collect + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "path" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/segmentio/ksuid" +) + +func HostHTTPLoadBalancer(c *HostCollector) (map[string][]byte, error) { + listenAddress := fmt.Sprintf("0.0.0.0:%d", c.Collect.HTTPLoadBalancer.Port) + + timeout := 60 * time.Minute + if c.Collect.HTTPLoadBalancer.Timeout != "" { + var err error + timeout, err = time.ParseDuration(c.Collect.HTTPLoadBalancer.Timeout) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse timeout %q", c.Collect.HTTPLoadBalancer.Timeout) + } + } + + requestToken := ksuid.New().Bytes() + responseToken := ksuid.New().Bytes() + + listenErr := make(chan error, 1) + + go func() { + mux := http.NewServeMux() + server := http.Server{ + Addr: listenAddress, + Handler: mux, + } + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + return + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return + } + if !bytes.Equal(body, requestToken) { + return + } + _, err = w.Write(responseToken) + if err != nil { + return + } + server.Shutdown(context.Background()) + }) + + err := http.ListenAndServe(listenAddress, mux) + if err != http.ErrServerClosed { + listenErr <- err + } + }() + + var networkStatus NetworkStatus + + stopAfter := time.Now().Add(timeout) + for { + if len(listenErr) > 0 { + err := <-listenErr + if strings.Contains(err.Error(), "address already in use") { + networkStatus = NetworkStatusAddressInUse + break + } + if strings.Contains(err.Error(), "permission denied") { + networkStatus = NetworkStatusBindPermissionDenied + break + } + log.Println(err.Error()) + networkStatus = NetworkStatusErrorOther + break + } + if time.Now().After(stopAfter) { + break + } + + networkStatus = attemptPOST(c.Collect.HTTPLoadBalancer.Address, requestToken, responseToken) + + if networkStatus == NetworkStatusErrorOther || networkStatus == NetworkStatusConnectionTimeout { + time.Sleep(50 * time.Millisecond) + continue + } + + break + } + + result := NetworkStatusResult{ + Status: networkStatus, + } + + b, err := json.Marshal(result) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal result") + } + + name := path.Join("httpLoadBalancer", "httpLoadBalancer.json") + if c.Collect.HTTPLoadBalancer.CollectorName != "" { + name = path.Join("httpLoadBalancer", fmt.Sprintf("%s.json", c.Collect.HTTPLoadBalancer.CollectorName)) + } + + return map[string][]byte{ + name: b, + }, nil +} + +func attemptPOST(address string, request []byte, response []byte) NetworkStatus { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + // Create a new transport every time to ensure a new TCP connection so the load balancer does + // not forward every request to the same backend + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 50 * time.Millisecond, + DualStack: true, + }).DialContext, + ForceAttemptHTTP2: true, + } + client := http.Client{ + Transport: transport, + } + + buf := bytes.NewBuffer(request) + req, err := http.NewRequestWithContext(ctx, "POST", address, buf) + if err != nil { + fmt.Println(err.Error()) + return NetworkStatusErrorOther + } + + resp, err := client.Do(req) + if err != nil { + if strings.Contains(err.Error(), "connection refused") { + return NetworkStatusConnectionRefused + } + if strings.Contains(err.Error(), "i/o timeout") { + return NetworkStatusConnectionTimeout + } + + return NetworkStatusErrorOther + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return NetworkStatusErrorOther + } + if !bytes.Equal(body, response) { + return NetworkStatusErrorOther + } + + return NetworkStatusConnected +} diff --git a/pkg/collect/host_network.go b/pkg/collect/host_network.go index 06cabfde..2bb9b84b 100644 --- a/pkg/collect/host_network.go +++ b/pkg/collect/host_network.go @@ -14,11 +14,12 @@ import ( type NetworkStatus string const ( - NetworkStatusAddressInUse = "address-in-use" - NetworkStatusConnectionRefused = "connection-refused" - NetworkStatusConnectionTimeout = "connection-timeout" - NetworkStatusConnected = "connected" - NetworkStatusErrorOther = "error" + NetworkStatusAddressInUse = "address-in-use" + NetworkStatusConnectionRefused = "connection-refused" + NetworkStatusConnectionTimeout = "connection-timeout" + NetworkStatusConnected = "connected" + NetworkStatusErrorOther = "error" + NetworkStatusBindPermissionDenied = "bind-permission-denied" ) type NetworkStatusResult struct { diff --git a/pkg/collect/host_tcploadbalancer.go b/pkg/collect/host_tcploadbalancer.go index d561620c..978c18d0 100644 --- a/pkg/collect/host_tcploadbalancer.go +++ b/pkg/collect/host_tcploadbalancer.go @@ -9,16 +9,6 @@ import ( "github.com/pkg/errors" ) -type ConnectionResult int - -const ( - ConnectionRefused ConnectionResult = iota - Connected - ConnectionTimeout - ConnectionAddressInUse - ErrorOther -) - func HostTCPLoadBalancer(c *HostCollector) (map[string][]byte, error) { listenAddress := fmt.Sprintf("0.0.0.0:%d", c.Collect.TCPLoadBalancer.Port) dialAddress := c.Collect.TCPLoadBalancer.Address