Merge pull request #320 from areed/host-http-load-balancer

Analyze HTTP load balancer

examples/preflight/host-http-load-balancer.yaml

@@ -0,0 +1,33 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: httploadbalancer
+spec:
+  collectors:
+    - httpLoadBalancer:
+        collectorName: httploadbalancer
+        port: 80
+        address: http://app.corporate.internal
+        timeout: 10s
+  analyzers:
+    - httpLoadBalancer:
+        collectorName: httploadbalancer
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 80 via load balancer was refused.
+          - fail:
+              when: "address-in-use"
+              message: Another process was already listening on port 80.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 80 via load balancer. Check your firewall.
+          - fail:
+              when: "bind-permission-denied"
+              message: Bind permission denied. Try running as root.
+          - fail:
+              when: "error"
+              message: Failed to connect to port 80 via load balancer.
+          - pass:
+              when: "connected"
+              message: Successfully connected to port 80 via load balancer.

pkg/analyze/analyzer.go

@@ -55,6 +55,13 @@ func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getColle
 		}
 		return []*AnalyzeResult{result}, nil
 	}
+	if hostAnalyzer.HTTPLoadBalancer != nil {
+		result, err := analyzeHostHTTPLoadBalancer(hostAnalyzer.HTTPLoadBalancer, getFile)
+		if err != nil {
+			return nil, err
+		}
+		return []*AnalyzeResult{result}, nil
+	}
 	if hostAnalyzer.DiskUsage != nil {
 		result, err := analyzeHostDiskUsage(hostAnalyzer.DiskUsage, getFile)
 		if err != nil {

pkg/analyze/host_httploadbalancer.go

@@ -0,0 +1,90 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+)
+
+func analyzeHostHTTPLoadBalancer(hostAnalyzer *troubleshootv1beta2.HTTPLoadBalancerAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	collectorName := hostAnalyzer.CollectorName
+	if collectorName == "" {
+		collectorName = "httpLoadBalancer"
+	}
+	fullPath := path.Join("httpLoadBalancer", fmt.Sprintf("%s.json", collectorName))
+
+	collected, err := getCollectedFileContents(fullPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read collected file name: %s", fullPath)
+	}
+	actual := collect.NetworkStatusResult{}
+	if err := json.Unmarshal(collected, &actual); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal collected")
+	}
+
+	result := AnalyzeResult{}
+
+	title := hostAnalyzer.CheckName
+	if title == "" {
+		title = "HTTP Load Balancer"
+	}
+	result.Title = title
+
+	for _, outcome := range hostAnalyzer.Outcomes {
+		if outcome.Fail != nil {
+			if outcome.Fail.When == "" {
+				result.IsFail = true
+				result.Message = outcome.Fail.Message
+				result.URI = outcome.Fail.URI
+
+				return &result, nil
+			}
+
+			if string(actual.Status) == outcome.Fail.When {
+				result.IsFail = true
+				result.Message = outcome.Fail.Message
+				result.URI = outcome.Fail.URI
+
+				return &result, nil
+			}
+		} else if outcome.Warn != nil {
+			if outcome.Warn.When == "" {
+				result.IsWarn = true
+				result.Message = outcome.Warn.Message
+				result.URI = outcome.Warn.URI
+
+				return &result, nil
+			}
+
+			if string(actual.Status) == outcome.Warn.When {
+				result.IsWarn = true
+				result.Message = outcome.Warn.Message
+				result.URI = outcome.Warn.URI
+
+				return &result, nil
+			}
+		} else if outcome.Pass != nil {
+			if outcome.Pass.When == "" {
+				result.IsPass = true
+				result.Message = outcome.Pass.Message
+				result.URI = outcome.Pass.URI
+
+				return &result, nil
+			}
+
+			if string(actual.Status) == outcome.Pass.When {
+				result.IsPass = true
+				result.Message = outcome.Pass.Message
+				result.URI = outcome.Pass.URI
+
+				return &result, nil
+			}
+		}
+	}
+
+	return &result, nil
+}

pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go

@@ -16,6 +16,12 @@ type TCPLoadBalancerAnalyze struct {
 	Outcomes      []*Outcome `json:"outcomes" yaml:"outcomes"`
 }
 
+type HTTPLoadBalancerAnalyze struct {
+	AnalyzeMeta   `json:",inline" yaml:",inline"`
+	CollectorName string     `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
+	Outcomes      []*Outcome `json:"outcomes" yaml:"outcomes"`
+}
+
 type TCPPortStatusAnalyze struct {
 	AnalyzeMeta   `json:",inline" yaml:",inline"`
 	CollectorName string     `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
@@ -47,7 +53,8 @@ type BlockDevicesAnalyze struct {
 type HostAnalyze struct {
 	CPU *CPUAnalyze `json:"cpu,omitempty" yaml:"cpu,omitempty"`
 	//
-	TCPLoadBalancer *TCPLoadBalancerAnalyze `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"`
+	TCPLoadBalancer  *TCPLoadBalancerAnalyze  `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"`
+	HTTPLoadBalancer *HTTPLoadBalancerAnalyze `json:"httpLoadBalancer,omitempty" yaml:"httpLoadBalancer,omitempty"`
 
 	DiskUsage *DiskUsageAnalyze `json:"diskUsage,omitempty" yaml:"diskUsage,omitempty"`
 

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

@@ -1097,6 +1097,33 @@ func (in *HTTPLoadBalancer) DeepCopy() *HTTPLoadBalancer {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HTTPLoadBalancerAnalyze) DeepCopyInto(out *HTTPLoadBalancerAnalyze) {
+	*out = *in
+	out.AnalyzeMeta = in.AnalyzeMeta
+	if in.Outcomes != nil {
+		in, out := &in.Outcomes, &out.Outcomes
+		*out = make([]*Outcome, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(Outcome)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPLoadBalancerAnalyze.
+func (in *HTTPLoadBalancerAnalyze) DeepCopy() *HTTPLoadBalancerAnalyze {
+	if in == nil {
+		return nil
+	}
+	out := new(HTTPLoadBalancerAnalyze)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HostAnalyze) DeepCopyInto(out *HostAnalyze) {
 	*out = *in
@@ -1110,6 +1137,11 @@ func (in *HostAnalyze) DeepCopyInto(out *HostAnalyze) {
 		*out = new(TCPLoadBalancerAnalyze)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.HTTPLoadBalancer != nil {
+		in, out := &in.HTTPLoadBalancer, &out.HTTPLoadBalancer
+		*out = new(HTTPLoadBalancerAnalyze)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.DiskUsage != nil {
 		in, out := &in.DiskUsage, &out.DiskUsage
 		*out = new(DiskUsageAnalyze)

pkg/collect/host_collector.go

@@ -24,6 +24,8 @@ func (c *HostCollector) RunCollectorSync() (result map[string][]byte, err error)
 		result, err = HostMemory(c)
 	} else if c.Collect.TCPLoadBalancer != nil {
 		result, err = HostTCPLoadBalancer(c)
+	} else if c.Collect.HTTPLoadBalancer != nil {
+		result, err = HostHTTPLoadBalancer(c)
 	} else if c.Collect.DiskUsage != nil {
 		result, err = HostDiskUsage(c)
 	} else if c.Collect.TCPPortStatus != nil {

pkg/collect/host_httploadbalancer.go

@@ -0,0 +1,164 @@
+package collect
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/segmentio/ksuid"
+)
+
+func HostHTTPLoadBalancer(c *HostCollector) (map[string][]byte, error) {
+	listenAddress := fmt.Sprintf("0.0.0.0:%d", c.Collect.HTTPLoadBalancer.Port)
+
+	timeout := 60 * time.Minute
+	if c.Collect.HTTPLoadBalancer.Timeout != "" {
+		var err error
+		timeout, err = time.ParseDuration(c.Collect.HTTPLoadBalancer.Timeout)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse timeout %q", c.Collect.HTTPLoadBalancer.Timeout)
+		}
+	}
+
+	requestToken := ksuid.New().Bytes()
+	responseToken := ksuid.New().Bytes()
+
+	listenErr := make(chan error, 1)
+
+	go func() {
+		mux := http.NewServeMux()
+		server := http.Server{
+			Addr:    listenAddress,
+			Handler: mux,
+		}
+
+		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			if r.Method != "POST" {
+				return
+			}
+			body, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				return
+			}
+			if !bytes.Equal(body, requestToken) {
+				return
+			}
+			_, err = w.Write(responseToken)
+			if err != nil {
+				return
+			}
+			server.Shutdown(context.Background())
+		})
+
+		err := http.ListenAndServe(listenAddress, mux)
+		if err != http.ErrServerClosed {
+			listenErr <- err
+		}
+	}()
+
+	var networkStatus NetworkStatus
+
+	stopAfter := time.Now().Add(timeout)
+	for {
+		if len(listenErr) > 0 {
+			err := <-listenErr
+			if strings.Contains(err.Error(), "address already in use") {
+				networkStatus = NetworkStatusAddressInUse
+				break
+			}
+			if strings.Contains(err.Error(), "permission denied") {
+				networkStatus = NetworkStatusBindPermissionDenied
+				break
+			}
+			log.Println(err.Error())
+			networkStatus = NetworkStatusErrorOther
+			break
+		}
+		if time.Now().After(stopAfter) {
+			break
+		}
+
+		networkStatus = attemptPOST(c.Collect.HTTPLoadBalancer.Address, requestToken, responseToken)
+
+		if networkStatus == NetworkStatusErrorOther || networkStatus == NetworkStatusConnectionTimeout {
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+
+		break
+	}
+
+	result := NetworkStatusResult{
+		Status: networkStatus,
+	}
+
+	b, err := json.Marshal(result)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal result")
+	}
+
+	name := path.Join("httpLoadBalancer", "httpLoadBalancer.json")
+	if c.Collect.HTTPLoadBalancer.CollectorName != "" {
+		name = path.Join("httpLoadBalancer", fmt.Sprintf("%s.json", c.Collect.HTTPLoadBalancer.CollectorName))
+	}
+
+	return map[string][]byte{
+		name: b,
+	}, nil
+}
+
+func attemptPOST(address string, request []byte, response []byte) NetworkStatus {
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	// Create a new transport every time to ensure a new TCP connection so the load balancer does
+	// not forward every request to the same backend
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   50 * time.Millisecond,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2: true,
+	}
+	client := http.Client{
+		Transport: transport,
+	}
+
+	buf := bytes.NewBuffer(request)
+	req, err := http.NewRequestWithContext(ctx, "POST", address, buf)
+	if err != nil {
+		fmt.Println(err.Error())
+		return NetworkStatusErrorOther
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		if strings.Contains(err.Error(), "connection refused") {
+			return NetworkStatusConnectionRefused
+		}
+		if strings.Contains(err.Error(), "i/o timeout") {
+			return NetworkStatusConnectionTimeout
+		}
+
+		return NetworkStatusErrorOther
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return NetworkStatusErrorOther
+	}
+	if !bytes.Equal(body, response) {
+		return NetworkStatusErrorOther
+	}
+
+	return NetworkStatusConnected
+}

pkg/collect/host_network.go

@@ -14,11 +14,12 @@ import (
 type NetworkStatus string
 
 const (
-	NetworkStatusAddressInUse      = "address-in-use"
-	NetworkStatusConnectionRefused = "connection-refused"
-	NetworkStatusConnectionTimeout = "connection-timeout"
-	NetworkStatusConnected         = "connected"
-	NetworkStatusErrorOther        = "error"
+	NetworkStatusAddressInUse         = "address-in-use"
+	NetworkStatusConnectionRefused    = "connection-refused"
+	NetworkStatusConnectionTimeout    = "connection-timeout"
+	NetworkStatusConnected            = "connected"
+	NetworkStatusErrorOther           = "error"
+	NetworkStatusBindPermissionDenied = "bind-permission-denied"
 )
 
 type NetworkStatusResult struct {

pkg/collect/host_tcploadbalancer.go

@@ -9,16 +9,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-type ConnectionResult int
-
-const (
-	ConnectionRefused ConnectionResult = iota
-	Connected
-	ConnectionTimeout
-	ConnectionAddressInUse
-	ErrorOther
-)
-
 func HostTCPLoadBalancer(c *HostCollector) (map[string][]byte, error) {
 	listenAddress := fmt.Sprintf("0.0.0.0:%d", c.Collect.TCPLoadBalancer.Port)
 	dialAddress := c.Collect.TCPLoadBalancer.Address