get container logs for unhealthy pods (#469)

* get container logs for unhealthy pods

Co-authored-by: divolgin <dmitriy@replicated.com>
Co-authored-by: divolgin <divolgin@users.noreply.github.com>

pkg/analyze/cluster_pod_statuses.go

@@ -42,8 +42,6 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 	allResults := []*AnalyzeResult{}
 
 	for _, pod := range pods {
-		podResults := []*AnalyzeResult{}
-
 		if pod.Status.Reason == "" {
 			pod.Status.Reason = k8sutil.GetPodStatusReason(&pod)
 		}
@@ -78,12 +76,23 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 				continue
 			}
 
+			operator := parts[0]
+			reason := parts[1]
 			match := false
-			switch parts[0] {
+
+			switch operator {
 			case "=", "==", "===":
-				match = parts[1] == string(pod.Status.Phase) || parts[1] == string(pod.Status.Reason)
+				if reason == "Healthy" {
+					match = !k8sutil.IsPodUnhealthy(&pod)
+				} else {
+					match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason)
+				}
 			case "!=", "!==":
-				match = parts[1] != string(pod.Status.Phase) && parts[1] != string(pod.Status.Reason)
+				if reason == "Healthy" {
+					match = k8sutil.IsPodUnhealthy(&pod)
+				} else {
+					match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason)
+				}
 			}
 
 			if !match {
@@ -132,10 +141,9 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 			}
 			r.Message = m.String()
 
-			podResults = append(podResults, &r)
+			allResults = append(allResults, &r)
+			break
 		}
-
-		allResults = append(allResults, podResults...)
 	}
 
 	return allResults, nil

pkg/collect/cluster_resources.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"gopkg.in/yaml.v2"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -67,16 +68,19 @@ func ClusterResources(c *Collector, clusterResourcesCollector *troubleshootv1bet
 	}
 
 	// pods
-	pods, podErrors, failedPods := pods(ctx, client, namespaceNames)
+	pods, podErrors, unhealthyPods := pods(ctx, client, namespaceNames)
 	for k, v := range pods {
 		output.SaveResult(c.BundlePath, path.Join("cluster-resources/pods", k), bytes.NewBuffer(v))
 	}
 	output.SaveResult(c.BundlePath, "cluster-resources/pods-errors.json", marshalErrors(podErrors))
 
-	for _, pod := range failedPods {
+	for _, pod := range unhealthyPods {
 		allContainers := append(pod.Spec.InitContainers, pod.Spec.Containers...)
 		for _, container := range allContainers {
-			logsRoot := path.Join(c.BundlePath, "cluster-resources", "pods", pod.Namespace, "logs")
+			logsRoot := ""
+			if c.BundlePath != "" {
+				logsRoot = path.Join(c.BundlePath, "cluster-resources", "pods", "logs", pod.Namespace)
+			}
 			limits := &troubleshootv1beta2.LogLimits{
 				MaxLines: 500,
 			}
@@ -86,7 +90,7 @@ func ClusterResources(c *Collector, clusterResourcesCollector *troubleshootv1bet
 				output.SaveResult(c.BundlePath, errPath, bytes.NewBuffer([]byte(err.Error())))
 			}
 			for k, v := range podLogs {
-				output[filepath.Join("cluster-resources", "pods", pod.Namespace, "logs", k)] = v
+				output[filepath.Join("cluster-resources", "pods", "logs", pod.Namespace, k)] = v
 			}
 		}
 	}
@@ -263,7 +267,7 @@ func getNamespace(ctx context.Context, client *kubernetes.Clientset, namespace s
 func pods(ctx context.Context, client *kubernetes.Clientset, namespaces []string) (map[string][]byte, map[string]string, []corev1.Pod) {
 	podsByNamespace := make(map[string][]byte)
 	errorsByNamespace := make(map[string]string)
-	failedPods := []corev1.Pod{}
+	unhealthyPods := []corev1.Pod{}
 
 	for _, namespace := range namespaces {
 		pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
@@ -279,15 +283,15 @@ func pods(ctx context.Context, client *kubernetes.Clientset, namespaces []string
 		}
 
 		for _, pod := range pods.Items {
-			if pod.Status.Phase == corev1.PodFailed {
-				failedPods = append(failedPods, pod)
+			if k8sutil.IsPodUnhealthy(&pod) {
+				unhealthyPods = append(unhealthyPods, pod)
 			}
 		}
 
 		podsByNamespace[namespace+".json"] = b
 	}
 
-	return podsByNamespace, errorsByNamespace, failedPods
+	return podsByNamespace, errorsByNamespace, unhealthyPods
 }
 
 func services(ctx context.Context, client *kubernetes.Clientset, namespaces []string) (map[string][]byte, map[string]string) {

pkg/collect/result.go

@@ -131,8 +131,13 @@ func (r CollectorResult) CloseWriter(bundlePath string, relativePath string, wri
 		return errors.Wrap(c.Close(), "failed to close writer")
 	}
 
-	if b, ok := writer.(*bytes.Buffer); ok {
-		r[relativePath] = b.Bytes()
+	if buff, ok := writer.(*bytes.Buffer); ok {
+		b := buff.Bytes()
+		if b == nil {
+			// nil means data is on disk, so make it an empty array
+			b = []byte{}
+		}
+		r[relativePath] = b
 		return nil
 	}
 

pkg/k8sutil/pod.go

@@ -6,6 +6,25 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
+type PodStatusReason string
+
+const (
+	PodStatusReasonRunning              PodStatusReason = "Running"
+	PodStatusReasonError                PodStatusReason = "Error"
+	PodStatusReasonNotReady             PodStatusReason = "NotReady"
+	PodStatusReasonUnknown              PodStatusReason = "Unknown"
+	PodStatusReasonShutdown             PodStatusReason = "Shutdown"
+	PodStatusReasonTerminating          PodStatusReason = "Terminating"
+	PodStatusReasonCrashLoopBackOff     PodStatusReason = "CrashLoopBackOff"
+	PodStatusReasonImagePullBackOff     PodStatusReason = "ImagePullBackOff"
+	PodStatusReasonContainerCreating    PodStatusReason = "ContainerCreating"
+	PodStatusReasonPending              PodStatusReason = "Pending"
+	PodStatusReasonCompleted            PodStatusReason = "Completed"
+	PodStatusReasonEvicted              PodStatusReason = "Evicted"
+	PodStatusReasonInitError            PodStatusReason = "Init:Error"
+	PodStatusReasonInitCrashLoopBackOff PodStatusReason = "Init:CrashLoopBackOff"
+)
+
 // reference: https://github.com/kubernetes/kubernetes/blob/e8fcd0de98d50f4019561a6b7a0287f5c059267a/pkg/printers/internalversion/printers.go#L741
 func GetPodStatusReason(pod *corev1.Pod) string {
 	reason := string(pod.Status.Phase)
@@ -88,3 +107,20 @@ func hasPodReadyCondition(conditions []corev1.PodCondition) bool {
 	}
 	return false
 }
+
+func IsPodUnhealthy(pod *corev1.Pod) bool {
+	if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodPending || pod.Status.Phase == corev1.PodUnknown {
+		return true
+	}
+
+	reason := GetPodStatusReason(pod)
+
+	switch PodStatusReason(reason) {
+	case PodStatusReasonRunning:
+		fallthrough
+	case PodStatusReasonCompleted:
+		return false
+	}
+
+	return true
+}