diff --git a/pkg/analyze/cluster_pod_statuses.go b/pkg/analyze/cluster_pod_statuses.go index 5af5b8cd..34c26169 100644 --- a/pkg/analyze/cluster_pod_statuses.go +++ b/pkg/analyze/cluster_pod_statuses.go @@ -42,8 +42,6 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi allResults := []*AnalyzeResult{} for _, pod := range pods { - podResults := []*AnalyzeResult{} - if pod.Status.Reason == "" { pod.Status.Reason = k8sutil.GetPodStatusReason(&pod) } @@ -78,12 +76,23 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi continue } + operator := parts[0] + reason := parts[1] match := false - switch parts[0] { + + switch operator { case "=", "==", "===": - match = parts[1] == string(pod.Status.Phase) || parts[1] == string(pod.Status.Reason) + if reason == "Healthy" { + match = !k8sutil.IsPodUnhealthy(&pod) + } else { + match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason) + } case "!=", "!==": - match = parts[1] != string(pod.Status.Phase) && parts[1] != string(pod.Status.Reason) + if reason == "Healthy" { + match = k8sutil.IsPodUnhealthy(&pod) + } else { + match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason) + } } if !match { @@ -132,10 +141,9 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi } r.Message = m.String() - podResults = append(podResults, &r) + allResults = append(allResults, &r) + break } - - allResults = append(allResults, podResults...) } return allResults, nil diff --git a/pkg/collect/cluster_resources.go b/pkg/collect/cluster_resources.go index f287743f..e93c90ac 100644 --- a/pkg/collect/cluster_resources.go +++ b/pkg/collect/cluster_resources.go @@ -11,6 +11,7 @@ import ( "strings" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/k8sutil" "gopkg.in/yaml.v2" authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" @@ -67,16 +68,19 @@ func ClusterResources(c *Collector, clusterResourcesCollector *troubleshootv1bet } // pods - pods, podErrors, failedPods := pods(ctx, client, namespaceNames) + pods, podErrors, unhealthyPods := pods(ctx, client, namespaceNames) for k, v := range pods { output.SaveResult(c.BundlePath, path.Join("cluster-resources/pods", k), bytes.NewBuffer(v)) } output.SaveResult(c.BundlePath, "cluster-resources/pods-errors.json", marshalErrors(podErrors)) - for _, pod := range failedPods { + for _, pod := range unhealthyPods { allContainers := append(pod.Spec.InitContainers, pod.Spec.Containers...) for _, container := range allContainers { - logsRoot := path.Join(c.BundlePath, "cluster-resources", "pods", pod.Namespace, "logs") + logsRoot := "" + if c.BundlePath != "" { + logsRoot = path.Join(c.BundlePath, "cluster-resources", "pods", "logs", pod.Namespace) + } limits := &troubleshootv1beta2.LogLimits{ MaxLines: 500, } @@ -86,7 +90,7 @@ func ClusterResources(c *Collector, clusterResourcesCollector *troubleshootv1bet output.SaveResult(c.BundlePath, errPath, bytes.NewBuffer([]byte(err.Error()))) } for k, v := range podLogs { - output[filepath.Join("cluster-resources", "pods", pod.Namespace, "logs", k)] = v + output[filepath.Join("cluster-resources", "pods", "logs", pod.Namespace, k)] = v } } } @@ -263,7 +267,7 @@ func getNamespace(ctx context.Context, client *kubernetes.Clientset, namespace s func pods(ctx context.Context, client *kubernetes.Clientset, namespaces []string) (map[string][]byte, map[string]string, []corev1.Pod) { podsByNamespace := make(map[string][]byte) errorsByNamespace := make(map[string]string) - failedPods := []corev1.Pod{} + unhealthyPods := []corev1.Pod{} for _, namespace := range namespaces { pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) @@ -279,15 +283,15 @@ func pods(ctx context.Context, client *kubernetes.Clientset, namespaces []string } for _, pod := range pods.Items { - if pod.Status.Phase == corev1.PodFailed { - failedPods = append(failedPods, pod) + if k8sutil.IsPodUnhealthy(&pod) { + unhealthyPods = append(unhealthyPods, pod) } } podsByNamespace[namespace+".json"] = b } - return podsByNamespace, errorsByNamespace, failedPods + return podsByNamespace, errorsByNamespace, unhealthyPods } func services(ctx context.Context, client *kubernetes.Clientset, namespaces []string) (map[string][]byte, map[string]string) { diff --git a/pkg/collect/result.go b/pkg/collect/result.go index 7536dc7a..822203ee 100644 --- a/pkg/collect/result.go +++ b/pkg/collect/result.go @@ -131,8 +131,13 @@ func (r CollectorResult) CloseWriter(bundlePath string, relativePath string, wri return errors.Wrap(c.Close(), "failed to close writer") } - if b, ok := writer.(*bytes.Buffer); ok { - r[relativePath] = b.Bytes() + if buff, ok := writer.(*bytes.Buffer); ok { + b := buff.Bytes() + if b == nil { + // nil means data is on disk, so make it an empty array + b = []byte{} + } + r[relativePath] = b return nil } diff --git a/pkg/k8sutil/pod.go b/pkg/k8sutil/pod.go index 0bd39da9..98a6fc0f 100644 --- a/pkg/k8sutil/pod.go +++ b/pkg/k8sutil/pod.go @@ -6,6 +6,25 @@ import ( corev1 "k8s.io/api/core/v1" ) +type PodStatusReason string + +const ( + PodStatusReasonRunning PodStatusReason = "Running" + PodStatusReasonError PodStatusReason = "Error" + PodStatusReasonNotReady PodStatusReason = "NotReady" + PodStatusReasonUnknown PodStatusReason = "Unknown" + PodStatusReasonShutdown PodStatusReason = "Shutdown" + PodStatusReasonTerminating PodStatusReason = "Terminating" + PodStatusReasonCrashLoopBackOff PodStatusReason = "CrashLoopBackOff" + PodStatusReasonImagePullBackOff PodStatusReason = "ImagePullBackOff" + PodStatusReasonContainerCreating PodStatusReason = "ContainerCreating" + PodStatusReasonPending PodStatusReason = "Pending" + PodStatusReasonCompleted PodStatusReason = "Completed" + PodStatusReasonEvicted PodStatusReason = "Evicted" + PodStatusReasonInitError PodStatusReason = "Init:Error" + PodStatusReasonInitCrashLoopBackOff PodStatusReason = "Init:CrashLoopBackOff" +) + // reference: https://github.com/kubernetes/kubernetes/blob/e8fcd0de98d50f4019561a6b7a0287f5c059267a/pkg/printers/internalversion/printers.go#L741 func GetPodStatusReason(pod *corev1.Pod) string { reason := string(pod.Status.Phase) @@ -88,3 +107,20 @@ func hasPodReadyCondition(conditions []corev1.PodCondition) bool { } return false } + +func IsPodUnhealthy(pod *corev1.Pod) bool { + if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodPending || pod.Status.Phase == corev1.PodUnknown { + return true + } + + reason := GetPodStatusReason(pod) + + switch PodStatusReason(reason) { + case PodStatusReasonRunning: + fallthrough + case PodStatusReasonCompleted: + return false + } + + return true +}