diff --git a/cmd/preflight/cli/run.go b/cmd/preflight/cli/run.go index e24778a9..ca77a6f9 100644 --- a/cmd/preflight/cli/run.go +++ b/cmd/preflight/cli/run.go @@ -203,8 +203,14 @@ func collectInCluster(preflightSpec *troubleshootv1beta2.Preflight, finishedCh c return nil, errors.Wrap(err, "failed to convert kube flags to rest config") } + namespace := v.GetString("namespace") + if namespace == "" { + kubeconfig := k8sutil.GetKubeconfig() + namespace, _, _ = kubeconfig.Namespace() + } + collectOpts := preflight.CollectOpts{ - Namespace: v.GetString("namespace"), + Namespace: namespace, IgnorePermissionErrors: v.GetBool("collect-without-permissions"), ProgressChan: progressCh, KubernetesRestConfig: restConfig, diff --git a/cmd/troubleshoot/cli/run.go b/cmd/troubleshoot/cli/run.go index 14ec51eb..f34a4331 100644 --- a/cmd/troubleshoot/cli/run.go +++ b/cmd/troubleshoot/cli/run.go @@ -42,11 +42,17 @@ func runTroubleshoot(v *viper.Viper, arg string) error { os.Exit(0) }() - k8sConfig, err := k8sutil.GetRESTConfig() + restConfig, err := k8sutil.GetRESTConfig() if err != nil { return errors.Wrap(err, "failed to convert kube flags to rest config") } + namespace := v.GetString("namespace") + if namespace == "" { + kubeconfig := k8sutil.GetKubeconfig() + namespace, _, _ = kubeconfig.Namespace() + } + var sinceTime *time.Time if v.GetString("since-time") != "" || v.GetString("since") != "" { sinceTime, err = parseTimeFlags(v) @@ -149,8 +155,8 @@ func runTroubleshoot(v *viper.Viper, arg string) error { createOpts := supportbundle.SupportBundleCreateOpts{ CollectorProgressCallback: collectorCB, CollectWithoutPermissions: v.GetBool("collect-without-permissions"), - KubernetesRestConfig: k8sConfig, - Namespace: v.GetString("namespace"), + KubernetesRestConfig: restConfig, + Namespace: namespace, ProgressChan: progressChan, SinceTime: sinceTime, } diff --git a/pkg/analyze/download.go b/pkg/analyze/download.go index e2ee1bbd..d39e0e31 100644 --- a/pkg/analyze/download.go +++ b/pkg/analyze/download.go @@ -34,7 +34,7 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy for _, analyzer := range analyzers { analyzeResult, err := Analyze(analyzer, fcp.getFileContents, fcp.getChildFileContents) if err != nil { - logger.Printf("an analyzer failed to run: %v\n", err) + logger.Printf("An analyzer failed to run: %v", err) continue } diff --git a/pkg/apis/troubleshoot/v1beta2/collector_shared.go b/pkg/apis/troubleshoot/v1beta2/collector_shared.go index 5e4c3ed3..b2165cd6 100644 --- a/pkg/apis/troubleshoot/v1beta2/collector_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/collector_shared.go @@ -100,6 +100,17 @@ type Copy struct { ContainerName string `json:"containerName,omitempty" yaml:"containerName,omitempty"` } +type CopyFromHost struct { + CollectorMeta `json:",inline" yaml:",inline"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` + Image string `json:"image" yaml:"image"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty" yaml:"imagePullPolicy,omitempty"` + ImagePullSecret *ImagePullSecrets `json:"imagePullSecret,omitempty" yaml:"imagePullSecret,omitempty"` + Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` + HostPath string `json:"hostPath" yaml:"hostPath"` +} + type HTTP struct { CollectorMeta `json:",inline" yaml:",inline"` Name string `json:"name,omitempty" yaml:"name,omitempty"` @@ -172,6 +183,7 @@ type Collect struct { Exec *Exec `json:"exec,omitempty" yaml:"exec,omitempty"` Data *Data `json:"data,omitempty" yaml:"data,omitempty"` Copy *Copy `json:"copy,omitempty" yaml:"copy,omitempty"` + CopyFromHost *CopyFromHost `json:"copyFromHost,omitempty" yaml:"copyFromHost,omitempty"` HTTP *HTTP `json:"http,omitempty" yaml:"http,omitempty"` Postgres *Database `json:"postgres,omitempty" yaml:"postgres,omitempty"` Mysql *Database `json:"mysql,omitempty" yaml:"mysql,omitempty"` @@ -350,6 +362,10 @@ func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSub }, NonResourceAttributes: nil, }) + } else if c.CopyFromHost != nil { + // TODO + } else if c.Collectd != nil { + // TODO } else if c.HTTP != nil { // NOOP } else if c.RegistryImages != nil && @@ -409,6 +425,10 @@ func (c *Collect) GetName() string { name = c.Copy.CollectorName selector = strings.Join(c.Copy.Selector, ",") } + if c.CopyFromHost != nil { + collector = "copy-from-host" + name = c.CopyFromHost.CollectorName + } if c.HTTP != nil { collector = "http" name = c.HTTP.CollectorName diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index 7a8f2fa3..e037cae9 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -592,6 +592,11 @@ func (in *Collect) DeepCopyInto(out *Collect) { *out = new(Copy) (*in).DeepCopyInto(*out) } + if in.CopyFromHost != nil { + in, out := &in.CopyFromHost, &out.CopyFromHost + *out = new(CopyFromHost) + (*in).DeepCopyInto(*out) + } if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP *out = new(HTTP) @@ -888,6 +893,27 @@ func (in *Copy) DeepCopy() *Copy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CopyFromHost) DeepCopyInto(out *CopyFromHost) { + *out = *in + out.CollectorMeta = in.CollectorMeta + if in.ImagePullSecret != nil { + in, out := &in.ImagePullSecret, &out.ImagePullSecret + *out = new(ImagePullSecrets) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CopyFromHost. +func (in *CopyFromHost) DeepCopy() *CopyFromHost { + if in == nil { + return nil + } + out := new(CopyFromHost) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomResourceDefinition) DeepCopyInto(out *CustomResourceDefinition) { *out = *in diff --git a/pkg/collect/collectd.go b/pkg/collect/collectd.go index 10d6ea6f..e130aa80 100644 --- a/pkg/collect/collectd.go +++ b/pkg/collect/collectd.go @@ -2,215 +2,21 @@ package collect import ( "context" - "path" - "path/filepath" - "time" - "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" - "github.com/replicatedhq/troubleshoot/pkg/logger" - "github.com/segmentio/ksuid" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" ) -func Collectd(c *Collector, collectdCollector *troubleshootv1beta2.Collectd) (map[string][]byte, error) { - ctx := context.Background() - label := ksuid.New().String() - namespace := collectdCollector.Namespace - - client, err := kubernetes.NewForConfig(c.ClientConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to create client from config") - } - - dsName, err := createDaemonSet(ctx, client, collectdCollector, namespace, label) - if dsName != "" { - defer func() { - if err := client.AppsV1().DaemonSets(namespace).Delete(ctx, dsName, metav1.DeleteOptions{}); err != nil { - logger.Printf("Failed to delete daemonset %s: %v\n", dsName, err) - } - }() - - if collectdCollector.ImagePullSecret != nil && collectdCollector.ImagePullSecret.Data != nil { - defer func() { - err := client.CoreV1().Secrets(namespace).Delete(ctx, collectdCollector.ImagePullSecret.Name, metav1.DeleteOptions{}) - if err != nil && !kuberneteserrors.IsNotFound(err) { - logger.Printf("Failed to delete secret %s: %v\n", collectdCollector.ImagePullSecret.Name, err) - } - }() - } - } - if err != nil { - return nil, errors.Wrap(err, "failed to create daemonset") - } - - if collectdCollector.Timeout == "" { - return collectRRDFiles(ctx, client, c, collectdCollector, label, namespace) - } - - timeout, err := time.ParseDuration(collectdCollector.Timeout) - if err != nil { - return nil, errors.Wrap(err, "failed to parse timeout") - } - - childCtx, cancel := context.WithCancel(ctx) - defer cancel() - - errCh := make(chan error, 1) - resultCh := make(chan map[string][]byte, 1) - go func() { - b, err := collectRRDFiles(childCtx, client, c, collectdCollector, label, namespace) - if err != nil { - errCh <- err - } else { - resultCh <- b - } - }() - - select { - case <-time.After(timeout): - return nil, errors.New("timeout") - case result := <-resultCh: - return result, nil - case err := <-errCh: - return nil, err - } -} - -func createDaemonSet(ctx context.Context, client *kubernetes.Clientset, rrdCollector *troubleshootv1beta2.Collectd, namespace string, label string) (string, error) { - pullPolicy := corev1.PullIfNotPresent - volumeType := corev1.HostPathDirectory - if rrdCollector.ImagePullPolicy != "" { - pullPolicy = corev1.PullPolicy(rrdCollector.ImagePullPolicy) - } - dsLabels := map[string]string{ - "rrd-collector": label, - } - - ds := appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "troubleshoot", - Namespace: namespace, - Labels: dsLabels, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: dsLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: dsLabels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - Containers: []corev1.Container{ - { - Image: rrdCollector.Image, - ImagePullPolicy: pullPolicy, - Name: "collector", - Command: []string{"sleep"}, - Args: []string{"1000000"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "rrd", - MountPath: "/rrd", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "rrd", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: rrdCollector.HostPath, - Type: &volumeType, - }, - }, - }, - }, - }, - }, - }, - } - - if rrdCollector.ImagePullSecret != nil && rrdCollector.ImagePullSecret.Data != nil { - secretName, err := createSecret(ctx, client, namespace, rrdCollector.ImagePullSecret) - if err != nil { - return "", errors.Wrap(err, "failed to create secret") - } - ds.Spec.Template.Spec.ImagePullSecrets = append(ds.Spec.Template.Spec.ImagePullSecrets, corev1.LocalObjectReference{Name: secretName}) - } - - createdDS, err := client.AppsV1().DaemonSets(namespace).Create(ctx, &ds, metav1.CreateOptions{}) - if err != nil { - return "", errors.Wrap(err, "failed to create daemonset") - } - - // This timeout is different from collector timeout. - // Time it takes to pull images should not count towards collector timeout. - childCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - for { - select { - case <-time.After(1 * time.Second): - case <-childCtx.Done(): - return createdDS.Name, errors.Wrap(ctx.Err(), "failed to wait for daemonset") - } - - ds, err := client.AppsV1().DaemonSets(namespace).Get(ctx, createdDS.Name, metav1.GetOptions{}) - if err != nil { - if !kuberneteserrors.IsNotFound(err) { - continue - } - return createdDS.Name, errors.Wrap(err, "failed to get daemonset") - } - - if ds.Status.DesiredNumberScheduled != ds.Status.NumberReady { - continue - } - - break - } - - return createdDS.Name, nil -} - -func collectRRDFiles(ctx context.Context, client *kubernetes.Clientset, c *Collector, rrdCollector *troubleshootv1beta2.Collectd, label string, namespace string) (map[string][]byte, error) { - labelSelector := map[string]string{ - "rrd-collector": label, - } - opts := metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labelSelector).String(), - } - - pods, err := client.CoreV1().Pods(namespace).List(ctx, opts) - if err != nil { - return nil, errors.Wrap(err, "list rrd collector pods") - } - - pathPrefix := path.Join("collectd", "rrd") - runOutput := map[string][]byte{} - for _, pod := range pods.Items { - stdout, stderr, err := getFilesFromPod(ctx, client, c, pod.Name, "", namespace, "/rrd") - if err != nil { - runOutput[path.Join(pathPrefix, pod.Spec.NodeName)+".error"] = []byte(err.Error()) - if len(stdout) > 0 { - runOutput[filepath.Join(pathPrefix, pod.Spec.NodeName)+".stdout"] = stdout - } - if len(stderr) > 0 { - runOutput[filepath.Join(pathPrefix, pod.Spec.NodeName)+".stderr"] = stderr - } - continue - } - - runOutput[path.Join(pathPrefix, pod.Spec.NodeName)+".tar"] = stdout - } - - return runOutput, nil +func Collectd(ctx context.Context, namespace string, clientConfig *restclient.Config, client kubernetes.Interface, collector *troubleshootv1beta2.Collectd) (map[string][]byte, error) { + return CopyFromHost(ctx, namespace, clientConfig, client, &troubleshootv1beta2.CopyFromHost{ + CollectorMeta: collector.CollectorMeta, + Name: "collectd/rrd", + Namespace: collector.Namespace, + Image: collector.Image, + ImagePullPolicy: collector.ImagePullPolicy, + ImagePullSecret: collector.ImagePullSecret, + Timeout: collector.Timeout, + HostPath: collector.HostPath, + }) } diff --git a/pkg/collect/collector.go b/pkg/collect/collector.go index 43953c66..3641ab9d 100644 --- a/pkg/collect/collector.go +++ b/pkg/collect/collector.go @@ -2,6 +2,7 @@ package collect import ( "context" + "runtime" "strconv" "github.com/pkg/errors" @@ -114,6 +115,14 @@ func (c *Collector) IsExcluded() bool { if isExcludedResult { return true } + } else if c.Collect.CopyFromHost != nil { + isExcludedResult, err := isExcluded(c.Collect.CopyFromHost.Exclude) + if err != nil { + return true + } + if isExcludedResult { + return true + } } else if c.Collect.HTTP != nil { isExcludedResult, err := isExcluded(c.Collect.HTTP.Exclude) if err != nil { @@ -175,10 +184,11 @@ func (c *Collector) IsExcluded() bool { return false } -func (c *Collector) RunCollectorSync(client kubernetes.Interface, globalRedactors []*troubleshootv1beta2.Redact) (result map[string][]byte, err error) { +func (c *Collector) RunCollectorSync(clientConfig *rest.Config, client kubernetes.Interface, globalRedactors []*troubleshootv1beta2.Redact) (result map[string][]byte, err error) { defer func() { if r := recover(); r != nil { - err = errors.Errorf("recovered from panic: %v", r) + _, file, line, _ := runtime.Caller(4) + err = errors.Errorf("recovered from panic at \"%s:%d\": %v", file, line, r) } }() @@ -206,6 +216,12 @@ func (c *Collector) RunCollectorSync(client kubernetes.Interface, globalRedactor result, err = Data(c, c.Collect.Data) } else if c.Collect.Copy != nil { result, err = Copy(c, c.Collect.Copy) + } else if c.Collect.CopyFromHost != nil { + namespace := c.Collect.CopyFromHost.Namespace + if namespace == "" { + namespace = c.Namespace + } + result, err = CopyFromHost(ctx, namespace, clientConfig, client, c.Collect.CopyFromHost) } else if c.Collect.HTTP != nil { result, err = HTTP(c, c.Collect.HTTP) } else if c.Collect.Postgres != nil { @@ -216,7 +232,11 @@ func (c *Collector) RunCollectorSync(client kubernetes.Interface, globalRedactor result, err = Redis(c, c.Collect.Redis) } else if c.Collect.Collectd != nil { // TODO: see if redaction breaks these - result, err = Collectd(c, c.Collect.Collectd) + namespace := c.Collect.Collectd.Namespace + if namespace == "" { + namespace = c.Namespace + } + result, err = Collectd(ctx, namespace, clientConfig, client, c.Collect.Collectd) } else if c.Collect.Ceph != nil { result, err = Ceph(c, c.Collect.Ceph) } else if c.Collect.Longhorn != nil { @@ -233,6 +253,7 @@ func (c *Collector) RunCollectorSync(client kubernetes.Interface, globalRedactor if c.Redact { result, err = redactMap(result, globalRedactors) + err = errors.Wrap(err, "failed to redact") } return diff --git a/pkg/collect/collector_test.go b/pkg/collect/collector_test.go index a62b25dc..4b828974 100644 --- a/pkg/collect/collector_test.go +++ b/pkg/collect/collector_test.go @@ -284,7 +284,7 @@ pwd=somethinggoeshere;`, Collect: tt.Collect, Redact: true, } - got, err := c.RunCollectorSync(nil, tt.Redactors) + got, err := c.RunCollectorSync(nil, nil, tt.Redactors) req.NoError(err) // convert to string to make differences easier to see @@ -344,7 +344,7 @@ pwd=somethinggoeshere;`, Collect: tt.Collect, Redact: false, } - got, err := c.RunCollectorSync(nil, tt.Redactors) + got, err := c.RunCollectorSync(nil, nil, tt.Redactors) req.NoError(err) // convert to string to make differences easier to see diff --git a/pkg/collect/copy.go b/pkg/collect/copy.go index 63351f40..1cfc9d2a 100644 --- a/pkg/collect/copy.go +++ b/pkg/collect/copy.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" ) @@ -63,7 +64,7 @@ func copyFiles(ctx context.Context, client *kubernetes.Clientset, c *Collector, containerName = copyCollector.ContainerName } - stdout, stderr, err := getFilesFromPod(ctx, client, c, pod.Name, containerName, pod.Namespace, copyCollector.ContainerPath) + stdout, stderr, err := getFilesFromPod(ctx, c.ClientConfig, client, pod.Name, containerName, pod.Namespace, copyCollector.ContainerPath) if err != nil { errors := map[string]string{ filepath.Join(copyCollector.ContainerPath, "error"): err.Error(), @@ -82,7 +83,7 @@ func copyFiles(ctx context.Context, client *kubernetes.Clientset, c *Collector, }, nil } -func getFilesFromPod(ctx context.Context, client *kubernetes.Clientset, c *Collector, podName string, containerName string, namespace string, containerPath string) ([]byte, []byte, error) { +func getFilesFromPod(ctx context.Context, clientConfig *restclient.Config, client kubernetes.Interface, podName string, containerName string, namespace string, containerPath string) ([]byte, []byte, error) { command := []string{"tar", "-C", filepath.Dir(containerPath), "-cf", "-", filepath.Base(containerPath)} req := client.CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace(namespace).SubResource("exec") scheme := runtime.NewScheme() @@ -100,7 +101,7 @@ func getFilesFromPod(ctx context.Context, client *kubernetes.Clientset, c *Colle TTY: false, }, parameterCodec) - exec, err := remotecommand.NewSPDYExecutor(c.ClientConfig, "POST", req.URL()) + exec, err := remotecommand.NewSPDYExecutor(clientConfig, "POST", req.URL()) if err != nil { return nil, nil, errors.Wrap(err, "failed to create SPDY executor") } diff --git a/pkg/collect/copy_from_host.go b/pkg/collect/copy_from_host.go new file mode 100644 index 00000000..ae7c8913 --- /dev/null +++ b/pkg/collect/copy_from_host.go @@ -0,0 +1,233 @@ +package collect + +import ( + "context" + "path/filepath" + "time" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/logger" + "github.com/segmentio/ksuid" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" +) + +// CopyFromHost is a function that copies a file or directory from a host or hosts to include in the bundle. +func CopyFromHost(ctx context.Context, namespace string, clientConfig *restclient.Config, client kubernetes.Interface, collector *troubleshootv1beta2.CopyFromHost) (map[string][]byte, error) { + labels := map[string]string{ + "app.kubernetes.io/managed-by": "troubleshoot.sh", + "troubleshoot.sh/collector": "copyfromhost", + "troubleshoot.sh/copyfromhost-id": ksuid.New().String(), + } + + hostPath := filepath.Clean(collector.HostPath) // strip trailing slash + + hostDir := filepath.Dir(hostPath) + fileName := filepath.Base(hostPath) + if hostDir == filepath.Dir(hostDir) { // is the parent directory the root? + hostDir = hostPath + fileName = "." + } + + _, cleanup, err := copyFromHostCreateDaemonSet(ctx, client, collector, hostDir, namespace, "troubleshoot-copyfromhost-", labels) + defer cleanup() + if err != nil { + return nil, errors.Wrap(err, "create daemonset") + } + + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + timeoutCtx := context.Background() + if collector.Timeout != "" { + timeout, err := time.ParseDuration(collector.Timeout) + if err != nil { + return nil, errors.Wrap(err, "parse timeout") + } + + if timeout > 0 { + childCtx, cancel = context.WithTimeout(childCtx, timeout) + defer cancel() + } + } + + errCh := make(chan error, 1) + resultCh := make(chan map[string][]byte, 1) + go func() { + var outputFilename string + if collector.Name != "" { + outputFilename = collector.Name + } else { + outputFilename = hostPath + } + b, err := copyFromHostGetFilesFromPods(childCtx, clientConfig, client, collector, fileName, outputFilename, labels, namespace) + if err != nil { + errCh <- err + } else { + resultCh <- b + } + }() + + select { + case <-timeoutCtx.Done(): + return nil, errors.New("timeout") + case result := <-resultCh: + return result, nil + case err := <-errCh: + if errors.Is(err, context.DeadlineExceeded) { + return nil, errors.New("timeout") + } + return nil, err + } +} + +func copyFromHostCreateDaemonSet(ctx context.Context, client kubernetes.Interface, collector *troubleshootv1beta2.CopyFromHost, hostPath string, namespace string, generateName string, labels map[string]string) (name string, cleanup func(), err error) { + pullPolicy := corev1.PullIfNotPresent + volumeType := corev1.HostPathDirectory + if collector.ImagePullPolicy != "" { + pullPolicy = corev1.PullPolicy(collector.ImagePullPolicy) + } + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: generateName, + Namespace: namespace, + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{ + { + Image: collector.Image, + ImagePullPolicy: pullPolicy, + Name: "collector", + Command: []string{"sleep"}, + Args: []string{"1000000"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/host", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath, + Type: &volumeType, + }, + }, + }, + }, + }, + }, + }, + } + + cleanupFuncs := []func(){} + cleanup = func() { + for _, fn := range cleanupFuncs { + fn() + } + } + + if collector.ImagePullSecret != nil && collector.ImagePullSecret.Data != nil { + secretName, err := createSecret(ctx, client, namespace, collector.ImagePullSecret) + if err != nil { + return "", cleanup, errors.Wrap(err, "create secret") + } + ds.Spec.Template.Spec.ImagePullSecrets = append(ds.Spec.Template.Spec.ImagePullSecrets, corev1.LocalObjectReference{Name: secretName}) + + cleanupFuncs = append(cleanupFuncs, func() { + err := client.CoreV1().Secrets(namespace).Delete(ctx, collector.ImagePullSecret.Name, metav1.DeleteOptions{}) + if err != nil && !kuberneteserrors.IsNotFound(err) { + logger.Printf("Failed to delete secret %s: %v", collector.ImagePullSecret.Name, err) + } + }) + } + + createdDS, err := client.AppsV1().DaemonSets(namespace).Create(ctx, &ds, metav1.CreateOptions{}) + if err != nil { + return "", cleanup, errors.Wrap(err, "create daemonset") + } + cleanupFuncs = append(cleanupFuncs, func() { + if err := client.AppsV1().DaemonSets(namespace).Delete(ctx, createdDS.Name, metav1.DeleteOptions{}); err != nil { + logger.Printf("Failed to delete daemonset %s: %v", createdDS.Name, err) + } + }) + + // This timeout is different from collector timeout. + // Time it takes to pull images should not count towards collector timeout. + childCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + for { + select { + case <-time.After(1 * time.Second): + case <-childCtx.Done(): + return createdDS.Name, cleanup, errors.Wrap(ctx.Err(), "wait for daemonset") + } + + ds, err := client.AppsV1().DaemonSets(namespace).Get(ctx, createdDS.Name, metav1.GetOptions{}) + if err != nil { + if !kuberneteserrors.IsNotFound(err) { + continue + } + return createdDS.Name, cleanup, errors.Wrap(err, "get daemonset") + } + + if ds.Status.DesiredNumberScheduled == 0 || ds.Status.DesiredNumberScheduled != ds.Status.NumberAvailable { + continue + } + + break + } + + return createdDS.Name, cleanup, nil +} + +func copyFromHostGetFilesFromPods(ctx context.Context, clientConfig *restclient.Config, client kubernetes.Interface, collector *troubleshootv1beta2.CopyFromHost, fileName string, outputFilename string, labelSelector map[string]string, namespace string) (map[string][]byte, error) { + opts := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelSelector).String(), + } + + pods, err := client.CoreV1().Pods(namespace).List(ctx, opts) + if err != nil { + return nil, errors.Wrap(err, "list pods") + } + + runOutput := map[string][]byte{} + for _, pod := range pods.Items { + outputNodeFilename := filepath.Join(outputFilename, pod.Spec.NodeName) + stdout, stderr, err := getFilesFromPod(ctx, clientConfig, client, pod.Name, "collector", namespace, filepath.Join("/host", fileName)) + if err != nil { + runOutput[filepath.Join(outputNodeFilename, "error.txt")] = []byte(err.Error()) + if len(stdout) > 0 { + runOutput[filepath.Join(outputNodeFilename, "stdout.txt")] = stdout + } + if len(stderr) > 0 { + runOutput[filepath.Join(outputNodeFilename, "stderr.txt")] = stderr + } + } else { + runOutput[filepath.Join(outputNodeFilename, "archive.tar")] = stdout + } + } + + return runOutput, nil +} diff --git a/pkg/collect/logs.go b/pkg/collect/logs.go index ae4931bd..73c64592 100644 --- a/pkg/collect/logs.go +++ b/pkg/collect/logs.go @@ -155,7 +155,7 @@ func getPodLogs(ctx context.Context, client *kubernetes.Clientset, pod corev1.Po func convertMaxAgeToTime(maxAge string) *metav1.Time { parsedDuration, err := time.ParseDuration(maxAge) if err != nil { - logger.Printf("unable to parse time duration %s\n", maxAge) + logger.Printf("Failed to parse time duration %s", maxAge) return nil } diff --git a/pkg/collect/run.go b/pkg/collect/run.go index fcd63b0c..3193b4dd 100644 --- a/pkg/collect/run.go +++ b/pkg/collect/run.go @@ -30,14 +30,14 @@ func Run(c *Collector, runCollector *troubleshootv1beta2.Run) (map[string][]byte defer func() { if err := client.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}); err != nil { - logger.Printf("Failed to delete pod %s: %v\n", pod.Name, err) + logger.Printf("Failed to delete pod %s: %v", pod.Name, err) } }() if runCollector.ImagePullSecret != nil && runCollector.ImagePullSecret.Data != nil { defer func() { for _, k := range pod.Spec.ImagePullSecrets { if err := client.CoreV1().Secrets(pod.Namespace).Delete(ctx, k.Name, metav1.DeleteOptions{}); err != nil { - logger.Printf("Failed to delete secret %s: %v\n", k.Name, err) + logger.Printf("Failed to delete secret %s: %v", k.Name, err) } } }() @@ -170,7 +170,7 @@ func runPod(ctx context.Context, client *kubernetes.Clientset, runCollector *tro return created, nil } -func createSecret(ctx context.Context, client *kubernetes.Clientset, namespace string, imagePullSecret *troubleshootv1beta2.ImagePullSecrets) (string, error) { +func createSecret(ctx context.Context, client kubernetes.Interface, namespace string, imagePullSecret *troubleshootv1beta2.ImagePullSecrets) (string, error) { if imagePullSecret.Data == nil { return "", nil } @@ -209,6 +209,9 @@ func createSecret(ctx context.Context, client *kubernetes.Clientset, namespace s Name: imagePullSecret.Name, GenerateName: "troubleshoot", Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "troubleshoot.sh", + }, }, Data: data, Type: corev1.SecretType(imagePullSecret.SecretType), diff --git a/pkg/k8sutil/config.go b/pkg/k8sutil/config.go index 1ba097a5..82d30545 100644 --- a/pkg/k8sutil/config.go +++ b/pkg/k8sutil/config.go @@ -4,6 +4,7 @@ import ( flag "github.com/spf13/pflag" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" ) var ( @@ -18,6 +19,10 @@ func AddFlags(flags *flag.FlagSet) { kubernetesConfigFlags.AddFlags(flags) } +func GetKubeconfig() clientcmd.ClientConfig { + return kubernetesConfigFlags.ToRawKubeConfigLoader() +} + func GetRESTConfig() (*rest.Config, error) { return kubernetesConfigFlags.ToRESTConfig() } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index aad78fd0..44acf7fd 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -1,13 +1,19 @@ package logger import ( - "fmt" + "log" + "os" ) var ( - quiet = false + logger *log.Logger + quiet = false ) +func init() { + logger = log.New(os.Stderr, "", log.LstdFlags) +} + func SetQuiet(s bool) { quiet = s } @@ -16,5 +22,5 @@ func Printf(format string, args ...interface{}) { if quiet { return } - fmt.Printf(format, args...) + logger.Printf(format, args...) } diff --git a/pkg/preflight/collect.go b/pkg/preflight/collect.go index 46ea0ea5..dbaf2074 100644 --- a/pkg/preflight/collect.go +++ b/pkg/preflight/collect.go @@ -168,7 +168,7 @@ func Collect(opts CollectOpts, p *troubleshootv1beta2.Preflight) (CollectResult, TotalCount: len(collectors), } - result, err := collector.RunCollectorSync(k8sClient, nil) + result, err := collector.RunCollectorSync(opts.KubernetesRestConfig, k8sClient, nil) if err != nil { opts.ProgressChan <- errors.Errorf("failed to run collector %s: %v\n", collector.GetDisplayName(), err) opts.ProgressChan <- CollectProgress{ diff --git a/pkg/supportbundle/collect.go b/pkg/supportbundle/collect.go index 3ae0a8d1..3c3c609e 100644 --- a/pkg/supportbundle/collect.go +++ b/pkg/supportbundle/collect.go @@ -84,7 +84,7 @@ func runCollectors(collectors []*troubleshootv1beta2.Collect, additionalRedactor opts.CollectorProgressCallback(opts.ProgressChan, collector.GetDisplayName()) - result, err := collector.RunCollectorSync(k8sClient, globalRedactors) + result, err := collector.RunCollectorSync(opts.KubernetesRestConfig, k8sClient, globalRedactors) if err != nil { opts.ProgressChan <- fmt.Errorf("failed to run collector %q: %v", collector.GetDisplayName(), err) continue