Running collectors without the CRD

Makefile

@@ -90,13 +90,13 @@ local-release: snapshot-release
 .PHONY: run-preflight
 run-preflight: preflight
 	./bin/preflight run \
-		--collector-image=localhost:32000/troubleshoot:alpha \
-		--collector-pullpolicy=Always \
 		--image=localhost:32000/troubleshoot:alpha \
-		--pullpolicy=Always
+		--pullpolicy=Always \
+		./config/samples/troubleshoot_v1beta1_preflight.yaml
 
 .PHONY: run-troubleshoot
 run-troubleshoot: troubleshoot
 	./bin/troubleshoot run \
 		--image=localhost:32000/troubleshoot:alpha \
-		--pullpolicy=Always
+		--pullpolicy=Always \
+		./config/samples/troubleshoot_v1beta1_collector.yaml

cmd/preflight/cli/run_nocrd.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -16,7 +15,7 @@ import (
 
 	analyzerunner "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
-	preflightrunner "github.com/replicatedhq/troubleshoot/pkg/preflight"
+	collectrunner "github.com/replicatedhq/troubleshoot/pkg/collect"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
@@ -71,7 +70,7 @@ func runPreflightsNoCRD(v *viper.Viper, arg string) error {
 	getCollectedFileContents := func(fileName string) ([]byte, error) {
 		contents, ok := allCollectedData[fileName]
 		if !ok {
-			return nil, errors.New("not found")
+			return nil, fmt.Errorf("file %s was not collected", fileName)
 		}
 
 		return contents, nil
@@ -81,7 +80,8 @@ func runPreflightsNoCRD(v *viper.Viper, arg string) error {
 	for _, analyzer := range preflight.Spec.Analyzers {
 		analyzeResult, err := analyzerunner.Analyze(analyzer, getCollectedFileContents)
 		if err != nil {
-			return err
+			fmt.Printf("an analyzer failed to run: %v\n", err)
+			continue
 		}
 
 		analyzeResults = append(analyzeResults, analyzeResult)
@@ -215,7 +215,7 @@ func runCollectors(v *viper.Viper, preflight troubleshootv1beta1.Preflight) (map
 	s := runtime.NewScheme()
 	s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.ConfigMap{})
 	for _, collector := range desiredCollectors {
-		_, pod, err := preflightrunner.CreateCollector(client, s, &owner, preflight.Name, v.GetString("namespace"), collector, v.GetString("image"), v.GetString("pullpolicy"))
+		_, pod, err := collectrunner.CreateCollector(client, s, &owner, preflight.Name, v.GetString("namespace"), "preflight", collector, v.GetString("image"), v.GetString("pullpolicy"))
 		if err != nil {
 			return nil, err
 		}

cmd/troubleshoot/cli/receive.go

@@ -74,6 +74,7 @@ func receiveSupportBundle(collectorJobNamespace string, collectorJobName string)
 			for filename, maybeContents := range files {
 				fileDir, fileName := filepath.Split(filename)
 				outPath := filepath.Join(bundlePath, fileDir)
+
 				if err := os.MkdirAll(outPath, 0777); err != nil {
 					return err
 				}

cmd/troubleshoot/cli/retrieve.go

@@ -3,7 +3,6 @@ package cli
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 
 	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
@@ -79,10 +78,3 @@ func Retrieve() *cobra.Command {
 
 	return cmd
 }
-
-func homeDir() string {
-	if h := os.Getenv("HOME"); h != "" {
-		return h
-	}
-	return os.Getenv("USERPROFILE") // windows
-}

cmd/troubleshoot/cli/run.go

@@ -1,17 +1,11 @@
 package cli
 
 import (
-	"errors"
-	"fmt"
 	"path/filepath"
-	"time"
 
 	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
-	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func Run() *cobra.Command {
@@ -34,91 +28,11 @@ troubleshoot run --collectors application --wait
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			troubleshootClient, err := createTroubleshootK8sClient()
-			if err != nil {
-				return err
+			if len(args) == 0 {
+				return runTroubleshootCRD(v)
 			}
 
-			collectorName := v.GetString("collectors")
-			if collectorName == "" {
-				collectors, err := troubleshootClient.Collectors(v.GetString("namespace")).List(metav1.ListOptions{})
-				if err != nil {
-					return err
-				}
-
-				if len(collectors.Items) == 1 {
-					collectorName = collectors.Items[0].Name
-				}
-			}
-
-			if collectorName == "" {
-				return errors.New("unknown collectors, try using the --collectors flags")
-			}
-
-			// generate a unique name
-			now := time.Now()
-			suffix := fmt.Sprintf("%d", now.Unix())
-
-			collectorJobName := fmt.Sprintf("%s-job-%s", collectorName, suffix[len(suffix)-4:])
-			collectorJob := troubleshootv1beta1.CollectorJob{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      collectorJobName,
-					Namespace: v.GetString("namespace"),
-				},
-				TypeMeta: metav1.TypeMeta{
-					APIVersion: "v1",
-					Kind:       "collectorjob.troubleshoot.replicated.com",
-				},
-				Spec: troubleshootv1beta1.CollectorJobSpec{
-					Collector: troubleshootv1beta1.CollectorRef{
-						Name:      collectorName,
-						Namespace: v.GetString("namespace"),
-					},
-					Image:           v.GetString("image"),
-					ImagePullPolicy: v.GetString("pullpolicy"),
-				},
-			}
-			if _, err := troubleshootClient.CollectorJobs(v.GetString("namespace")).Create(&collectorJob); err != nil {
-				return err
-			}
-
-			// Poll the status of the Custom Resource for it to include a callback
-			var found *troubleshootv1beta1.CollectorJob
-			start := time.Now()
-			for {
-				current, err := troubleshootClient.CollectorJobs(v.GetString("namespace")).Get(collectorJobName, metav1.GetOptions{})
-				if err != nil && kuberneteserrors.IsNotFound(err) {
-					continue
-				} else if err != nil {
-					return err
-				}
-
-				if current.Status.IsServerReady {
-					found = current
-					break
-				}
-
-				if time.Now().Sub(start) > time.Duration(time.Second*10) {
-					return errors.New("collectorjob failed to start")
-				}
-
-				time.Sleep(time.Millisecond * 200)
-			}
-
-			// Connect to the callback
-			stopChan, err := k8sutil.PortForward(v.GetString("kubecontext"), 8000, 8000, found.Status.ServerPodNamespace, found.Status.ServerPodName)
-			if err != nil {
-				return err
-			}
-
-			if err := receiveSupportBundle(found.Namespace, found.Name); err != nil {
-				return err
-			}
-
-			// Write
-
-			close(stopChan)
-			return nil
+			return runTroubleshootNoCRD(v, args[0])
 		},
 	}
 
@@ -134,3 +48,16 @@ troubleshoot run --collectors application --wait
 
 	return cmd
 }
+
+func ensureCollectorInList(list []*troubleshootv1beta1.Collect, collector troubleshootv1beta1.Collect) []*troubleshootv1beta1.Collect {
+	for _, inList := range list {
+		if collector.ClusterResources != nil && inList.ClusterResources != nil {
+			return list
+		}
+		if collector.ClusterInfo != nil && inList.ClusterInfo != nil {
+			return list
+		}
+	}
+
+	return append(list, &collector)
+}

cmd/troubleshoot/cli/run_crd.go

@@ -0,0 +1,101 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
+	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
+	"github.com/spf13/viper"
+	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func runTroubleshootCRD(v *viper.Viper) error {
+	troubleshootClient, err := createTroubleshootK8sClient()
+	if err != nil {
+		return err
+	}
+
+	collectorName := v.GetString("collectors")
+	if collectorName == "" {
+		collectors, err := troubleshootClient.Collectors(v.GetString("namespace")).List(metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+
+		if len(collectors.Items) == 1 {
+			collectorName = collectors.Items[0].Name
+		}
+	}
+
+	if collectorName == "" {
+		return errors.New("unknown collectors, try using the --collectors flags")
+	}
+
+	// generate a unique name
+	now := time.Now()
+	suffix := fmt.Sprintf("%d", now.Unix())
+
+	collectorJobName := fmt.Sprintf("%s-job-%s", collectorName, suffix[len(suffix)-4:])
+	collectorJob := troubleshootv1beta1.CollectorJob{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      collectorJobName,
+			Namespace: v.GetString("namespace"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "collectorjob.troubleshoot.replicated.com",
+		},
+		Spec: troubleshootv1beta1.CollectorJobSpec{
+			Collector: troubleshootv1beta1.CollectorRef{
+				Name:      collectorName,
+				Namespace: v.GetString("namespace"),
+			},
+			Image:           v.GetString("image"),
+			ImagePullPolicy: v.GetString("pullpolicy"),
+		},
+	}
+	if _, err := troubleshootClient.CollectorJobs(v.GetString("namespace")).Create(&collectorJob); err != nil {
+		return err
+	}
+
+	// Poll the status of the Custom Resource for it to include a callback
+	var found *troubleshootv1beta1.CollectorJob
+	start := time.Now()
+	for {
+		current, err := troubleshootClient.CollectorJobs(v.GetString("namespace")).Get(collectorJobName, metav1.GetOptions{})
+		if err != nil && kuberneteserrors.IsNotFound(err) {
+			continue
+		} else if err != nil {
+			return err
+		}
+
+		if current.Status.IsServerReady {
+			found = current
+			break
+		}
+
+		if time.Now().Sub(start) > time.Duration(time.Second*10) {
+			return errors.New("collectorjob failed to start")
+		}
+
+		time.Sleep(time.Millisecond * 200)
+	}
+
+	// Connect to the callback
+	stopChan, err := k8sutil.PortForward(v.GetString("kubecontext"), 8000, 8000, found.Status.ServerPodNamespace, found.Status.ServerPodName)
+	if err != nil {
+		return err
+	}
+
+	if err := receiveSupportBundle(found.Namespace, found.Name); err != nil {
+		return err
+	}
+
+	// Write
+
+	close(stopChan)
+	return nil
+}

cmd/troubleshoot/cli/run_nocrd.go

@@ -0,0 +1,291 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/mholt/archiver"
+	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
+	collectrunner "github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+func runTroubleshootNoCRD(v *viper.Viper, arg string) error {
+	collectorContent := ""
+	if !isURL(arg) {
+		if _, err := os.Stat(arg); os.IsNotExist(err) {
+			return fmt.Errorf("%s was not found", arg)
+		}
+
+		b, err := ioutil.ReadFile(arg)
+		if err != nil {
+			return err
+		}
+
+		collectorContent = string(b)
+	} else {
+		resp, err := http.Get(arg)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		collectorContent = string(body)
+	}
+
+	collector := troubleshootv1beta1.Collector{}
+	if err := yaml.Unmarshal([]byte(collectorContent), &collector); err != nil {
+		return fmt.Errorf("unable to parse %s collectors", arg)
+	}
+
+	archivePath, err := runCollectors(v, collector)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s\n", archivePath)
+
+	return nil
+}
+
+func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector) (string, error) {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return "", err
+	}
+
+	client, err := client.New(cfg, client.Options{})
+	if err != nil {
+		return "", err
+	}
+	clientset, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return "", err
+	}
+	restClient := clientset.CoreV1().RESTClient()
+
+	// deploy an object that "owns" everything to aid in cleanup
+	owner := corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("troubleshoot-%s-owner", collector.Name),
+			Namespace: v.GetString("namespace"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		Data: make(map[string]string),
+	}
+	if err := client.Create(context.Background(), &owner); err != nil {
+		return "", err
+	}
+	defer func() {
+		if err := client.Delete(context.Background(), &owner); err != nil {
+			fmt.Println("failed to clean up preflight.")
+		}
+	}()
+
+	// deploy all collectors
+	desiredCollectors := make([]*troubleshootv1beta1.Collect, 0, 0)
+	for _, definedCollector := range collector.Spec {
+		desiredCollectors = append(desiredCollectors, definedCollector)
+	}
+	desiredCollectors = ensureCollectorInList(desiredCollectors, troubleshootv1beta1.Collect{ClusterInfo: &troubleshootv1beta1.ClusterInfo{}})
+	desiredCollectors = ensureCollectorInList(desiredCollectors, troubleshootv1beta1.Collect{ClusterResources: &troubleshootv1beta1.ClusterResources{}})
+
+	podsCreated := make([]*corev1.Pod, 0, 0)
+	podsDeleted := make([]*corev1.Pod, 0, 0)
+
+	collectorDirs := []string{}
+
+	bundlePath, err := ioutil.TempDir("", "troubleshoot")
+	if err != nil {
+		return "", err
+	}
+	// defer os.RemoveAll(bundlePath)
+
+	resyncPeriod := time.Second
+	ctx := context.Background()
+	watchList := cache.NewListWatchFromClient(restClient, "pods", "", fields.Everything())
+	_, controller := cache.NewInformer(watchList, &corev1.Pod{}, resyncPeriod,
+		cache.ResourceEventHandlerFuncs{
+			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+				newPod, ok := newObj.(*corev1.Pod)
+				if !ok {
+					return
+				}
+				oldPod, ok := oldObj.(*corev1.Pod)
+				if !ok {
+					return
+				}
+				labels := newPod.Labels
+
+				troubleshootRole, ok := labels["troubleshoot-role"]
+				if !ok || troubleshootRole != "troubleshoot" {
+					return
+				}
+
+				collectorName, ok := labels["troubleshoot"]
+				if !ok || collectorName != collector.Name {
+					return
+				}
+
+				if oldPod.Status.Phase == newPod.Status.Phase {
+					return
+				}
+
+				if newPod.Status.Phase != corev1.PodSucceeded {
+					return
+				}
+
+				podLogOpts := corev1.PodLogOptions{}
+
+				req := clientset.CoreV1().Pods(newPod.Namespace).GetLogs(newPod.Name, &podLogOpts)
+				podLogs, err := req.Stream()
+				if err != nil {
+					fmt.Println("get stream")
+					return
+				}
+				defer podLogs.Close()
+
+				buf := new(bytes.Buffer)
+				_, err = io.Copy(buf, podLogs)
+				if err != nil {
+					fmt.Println("copy logs")
+					return
+				}
+
+				collectorDir, err := parseAndSaveCollectorOutput(buf.String(), bundlePath)
+				if err != nil {
+					fmt.Printf("parse collected data: %v\n", err)
+					return
+				}
+
+				collectorDirs = append(collectorDirs, collectorDir)
+
+				if err := client.Delete(context.Background(), newPod); err != nil {
+					fmt.Println("delete pod")
+				}
+				podsDeleted = append(podsDeleted, newPod)
+			},
+		})
+	go func() {
+		controller.Run(ctx.Done())
+	}()
+
+	s := runtime.NewScheme()
+	s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.ConfigMap{})
+	for _, collect := range desiredCollectors {
+		_, pod, err := collectrunner.CreateCollector(client, s, &owner, collector.Name, v.GetString("namespace"), "troubleshoot", collect, v.GetString("image"), v.GetString("pullpolicy"))
+		if err != nil {
+			return "", err
+		}
+		podsCreated = append(podsCreated, pod)
+	}
+
+	start := time.Now()
+	for {
+		if start.Add(time.Second * 30).Before(time.Now()) {
+			fmt.Println("timeout running troubleshoot")
+			return "", err
+		}
+
+		if len(podsDeleted) == len(podsCreated) {
+			break
+		}
+
+		time.Sleep(time.Millisecond * 200)
+	}
+
+	ctx.Done()
+
+	tarGz := archiver.TarGz{
+		Tar: &archiver.Tar{
+			ImplicitTopLevelFolder: false,
+		},
+	}
+
+	paths := make([]string, 0, 0)
+	for _, collectorDir := range collectorDirs {
+		paths = append(paths, collectorDir)
+	}
+
+	if err := tarGz.Archive(paths, "support-bundle.tar.gz"); err != nil {
+		return "", err
+	}
+
+	return "support-bundle.tar.gz", nil
+}
+
+func parseAndSaveCollectorOutput(output string, bundlePath string) (string, error) {
+	dir := ""
+
+	input := make(map[string]interface{})
+	if err := json.Unmarshal([]byte(output), &input); err != nil {
+		return "", err
+	}
+
+	for filename, maybeContents := range input {
+		fileDir, fileName := filepath.Split(filename)
+		outPath := filepath.Join(bundlePath, fileDir)
+		dir = outPath
+
+		if err := os.MkdirAll(outPath, 0777); err != nil {
+			return "", err
+		}
+
+		switch maybeContents.(type) {
+		case string:
+			decoded, err := base64.StdEncoding.DecodeString(maybeContents.(string))
+			if err != nil {
+				return "", err
+			}
+
+			if err := writeFile(filepath.Join(outPath, fileName), decoded); err != nil {
+				return "", err
+			}
+
+		case map[string]interface{}:
+			for k, v := range maybeContents.(map[string]interface{}) {
+				s, _ := filepath.Split(filepath.Join(outPath, fileName, k))
+				if err := os.MkdirAll(s, 0777); err != nil {
+					return "", err
+				}
+
+				decoded, err := base64.StdEncoding.DecodeString(v.(string))
+				if err != nil {
+					return "", err
+				}
+				if err := writeFile(filepath.Join(outPath, fileName, k), decoded); err != nil {
+					return "", err
+				}
+			}
+		}
+	}
+
+	return dir, nil
+}

cmd/troubleshoot/cli/util.go

@@ -1,11 +1,30 @@
 package cli
 
 import (
+	"net/url"
+	"os"
+
 	troubleshootclientv1beta1 "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1"
 	"github.com/spf13/viper"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+func homeDir() string {
+	if h := os.Getenv("HOME"); h != "" {
+		return h
+	}
+	return os.Getenv("USERPROFILE") // windows
+}
+
+func isURL(str string) bool {
+	_, err := url.ParseRequestURI(str)
+	if err != nil {
+		return false
+	}
+
+	return true
+}
+
 func createTroubleshootK8sClient() (*troubleshootclientv1beta1.TroubleshootV1beta1Client, error) {
 	v := viper.GetViper()
 

config/crds/troubleshoot.replicated.com_collectors.yaml

@@ -396,6 +396,19 @@ spec:
                   type: object
                 clusterResources:
                   type: object
+                secret:
+                  properties:
+                    includeValue:
+                      type: boolean
+                    key:
+                      type: string
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                  required:
+                  - name
+                  type: object
               type: object
             type: array
           status:

config/crds/troubleshoot.replicated.com_preflights.yaml

@@ -520,6 +520,53 @@ spec:
                       - ingressName
                       - namespace
                       type: object
+                    secret:
+                      properties:
+                        checkName:
+                          type: string
+                        key:
+                          type: string
+                        namespace:
+                          type: string
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        secretName:
+                          type: string
+                      required:
+                      - outcomes
+                      - secretName
+                      - namespace
+                      type: object
                     storageClass:
                       properties:
                         checkName:
@@ -571,6 +618,19 @@ spec:
                       type: object
                     clusterResources:
                       type: object
+                    secret:
+                      properties:
+                        includeValue:
+                          type: boolean
+                        key:
+                          type: string
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      required:
+                      - name
+                      type: object
                   type: object
                 type: array
             type: object

config/crds/zz_generated.deepcopy.go

@@ -31,6 +31,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) {
 		*out = new(Ingress)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Secret != nil {
+		in, out := &in.Secret, &out.Secret
+		*out = new(AnalyzeSecret)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Analyze.
@@ -58,6 +63,33 @@ func (in *AnalyzeMeta) DeepCopy() *AnalyzeMeta {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AnalyzeSecret) DeepCopyInto(out *AnalyzeSecret) {
+	*out = *in
+	out.AnalyzeMeta = in.AnalyzeMeta
+	if in.Outcomes != nil {
+		in, out := &in.Outcomes, &out.Outcomes
+		*out = make([]*Outcome, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(Outcome)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnalyzeSecret.
+func (in *AnalyzeSecret) DeepCopy() *AnalyzeSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(AnalyzeSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Analyzer) DeepCopyInto(out *Analyzer) {
 	*out = *in
@@ -306,6 +338,11 @@ func (in *Collect) DeepCopyInto(out *Collect) {
 		*out = new(ClusterResources)
 		**out = **in
 	}
+	if in.Secret != nil {
+		in, out := &in.Secret, &out.Secret
+		*out = new(Secret)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Collect.
@@ -852,6 +889,21 @@ func (in *PreflightStatus) DeepCopy() *PreflightStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Secret) DeepCopyInto(out *Secret) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret.
+func (in *Secret) DeepCopy() *Secret {
+	if in == nil {
+		return nil
+	}
+	out := new(Secret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SingleOutcome) DeepCopyInto(out *SingleOutcome) {
 	*out = *in

config/samples/troubleshoot_v1beta1_collector.yaml

@@ -3,5 +3,9 @@ kind: Collector
 metadata:
   name: collector-sample
 spec:
-  - clusterInfo: {}
-  - clusterResources: {}
+  # - clusterInfo: {}
+  # - clusterResources: {}
+  - secret:
+      name: illmannered-cricket-mysql
+      namespace: default
+      key: mysql-password

config/samples/troubleshoot_v1beta1_preflight.yaml

@@ -24,6 +24,17 @@ spec:
               message: The micr0k8s storage class thing was not found
           - pass:
               message: All good on storage classes
+    - secret:
+        checkName: PG URI
+        secretName: postgres
+        namespace: default
+        key: uri
+        outcomes:
+          - fail:
+              message: You don't have a pg uri secret
+          - pass:
+              message: Probably a green light connecting to pg
+
     # - manifests:
     #     - secret:
     #         namespace: default

pkg/analyze/analyzer.go

@@ -29,6 +29,9 @@ func Analyze(analyzer *troubleshootv1beta1.Analyze, getCollectedFileContents fun
 	if analyzer.Ingress != nil {
 		return analyzeIngress(analyzer.Ingress, getCollectedFileContents)
 	}
+	if analyzer.Secret != nil {
+		return analyzeSecret(analyzer.Secret, getCollectedFileContents)
+	}
 
 	return nil, errors.New("invalid analyzer")
 }

pkg/analyze/secret.go

@@ -0,0 +1,65 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"fmt"
+
+	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+)
+
+func analyzeSecret(analyzer *troubleshootv1beta1.AnalyzeSecret, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	secretData, err := getCollectedFileContents(fmt.Sprintf("secrets/%s/%s.json", analyzer.Namespace, analyzer.SecretName))
+	if err != nil {
+		return nil, err
+	}
+
+	var foundSecret collect.FoundSecret
+	if err := json.Unmarshal(secretData, &foundSecret); err != nil {
+		return nil, err
+	}
+
+	title := analyzer.CheckName
+	if title == "" {
+		title = fmt.Sprintf("Secret %s", analyzer.SecretName)
+	}
+
+	result := AnalyzeResult{
+		Title: title,
+	}
+
+	var failOutcome *troubleshootv1beta1.Outcome
+	for _, outcome := range analyzer.Outcomes {
+		if outcome.Fail != nil {
+			failOutcome = outcome
+		}
+	}
+
+	if !foundSecret.SecretExists {
+		result.IsFail = true
+		result.Message = failOutcome.Fail.Message
+		result.URI = failOutcome.Fail.URI
+
+		return &result, nil
+	}
+
+	if analyzer.Key != "" {
+		if foundSecret.Key != analyzer.Key || !foundSecret.KeyExists {
+			result.IsFail = true
+			result.Message = failOutcome.Fail.Message
+			result.URI = failOutcome.Fail.URI
+
+			return &result, nil
+		}
+	}
+
+	result.IsPass = true
+	for _, outcome := range analyzer.Outcomes {
+		if outcome.Pass != nil {
+			result.Message = outcome.Pass.Message
+			result.URI = outcome.Pass.URI
+		}
+	}
+
+	return &result, nil
+}

pkg/apis/troubleshoot/v1beta1/analyzer_shared.go

@@ -36,6 +36,14 @@ type Ingress struct {
 	Namespace   string     `json:"namespace" yaml:"namespace"`
 }
 
+type AnalyzeSecret struct {
+	AnalyzeMeta `json:",inline" yaml:",inline"`
+	Outcomes    []*Outcome `json:"outcomes" yaml:"outcomes"`
+	SecretName  string     `json:"secretName" yaml:"secretName"`
+	Namespace   string     `json:"namespace" yaml:"namespace"`
+	Key         string     `json:"key,omitempty" yaml:"key,omitempty"`
+}
+
 type AnalyzeMeta struct {
 	CheckName string `json:"checkName,omitempty" yaml:"checkName,omitempty"`
 }
@@ -45,4 +53,5 @@ type Analyze struct {
 	StorageClass             *StorageClass             `json:"storageClass,omitempty" yaml:"storageClass,omitempty"`
 	CustomResourceDefinition *CustomResourceDefinition `json:"customResourceDefinition,omitempty" yaml:"customResourceDefinition,omitempty"`
 	Ingress                  *Ingress                  `json:"ingress,omitempty" yaml:"ingress,omitempty"`
+	Secret                   *AnalyzeSecret            `json:"secret,omitempty" yaml:"secret,omitempty"`
 }

pkg/apis/troubleshoot/v1beta1/collector_shared.go

@@ -6,7 +6,15 @@ type ClusterInfo struct {
 type ClusterResources struct {
 }
 
+type Secret struct {
+	Name         string `json:"name" yaml:"name"`
+	Namespace    string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
+	Key          string `json:"key,omitempty" yaml:"key,omitempty"`
+	IncludeValue bool   `json:"includeValue,omitempty" yaml:"includeValue,omitempty"`
+}
+
 type Collect struct {
 	ClusterInfo      *ClusterInfo      `json:"clusterInfo,omitempty" yaml:"clusterInfo,omitempty"`
 	ClusterResources *ClusterResources `json:"clusterResources,omitempty" yaml:"clusterResources,omitempty"`
+	Secret           *Secret           `json:"secret,omitempty" yaml:"secret,omitempty"`
 }

pkg/apis/troubleshoot/v1beta1/collector_types.go

@@ -32,10 +32,10 @@ type CollectorStatus struct {
 // Collector is the Schema for the collectors API
 // +k8s:openapi-gen=true
 type Collector struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
+	metav1.TypeMeta   `json:",inline" yaml:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
 
-	Spec   []*Collect      `json:"spec,omitempty"`
+	Spec   []*Collect      `json:"spec,omitempty" yaml:"spec,omitempty"`
 	Status CollectorStatus `json:"status,omitempty"`
 }
 

pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go

@@ -47,6 +47,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) {
 		*out = new(Ingress)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Secret != nil {
+		in, out := &in.Secret, &out.Secret
+		*out = new(AnalyzeSecret)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Analyze.
@@ -74,6 +79,33 @@ func (in *AnalyzeMeta) DeepCopy() *AnalyzeMeta {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AnalyzeSecret) DeepCopyInto(out *AnalyzeSecret) {
+	*out = *in
+	out.AnalyzeMeta = in.AnalyzeMeta
+	if in.Outcomes != nil {
+		in, out := &in.Outcomes, &out.Outcomes
+		*out = make([]*Outcome, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(Outcome)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnalyzeSecret.
+func (in *AnalyzeSecret) DeepCopy() *AnalyzeSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(AnalyzeSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Analyzer) DeepCopyInto(out *Analyzer) {
 	*out = *in
@@ -322,6 +354,11 @@ func (in *Collect) DeepCopyInto(out *Collect) {
 		*out = new(ClusterResources)
 		**out = **in
 	}
+	if in.Secret != nil {
+		in, out := &in.Secret, &out.Secret
+		*out = new(Secret)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Collect.
@@ -868,6 +905,21 @@ func (in *PreflightStatus) DeepCopy() *PreflightStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Secret) DeepCopyInto(out *Secret) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret.
+func (in *Secret) DeepCopy() *Secret {
+	if in == nil {
+		return nil
+	}
+	out := new(Secret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SingleOutcome) DeepCopyInto(out *SingleOutcome) {
 	*out = *in

pkg/collect/collector.go

@@ -19,9 +19,13 @@ func (c *Collector) RunCollectorSync() error {
 
 	if collect.ClusterInfo != nil {
 		return ClusterInfo()
-	} else if collect.ClusterResources != nil {
+	}
+	if collect.ClusterResources != nil {
 		return ClusterResources()
 	}
+	if collect.Secret != nil {
+		return Secret(collect.Secret)
+	}
 
 	return errors.New("no spec found to run")
 }

pkg/collect/runner.go

@@ -1,4 +1,4 @@
-package preflight
+package collect
 
 import (
 	"context"
@@ -15,13 +15,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
-func CreateCollector(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect, image string, pullPolicy string) (*corev1.ConfigMap, *corev1.Pod, error) {
-	configMap, err := createCollectorSpecConfigMap(client, scheme, ownerRef, preflightJobName, preflightJobNamespace, collect)
+func CreateCollector(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, jobName string, jobNamespace string, jobType string, collect *troubleshootv1beta1.Collect, image string, pullPolicy string) (*corev1.ConfigMap, *corev1.Pod, error) {
+	configMap, err := createCollectorSpecConfigMap(client, scheme, ownerRef, jobName, jobNamespace, collect)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	pod, err := createCollectorPod(client, scheme, ownerRef, preflightJobName, preflightJobNamespace, collect, configMap, image, pullPolicy)
+	pod, err := createCollectorPod(client, scheme, ownerRef, jobName, jobNamespace, jobType, collect, configMap, image, pullPolicy)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -29,12 +29,11 @@ func CreateCollector(client client.Client, scheme *runtime.Scheme, ownerRef meta
 	return configMap, pod, nil
 }
 
-func createCollectorSpecConfigMap(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect) (*corev1.ConfigMap, error) {
-	name := fmt.Sprintf("%s-%s", preflightJobName, idForCollector(collect))
-
+func createCollectorSpecConfigMap(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, jobName string, jobNamespace string, collect *troubleshootv1beta1.Collect) (*corev1.ConfigMap, error) {
+	name := fmt.Sprintf("%s-%s", jobName, idForCollector(collect))
 	namespacedName := types.NamespacedName{
 		Name:      name,
-		Namespace: preflightJobNamespace,
+		Namespace: jobNamespace,
 	}
 
 	found := &corev1.ConfigMap{}
@@ -54,7 +53,7 @@ func createCollectorSpecConfigMap(client client.Client, scheme *runtime.Scheme,
 	configMap := corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
-			Namespace: preflightJobNamespace,
+			Namespace: jobNamespace,
 		},
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -76,12 +75,12 @@ func createCollectorSpecConfigMap(client client.Client, scheme *runtime.Scheme,
 	return &configMap, nil
 }
 
-func createCollectorPod(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect, configMap *corev1.ConfigMap, image string, pullPolicy string) (*corev1.Pod, error) {
-	name := fmt.Sprintf("%s-%s", preflightJobName, idForCollector(collect))
+func createCollectorPod(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, jobName string, jobNamespace string, jobType string, collect *troubleshootv1beta1.Collect, configMap *corev1.ConfigMap, image string, pullPolicy string) (*corev1.Pod, error) {
+	name := fmt.Sprintf("%s-%s", jobName, idForCollector(collect))
 
 	namespacedName := types.NamespacedName{
 		Name:      name,
-		Namespace: preflightJobNamespace,
+		Namespace: jobNamespace,
 	}
 
 	found := &corev1.Pod{}
@@ -101,13 +100,14 @@ func createCollectorPod(client client.Client, scheme *runtime.Scheme, ownerRef m
 	}
 
 	podLabels := make(map[string]string)
-	podLabels["preflight"] = preflightJobName
-	podLabels["troubleshoot-role"] = "preflight"
+
+	podLabels[jobType] = jobName
+	podLabels["troubleshoot-role"] = jobType
 
 	pod := corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
-			Namespace: preflightJobNamespace,
+			Namespace: jobNamespace,
 			Labels:    podLabels,
 		},
 		TypeMeta: metav1.TypeMeta{
@@ -163,13 +163,15 @@ func createCollectorPod(client client.Client, scheme *runtime.Scheme, ownerRef m
 	return &pod, nil
 }
 
-// Todo these will overlap with troubleshoot containers running at the same time
 func idForCollector(collector *troubleshootv1beta1.Collect) string {
 	if collector.ClusterInfo != nil {
 		return "cluster-info"
-	} else if collector.ClusterResources != nil {
+	}
+	if collector.ClusterResources != nil {
 		return "cluster-resources"
 	}
-
+	if collector.Secret != nil {
+		return fmt.Sprintf("secret-%s%s", collector.Secret.Namespace, collector.Secret.Name)
+	}
 	return ""
 }

pkg/collect/secret.go

@@ -0,0 +1,94 @@
+package collect
+
+import (
+	"encoding/json"
+	"fmt"
+
+	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+type FoundSecret struct {
+	Namespace    string `json:"namespace"`
+	Name         string `json:"name"`
+	Key          string `json:"key"`
+	SecretExists bool   `json:"secretExists"`
+	KeyExists    bool   `json:"keyExists"`
+	Value        string `json:"value,omitempty"`
+}
+type SecretOutput struct {
+	FoundSecret map[string][]byte `json:"secrets/,omitempty"`
+}
+
+func Secret(secretCollector *troubleshootv1beta1.Secret) error {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	client, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	secret, encoded, err := secret(client, secretCollector)
+	if err != nil {
+		return err
+	}
+
+	secretOutput := SecretOutput{
+		FoundSecret: map[string][]byte{
+			fmt.Sprintf("%s/%s.json", secret.Namespace, secret.Name): encoded,
+		},
+	}
+
+	b, err := json.MarshalIndent(secretOutput, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s\n", b)
+
+	return nil
+}
+
+func secret(client *kubernetes.Clientset, secretCollector *troubleshootv1beta1.Secret) (*FoundSecret, []byte, error) {
+	found, err := client.CoreV1().Secrets(secretCollector.Namespace).Get(secretCollector.Name, metav1.GetOptions{})
+	if err != nil {
+		missingSecret := FoundSecret{
+			Namespace:    secretCollector.Namespace,
+			Name:         secretCollector.Name,
+			SecretExists: false,
+		}
+
+		b, err := json.MarshalIndent(missingSecret, "", "  ")
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return &missingSecret, b, err
+	}
+
+	keyExists := false
+	if secretCollector.Key != "" {
+		if _, ok := found.Data[secretCollector.Key]; ok {
+			keyExists = true
+		}
+	}
+
+	secret := FoundSecret{
+		Namespace:    found.Namespace,
+		Name:         found.Name,
+		SecretExists: true,
+		KeyExists:    keyExists,
+	}
+
+	b, err := json.MarshalIndent(secret, "", "  ")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &secret, b, nil
+}

pkg/controller/collectorjob/collectorjob_controller.go

@@ -434,6 +434,8 @@ func idForCollector(collector *troubleshootv1beta1.Collect) string {
 		return "cluster-info"
 	} else if collector.ClusterResources != nil {
 		return "cluster-resources"
+	} else if collector.Secret != nil {
+		return fmt.Sprintf("secret-%s%s", collector.Secret.Namespace, collector.Secret.Name)
 	}
 
 	return ""

pkg/controller/preflightjob/collectors.go

@@ -2,9 +2,10 @@ package preflightjob
 
 import (
 	"context"
+	"fmt"
 
 	troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1"
-	"github.com/replicatedhq/troubleshoot/pkg/preflight"
+	collectrunner "github.com/replicatedhq/troubleshoot/pkg/collect"
 )
 
 func (r *ReconcilePreflightJob) reconcilePreflightCollectors(instance *troubleshootv1beta1.PreflightJob, preflight *troubleshootv1beta1.Preflight) error {
@@ -57,7 +58,7 @@ func (r *ReconcilePreflightJob) reconcileOnePreflightCollector(instance *trouble
 		return nil
 	}
 
-	_, _, err := preflight.CreateCollector(r.Client, r.scheme, instance, instance.Name, instance.Namespace, collect, instance.Spec.Image, instance.Spec.ImagePullPolicy)
+	_, _, err := collectrunner.CreateCollector(r.Client, r.scheme, instance, instance.Name, instance.Namespace, "preflight", collect, instance.Spec.Image, instance.Spec.ImagePullPolicy)
 	if err != nil {
 		return err
 	}
@@ -94,6 +95,8 @@ func idForCollector(collector *troubleshootv1beta1.Collect) string {
 		return "cluster-info"
 	} else if collector.ClusterResources != nil {
 		return "cluster-resources"
+	} else if collector.Secret != nil {
+		return fmt.Sprintf("secret-%s%s", collector.Secret.Namespace, collector.Secret.Name)
 	}
 
 	return ""

pkg/server/collector.go

@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"encoding/base64"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 
@@ -36,8 +35,6 @@ func putCollectorOutput(c *gin.Context) {
 	}
 
 	collectorQueue[collectorID] = body
-
-	fmt.Printf("collectorQueue = %#v\n", collectorQueue)
 	c.Status(201)
 }