fix (support-bundle): Ensure specs are merged correctly (#1181)

When the support-bundle cli is used with --load-cluster-specs,
not all discovered specs are merged into the spec used to collect
data.

Fixes: #1179

.github/PULL_REQUEST_TEMPLATE.md

@@ -9,13 +9,13 @@ Fixes: #414
 
 ## Checklist
 
-- [ ] New and existing tests pass locally with the changes.
+- [ ] New and existing tests pass locally with introduced changes.
 - [ ] Tests for the changes have been added (for bug fixes / features)
 - [ ] The commit message(s) are informative and highlight any breaking changes
+- [ ] Any documentation required has been added/updated. For changes to https://troubleshoot.sh/ create a PR [here](https://github.com/replicatedhq/troubleshoot.sh/pulls)
 
 ## Does this PR introduce a breaking change?
 - [ ] Yes
 - [ ] No
 
 <!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. -->
-

.github/workflows/build-test-deploy.yaml

@@ -1,5 +1,10 @@
 on:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
     branches:
       - main
   push:
@@ -8,15 +13,22 @@ on:
     tags:
       - "v*.*.*"
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  fail_if_pull_request_is_draft:
+    if: github.event.pull_request.draft == true
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Fails in order to indicate that pull request needs to be marked as ready to review and unit tests workflow needs to pass.
+        run: exit 1
+
   test:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
@@ -30,12 +42,49 @@ jobs:
 
       - run: make test
 
+  test-integration:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.19"
+
+      - name: setup env
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+
+      - uses: actions/checkout@v3
+      - uses: replicatedhq/action-k3s@main
+        id: k3s
+        with:
+          version: v1.23.6-k3s1
+
+      - run: make test-integration
+
+  ensure-schemas-are-generated:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.19"
+      - name: setup env
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: make check-schemas
+
   compile-preflight:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -63,10 +112,17 @@ jobs:
           path: bin/
       - run: chmod +x bin/preflight
       - run: |
+          set +e
           ./bin/preflight --interactive=false --format=json https://preflight.replicated.com > result.json
+          EXIT_CODE=$?
           cat result.json
 
           EXIT_STATUS=0
+          if [ $EXIT_CODE -ne 3 ]; then
+            echo "Expected exit code of 3 (some checks failed), got $EXIT_CODE"
+            EXIT_STATUS=1
+          fi
+
           if grep -q "was not collected" result.json; then
             echo "Some files were not collected"
             EXIT_STATUS=1
@@ -109,9 +165,9 @@ jobs:
   compile-supportbundle:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -163,9 +219,9 @@ jobs:
   compile-collect:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -181,22 +237,31 @@ jobs:
   goreleaser-test:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v') != true
+    strategy:
+      matrix:
+        goarch: [amd64, arm64]
+        goos: [darwin, linux, windows]
+        include:
+          - goarch: arm
+            goos: linux
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-      - name: Unshallow
-        run: git fetch --prune --unshallow
-
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: "v0.183.0"
-          args: build --rm-dist --snapshot --config deploy/.goreleaser.yaml
+          args: build --rm-dist --snapshot --config deploy/.goreleaser.yaml --single-target
+        env:
+          GOARCH: ${{ matrix.goarch }}
+          GOOS: ${{ matrix.goos }}
 
   goreleaser:
     runs-on: ubuntu-latest
@@ -208,16 +273,15 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - uses: azure/docker-login@v1
         with:
-          username: repldeploy2
+          username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Unshallow
-        run: git fetch --prune --unshallow
-
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
@@ -239,19 +303,18 @@ jobs:
           COSIGN_KEY: ${{ secrets.COSIGN_KEY }}
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: "v0.183.0"
           args: release --rm-dist --config deploy/.goreleaser.yaml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-
       - name: Update new preflight version in krew-index
-        uses: rajatjindal/krew-release-bot@v0.0.38
+        uses: rajatjindal/krew-release-bot@v0.0.46
         with:
           krew_template_file: deploy/krew/preflight.yaml
       - name: Update new support-bundle version in krew-index
-        uses: rajatjindal/krew-release-bot@v0.0.38
+        uses: rajatjindal/krew-release-bot@v0.0.46
         with:
           krew_template_file: deploy/krew/support-bundle.yaml

.github/workflows/daily-scan.yaml

@@ -22,6 +22,6 @@ jobs:
           severity: 'HIGH,CRITICAL'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/fossa.yaml

@@ -0,0 +1,49 @@
+name: FOSSA license scan
+
+on:
+  pull_request_target: # this is safe as these scans do not execute provided code
+    branches:
+      - main
+    paths:
+      - go.sum
+
+  push:
+    branches:
+      - main
+
+jobs:
+  fossa-scan-pr:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: refs/pull/${{ github.event.number }}/merge
+      - name: "Install FOSSA" 
+        uses: replicatedhq/action-fossa/install@main
+      - name: "Run FOSSA Scan"
+        uses: replicatedhq/action-fossa/scan@main
+        with:
+          api-key: ${{ secrets.FOSSA_API_KEY }}
+          diff: true
+          diff-ref: ${{ github.event.pull_request.base.sha }}
+          debug: true
+
+  fossa-scan-merge:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - id: previous
+        run: echo "sha=$(git rev-parse HEAD~1)" >> "${GITHUB_OUTPUT}"
+      - name: "Install FOSSA" 
+        uses: replicatedhq/action-fossa/install@main
+      - name: "Run FOSSA Scan"
+        uses: replicatedhq/action-fossa/scan@main
+        with:
+          api-key: ${{ secrets.FOSSA_API_KEY }}
+          diff: true
+          diff-ref: ${{ steps.previous.outputs.sha }}
+          debug: true

.github/workflows/pr-license-scan.yaml

@@ -1,18 +0,0 @@
-name: PR license scan
-
-on:
-  pull_request_target: # this is safe as these scans do not execute provided code
-
-jobs:
-  fossa-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: refs/pull/${{ github.event.number }}/merge
-      - name: "Install FOSSA" 
-        uses: replicatedhq/action-fossa/install@main
-      - name: "Run FOSSA Scan"
-        uses: replicatedhq/action-fossa/scan@main
-        with:
-          api-key: ${{ secrets.FOSSA_API_KEY }}

.github/workflows/require-pr-labels.yaml

@@ -1,25 +0,0 @@
-name: Require pull request labels
-
-permissions:
-  pull-requests: read
-
-on:
-  pull_request:
-    types:
-      - opened
-      - labeled
-      - unlabeled
-      - synchronize
-
-jobs:
-  require-pr-labels:
-    uses: replicatedhq/reusable-workflows/.github/workflows/pr-enforce-labels.yaml@main
-  require-bug-labels:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: mheap/github-action-required-labels@v2
-        if: ${{ github.event.label.name == 'type::bug' }}
-        with:
-          mode: exactly
-          count: 1
-          labels: "bug::normal, bug::regression"

.gitignore

@@ -41,3 +41,4 @@ sbom/
 
 # Ignore generated support bundles
 *.tar.gz
+!testdata/supportbundle/*.tar.gz

CONTRIBUTING.md

@@ -15,7 +15,7 @@ There are [community meetings](https://calendar.google.com/calendar/u/0?cid=Y19m
 
 ## Design Principles
 
-When implementing a new feature please review the [design principles](./design/design-principles.md) to help guide the approach.
+When implementing a new feature please review the [design principles](./docs/design/design-principles.md) to help guide the approach.
 
 ## Development Environment
 
@@ -38,6 +38,7 @@ To run the tests locally run the following:
 
 ```bash
 make test
+make test RUN=TestClusterResources_Merge
 ```
 
 Additionally, e2e tests can be run with:
@@ -46,7 +47,25 @@ Additionally, e2e tests can be run with:
 make support-bundle preflight e2e-test
 ```
 
-A Kubernetes cluster as well as `jq` are required to run e2e tests.
+A running Kubernetes cluster as well as `jq` are required to run e2e tests.
+
+### Profiling
+
+You are able to collect CPU & memory runtime properties and store the data for analysis in a file. To do so, pass in the file paths using `--cpuprofile` and `--memprofile` flags in the CLI. Once you have your data collected, you can analyse it using [pprof visualization tool](https://github.com/google/pprof/blob/main/doc/README.md). Here is how
+
+Run support bundle and with CPU & memory profile flags
+```sh
+./bin/support-bundle examples/support-bundle/sample-supportbundle.yaml --cpuprofile=cpu.prof --memprofile=mem.prof
+```
+
+Visualize using [pprof](https://github.com/google/pprof/blob/main/doc/README.md)
+```sh
+go tool pprof -http=":8000" cpu.prof
+
+go tool pprof -http=":8001" mem.prof
+```
+
+More on profiling please visit https://go.dev/doc/diagnostics#profiling
 
 ## Contribution workflow
 
@@ -85,4 +104,4 @@ Commit messages should follow the general guidelines:
 - Messages should be descriptive:
   - First line, 50 chars or less, as a heading/title that people can find
   - Then a paragraph explaining things
-- Consider a footer with links to which bugs they fix etc, bearing in mind that Github does some of this magic already
+- Consider a footer with links to which bugs they fix etc, bearing in mind that Github does some of this magic already

Makefile

@@ -2,10 +2,11 @@
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 
-SHELL := /bin/bash -o pipefail
+SHELL := bash -o pipefail
 VERSION_PACKAGE = github.com/replicatedhq/troubleshoot/pkg/version
 VERSION ?=`git describe --tags --dirty`
 DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+RUN?=""
 
 GIT_TREE = $(shell git rev-parse --is-inside-work-tree 2>/dev/null)
 ifneq "$(GIT_TREE)" ""
@@ -34,8 +35,10 @@ define LDFLAGS
 endef
 
 BUILDFLAGS = -tags "netgo containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" -installsuffix netgo
+BUILDPATHS = ./pkg/... ./cmd/... ./internal/...
+TESTFLAGS ?= -v -coverprofile cover.out
 
-all: test support-bundle preflight collect
+all: test support-bundle preflight collect analyze
 
 .PHONY: ffi
 ffi: fmt vet
@@ -43,7 +46,17 @@ ffi: fmt vet
 
 .PHONY: test
 test: generate fmt vet
-	go test ${BUILDFLAGS} ./pkg/... ./cmd/... -coverprofile cover.out
+	if [ -n $(RUN) ]; then \
+		go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS} -run $(RUN); \
+	else \
+		go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS}; \
+	fi
+
+# Go tests that require a K8s instance
+# TODOLATER: merge with test, so we get unified coverage reports? it'll add 21~sec to the test job though...
+.PHONY: test-integration
+test-integration:
+	go test -v --tags "integration exclude_graphdriver_devicemapper exclude_graphdriver_btrfs" ${BUILDPATHS}
 
 .PHONY: preflight-e2e-test
 preflight-e2e-test:
@@ -71,11 +84,11 @@ collect:
 
 .PHONY: fmt
 fmt:
-	go fmt ./pkg/... ./cmd/...
+	go fmt ${BUILDPATHS}
 
 .PHONY: vet
 vet:
-	go vet ${BUILDFLAGS} ./pkg/... ./cmd/...
+	go vet ${BUILDFLAGS} ${BUILDPATHS}
 
 .PHONY: generate
 generate: controller-gen client-gen
@@ -95,6 +108,13 @@ openapischema: controller-gen
 	controller-gen crd +output:dir=./config/crds  paths=./pkg/apis/troubleshoot/v1beta1
 	controller-gen crd +output:dir=./config/crds  paths=./pkg/apis/troubleshoot/v1beta2
 
+check-schemas: generate schemas
+	@if [ -n "$(shell git status --short)" ]; then \
+    	echo -e "\033[31mThe git repo is dirty :( Ensure all generated files are committed e.g CRD schema files\033[0;m"; \
+    	git status --short; \
+    	exit 1; \
+	fi
+
 .PHONY: schemas
 schemas: fmt vet openapischema
 	go build ${LDFLAGS} -o bin/schemagen github.com/replicatedhq/troubleshoot/cmd/schemagen
@@ -106,13 +126,13 @@ docs: fmt vet
 	./bin/docsgen
 
 controller-gen:
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.2
 CONTROLLER_GEN=$(shell which controller-gen)
 
 .PHONY: client-gen
 client-gen:
-ifeq (, $(shell which client-gen))
-	go install k8s.io/code-generator/cmd/client-gen@v0.22.2
+ifeq (, $(shell which client-gen 2>/dev/null))
+	go install k8s.io/code-generator/cmd/client-gen@v0.26.1
 CLIENT_GEN=$(shell go env GOPATH)/bin/client-gen
 else
 CLIENT_GEN=$(shell which client-gen)
@@ -121,11 +141,11 @@ endif
 .PHONY: release
 release: export GITHUB_TOKEN = $(shell echo ${GITHUB_TOKEN_TROUBLESHOOT})
 release:
-	curl -sL https://git.io/goreleaser | bash -s -- --rm-dist --config deploy/.goreleaser.yaml
+	curl -sL https://git.io/goreleaser | bash -s -- --rm-dist --config deploy/.goreleaser.yml
 
 .PHONY: snapshot-release
 snapshot-release:
-	curl -sL https://git.io/goreleaser | bash -s -- --rm-dist --snapshot --config deploy/.goreleaser.snapshot.yaml
+	curl -sL https://git.io/goreleaser | bash -s -- --rm-dist --snapshot --config deploy/.goreleaser.snapshot.yml
 	docker push replicated/troubleshoot:alpha
 	docker push replicated/preflight:alpha
 
@@ -194,9 +214,9 @@ scan:
 		./
 
 .PHONY: lint
-lint:
-	golangci-lint run -c .golangci.yaml
+lint: fmt vet
+	golangci-lint run --new -c .golangci.yaml ${BUILDPATHS}
 
 .PHONY: lint-and-fix
-lint-and-fix:
-	golangci-lint run --fix -c .golangci.yaml
+lint-and-fix: fmt vet
+	golangci-lint run --new --fix -c .golangci.yaml ${BUILDPATHS}

README.md

@@ -4,6 +4,8 @@ Replicated Troubleshoot is a framework for collecting, redacting, and analyzing
 
 Troubleshoot provides two CLI tools as kubectl plugins (using [Krew](https://krew.dev)): `kubectl preflight` and `kubectl support-bundle`. Preflight provides pre-installation cluster conformance testing and validation (preflight checks) and support-bundle provides post-installation troubleshooting and diagnostics (support bundles).
 
+To know more about troubleshoot, please visit: https://troubleshoot.sh/
+
 ## Preflight Checks
 Preflight checks are an easy-to-run set of conformance tests that can be written to verify that specific requirements in a cluster are met.
 
@@ -13,12 +15,12 @@ To run a sample preflight check from a sample application, install the preflight
 curl https://krew.sh/preflight | bash
 ```
  and run, where https://preflight.replicated.com provides an **example** preflight spec:
- 
+
 ```
 kubectl preflight https://preflight.replicated.com
 ```
 
-**NOTE** this is an example. Do **not** use to validate real scenarios. 
+**NOTE** this is an example. Do **not** use to validate real scenarios.
 
 For more details on creating the custom resource files that drive preflight checks, visit [creating preflight checks](https://troubleshoot.sh/docs/preflight/introduction/).
 
@@ -32,12 +34,12 @@ To collect a sample support bundle, install the troubleshoot kubectl plugin:
 curl https://krew.sh/support-bundle | bash
 ```
  and run, where https://support-bundle.replicated.com provides an **example** support bundle spec:
- 
+
 ```
 kubectl support-bundle https://support-bundle.replicated.com
 ```
 
-**NOTE** this is an example. Do **not** use to validate real scenarios. 
+**NOTE** this is an example. Do **not** use to validate real scenarios.
 
 For more details on creating the custom resource files that drive support-bundle collection, visit [creating collectors](https://troubleshoot.sh/docs/collect/) and [creating analyzers](https://troubleshoot.sh/docs/analyze/).
 
@@ -45,17 +47,21 @@ And see our other tool [sbctl](https://github.com/replicatedhq/sbctl) that makes
 
 # Community
 
-For questions about using Troubleshoot, there's a [Replicated Community](https://help.replicated.com/community) forum, and a [#app-troubleshoot channel in Kubernetes Slack](https://kubernetes.slack.com/channels/app-troubleshoot).
+For questions about using Troubleshoot, how to contribute and engaging with the project in any other way, please refer to the following resources and channels.
 
-# Software Bill of Materials 
-A signed SBOM  that includes Troubleshoot dependencies is included in each release. 
-- **troubleshoot-sbom.tgz** contains a software bill of materials for Troubleshoot. 
+- [Replicated Community](https://help.replicated.com/community) forum
+- [#app-troubleshoot channel in Kubernetes Slack](https://kubernetes.slack.com/channels/app-troubleshoot)
+- [#Community meetings calendar](https://calendar.google.com/calendar/u/0?cid=Y19mMGx1aGhiZGtscGllOGo5dWpicXMwNnN1a0Bncm91cC5jYWxlbmRhci5nb29nbGUuY29t). This happen monthly but dates may change and would be kept upto date in the calendar.
+
+# Software Bill of Materials
+A signed SBOM  that includes Troubleshoot dependencies is included in each release.
+- **troubleshoot-sbom.tgz** contains a software bill of materials for Troubleshoot.
 - **troubleshoot-sbom.tgz.sig** is the digital signature for troubleshoot-sbom.tgz
 - **key.pub** is the public key from the key pair used to sign troubleshoot-sbom.tgz
 
 The following example illustrates using [cosign](https://github.com/sigstore/cosign) to verify that **troubleshoot-sbom.tgz** has
 not been tampered with.
 ```
-$ cosign verify-blob -key key.pub -signature troubleshoot-sbom.tgz.sig troubleshoot-sbom.tgz
+$ cosign verify-blob --key key.pub --signature troubleshoot-sbom.tgz.sig troubleshoot-sbom.tgz
 Verified OK
 ```

cmd/analyze/cli/root.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -23,17 +23,22 @@ func RootCmd() *cobra.Command {
 			v := viper.GetViper()
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-
 			return runAnalyzers(v, args[0])
 		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
+		},
 	}
 
 	cobra.OnInitialize(initConfig)
@@ -45,8 +50,14 @@ func RootCmd() *cobra.Command {
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
 	k8sutil.AddFlags(cmd.Flags())
 
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 

cmd/collect/cli/root.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -23,16 +23,22 @@ func RootCmd() *cobra.Command {
 			v := viper.GetViper()
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
 			return runCollect(v, args[0])
 		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
+		},
 	}
 
 	cobra.OnInitialize(initConfig)
@@ -58,6 +64,12 @@ func RootCmd() *cobra.Command {
 
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 

cmd/collect/cli/run.go

@@ -91,16 +91,15 @@ func runCollect(v *viper.Viper, arg string) error {
 	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 
-	additionalRedactors := &troubleshootv1beta2.Redactor{}
-	for idx, redactor := range v.GetStringSlice("redactors") {
-		redactorObj, err := supportbundle.GetRedactorFromURI(redactor)
-		if err != nil {
-			return errors.Wrapf(err, "failed to get redactor spec %s, #%d", redactor, idx)
-		}
+	redactors, err := supportbundle.GetRedactorsFromURIs(v.GetStringSlice("redactors"))
+	if err != nil {
+		return errors.Wrap(err, "failed to get redactors")
+	}
 
-		if redactorObj != nil {
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorObj.Spec.Redactors...)
-		}
+	additionalRedactors := &troubleshootv1beta2.Redactor{
+		Spec: troubleshootv1beta2.RedactorSpec{
+			Redactors: redactors,
+		},
 	}
 
 	for i, additionalDoc := range multidocs {

cmd/preflight/cli/oci_fetch.go

@@ -0,0 +1,26 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/replicatedhq/troubleshoot/pkg/oci"
+	"github.com/spf13/cobra"
+)
+
+func OciFetchCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "oci-fetch [URI]",
+		Args:  cobra.MinimumNArgs(1),
+		Short: "Fetch a preflight from an OCI registry and print it to standard out",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uri := args[0]
+			data, err := oci.PullPreflightFromOCI(uri)
+			if err != nil {
+				return err
+			}
+			fmt.Println(string(data))
+			return nil
+		},
+	}
+	return cmd
+}

cmd/preflight/cli/root.go

@@ -1,12 +1,18 @@
 package cli
 
 import (
+	"errors"
+	"fmt"
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/traces"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/replicatedhq/troubleshoot/pkg/preflight"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
@@ -19,35 +25,79 @@ func RootCmd() *cobra.Command {
 		Short: "Run and retrieve preflight checks in a cluster",
 		Long: `A preflight check is a set of validations that can and should be run to ensure
 that a cluster meets the requirements to run an application.`,
-		SilenceUsage: true,
+		SilenceUsage:  true,
+		SilenceErrors: true,
 		PreRun: func(cmd *cobra.Command, args []string) {
 			v := viper.GetViper()
+			v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
-			return preflight.RunPreflights(v.GetBool("interactive"), v.GetString("output"), v.GetString("format"), args[0])
+			closer, err := traces.ConfigureTracing("preflight")
+			if err != nil {
+				// Do not fail running preflights if tracing fails
+				klog.Errorf("Failed to initialize open tracing provider: %v", err)
+			} else {
+				defer closer()
+			}
+
+			err = preflight.RunPreflights(v.GetBool("interactive"), v.GetString("output"), v.GetString("format"), args)
+			if v.GetBool("debug") || v.IsSet("v") {
+				fmt.Printf("\n%s", traces.GetExporterInstance().GetSummary())
+			}
+
+			return err
+		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
 		},
 	}
 
 	cobra.OnInitialize(initConfig)
 
 	cmd.AddCommand(VersionCmd())
+	cmd.AddCommand(OciFetchCmd())
 	preflight.AddFlags(cmd.PersistentFlags())
 
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 
 func InitAndExecute() {
-	if err := RootCmd().Execute(); err != nil {
+	cmd := RootCmd()
+	err := cmd.Execute()
+
+	if err != nil {
+		var exitErr types.ExitError
+		if errors.As(err, &exitErr) {
+			// We need to do this, there's situations where we need the non-zero exit code (which comes as part of the custom error struct)
+			// but there's no actual error, just an exit code.
+			// If there's also an error to output (eg. invalid format etc) then print it as well
+			if exitErr.ExitStatus() != constants.EXIT_CODE_FAIL && exitErr.ExitStatus() != constants.EXIT_CODE_WARN {
+				cmd.PrintErrln("Error:", err.Error())
+			}
+
+			os.Exit(exitErr.ExitStatus())
+		}
+
+		// Fallback, should almost never be used (the above Exit() should handle almost all situations
+		cmd.PrintErrln("Error:", err.Error())
 		os.Exit(1)
 	}
 }

cmd/troubleshoot/cli/analyze.go

@@ -10,7 +10,6 @@ import (
 	"github.com/replicatedhq/troubleshoot/cmd/util"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
@@ -23,15 +22,11 @@ func Analyze() *cobra.Command {
 		Short: "analyze a support bundle",
 		Long:  `Analyze a support bundle using the Analyzer definitions provided`,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			viper.BindPFlag("bundle", cmd.Flags().Lookup("bundle"))
-			viper.BindPFlag("output", cmd.Flags().Lookup("output"))
-			viper.BindPFlag("quiet", cmd.Flags().Lookup("quiet"))
+			viper.BindPFlags(cmd.Flags())
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-
 			specPath := args[0]
 			analyzerSpec, err := downloadAnalyzerSpec(specPath)
 			if err != nil {
@@ -77,8 +72,6 @@ func Analyze() *cobra.Command {
 	cmd.Flags().MarkHidden("compatibility")
 	cmd.Flags().Bool("quiet", false, "enable/disable error messaging and only show parseable output")
 
-	viper.BindPFlags(cmd.Flags())
-
 	return cmd
 }
 

cmd/troubleshoot/cli/redact.go

@@ -0,0 +1,82 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/replicatedhq/troubleshoot/pkg/supportbundle"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func Redact() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "redact [urls...]",
+		Args:  cobra.MinimumNArgs(1), // TODO
+		Short: "Redact information from a generated support bundle archive",
+		Long: `Redaction is the process of masking sensitive information from collected data in a support bundle.
+This is done using rules defined in the list of redactor manifests provided in the [urls...] command line
+argument. Default built in redactors will also be run, but these would have been run when the support
+bundle was generated. After redaction, the support bundle is archived once more. The resulting file will
+be stored in the current directory in the path provided by the --output flag.
+
+The [urls...] argument is a list of either oci://.., http://.., https://.. or local paths to yaml files.
+
+For more information on redactors visit https://troubleshoot.sh/docs/redact/
+		`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return viper.BindPFlags(cmd.Flags())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := viper.GetViper()
+
+			// 1. Decode redactors from provided URLs
+			redactors, err := supportbundle.GetRedactorsFromURIs(args)
+			if err != nil {
+				return err
+			}
+
+			// 2. Download the bundle and extract it
+			tmpDir, bundleDir, err := analyzer.DownloadAndExtractSupportBundle(v.GetString("bundle"))
+			if err != nil {
+				return err
+			}
+			defer os.RemoveAll(tmpDir)
+
+			// 3. Represent bundle as a CollectorResult
+			collectorResult, err := collect.CollectorResultFromBundle(bundleDir)
+			if err != nil {
+				return err
+			}
+
+			// 4. Perform redaction on the bundle
+			err = collect.RedactResult(bundleDir, collectorResult, redactors)
+			if err != nil {
+				return errors.Wrap(err, "failed to redact support bundle")
+			}
+
+			// 5. Compress the bundle once more after redacting
+			output := v.GetString("output")
+			if output == "" {
+				output = fmt.Sprintf("redacted-support-bundle-%s.tar.gz", time.Now().Format("2006-01-02T15_04_05"))
+			}
+			err = collectorResult.ArchiveSupportBundle(bundleDir, output)
+			if err != nil {
+				return errors.Wrap(err, "failed to create support bundle archive")
+			}
+			fmt.Println("Redacted support bundle:", output)
+			return nil
+		},
+	}
+
+	cmd.Flags().String("bundle", "", "file path of the support bundle archive to redact")
+	cmd.MarkFlagRequired("bundle")
+	cmd.Flags().BoolP("quiet", "q", false, "enable/disable error messaging and only show parseable output")
+	cmd.Flags().StringP("output", "o", "", "file path of where to save the redacted support bundle archive (default \"redacted-support-bundle-YYYY-MM-DDTHH_MM_SS.tar.gz\")")
+
+	return cmd
+}

cmd/troubleshoot/cli/root.go

@@ -1,11 +1,12 @@
 package cli
 
 import (
-	"io/ioutil"
+	"fmt"
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/traces"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -15,43 +16,64 @@ import (
 
 func RootCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "support-bundle [url]",
+		Use:   "support-bundle [urls...]",
 		Args:  cobra.MinimumNArgs(0),
 		Short: "Generate a support bundle",
 		Long: `A support bundle is an archive of files, output, metrics and state
 from a server that can be used to assist when troubleshooting a Kubernetes cluster.`,
 		SilenceUsage: true,
-		PreRun: func(cmd *cobra.Command, args []string) {
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			v := viper.GetViper()
+			v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-			return runTroubleshoot(v, args)
+			closer, err := traces.ConfigureTracing("support-bundle")
+			if err != nil {
+				// Do not fail running support-bundle if tracing fails
+				klog.Errorf("Failed to initialize open tracing provider: %v", err)
+			} else {
+				defer closer()
+			}
+
+			err = runTroubleshoot(v, args)
+			if v.GetBool("debug") || v.IsSet("v") {
+				fmt.Printf("\n%s", traces.GetExporterInstance().GetSummary())
+			}
+
+			return err
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
 		},
 	}
 
 	cobra.OnInitialize(initConfig)
 
 	cmd.AddCommand(Analyze())
+	cmd.AddCommand(Redact())
 	cmd.AddCommand(VersionCmd())
 
 	cmd.Flags().StringSlice("redactors", []string{}, "names of the additional redactors to use")
 	cmd.Flags().Bool("redact", true, "enable/disable default redactions")
 	cmd.Flags().Bool("interactive", true, "enable/disable interactive mode")
 	cmd.Flags().Bool("collect-without-permissions", true, "always generate a support bundle, even if it some require additional permissions")
-	cmd.Flags().StringSliceP("selector", "l", []string{"troubleshoot.io/kind=supportbundle-spec"}, "selector to filter on for loading additional support bundle specs found in secrets within the cluster")
+	cmd.Flags().StringSliceP("selector", "l", []string{"troubleshoot.io/kind=support-bundle"}, "selector to filter on for loading additional support bundle specs found in secrets within the cluster")
 	cmd.Flags().Bool("load-cluster-specs", false, "enable/disable loading additional troubleshoot specs found within the cluster. required when no specs are provided on the command line")
 	cmd.Flags().String("since-time", "", "force pod logs collectors to return logs after a specific date (RFC3339)")
 	cmd.Flags().String("since", "", "force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.")
 	cmd.Flags().StringP("output", "o", "", "specify the output file path for the support bundle")
-	cmd.Flags().Bool("debug", false, "enable debug logging")
+	cmd.Flags().Bool("debug", false, "enable debug logging. This is equivalent to --v=0")
 
 	// hidden in favor of the `insecure-skip-tls-verify` flag
 	cmd.Flags().Bool("allow-insecure-connections", false, "when set, do not verify TLS certs when retrieving spec and reporting results")
@@ -61,12 +83,14 @@ from a server that can be used to assist when troubleshooting a Kubernetes clust
 	// This flag makes sure we can also disable this and fall back to the default spec.
 	cmd.Flags().Bool("no-uri", false, "When this flag is used, Troubleshoot does not attempt to retrieve the bundle referenced by the uri: field in the spec.`")
 
-	viper.BindPFlags(cmd.Flags())
-
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 
@@ -80,11 +104,3 @@ func initConfig() {
 	viper.SetEnvPrefix("TROUBLESHOOT")
 	viper.AutomaticEnv()
 }
-
-func writeFile(filename string, contents []byte) error {
-	if err := ioutil.WriteFile(filename, contents, 0644); err != nil {
-		return err
-	}
-
-	return nil
-}

cmd/troubleshoot/cli/run.go

@@ -1,20 +1,20 @@
 package cli
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	cursor "github.com/ahmetalpbalkan/go-cursor"
 	"github.com/fatih/color"
-	"github.com/manifoldco/promptui"
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
@@ -24,18 +24,20 @@ import (
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
 	"github.com/replicatedhq/troubleshoot/pkg/httputil"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/replicatedhq/troubleshoot/pkg/specs"
 	"github.com/replicatedhq/troubleshoot/pkg/supportbundle"
 	"github.com/spf13/viper"
 	spin "github.com/tj/go-spin"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
 )
 
 func runTroubleshoot(v *viper.Viper, arg []string) error {
-	if v.GetBool("load-cluster-specs") == false && len(arg) < 1 {
+	if !v.GetBool("load-cluster-specs") && len(arg) < 1 {
 		return errors.New("flag load-cluster-specs must be set if no specs are provided on the command line")
 	}
 
@@ -82,7 +84,7 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 
 	// Defining `v` below will render using `v` in reference to Viper unusable.
 	// Therefore refactoring `v` to `val` will make sure we can still use it.
-	for i, val := range arg {
+	for _, val := range arg {
 
 		collectorContent, err := supportbundle.LoadSupportBundleSpec(val)
 		if err != nil {
@@ -98,11 +100,7 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 			return errors.Wrap(err, "failed to parse support bundle spec")
 		}
 
-		if i == 0 {
-			mainBundle = supportBundle
-		} else {
-			mainBundle = supportbundle.ConcatSpec(mainBundle, supportBundle)
-		}
+		mainBundle = supportbundle.ConcatSpec(mainBundle, supportBundle)
 
 		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
 		if err != nil {
@@ -112,94 +110,15 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 	}
 
 	if v.GetBool("load-cluster-specs") {
-		labelSelector := strings.Join(v.GetStringSlice("selector"), ",")
-
-		parsedSelector, err := labels.Parse(labelSelector)
+		sbFromCluster, redactorsFromCluster, err := loadClusterSpecs()
 		if err != nil {
-			return errors.Wrap(err, "unable to parse selector")
+			return err
 		}
-
-		namespace := ""
-		if v.GetString("namespace") != "" {
-			namespace = v.GetString("namespace")
-		}
-
-		config, err := k8sutil.GetRESTConfig()
-		if err != nil {
-			return errors.Wrap(err, "failed to convert kube flags to rest config")
-		}
-
-		client, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			return errors.Wrap(err, "failed to convert create k8s client")
-		}
-
-		var bundlesFromCluster []string
-
-		// Search cluster for Troubleshoot objects in cluster
-		bundlesFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), namespace, specs.SupportBundleKey)
-		if err != nil {
-			logger.Printf("failed to load support bundle spec from secrets: %s", err)
-		}
-		bundlesFromCluster = append(bundlesFromCluster, bundlesFromSecrets...)
-
-		bundlesFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), namespace, specs.SupportBundleKey)
-		if err != nil {
-			logger.Printf("failed to load support bundle spec from secrets: %s", err)
-		}
-		bundlesFromCluster = append(bundlesFromCluster, bundlesFromConfigMaps...)
-
-		for _, bundle := range bundlesFromCluster {
-			multidocs := strings.Split(string(bundle), "\n---\n")
-			parsedBundleFromSecret, err := supportbundle.ParseSupportBundleFromDoc([]byte(multidocs[0]))
-			if err != nil {
-				logger.Printf("failed to parse support bundle spec:  %s", err)
-				continue
-			}
-
-			if mainBundle == nil {
-				mainBundle = parsedBundleFromSecret
-			} else {
-				mainBundle = supportbundle.ConcatSpec(mainBundle, parsedBundleFromSecret)
-			}
-
-			parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
-			if err != nil {
-				logger.Printf("failed to parse redactors from doc:  %s", err)
-				continue
-			}
-
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, parsedRedactors...)
-		}
-
-		var redactorsFromCluster []string
-
-		// Search cluster for Troubleshoot objects in ConfigMaps
-		redactorsFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), namespace, specs.RedactorKey)
-		if err != nil {
-			logger.Printf("failed to load redactor specs from config maps: %s", err)
-		}
-		redactorsFromCluster = append(redactorsFromCluster, redactorsFromSecrets...)
-
-		redactorsFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), namespace, specs.RedactorKey)
-		if err != nil {
-			logger.Printf("failed to load redactor specs from config maps: %s", err)
-		}
-		redactorsFromCluster = append(redactorsFromCluster, redactorsFromConfigMaps...)
-
-		for _, redactor := range redactorsFromCluster {
-			multidocs := strings.Split(string(redactor), "\n---\n")
-			parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
-			if err != nil {
-				logger.Printf("failed to parse redactors from doc:  %s", err)
-			}
-
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, parsedRedactors...)
-		}
-
-		if mainBundle == nil {
+		if sbFromCluster == nil {
 			return errors.New("no specs found in cluster")
 		}
+		mainBundle = supportbundle.ConcatSpec(mainBundle, sbFromCluster)
+		additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorsFromCluster.Spec.Redactors...)
 	}
 
 	if mainBundle == nil {
@@ -208,45 +127,48 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 		return errors.New("no collectors specified in support bundle")
 	}
 
-	for idx, redactor := range v.GetStringSlice("redactors") {
-		redactorObj, err := supportbundle.GetRedactorFromURI(redactor)
-		if err != nil {
-			return errors.Wrapf(err, "failed to get redactor spec %s, #%d", redactor, idx)
-		}
-
-		if redactorObj != nil {
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorObj.Spec.Redactors...)
-		}
+	redactors, err := supportbundle.GetRedactorsFromURIs(v.GetStringSlice("redactors"))
+	if err != nil {
+		return errors.Wrap(err, "failed to get redactors")
 	}
+	additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactors...)
 
-	var collectorCB func(chan interface{}, string)
-	progressChan := make(chan interface{}) // non-zero buffer can result in missed messages
-	finishedCh := make(chan bool, 1)
-	isFinishedChClosed := false
+	var wg sync.WaitGroup
+	collectorCB := func(c chan interface{}, msg string) { c <- msg }
+	progressChan := make(chan interface{})
+	isProgressChanClosed := false
+	defer func() {
+		if !isProgressChanClosed {
+			close(progressChan)
+		}
+		wg.Wait()
+	}()
 
 	if !interactive {
 		// TODO (dans): custom warning handler to capture warning in `analysisOutput`
 		restConfig.WarningHandler = rest.NoWarnings{}
-		collectorCB = func(ch chan interface{}, name string) {
-			return
-		}
 
 		// TODO (dans): maybe log to file
+		wg.Add(1)
 		go func() {
-			for {
-				select {
-				case _ = <-progressChan:
-					// do nothing
-				}
+			defer wg.Done()
+			for msg := range progressChan {
+				klog.Infof("Collecting support bundle: %v", msg)
 			}
 		}()
 	} else {
 		s := spin.New()
+		wg.Add(1)
 		go func() {
+			defer wg.Done()
 			currentDir := ""
 			for {
 				select {
-				case msg := <-progressChan:
+				case msg, ok := <-progressChan:
+					if !ok {
+						fmt.Printf("\r%s\r", cursor.ClearEntireLine())
+						return
+					}
 					switch msg := msg.(type) {
 					case error:
 						c := color.New(color.FgHiRed)
@@ -254,9 +176,6 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 					case string:
 						currentDir = filepath.Base(msg)
 					}
-				case <-finishedCh:
-					fmt.Printf("\r%s\r", cursor.ClearEntireLine())
-					return
 				case <-time.After(time.Millisecond * 100):
 					if currentDir == "" {
 						fmt.Printf("\r%s \033[36mCollecting support bundle\033[m %s", cursor.ClearEntireLine(), s.Next())
@@ -266,16 +185,6 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 				}
 			}
 		}()
-		defer func() {
-			if !isFinishedChClosed {
-				close(finishedCh)
-			}
-		}()
-
-		collectorCB = func(c chan interface{}, msg string) {
-			c <- fmt.Sprintf("%s", msg)
-		}
-
 	}
 
 	createOpts := supportbundle.SupportBundleCreateOpts{
@@ -301,11 +210,12 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to run collect and analyze process")
 	}
+
+	close(progressChan) // this removes the spinner in interactive mode
+	isProgressChanClosed = true
+
 	if len(response.AnalyzerResults) > 0 {
 		if interactive {
-			close(finishedCh) // this removes the spinner
-			isFinishedChClosed = true
-
 			if err := showInteractiveResults(mainBundle.Name, response.AnalyzerResults); err != nil {
 				interactive = false
 			}
@@ -333,7 +243,7 @@ the %s Admin Console to begin analysis.`
 			return nil
 		}
 
-		fmt.Printf("%s\n", response.ArchivePath)
+		fmt.Printf("\n%s\n", response.ArchivePath)
 		return nil
 	}
 
@@ -349,12 +259,163 @@ the %s Admin Console to begin analysis.`
 	return nil
 }
 
-func getExpectedContentType(uploadURL string) string {
-	parsedURL, err := url.Parse(uploadURL)
+// loadClusterSpecs loads the support bundle and redactor specs from the cluster
+// based on troubleshoot.io/kind=support-bundle label selector. We search for secrets
+// and configmaps with the label selector and parse the data as a support bundle. If the
+// user does not have sufficient permissions to list & read secrets and configmaps from
+// all namespaces, we will fallback to trying each namespace individually, and eventually
+// default to the configured kubeconfig namespace.
+func loadClusterSpecs() (*troubleshootv1beta2.SupportBundle, *troubleshootv1beta2.Redactor, error) {
+	redactors := &troubleshootv1beta2.Redactor{}
+
+	v := viper.GetViper() // It's singleton, so we can use it anywhere
+
+	klog.Info("Discover troubleshoot specs from cluster")
+
+	labelSelector := strings.Join(v.GetStringSlice("selector"), ",")
+
+	parsedSelector, err := labels.Parse(labelSelector)
 	if err != nil {
-		return ""
+		return nil, nil, errors.Wrap(err, "unable to parse selector")
 	}
-	return parsedURL.Query().Get("Content-Type")
+
+	config, err := k8sutil.GetRESTConfig()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to convert kube flags to rest config")
+	}
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to convert create k8s client")
+	}
+
+	// List of namespaces we want to search for secrets and configmaps with support bundle specs
+	namespaces := []string{}
+	ctx := context.Background()
+
+	if v.GetString("namespace") != "" {
+		// Just progress with the namespace provided
+		namespaces = []string{v.GetString("namespace")}
+	} else {
+		// Check if I can read secrets and configmaps in all namespaces
+		ican, err := k8sutil.CanIListAndGetAllSecretsAndConfigMaps(ctx, client)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "failed to check if I can read secrets and configmaps")
+		}
+		klog.V(1).Infof("Can I read any secrets and configmaps: %v", ican)
+
+		if ican {
+			// I can read secrets and configmaps in all namespaces
+			// No need to iterate over all namespaces
+			namespaces = []string{""}
+		} else {
+			// Get list of all namespaces and try to find specs from each namespace
+			nsList, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+			if err != nil {
+				if k8serrors.IsForbidden(err) {
+					kubeconfig := k8sutil.GetKubeconfig()
+					ns, _, err := kubeconfig.Namespace()
+					if err != nil {
+						return nil, nil, errors.Wrap(err, "failed to get namespace from kubeconfig")
+					}
+					// If we are not allowed to list namespaces, just use the default namespace
+					// configured in the kubeconfig
+					namespaces = []string{ns}
+				} else {
+					return nil, nil, errors.Wrap(err, "failed to list namespaces")
+				}
+			}
+
+			for _, ns := range nsList.Items {
+				namespaces = append(namespaces, ns.Name)
+			}
+		}
+	}
+
+	var bundlesFromCluster []string
+
+	// Search cluster for support bundle specs
+	klog.V(1).Infof("Search support bundle specs from [%q] namespaces using %q selector", strings.Join(namespaces, ", "), parsedSelector.String())
+	for _, ns := range namespaces {
+		bundlesFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), ns, specs.SupportBundleKey)
+		if err != nil {
+			if !k8serrors.IsForbidden(err) {
+				klog.Errorf("failed to load support bundle spec from secrets: %s", err)
+			} else {
+				klog.Warningf("Reading secrets from %q namespace forbidden", ns)
+			}
+		}
+		bundlesFromCluster = append(bundlesFromCluster, bundlesFromSecrets...)
+
+		bundlesFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), ns, specs.SupportBundleKey)
+		if err != nil {
+			if !k8serrors.IsForbidden(err) {
+				klog.Errorf("failed to load support bundle spec from configmap: %s", err)
+			} else {
+				klog.Warningf("Reading configmaps from %q namespace forbidden", ns)
+			}
+		}
+		bundlesFromCluster = append(bundlesFromCluster, bundlesFromConfigMaps...)
+	}
+
+	parsedBundle := &troubleshootv1beta2.SupportBundle{}
+
+	for _, bundle := range bundlesFromCluster {
+		multidocs := strings.Split(string(bundle), "\n---\n")
+		bundleFromDoc, err := supportbundle.ParseSupportBundleFromDoc([]byte(multidocs[0]))
+		if err != nil {
+			klog.Errorf("failed to parse support bundle spec:  %s", err)
+			continue
+		}
+
+		parsedBundle = supportbundle.ConcatSpec(parsedBundle, bundleFromDoc)
+
+		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
+		if err != nil {
+			klog.Errorf("failed to parse redactors from doc:  %s", err)
+			continue
+		}
+
+		redactors.Spec.Redactors = append(redactors.Spec.Redactors, parsedRedactors...)
+	}
+
+	var redactorsFromCluster []string
+
+	// Search cluster for redactor specs
+	klog.V(1).Infof("Search redactor specs from [%q] namespaces using %q selector", strings.Join(namespaces, ", "), parsedSelector.String())
+	for _, ns := range namespaces {
+		redactorsFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), ns, specs.RedactorKey)
+		if err != nil {
+			if !k8serrors.IsForbidden(err) {
+				klog.Errorf("failed to load support bundle spec from secrets: %s", err)
+			} else {
+				klog.Warningf("Reading secrets from %q namespace forbidden", ns)
+			}
+		}
+		redactorsFromCluster = append(redactorsFromCluster, redactorsFromSecrets...)
+
+		redactorsFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), ns, specs.RedactorKey)
+		if err != nil {
+			if !k8serrors.IsForbidden(err) {
+				klog.Errorf("failed to load support bundle spec from configmap: %s", err)
+			} else {
+				klog.Warningf("Reading configmaps from %q namespace forbidden", ns)
+			}
+		}
+		redactorsFromCluster = append(redactorsFromCluster, redactorsFromConfigMaps...)
+	}
+
+	for _, redactor := range redactorsFromCluster {
+		multidocs := strings.Split(string(redactor), "\n---\n")
+		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
+		if err != nil {
+			klog.Errorf("failed to parse redactors from doc:  %s", err)
+		}
+
+		redactors.Spec.Redactors = append(redactors.Spec.Redactors, parsedRedactors...)
+	}
+
+	return parsedBundle, redactors, nil
 }
 
 func parseTimeFlags(v *viper.Viper) (*time.Time, error) {
@@ -382,29 +443,6 @@ func parseTimeFlags(v *viper.Viper) (*time.Time, error) {
 	return &sinceTime, nil
 }
 
-func shouldRetryRequest(err error) bool {
-	if strings.Contains(err.Error(), "x509") && canTryInsecure() {
-		httputil.AddTransport(&http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		})
-		return true
-	}
-	return false
-}
-
-func canTryInsecure() bool {
-	if !isatty.IsTerminal(os.Stdout.Fd()) {
-		return false
-	}
-	prompt := promptui.Prompt{
-		Label:     "Connection appears to be insecure. Would you like to attempt to create a support bundle anyway?",
-		IsConfirm: true,
-	}
-
-	_, err := prompt.Run()
-	return err == nil
-}
-
 type analysisOutput struct {
 	Analysis    []*analyzer.AnalyzeResult
 	ArchivePath string

cmd/troubleshoot/cli/util.go

@@ -1,24 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pkg/errors"
-)
-
-func findFileName(basename, extension string) (string, error) {
-	n := 1
-	name := basename
-	for {
-		filename := name + "." + extension
-		if _, err := os.Stat(filename); os.IsNotExist(err) {
-			return filename, nil
-		} else if err != nil {
-			return "", errors.Wrap(err, "check file exists")
-		}
-
-		name = fmt.Sprintf("%s (%d)", basename, n)
-		n = n + 1
-	}
-}

cmd/troubleshoot/cli/version.go

@@ -3,13 +3,8 @@ package cli
 import (
 	"fmt"
 
-	"io/ioutil"
-	"path/filepath"
-
-	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/version"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 )
 
 func VersionCmd() *cobra.Command {
@@ -25,27 +20,3 @@ func VersionCmd() *cobra.Command {
 	}
 	return cmd
 }
-
-const VersionFilename = "version.yaml"
-
-func writeVersionFile(path string) error {
-	version := troubleshootv1beta2.SupportBundleVersion{
-		ApiVersion: "troubleshoot.sh/v1beta2",
-		Kind:       "SupportBundle",
-		Spec: troubleshootv1beta2.SupportBundleVersionSpec{
-			VersionNumber: version.Version(),
-		},
-	}
-	b, err := yaml.Marshal(version)
-	if err != nil {
-		return err
-	}
-
-	filename := filepath.Join(path, VersionFilename)
-	err = ioutil.WriteFile(filename, b, 0644)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

cmd/util/profilers.go

@@ -0,0 +1,83 @@
+package util
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	cpuProfileFile *os.File
+)
+
+// StartProfiling starts profiling CPU and memory usage if either --cpuprofile or
+// --memprofile flags were set and bound to viper configurations respectively.
+func StartProfiling() error {
+	v := viper.GetViper()
+	if v.GetString("cpuprofile") != "" {
+		var err error
+		cpuProfileFile, err = os.Create(v.GetString("cpuprofile"))
+		if err != nil {
+			return fmt.Errorf("could not create CPU profile: %w", err)
+		}
+		if err := pprof.StartCPUProfile(cpuProfileFile); err != nil {
+			cpuProfileFile.Close()
+			cpuProfileFile = nil
+			return fmt.Errorf("could not start CPU profile: %w", err)
+		}
+	}
+	return nil
+}
+
+// StopProfiling stops profiling CPU and memory usage and writes the results to
+// the files specified by --cpuprofile and --memprofile flags respectively.
+func StopProfiling() error {
+	v := viper.GetViper()
+	errs := []string{}
+
+	// Stop CPU profiling if it was started
+	if cpuProfileFile != nil {
+		pprof.StopCPUProfile()
+		err := cpuProfileFile.Close()
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+		cpuProfileFile = nil
+	}
+
+	if v.GetString("memprofile") != "" {
+		f, err := os.Create(v.GetString("memprofile"))
+		if err != nil {
+			errs = append(errs, err.Error())
+			goto BREAK_FROM_MEMPROFILE_IF
+		}
+		runtime.GC() // get up-to-date statistics
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			errs = append(errs, err.Error())
+			goto BREAK_FROM_MEMPROFILE_IF
+		}
+		err = f.Close()
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+	}
+
+BREAK_FROM_MEMPROFILE_IF:
+	if len(errs) > 0 {
+		return fmt.Errorf("errors while stopping profiling: [%s]", strings.Join(errs, ", "))
+	}
+
+	return nil
+}
+
+// AddProfilingFlags adds the --cpuprofile and --memprofile flags to the given command.
+func AddProfilingFlags(cmd *cobra.Command) {
+	// Persistent flags to make available to subcommands
+	cmd.PersistentFlags().String("cpuprofile", "", "File path to write cpu profiling data")
+	cmd.PersistentFlags().String("memprofile", "", "File path to write memory profiling data")
+}

cmd/util/util.go

@@ -4,6 +4,9 @@ import (
 	"net/url"
 	"os"
 	"strings"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 func HomeDir() string {
@@ -23,7 +26,7 @@ func IsURL(str string) bool {
 }
 
 func AppName(name string) string {
-	words := strings.Split(strings.Title(strings.Replace(name, "-", " ", -1)), " ")
+	words := strings.Split(cases.Title(language.English).String(strings.ReplaceAll(name, "-", " ")), " ")
 	casedWords := []string{}
 	for i, word := range words {
 		if strings.ToLower(word) == "ai" {

config/crds/troubleshoot.replicated.com_analyzers.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: analyzers.troubleshoot.replicated.com
 spec:
@@ -746,9 +745,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_collectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: collectors.troubleshoot.replicated.com
 spec:
@@ -329,9 +328,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_preflights.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: preflights.troubleshoot.replicated.com
 spec:
@@ -1003,9 +1002,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_redactors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: redactors.troubleshoot.replicated.com
 spec:
@@ -79,9 +78,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_supportbundles.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: supportbundles.troubleshoot.replicated.com
 spec:
@@ -1032,9 +1031,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_analyzers.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: analyzers.troubleshoot.sh
 spec:
@@ -142,6 +141,71 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    clusterResource:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        clusterScoped:
+                          type: boolean
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        expectedValue:
+                          type: string
+                        kind:
+                          type: string
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                        yamlPath:
+                          type: string
+                      required:
+                      - clusterScoped
+                      - kind
+                      - name
+                      - outcomes
+                      - yamlPath
+                      type: object
                     clusterVersion:
                       properties:
                         annotations:
@@ -710,6 +774,58 @@ spec:
                       - namespace
                       - outcomes
                       type: object
+                    mssql:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        fileName:
+                          type: string
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - collectorName
+                      - outcomes
+                      type: object
                     mysql:
                       properties:
                         annotations:
@@ -774,6 +890,8 @@ spec:
                           type: BoolString
                         filters:
                           properties:
+                            architecture:
+                              type: string
                             cpuAllocatable:
                               type: string
                             cpuCapacity:
@@ -1269,6 +1387,10 @@ spec:
                           type: string
                         exclude:
                           type: BoolString
+                        excludeFiles:
+                          items:
+                            type: string
+                          type: array
                         fileName:
                           type: string
                         ignoreIfNoFiles:
@@ -1986,6 +2108,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -2231,6 +2402,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
             type: object
@@ -2240,9 +2460,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_hostcollectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: hostcollectors.troubleshoot.sh
 spec:
@@ -634,6 +633,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -879,6 +927,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
               collectors:
@@ -905,6 +1002,17 @@ spec:
                       - certificatePath
                       - keyPath
                       type: object
+                    copy:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        path:
+                          type: string
+                      required:
+                      - path
+                      type: object
                     cpu:
                       properties:
                         collectorName:
@@ -1128,6 +1236,20 @@ spec:
                       - args
                       - command
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         amzn:
@@ -1150,6 +1272,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        centos9:
+                          items:
+                            type: string
+                          type: array
                         collectorName:
                           type: string
                         exclude:
@@ -1166,6 +1292,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        ol9:
+                          items:
+                            type: string
+                          type: array
                         rhel:
                           items:
                             type: string
@@ -1178,6 +1308,22 @@ spec:
                           items:
                             type: string
                           type: array
+                        rhel9:
+                          items:
+                            type: string
+                          type: array
+                        rocky:
+                          items:
+                            type: string
+                          type: array
+                        rocky8:
+                          items:
+                            type: string
+                          type: array
+                        rocky9:
+                          items:
+                            type: string
+                          type: array
                         ubuntu:
                           items:
                             type: string
@@ -1244,6 +1390,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
             type: object
@@ -1253,9 +1412,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_hostpreflights.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: hostpreflights.troubleshoot.sh
 spec:
@@ -634,6 +633,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -879,6 +927,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
               collectors:
@@ -905,6 +1002,17 @@ spec:
                       - certificatePath
                       - keyPath
                       type: object
+                    copy:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        path:
+                          type: string
+                      required:
+                      - path
+                      type: object
                     cpu:
                       properties:
                         collectorName:
@@ -1128,6 +1236,20 @@ spec:
                       - args
                       - command
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         amzn:
@@ -1150,6 +1272,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        centos9:
+                          items:
+                            type: string
+                          type: array
                         collectorName:
                           type: string
                         exclude:
@@ -1166,6 +1292,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        ol9:
+                          items:
+                            type: string
+                          type: array
                         rhel:
                           items:
                             type: string
@@ -1178,6 +1308,22 @@ spec:
                           items:
                             type: string
                           type: array
+                        rhel9:
+                          items:
+                            type: string
+                          type: array
+                        rocky:
+                          items:
+                            type: string
+                          type: array
+                        rocky8:
+                          items:
+                            type: string
+                          type: array
+                        rocky9:
+                          items:
+                            type: string
+                          type: array
                         ubuntu:
                           items:
                             type: string
@@ -1244,6 +1390,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
               remoteCollectors:
@@ -1463,6 +1622,20 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         collectorName:
@@ -1519,6 +1692,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
             type: object
@@ -1528,9 +1714,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_redactors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: redactors.troubleshoot.sh
 spec:
@@ -79,9 +78,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_remotecollectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: remotecollectors.troubleshoot.sh
 spec:
@@ -284,6 +283,20 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         collectorName:
@@ -340,6 +353,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
               nodeSelector:
@@ -353,9 +379,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

deploy/.goreleaser.yaml

@@ -5,13 +5,19 @@ release:
     name: troubleshoot
 builds:
   - id: preflight
+    # NOTE: if you add any additional goos/goarch values, ensure you update ../.github/workflows/build-test-deploy.yaml
+    # specifically the matrix values for goreleaser-test
     goos:
       - linux
       - darwin
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/preflight/main.go
@@ -30,7 +36,11 @@ builds:
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/troubleshoot/main.go
@@ -49,7 +59,11 @@ builds:
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/collect/main.go
@@ -125,28 +139,20 @@ archives:
 dockers:
   - dockerfile: ./deploy/Dockerfile.troubleshoot
     image_templates:
-      - "docker.io/replicated/troubleshoot:latest"
-      - "docker.io/replicated/troubleshoot:{{ .Major }}"
-      - "docker.io/replicated/troubleshoot:{{ .Major }}.{{ .Minor }}"
-      - "docker.io/replicated/troubleshoot:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
-      - "ghcr.io/replicatedhq/troubleshoot:latest"
-      - "ghcr.io/replicatedhq/troubleshoot:{{ .Major }}"
-      - "ghcr.io/replicatedhq/troubleshoot:{{ .Major }}.{{ .Minor }}"
-      - "ghcr.io/replicatedhq/troubleshoot:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
+      - "replicated/troubleshoot:latest"
+      - "replicated/troubleshoot:{{ .Major }}"
+      - "replicated/troubleshoot:{{ .Major }}.{{ .Minor }}"
+      - "replicated/troubleshoot:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
     ids:
       - support-bundle
       - preflight
       - collect
   - dockerfile: ./deploy/Dockerfile.troubleshoot
     image_templates:
-      - "docker.io/replicated/preflight:latest"
-      - "docker.io/replicated/preflight:{{ .Major }}"
-      - "docker.io/replicated/preflight:{{ .Major }}.{{ .Minor }}"
-      - "docker.io/replicated/preflight:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
-      - "ghcr.io/replicatedhq/preflight:latest"
-      - "ghcr.io/replicatedhq/preflight:{{ .Major }}"
-      - "ghcr.io/replicatedhq/preflight:{{ .Major }}.{{ .Minor }}"
-      - "ghcr.io/replicatedhq/preflight:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
+      - "replicated/preflight:latest"
+      - "replicated/preflight:{{ .Major }}"
+      - "replicated/preflight:{{ .Major }}.{{ .Minor }}"
+      - "replicated/preflight:{{ .Major }}.{{ .Minor }}.{{ .Patch }}"
     ids:
       - support-bundle
       - preflight

deploy/krew/preflight.yaml

@@ -16,6 +16,28 @@ spec:
     - from: LICENSE
       to: .
     bin: preflight
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm64
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/preflight_linux_arm64.tar.gz" .TagName }}
+    files:
+    - from: preflight
+      to: .
+    - from: LICENSE
+      to: .
+    bin: preflight
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/preflight_linux_arm.tar.gz" .TagName }}
+    files:
+    - from: preflight
+      to: .
+    - from: LICENSE
+      to: .
+    bin: preflight
   - selector:
       matchLabels:
         os: darwin
@@ -37,7 +59,7 @@ spec:
       to: .
     - from: LICENSE
       to: .
-    bin: preflight    
+    bin: preflight
   - selector:
       matchLabels:
         os: windows

deploy/krew/support-bundle.yaml

@@ -16,6 +16,28 @@ spec:
     - from: LICENSE
       to: .
     bin: support-bundle
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm64
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/support-bundle_linux_arm64.tar.gz" .TagName }}
+    files:
+    - from: support-bundle
+      to: .
+    - from: LICENSE
+      to: .
+    bin: support-bundle
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/support-bundle_linux_arm.tar.gz" .TagName }}
+    files:
+    - from: support-bundle
+      to: .
+    - from: LICENSE
+      to: .
+    bin: support-bundle
   - selector:
       matchLabels:
         os: darwin
@@ -37,7 +59,7 @@ spec:
       to: .
     - from: LICENSE
       to: .
-    bin: support-bundle    
+    bin: support-bundle
   - selector:
       matchLabels:
         os: windows

design/proposal-concurrent-collectors.md

@@ -0,0 +1,68 @@
+# Run collectors concurrently
+ 
+## Goals
+
+Increase the speed at which collectors complete.
+
+## Non Goals
+
+
+## Background
+
+Currently all collectors run sequentially, in the order in which they are specified in the spec.  This is implied, and has not been guaranteed.  With the addition of the ability to supply multiple specs, the order is no longer predicatble.
+
+Although sequence is not guaranteed for any collectors, #768 ensures that the `clusterResources` collector runs first, so that the list of resources does not include those that are started during the Troubleshoot collection sequence.
+
+The performance for support-bundle and preflight runs is important to the end user experience of the product.  There are examples where a support bundle takes multiple minutes to complete, and folks are left wondering if the process has hung or crashed.  Preflights that take too long will simply be skipped, which leads to problems with installation being difficult to identify.
+
+## High-Level Design
+
+Switch collectors and host collectors to use goroutines to collect (concurrently).  This should run the list of collectors of `clusterResources` type first, then the remainder concurrently when `clusterResources` is complete.
+
+## Detailed Design
+
+Tasks that can be split to separate merges (in order):
+
+* Switch all collectors to use goroutines to collect (concurrently).
+  * In `supportbundle.runCollectors()`, refactor to use a `go func()` pattern to run all the collectors that are not `clusterResources` concurrently.  Check that the first in the list is `clusterResources` and run that separately, prior to the others.
+  * This task should introduce an optional concurrency limit field, default to, say, 10, which limits the number of goroutines/collectors that can run at any given time.
+  * This task should improve the collection speed of in-cluster collectors and preflights
+
+* Switch Host Collectors to use goroutines for concurrent collection.
+  * In `supportbundle.runHostCollectors()`, refactor to use a `go func()` pattern for each collector.
+  * No need to honor the same concurrency limit as host collectors do not load the Kubernetes API
+
+* Switch Host Preflights to use goroutines - `preflight.CollectHost()` runs `go func()` for each collector.
+
+* Switch in-cluster Preflights to use goroutines - `preflight.Collect()` runs `go func()` for each collector.
+  * Ensure `clusterResources` runs in advance of the others so that pods started by collectors do not produce an error in the results.
+
+## Limitations
+
+Relies on splitting up the list of collectors into `clusterResources` and others.
+
+Collisions between specs (e.g. if `runPod` has multiple specs with the same name) are not handled.  At the current stage with collectors running sequentially, if two collectors write to the same file the second overwrites the data from the first.  If collectors run concurrently, there can be issues creating multiple pods with the same name, and issues with concurrent access to the target file.  This issue is tracked in [#895](https://github.com/replicatedhq/troubleshoot/issues/895)
+
+This proposal excludes Remote Collectors.
+## Assumptions
+
+* The Kubernetes API can handle the load of collectors running concurrently, including when the pod logs collector runs.
+
+## Testing
+
+Any new function will need unit tests.
+
+The existing Collect functions will need unit tests altered (or added).
+
+We will need to ensure that `clusterResources` is collected prior to others.
+
+## Documentation
+
+There should be a note in https://troubleshoot.sh/docs/collect/ ensuring that folks do not expect the collectors to run in any particular order.
+
+## Alternatives Considered
+
+None.
+## Security Considerations
+
+None identified.

docs/design/proposal-sbctl-integration.md

@@ -0,0 +1,120 @@
+# Integrate SBCTL with Troubleshoot
+
+## Goals
+
+Make sbctl easier to maintain by consolidating code bases.
+
+Reduce code duplication.
+
+Improve end-user experience and improve discoverability of sbctl.
+
+## Non Goals
+
+## Background
+sbctl is our command line tool for examining K8s resources inside of Support Bundles generated by our troubleshoot project. With both projects having separate code bases we see the following problems:
+
+* Most users are familiar with the binaries from troubleshoot (support-bundle, preflight, etc.), but are unaware of the existince of sbctl because it's a separate project
+* sbctl and troubleshoot are naturally coupled together due to sbctl's reliance on support bundles to operate. Them being in separate repos leads to duplication of code in both projects increasing maintenance burden.
+
+## High-Level Design
+Because the functionality of sbctl and Troubleshoot are so tightly coupled together, they should live in the same repository. Moreover, as opposed to maintaing a separate binary for sbctl, all sbctl commands should become subcommands of `support-bundle`
+
+## Detailed Design
+The `serve` and `shell` commands become subcommands of the `support-bundle` CLI.
+
+```
+./bin/support-bundle --help
+A support bundle is an archive of files, output, metrics and state
+from a server that can be used to assist when troubleshooting a Kubernetes cluster.
+
+Usage:
+  support-bundle [url] [flags]
+  support-bundle [command]
+
+Available Commands:
+  analyze     analyze a support bundle
+  serve       Start API server
+  shell       Start interractive shell
+  completion  Generate the autocompletion script for the specified shell
+  ...
+```
+
+Similiar to the [Analyze](https://github.com/replicatedhq/troubleshoot/blob/main/cmd/troubleshoot/cli/analyze.go) subcommand, we add `serve.go` and `shell.go` to `cmd/troubleshoot/cli`
+
+### Standardize Cluster Resources Collector
+* In `pkg/collect` standardize the root directory location of the Cluster Resources collector with a variable instead of repeatedly specifying it as a string - https://github.com/replicatedhq/troubleshoot/blob/main/pkg/collect/cluster_resources.go#L121.
+* Import the directory the Cluster Resources collector uses from `pkg/collect` for usage in the APIs - https://github.com/replicatedhq/sbctl/blob/main/pkg/api/server.go#L220. This is an example of the way we can start making the sbctl code more efficient with this change.
+
+cluster_resources.go
+```go
+const ClusterResourcesPath = `cluster-resources`
+...
+
+path := filepath.Join(ClusterResourcesPath, "namespaces.json")
+output.SaveResult(c.BundlePath, path, bytes.NewBuffer(namespaces))
+```
+
+sbctl-server.go
+```go
+import (
+    ...
+    "github.com/replicatedhq/troubleshoot/pkg/collect"
+    ...
+)
+...
+
+dirName := filepath.Join(collect.ClusterResourcesPath, fmt.Sprintf("%s", resource))
+```
+
+### Package Structure
+```
+cmd
+    serve/
+    shell/
+pkg
+    serve/
+    shell/
+tests           # API tests, integration tests etc reside here.
+    serve/      # sbctl BDD tests.
+        # all tests and test fixtures moved here
+```
+
+### Makefile Changes
+
+We'll need to add make targets specific to these commands similar to the current ones
+```
+make serve run-serve
+make shell run-shell
+make ginkgo
+```
+
+### Docs 
+In troubleshoot.sh docs, add a new sub section under "Support Bundle" describing then new serve & shell subcommands.
+
+### Other Changes
+We would need to merge some of the support bundle functionality we see in both projects like so
+
+- Create a new sbutil module to store reusable support bundle functionality. At least `analyze` & `serve` modules will depend on this module
+
+```
+pkg
+    sbutil/
+```
+
+- Merge `troubleshoot/pkg/analyze/download.ExtractTroubleshootBundle` & `sbctl/pkg/sbctl/support-bundle.ExtractBundle` that duplicate support bundle archive extraction
+- Build on `sbctl/pkg/sbctl/support-bundle.ClusterData` to have it expose well known paths (ClusterResourcesDir, Version files, analysis results etc) and unmarshalled resources (this bit can be built on a need to have basis) in a support bundle especially once [Add generic kubernetes resource analyzer #780 PR](https://github.com/replicatedhq/troubleshoot/pull/780) is merged.
+- Build on `sbctl/pkg/sbctl/support-bundle.FindClusterData` to perform pre-validation of a support bundle to ensure it is valid before running any applications.
+
+## Limitations
+
+## Assumptions
+
+## Testing
+We will need to have a separate directory for sbctl (ginkgo BDD) tests. The test harness used is ginkgo cli instead of go test and there is no straight forward way of separating these tests. Also, from a directory structure point of view, such integration tests usually reside in modules in a root level tests directory. Changes to the test code need to be minimal, ideally no change, cause they will be used to ensure any refactored code does not break. We could now easily introduce tests which ensure that changes to collectors don't break APIs in sbctl.
+
+## Alternatives Considered
+* Have troubleshoot as a library dependency in sbctl keeping it as a separate binary and repo
+* Move sbctl to the Troubleshoot library "as is" and continue maintaing a separate binary
+* Leave things as they are
+
+## Security Considerations

docs/preflight.md

@@ -17,7 +17,7 @@ preflight [url] [flags]
       --as string                      Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
       --as-group stringArray           Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
       --as-uid string                  UID to impersonate for the operation.
-      --cache-dir string               Default cache directory (default "/Users/xavpaice/.kube/cache")
+      --cache-dir string               Default cache directory (default "$HOME/.kube/cache")
       --certificate-authority string   Path to a cert file for the certificate authority
       --client-certificate string      Path to a client certificate file for TLS
       --client-key string              Path to a client key file for TLS
@@ -26,12 +26,14 @@ preflight [url] [flags]
       --collector-image string         the full name of the collector image to use
       --collector-pullpolicy string    the pull policy of the collector image
       --context string                 The name of the kubeconfig context to use
+      --cpuprofile string              File path to write cpu profiling data
       --debug                          enable debug logging
       --format string                  output format, one of human, json, yaml. only used when interactive is set to false (default "human")
   -h, --help                           help for preflight
       --insecure-skip-tls-verify       If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
       --interactive                    interactive preflights (default true)
       --kubeconfig string              Path to the kubeconfig file to use for CLI requests.
+      --memprofile string              File path to write memory profiling data
   -n, --namespace string               If present, the namespace scope for this CLI request
   -o, --output string                  specify the output file path for the preflight checks
       --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
@@ -48,4 +50,4 @@ preflight [url] [flags]
 
 * [preflight version](preflight_version.md)	 - Print the current version and exit
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/preflight_version.md

@@ -22,9 +22,11 @@ preflight version [flags]
       --collect-without-permissions   always run preflight checks even if some require permissions that preflight does not have (default true)
       --collector-image string        the full name of the collector image to use
       --collector-pullpolicy string   the pull policy of the collector image
+      --cpuprofile string             File path to write cpu profiling data
       --debug                         enable debug logging
       --format string                 output format, one of human, json, yaml. only used when interactive is set to false (default "human")
       --interactive                   interactive preflights (default true)
+      --memprofile string             File path to write memory profiling data
   -o, --output string                 specify the output file path for the preflight checks
       --selector string               selector (label query) to filter remote collection nodes on.
       --since string                  force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.
@@ -35,4 +37,4 @@ preflight version [flags]
 
 * [preflight](preflight.md)	 - Run and retrieve preflight checks in a cluster
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle.md

@@ -8,7 +8,7 @@ A support bundle is an archive of files, output, metrics and state
 from a server that can be used to assist when troubleshooting a Kubernetes cluster.
 
 ```
-support-bundle [url] [flags]
+support-bundle [urls...] [flags]
 ```
 
 ### Options
@@ -17,26 +17,28 @@ support-bundle [url] [flags]
       --as string                      Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
       --as-group stringArray           Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
       --as-uid string                  UID to impersonate for the operation.
-      --cache-dir string               Default cache directory (default "/Users/xavpaice/.kube/cache")
+      --cache-dir string               Default cache directory (default "$HOME/.kube/cache")
       --certificate-authority string   Path to a cert file for the certificate authority
       --client-certificate string      Path to a client certificate file for TLS
       --client-key string              Path to a client key file for TLS
       --cluster string                 The name of the kubeconfig cluster to use
       --collect-without-permissions    always generate a support bundle, even if it some require additional permissions (default true)
       --context string                 The name of the kubeconfig context to use
+      --cpuprofile string              File path to write cpu profiling data
       --debug                          enable debug logging
   -h, --help                           help for support-bundle
       --insecure-skip-tls-verify       If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
       --interactive                    enable/disable interactive mode (default true)
       --kubeconfig string              Path to the kubeconfig file to use for CLI requests.
       --load-cluster-specs             enable/disable loading additional troubleshoot specs found within the cluster. required when no specs are provided on the command line
+      --memprofile string              File path to write memory profiling data
   -n, --namespace string               If present, the namespace scope for this CLI request
       --no-uri                         When this flag is used, Troubleshoot does not attempt to retrieve the bundle referenced by the uri: field in the spec.`
   -o, --output string                  specify the output file path for the support bundle
       --redact                         enable/disable default redactions (default true)
       --redactors strings              names of the additional redactors to use
       --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
-  -l, --selector strings               selector to filter on for loading additional support bundle specs found in secrets within the cluster (default [troubleshoot.io/kind=supportbundle-spec])
+  -l, --selector strings               selector to filter on for loading additional support bundle specs found in secrets within the cluster (default [troubleshoot.io/kind=support-bundle])
   -s, --server string                  The address and port of the Kubernetes API server
       --since string                   force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.
       --since-time string              force pod logs collectors to return logs after a specific date (RFC3339)
@@ -48,6 +50,7 @@ support-bundle [url] [flags]
 ### SEE ALSO
 
 * [support-bundle analyze](support-bundle_analyze.md)	 - analyze a support bundle
+* [support-bundle redact](support-bundle_redact.md)	 - Redact information from a generated support bundle archive
 * [support-bundle version](support-bundle_version.md)	 - Print the current version and exit
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle_analyze.md

@@ -19,8 +19,15 @@ support-bundle analyze [url] [flags]
       --quiet           enable/disable error messaging and only show parseable output
 ```
 
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
 ### SEE ALSO
 
 * [support-bundle](support-bundle.md)	 - Generate a support bundle
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle_redact.md

@@ -0,0 +1,42 @@
+## support-bundle redact
+
+Redact information from a generated support bundle archive
+
+### Synopsis
+
+Redaction is the process of masking sensitive information from collected data in a support bundle.
+This is done using rules defined in the list of redactor manifests provided in the [urls...] command line
+argument. Default built in redactors will also be run, but these would have been run when the support
+bundle was generated. After redaction, the support bundle is archived once more. The resulting file will
+be stored in the current directory in the path provided by the --output flag.
+
+The [urls...] argument is a list of either oci://.., http://.., https://.. or local paths to yaml files.
+
+For more information on redactors visit https://troubleshoot.sh/docs/redact/
+
+
+```
+support-bundle redact [urls...] [flags]
+```
+
+### Options
+
+```
+      --bundle string   file path of the support bundle archive to redact
+  -h, --help            help for redact
+  -o, --output string   file path of where to save the redacted support bundle archive (default "redacted-support-bundle-YYYY-MM-DDTHH_MM_SS.tar.gz")
+  -q, --quiet           enable/disable error messaging and only show parseable output
+```
+
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
+### SEE ALSO
+
+* [support-bundle](support-bundle.md)	 - Generate a support bundle
+
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle_version.md

@@ -16,8 +16,15 @@ support-bundle version [flags]
   -h, --help   help for version
 ```
 
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
 ### SEE ALSO
 
 * [support-bundle](support-bundle.md)	 - Generate a support bundle
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

examples/collect/host/udp-port.yaml

@@ -0,0 +1,9 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostCollector
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472

examples/collect/remote/udp-port.yaml

@@ -0,0 +1,9 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostCollector
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472

examples/preflight/host/subnet-available.yaml

@@ -0,0 +1,19 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: subnet-available
+spec:
+  collectors:
+    # would output yes/no depending if there is a /22 available in 10.0.0.0/8
+    - subnetAvailable:
+        CIDRRangeAlloc: "10.0.0.0/8"
+        desiredCIDR: 22
+  analyzers:
+    - subnetAvailable:
+        outcomes:
+          - fail:
+            when: "no-subnet-available"
+            message: failed to find available subnet
+          - pass:
+            when: "a-subnet-is-available"
+            message: available /22 subnet found

examples/preflight/host/system-packages.yaml

@@ -21,6 +21,15 @@ spec:
         rhel8:
           - nfs-utils
           - openssl
+        rhel9:
+          - nfs-utils
+          - openssl
+        rocky8:
+          - nfs-utils
+          - openssl
+        rocky9:
+          - nfs-utils
+          - openssl
         centos:
           - iscsi-initiator-utils
         centos7:
@@ -28,6 +37,9 @@ spec:
         centos8:
           - nfs-utils
           - openssl
+        centos9:
+          - nfs-utils
+          - openssl
         ol:
           - lvm2
         ol7:
@@ -35,6 +47,8 @@ spec:
           - openssl
         ol8:
           - nfs-utils
+        ol9:
+          - nfs-utils
         amzn:
           - libzstd
         amzn2:

examples/preflight/host/udp-port.yaml

@@ -0,0 +1,24 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472
+  analyzers:
+    - udpPortStatus:
+        collectorName: flannel
+        outcomes:
+          - fail:
+              when: "address-in-use"
+              message: Another process was already listening on port 8472.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 8472 is open
+          - warn:
+              message: Unexpected port status

examples/preflight/mssql.yaml

@@ -0,0 +1,70 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: Preflight
+metadata:
+  name: sample
+spec:
+  collectors:
+    - mssql:
+        collectorName: mssql
+        uri: sqlserver://SA:nac3lle-ch@stise-bOMBay-wag@db.lab.shortrib.net:1433/master
+  analyzers:
+    - mssql:
+        checkName: Must be SQLServer 15.x or later
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version < 15.x"
+              message: The SQLServer must be at least version 15
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must be SQLServer 14 or earlier
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version > 14.x"
+              message: The SQLServer must be at most version 14
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must be SQLServer 15.0.2000.1565 
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version == 15.0.2000.1565"
+              message: The SQLServer must be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must not be SQLServer 15.0.2000.1565 
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version != 15.0.2000.1565"
+              message: The SQLServer must not be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: I don't want to connect to your SQL Server
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == true"
+              message: Can connect to your SQL Server
+          - fail:
+              when: "version != 15.0.2000.1565"
+              message: The SQLServer must be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out

examples/preflight/node-resources.yaml

@@ -16,20 +16,20 @@ spec:
           - pass:
               message: This cluster has enough nodes.
     - nodeResources:
-        checkName: Must have 3 nodes with at least 6 cores
+        checkName: Must have 3 nodes with at least 6 cores each
         filters:
           cpuCapacity: "6"
         outcomes:
           - fail:
-              when: "< 3"
+              when: "count() < 3"
               message: This application requires at least 3 nodes with 6 cores each
           - pass:
-              message: This cluster has enough nodes with enough codes
+              message: This cluster has enough nodes with enough cores
     - nodeResources:
-        checkName: Must have 1 node with 16 GB (available) memory and 5 cores (on a single node)
+        checkName: Must have 1 node with 16 GB (available) memory and 10 cores (on a single node)
         filters:
           allocatableMemory: 16Gi
-          cpuCapacity: "5"
+          cpuCapacity: "10"
         outcomes:
           - fail:
               when: "< 1"
@@ -37,13 +37,32 @@ spec:
           - pass:
               message: This cluster has a node with enough memory.
     - nodeResources:
-        checkName: There must be at least 3 nodes in the cluster
+        checkName: Must have 1 node with 16 GB (available) memory and 4 cores of amd64 arch (on a single node)
+        filters:
+          allocatableMemory: 16Gi
+          cpuArchitecture: amd64
+          cpuCapacity: "4"
         outcomes:
           - fail:
-              when: "count() < 3"
-              message: This application requires at least 3 nodes in the cluster
+              when: "< 1"
+              message: This application requires at least 1 node with amd64 arch and 16GB available memory
           - pass:
-              message: This cluster haas sufficient nodes
+              message: This cluster has a node with the correct architecture and enough memory.
+    - nodeResources:
+        checkName: Must have 1 manager node with 16 GB (available) memory and 4 cores of amd64 arch
+        filters:
+          selector:
+            matchLabel:
+              node-role.kubernetes.io/master: ""
+          allocatableMemory: 16Gi
+          cpuArchitecture: amd64
+          cpuCapacity: "6"
+        outcomes:
+          - fail:
+              when: "< 1"
+              message: This application requires at least 1 manager node with amd64 arch, 16GB available memory, and the label `node-role.kubernetes.io/master=""`
+          - pass:
+              message: This cluster has a node with the correct architecture and enough memory.
     - nodeResources:
         checkName: There must be a total of at least 32Gi of memory on all nodes
         outcomes:
@@ -51,4 +70,14 @@ spec:
               when: "sum(memoryCapacity) < 32Gi"
               message: This application requires that 32Gi or more memory be available to the cluster
           - pass:
-              message: This cluster haas sufficient memory
+              message: This cluster has sufficient memory
+    - nodeResources:
+        checkName: There must be a total of at least 32Gi of memory on amd64 nodes
+        filters:
+          cpuArchitecture: amd64
+        outcomes:
+          - fail:
+              when: "sum(memoryCapacity) < 32Gi"
+              message: This application requires that 32Gi or more memory on amd64 nodes be available to the cluster
+          - pass:
+              message: This cluster has sufficient memory

examples/preflight/postgres.yaml

@@ -20,4 +20,3 @@ spec:
               message: The postgres server must be at least version 10
           - pass:
               message: The postgres connection checks out
-  

examples/preflight/redis.yaml

@@ -6,7 +6,9 @@ spec:
   collectors:
     - redis:
         collectorName: my-redis
-        uri: rediss://default:password@hostname:6379
+        uri: rediss://default:replicated@server:6380
+        tls:
+          skipVerify: true
   analyzers:
     - redis:
         checkName: Must be redis 5.x or later
@@ -14,10 +16,9 @@ spec:
         outcomes:
           - fail:
               when: "connected == false"
-              message: Cannot connect to postgres server
+              message: Cannot connect to redis server
           - fail:
               when: "version < 5.0.0"
               message: The redis server must be at least version 5
           - pass:
               message: The redis connection checks out
-  

examples/preflight/remote/udp-port.yaml

@@ -0,0 +1,24 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472
+  analyzers:
+    - udpPortStatus:
+        collectorName: flannel
+        outcomes:
+          - fail:
+              when: "address-in-use"
+              message: Another process was already listening on port 8472.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 8472 is open
+          - warn:
+              message: Unexpected port status

examples/redact/e2e.yaml

@@ -0,0 +1,10 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: Redactor
+metadata:
+  name: e2e-redactor
+spec:
+  redactors:
+    - name: redact-static-text
+      removals:
+        values:
+        - static

examples/support-bundle/db-collector.yaml

@@ -1,7 +1,7 @@
 apiVersion: troubleshoot.sh/v1beta2
 kind: SupportBundle
 metadata:
-  name: mysql
+  name: dbs-collector
 spec:
   collectors:
     - mysql:
@@ -15,3 +15,6 @@ spec:
           - innodb_large_prefix
           - innodb_strict_mode
           - log_bin_trust_function_creators
+    - redis:
+        collectorName: my-redis
+        uri: rediss://default:replicated@server:6380

examples/support-bundle/sample-supportbundle.yaml

@@ -3,6 +3,8 @@ kind: SupportBundle
 metadata:
   name: example
 spec:
+  hostCollectors:
+    - hostOS: {}
   collectors:
     - logs:
         selector:
@@ -12,6 +14,7 @@ spec:
         limits:
           maxAge: 720h # 30*24
           maxLines: 10000
+          maxBytes: 5000000
     - logs:
         collectorName: all-logs
         name: all-logs

go.mod

@@ -4,150 +4,160 @@ go 1.19
 
 require (
 	github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412
-	github.com/blang/semver v3.5.1+incompatible
-	github.com/containers/image/v5 v5.23.1
-	github.com/docker/distribution v2.8.1+incompatible
-	github.com/fatih/color v1.13.0
-	github.com/go-logr/logr v1.2.3
+	github.com/apparentlymart/go-cidr v1.1.0
+	github.com/blang/semver/v4 v4.0.0
+	github.com/containers/image/v5 v5.25.0
+	github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
+	github.com/fatih/color v1.15.0
+	github.com/go-logr/logr v1.2.4
 	github.com/go-redis/redis/v7 v7.4.1
-	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-sql-driver/mysql v1.7.1
 	github.com/gobwas/glob v0.2.3
-	github.com/godbus/dbus v4.1.0+incompatible
+	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/handlers v1.5.1
-	github.com/hashicorp/go-getter v1.6.2
+	github.com/hashicorp/go-getter v1.7.1
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/lib/pq v1.10.7
+	github.com/jackc/pgx/v5 v5.3.1
 	github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e
 	github.com/manifoldco/promptui v0.9.0
-	github.com/mattn/go-isatty v0.0.16
+	github.com/mattn/go-isatty v0.0.18
 	github.com/mholt/archiver/v3 v3.5.1
-	github.com/opencontainers/image-spec v1.1.0-rc2
+	github.com/microsoft/go-mssqldb v0.21.0
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/pkg/errors v0.9.1
 	github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil v3.21.11+incompatible
+	github.com/shirou/gopsutil/v3 v3.23.4
 	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.6.1
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.14.0
-	github.com/stretchr/testify v1.8.1
+	github.com/spf13/viper v1.15.0
+	github.com/stretchr/testify v1.8.3
 	github.com/tj/go-spin v1.1.0
-	golang.org/x/sync v0.1.0
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/sdk v1.15.1
+	golang.org/x/sync v0.2.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.25.4
-	k8s.io/apiextensions-apiserver v0.25.0
-	k8s.io/apimachinery v0.25.4
-	k8s.io/apiserver v0.25.4
-	k8s.io/cli-runtime v0.25.4
-	k8s.io/client-go v0.25.4
-	k8s.io/klog/v2 v2.80.1
-	oras.land/oras-go v1.2.1
-	sigs.k8s.io/controller-runtime v0.13.1
+	k8s.io/api v0.27.2
+	k8s.io/apiextensions-apiserver v0.27.0
+	k8s.io/apimachinery v0.27.2
+	k8s.io/apiserver v0.27.2
+	k8s.io/cli-runtime v0.27.2
+	k8s.io/client-go v0.27.2
+	k8s.io/klog/v2 v2.100.1
+	oras.land/oras-go v1.2.3
+	sigs.k8s.io/controller-runtime v0.14.6
 )
 
 require (
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
-	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
+	github.com/golang-sql/sqlexp v0.1.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sylabs/sif/v2 v2.8.1 // indirect
+	github.com/shoenig/go-m1cpu v0.1.5 // indirect
+	github.com/sylabs/sif/v2 v2.11.1 // indirect
+	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 )
 
 require (
-	cloud.google.com/go v0.104.0 // indirect
-	cloud.google.com/go/compute v1.12.1 // indirect
-	cloud.google.com/go/iam v0.3.0 // indirect
-	cloud.google.com/go/storage v1.23.0 // indirect
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go/iam v0.13.0 // indirect
+	cloud.google.com/go/storage v1.28.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/BurntSushi/toml v1.2.0 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.4 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
 	github.com/andybalholm/brotli v1.0.1 // indirect
-	github.com/aws/aws-sdk-go v1.43.16 // indirect
+	github.com/aws/aws-sdk-go v1.44.122 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
-	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
-	github.com/containers/ocicrypt v1.1.5 // indirect
-	github.com/containers/storage v1.43.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/containerd/containerd v1.7.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
+	github.com/containers/ocicrypt v1.1.7 // indirect
+	github.com/containers/storage v1.46.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/cli v20.10.17+incompatible // indirect
-	github.com/docker/docker v20.10.18+incompatible // indirect
+	github.com/docker/cli v23.0.1+incompatible // indirect
+	github.com/docker/docker v23.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-	github.com/felixge/httpsnoop v1.0.1 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-intervals v0.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
-	github.com/hashicorp/go-version v1.3.0 // indirect
+	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -156,57 +166,55 @@ require (
 	github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d // indirect
 	github.com/nwaples/rardecode v1.1.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
-	github.com/opencontainers/selinux v1.10.2 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
-	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.opencensus.io v0.23.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 // indirect
-	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
-	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
-	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
-	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
-	golang.org/x/text v0.4.0 // indirect
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/crypto v0.8.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/term v0.7.0 // indirect
+	golang.org/x/text v0.9.0
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.102.0 // indirect
+	google.golang.org/api v0.114.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
-	google.golang.org/grpc v1.50.1 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.54.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
-	periph.io/x/host/v3 v3.8.0
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
+	periph.io/x/host/v3 v3.8.2
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.2 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

internal/testutils/utils.go

@@ -0,0 +1,62 @@
+package testutils
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func GetTestFixture(t *testing.T, path string) string {
+	t.Helper()
+	p := filepath.Join("../../testdata", path)
+	b, err := os.ReadFile(p)
+	require.NoError(t, err)
+	return string(b)
+}
+
+// FileDir returns the directory of the current source file.
+func FileDir() string {
+	_, filename, _, _ := runtime.Caller(0)
+	return filepath.Dir(filename)
+}
+
+// Generates a temporary filename
+func TempFilename(prefix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(os.TempDir(), fmt.Sprintf("%s_%s", prefix, hex.EncodeToString(randBytes)))
+}
+
+func CreateTestFile(t *testing.T, path string) {
+	t.Helper()
+
+	CreateTestFileWithData(t, path, "Garbage for "+path)
+}
+
+func CreateTestFileWithData(t *testing.T, path, data string) {
+	t.Helper()
+
+	dir := filepath.Dir(path)
+	err := os.MkdirAll(dir, 0755)
+	require.NoError(t, err)
+	err = os.WriteFile(path, []byte(data), 0644)
+	require.NoError(t, err)
+}
+
+func LogJSON(t *testing.T, v interface{}) {
+	t.Helper()
+
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		t.Log(v)
+	} else {
+		t.Log(string(b))
+	}
+}

internal/traces/exporter.go

@@ -0,0 +1,273 @@
+package traces
+
+import (
+	"context"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+)
+
+var (
+	_        trace.SpanExporter = (*Exporter)(nil)
+	once     sync.Once
+	exporter *Exporter
+	printer  = message.NewPrinter(language.English)
+)
+
+const legend = "Suceeded (S), eXcluded (X), Failed (F)\n"
+
+// FUTURE WORK: This exporter should only be used by troubleshoot CLIs
+// until the following issue is addressed:
+// 1. The cache of spans grows infinitely at the moment. This is OK for short lived
+//    invocations such as CLI applications running one-shot commands. For long running
+//    applications, this will be a problem.
+//		* We can drop non-troubleshoot spans from the cache. There is an OTEP to add
+//        "inheritable attributes" to spans which is unsupported as of today. Once/if
+//		  it gets accepted and implemented, it will allow us to add a "troubleshoot"
+//		  attribute to the "root" span which gets propagated to downstream spans which
+//		  we can use to filter out non-troubleshoot spans.
+//		  https://github.com/dynatrace-oss-contrib/oteps/blob/feature/context-scoped-attributes/text/0207-context-scoped-attributes.md
+//		  https://github.com/open-telemetry/opentelemetry-specification/issues/1337
+//		* We can "search" for the "root" span by traversing up the span tree and drop
+//		  all spans that are not descendants of the "root" span. This can be slow, but
+//		  we can do it in a background goroutine (adds a bit of complexity).
+// 2. At the moment, the summary of an execution is the only case this exporter
+// 	  is being used for.
+
+// GetExporterInstance creates a singleton exporter instance
+func GetExporterInstance() *Exporter {
+	once.Do(func() {
+		exporter = &Exporter{
+			allSpans: make([]trace.ReadOnlySpan, 1024),
+		}
+	})
+	return exporter
+}
+
+// Exporter is an implementation of trace.SpanExporter that writes to a destination.
+type Exporter struct {
+	spansMu  sync.Mutex
+	allSpans []trace.ReadOnlySpan
+
+	stoppedMu sync.RWMutex
+	stopped   bool
+}
+
+// ExportSpans writes spans to an in-memory cache
+// This function can/will be called on every span.End() at worst.
+func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error {
+	// This is a no-op if the context is canceled.
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	e.stoppedMu.RLock()
+	stopped := e.stopped
+	e.stoppedMu.RUnlock()
+	if stopped {
+		return nil
+	}
+
+	if len(spans) == 0 {
+		return nil
+	}
+
+	e.spansMu.Lock()
+	defer e.spansMu.Unlock()
+
+	// Cache received span updates allSpans
+	e.allSpans = append(e.allSpans, spans...)
+
+	return nil
+}
+
+func isType(stub *tracetest.SpanStub, t string) bool {
+	if stub == nil {
+		return false
+	}
+
+	for _, attr := range stub.Attributes {
+		if string(attr.Key) == "type" && strings.Contains(attr.Value.AsString(), t) {
+			return true
+		}
+	}
+	return false
+}
+
+func formatStubName(stub *tracetest.SpanStub) string {
+	if stub == nil {
+		return ""
+	}
+
+	for _, attr := range stub.Attributes {
+		if stub.Status.Code == codes.Error {
+			return stub.Name + " (F)"
+		}
+		if string(attr.Key) == constants.EXCLUDED && attr.Value.AsBool() {
+			return stub.Name + " (X)"
+		}
+	}
+	return stub.Name + " (S)" // Assume success
+}
+
+// maxInt returns the larger of x or y.
+func maxInt(x, y int) int {
+	if x < y {
+		return y
+	}
+	return x
+}
+
+// GetSummary returns the runtime summary of the execution
+// so far. Call this function after your "root" span has ended
+// and the program operations needing tracing have completed.
+func (e *Exporter) GetSummary() string {
+	e.spansMu.Lock()
+	stubs := tracetest.SpanStubsFromReadOnlySpans(e.allSpans)
+	e.spansMu.Unlock()
+
+	// No spans to log
+	if len(stubs) == 0 {
+		return ""
+	}
+
+	// TODO: We may want to collect more information about the
+	// execution of the program. For example, we can collect
+	// the number of times a collector was executed, whether
+	// it was successful or not, if it was skipped, etc.
+	collectors := make(map[string]time.Duration)
+	redactors := make(map[string]time.Duration)
+	analysers := make(map[string]time.Duration)
+
+	totalDuration := time.Duration(0)
+
+	for i := range stubs {
+		stub := &stubs[i]
+
+		// Summary of span stubs
+		duration := stub.EndTime.Sub(stub.StartTime)
+		stubName := formatStubName(stub)
+		switch {
+		case stub.Name == constants.TROUBLESHOOT_ROOT_SPAN_NAME:
+			totalDuration = duration
+		case isType(stub, "Collect"):
+			collectors[stubName] = duration
+		case isType(stub, "Redactors"):
+			redactors[stub.Name] = duration
+		case isType(stub, "Analyze"):
+			analysers[stubName] = duration
+		default:
+			continue
+		}
+	}
+
+	sb := strings.Builder{}
+
+	collectorsSummary(collectors, &sb)
+	redactorsSummary(redactors, &sb)
+	analysersSummary(analysers, &sb)
+	sb.WriteString(printer.Sprintf("\nDuration: %dms\n", totalDuration/time.Millisecond))
+
+	return sb.String()
+}
+
+// summary of collector runtimes
+func collectorsSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("============ Collectors summary =============\n")
+	sb.WriteString(legend)
+	sb.WriteString("=============================================\n")
+	if len(summary) == 0 {
+		sb.WriteString("No collectors executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+// summary of redactor runtime
+func redactorsSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("\n============ Redactors summary =============\n")
+	if len(summary) == 0 {
+		sb.WriteString("No redactors executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+// summary of analyser runtime
+func analysersSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("\n============= Analyzers summary =============\n")
+	sb.WriteString(legend)
+	sb.WriteString("=============================================\n")
+	if len(summary) == 0 {
+		sb.WriteString("No analyzers executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+func sortedKeysAndPadding(summary map[string]time.Duration) (int, []string) {
+	keys := make([]string, 0, len(summary))
+	padding := 0
+	for k := range summary {
+		padding = maxInt(padding, len(k))
+		keys = append(keys, k)
+	}
+	sort.SliceStable(keys, func(l, r int) bool {
+		return summary[keys[l]] > summary[keys[r]]
+	})
+	return padding, keys
+}
+
+// Shutdown is called to stop the exporter, it preforms no action.
+func (e *Exporter) Shutdown(ctx context.Context) error {
+	e.stoppedMu.Lock()
+	e.stopped = true
+	e.stoppedMu.Unlock()
+
+	e.Reset()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+	return nil
+}
+
+func (e *Exporter) Reset() {
+	e.spansMu.Lock()
+	e.allSpans = e.allSpans[:0] // clear the slice
+	e.spansMu.Unlock()
+}
+
+// MarshalLog is the marshaling function used by the logging system to represent this exporter.
+func (e *Exporter) MarshalLog() interface{} {
+	return struct {
+		Type string
+	}{
+		Type: "troubleshoot",
+	}
+}

internal/traces/exporter_test.go

@@ -0,0 +1,187 @@
+package traces
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestExporter_GetSummary(t *testing.T) {
+	logger.SetQuiet(true)
+
+	tests := []struct {
+		name  string
+		spans tracetest.SpanStubs
+		want  string
+	}{
+		{
+			name:  "with no spans",
+			spans: tracetest.SpanStubs{},
+			want:  "",
+		},
+		{
+			name: "with root span only",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name:      constants.TROUBLESHOOT_ROOT_SPAN_NAME,
+					StartTime: time.Now(),
+					EndTime:   time.Now().Add(time.Second),
+				},
+			},
+			want: "Duration: 1,000ms",
+		},
+		{
+			name: "with collectors",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "all-logs", StartTime: time.Now(), EndTime: time.Now().Add(time.Minute),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectLogs"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "host-os", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "excluded-collector", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond * 2),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+						attribute.Bool("excluded", true),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "failed-collector", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+					},
+					Status: trace.Status{
+						Code:        codes.Error,
+						Description: "some error",
+					},
+				},
+			},
+			want: `
+============ Collectors summary =============
+Suceeded (S), eXcluded (X), Failed (F)
+=============================================
+all-logs (S)           : 60,000ms
+host-os (S)            : 1,000ms
+excluded-collector (X) : 2ms
+failed-collector (F)   : 1ms`,
+		},
+		{
+			name: "with analyzers",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "cluster-version", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*analyzer.AnalyzeClusterVersion"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "host-cpu", StartTime: time.Now(), EndTime: time.Now().Add(time.Minute),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*analyzer.AnalyzeHostCPU"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "excluded-analyser", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond * 2),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.AnalyzeHostCPU"),
+						attribute.Bool("excluded", true),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "failed-analyser", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.AnalyzeHostCPU"),
+					},
+					Status: trace.Status{
+						Code:        codes.Error,
+						Description: "some error",
+					},
+				},
+			},
+			want: `
+============= Analyzers summary =============
+Suceeded (S), eXcluded (X), Failed (F)
+=============================================
+host-cpu (S)          : 60,000ms
+cluster-version (S)   : 1,000ms
+excluded-analyser (X) : 2ms
+failed-analyser (F)   : 1ms`,
+		},
+		{
+			name: "with redactors",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "cluster redactor", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "Redactors"),
+					},
+				},
+			},
+			want: `
+============ Redactors summary =============
+cluster redactor : 1,000ms`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e := &Exporter{}
+
+			ctx := context.Background()
+			err := e.ExportSpans(ctx, tt.spans.Snapshots())
+			require.NoError(t, err)
+
+			assert.Contains(t, e.GetSummary(), strings.TrimSpace(tt.want))
+		})
+	}
+}
+
+func TestExporter_ExportSpansWithDoneContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	e := &Exporter{}
+	spans := tracetest.SpanStubs{}
+
+	assert.EqualError(t, e.ExportSpans(ctx, spans.Snapshots()), context.Canceled.Error())
+}
+
+func TestExporter_Shutdown(t *testing.T) {
+	e := &Exporter{}
+
+	ctx := context.Background()
+	spans := tracetest.SpanStubs{}
+	for i := 0; i < 5; i++ {
+		spans = append(spans, tracetest.SpanStub{Name: fmt.Sprintf("span-%d", i)})
+	}
+
+	err := e.ExportSpans(ctx, spans.Snapshots())
+	require.NoError(t, err)
+
+	assert.Len(t, e.allSpans, 5)
+
+	require.NoError(t, e.Shutdown(ctx))
+	assert.Len(t, e.allSpans, 0)
+
+	err = e.ExportSpans(ctx, spans.Snapshots())
+	require.NoError(t, err)
+
+	assert.Len(t, e.allSpans, 0)
+}

internal/traces/otel.go

@@ -0,0 +1,60 @@
+package traces
+
+import (
+	"context"
+
+	"github.com/replicatedhq/troubleshoot/pkg/version"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+	"k8s.io/klog/v2"
+)
+
+// ConfigureTracing configures the OpenTelemetry trace provider for CLI
+// commands. Projects using troubleshoot as a library would need to register
+// troubleshoot's exporter like so.
+//
+//	var tp *trace.TracerProvider	// client application's trace provider
+//	tp.RegisterSpanProcessor(
+//		trace.NewSimpleSpanProcessor(
+//			traces.GetExporterInstance(),	// Troubleshoot's exporter
+//		),
+//	)
+//
+// The client application is responsible for constructing the trace provider
+// and registering the exporter. Multiple exporters can be registered.
+func ConfigureTracing(processName string) (func(), error) {
+	r, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			resource.Default().SchemaURL(),
+			semconv.ProcessCommandKey.String(processName),
+			semconv.ProcessRuntimeVersionKey.String(version.Version()),
+			attribute.String("environment", "cli"),
+		),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Trace provider for support bundle cli. Each application is required
+	// to have its own trace provider.
+	tp := trace.NewTracerProvider(
+		trace.WithSampler(trace.AlwaysSample()),
+		trace.WithSyncer(
+			GetExporterInstance(),
+		),
+		trace.WithResource(r),
+	)
+
+	otel.SetTracerProvider(tp)
+
+	return func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			klog.Errorf("Failed to shutdown trace provider: %v", err)
+		}
+	}, nil
+}

pkg/analyze/analyzer.go

@@ -1,14 +1,21 @@
 package analyzer
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"strconv"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/multitype"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 )
 
 type AnalyzeResult struct {
@@ -27,7 +34,7 @@ type AnalyzeResult struct {
 }
 
 type getCollectedFileContents func(string) ([]byte, error)
-type getChildCollectedFileContents func(string) (map[string][]byte, error)
+type getChildCollectedFileContents func(string, []string) (map[string][]byte, error)
 
 func isExcluded(excludeVal *multitype.BoolOrString) (bool, error) {
 	if excludeVal == nil {
@@ -50,14 +57,25 @@ func isExcluded(excludeVal *multitype.BoolOrString) (bool, error) {
 	return parsed, nil
 }
 
-func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) []*AnalyzeResult {
+func HostAnalyze(
+	ctx context.Context,
+	hostAnalyzer *troubleshootv1beta2.HostAnalyze,
+	getFile getCollectedFileContents,
+	findFiles getChildCollectedFileContents,
+) []*AnalyzeResult {
 	analyzer, ok := GetHostAnalyzer(hostAnalyzer)
 	if !ok {
 		return NewAnalyzeResultError(analyzer, errors.New("invalid host analyzer"))
 	}
 
+	_, span := otel.Tracer(constants.LIB_TRACER_NAME).Start(ctx, analyzer.Title())
+	span.SetAttributes(attribute.String("type", reflect.TypeOf(analyzer).String()))
+	defer span.End()
+
 	isExcluded, _ := analyzer.IsExcluded()
 	if isExcluded {
+		klog.Infof("excluding %q analyzer", analyzer.Title())
+		span.SetAttributes(attribute.Bool(constants.EXCLUDED, true))
 		return nil
 	}
 
@@ -83,425 +101,51 @@ func NewAnalyzeResultError(analyzer HostAnalyzer, err error) []*AnalyzeResult {
 	}}
 }
 
-func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+func Analyze(
+	ctx context.Context,
+	analyzer *troubleshootv1beta2.Analyze,
+	getFile getCollectedFileContents,
+	findFiles getChildCollectedFileContents,
+) ([]*AnalyzeResult, error) {
 	if analyzer == nil {
 		return nil, errors.New("nil analyzer")
 	}
 
-	if analyzer.ClusterVersion != nil {
-		isExcluded, err := isExcluded(analyzer.ClusterVersion.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeClusterVersion(analyzer.ClusterVersion, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ClusterVersion.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.StorageClass != nil {
-		isExcluded, err := isExcluded(analyzer.StorageClass.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeStorageClass(analyzer.StorageClass, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.StorageClass.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.CustomResourceDefinition != nil {
-		isExcluded, err := isExcluded(analyzer.CustomResourceDefinition.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeCustomResourceDefinition(analyzer.CustomResourceDefinition, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.CustomResourceDefinition.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Ingress != nil {
-		isExcluded, err := isExcluded(analyzer.Ingress.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeIngress(analyzer.Ingress, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Ingress.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Secret != nil {
-		isExcluded, err := isExcluded(analyzer.Secret.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeSecret(analyzer.Secret, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Secret.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.ConfigMap != nil {
-		isExcluded, err := isExcluded(analyzer.ConfigMap.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeConfigMap(analyzer.ConfigMap, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ConfigMap.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.ImagePullSecret != nil {
-		isExcluded, err := isExcluded(analyzer.ImagePullSecret.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeImagePullSecret(analyzer.ImagePullSecret, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ImagePullSecret.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.DeploymentStatus != nil {
-		isExcluded, err := isExcluded(analyzer.DeploymentStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeDeploymentStatus(analyzer.DeploymentStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.DeploymentStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.StatefulsetStatus != nil {
-		isExcluded, err := isExcluded(analyzer.StatefulsetStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeStatefulsetStatus(analyzer.StatefulsetStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.StatefulsetStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.JobStatus != nil {
-		isExcluded, err := isExcluded(analyzer.JobStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeJobStatus(analyzer.JobStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.JobStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ReplicaSetStatus != nil {
-		isExcluded, err := isExcluded(analyzer.ReplicaSetStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeReplicaSetStatus(analyzer.ReplicaSetStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.ReplicaSetStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ClusterPodStatuses != nil {
-		isExcluded, err := isExcluded(analyzer.ClusterPodStatuses.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := clusterPodStatuses(analyzer.ClusterPodStatuses, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.ClusterPodStatuses.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ContainerRuntime != nil {
-		isExcluded, err := isExcluded(analyzer.ContainerRuntime.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeContainerRuntime(analyzer.ContainerRuntime, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ContainerRuntime.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Distribution != nil {
-		isExcluded, err := isExcluded(analyzer.Distribution.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeDistribution(analyzer.Distribution, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Distribution.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.NodeResources != nil {
-		isExcluded, err := isExcluded(analyzer.NodeResources.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeNodeResources(analyzer.NodeResources, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.NodeResources.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.TextAnalyze != nil {
-		isExcluded, err := isExcluded(analyzer.TextAnalyze.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeTextAnalyze(analyzer.TextAnalyze, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.TextAnalyze.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.YamlCompare != nil {
-		isExcluded, err := isExcluded(analyzer.YamlCompare.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeYamlCompare(analyzer.YamlCompare, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.YamlCompare.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.JsonCompare != nil {
-		isExcluded, err := isExcluded(analyzer.JsonCompare.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeJsonCompare(analyzer.JsonCompare, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.JsonCompare.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Postgres != nil {
-		isExcluded, err := isExcluded(analyzer.Postgres.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzePostgres(analyzer.Postgres, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Postgres.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Mysql != nil {
-		isExcluded, err := isExcluded(analyzer.Mysql.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeMysql(analyzer.Mysql, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Mysql.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Redis != nil {
-		isExcluded, err := isExcluded(analyzer.Redis.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeRedis(analyzer.Redis, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Redis.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.CephStatus != nil {
-		isExcluded, err := isExcluded(analyzer.CephStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := cephStatus(analyzer.CephStatus, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.CephStatus.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Longhorn != nil {
-		isExcluded, err := isExcluded(analyzer.Longhorn.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := longhorn(analyzer.Longhorn, getFile, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.Longhorn.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
+	analyzerInst := getAnalyzer(analyzer)
+	if analyzerInst == nil {
+		return []*AnalyzeResult{{
+			IsFail:  true,
+			Title:   "nonexistent analyzer",
+			Message: "Analyzer not found",
+		}}, nil
 	}
 
-	if analyzer.RegistryImages != nil {
-		isExcluded, err := isExcluded(analyzer.RegistryImages.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeRegistry(analyzer.RegistryImages, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.RegistryImages.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
+	_, span := otel.Tracer(constants.LIB_TRACER_NAME).Start(ctx, analyzerInst.Title())
+	span.SetAttributes(attribute.String("type", reflect.TypeOf(analyzerInst).String()))
+	defer span.End()
+
+	isExcluded, err := analyzerInst.IsExcluded()
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
+	}
+	if isExcluded {
+		klog.Infof("excluding %q analyzer", analyzerInst.Title())
+		span.SetAttributes(attribute.Bool(constants.EXCLUDED, true))
+		return nil, nil
 	}
 
-	if analyzer.WeaveReport != nil {
-		isExcluded, err := isExcluded(analyzer.WeaveReport.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeWeaveReport(analyzer.WeaveReport, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.WeaveReport.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
+	results, err := analyzerInst.Analyze(getFile, findFiles)
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
 	}
 
-	if analyzer.Sysctl != nil {
-		isExcluded, err := isExcluded(analyzer.Sysctl.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeSysctl(analyzer.Sysctl, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		if result == nil {
-			return []*AnalyzeResult{}, nil
-		}
-		result.Strict = analyzer.Sysctl.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
+	if results == nil {
+		results = []*AnalyzeResult{}
 	}
 
-	return nil, errors.New("invalid analyzer")
+	return results, nil
 }
 
 func GetExcludeFlag(analyzer *troubleshootv1beta2.Analyze) *multitype.BoolOrString {
@@ -525,3 +169,93 @@ func GetExcludeFlag(analyzer *troubleshootv1beta2.Analyze) *multitype.BoolOrStri
 
 	return nil
 }
+
+type Analyzer interface {
+	Title() string
+	IsExcluded() (bool, error)
+	Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error)
+}
+
+func getAnalyzer(analyzer *troubleshootv1beta2.Analyze) Analyzer {
+	switch {
+	case analyzer.ClusterVersion != nil:
+		return &AnalyzeClusterVersion{analyzer: analyzer.ClusterVersion}
+	case analyzer.StorageClass != nil:
+		return &AnalyzeStorageClass{analyzer: analyzer.StorageClass}
+	case analyzer.CustomResourceDefinition != nil:
+		return &AnalyzeCustomResourceDefinition{analyzer: analyzer.CustomResourceDefinition}
+	case analyzer.Ingress != nil:
+		return &AnalyzeIngress{analyzer: analyzer.Ingress}
+	case analyzer.Secret != nil:
+		return &AnalyzeSecret{analyzer: analyzer.Secret}
+	case analyzer.ConfigMap != nil:
+		return &AnalyzeConfigMap{analyzer: analyzer.ConfigMap}
+	case analyzer.ImagePullSecret != nil:
+		return &AnalyzeImagePullSecret{analyzer: analyzer.ImagePullSecret}
+	case analyzer.DeploymentStatus != nil:
+		return &AnalyzeDeploymentStatus{analyzer: analyzer.DeploymentStatus}
+	case analyzer.StatefulsetStatus != nil:
+		return &AnalyzeStatefulsetStatus{analyzer: analyzer.StatefulsetStatus}
+	case analyzer.JobStatus != nil:
+		return &AnalyzeJobStatus{analyzer: analyzer.JobStatus}
+	case analyzer.ReplicaSetStatus != nil:
+		return &AnalyzeReplicaSetStatus{analyzer: analyzer.ReplicaSetStatus}
+	case analyzer.ClusterPodStatuses != nil:
+		return &AnalyzeClusterPodStatuses{analyzer: analyzer.ClusterPodStatuses}
+	case analyzer.ContainerRuntime != nil:
+		return &AnalyzeContainerRuntime{analyzer: analyzer.ContainerRuntime}
+	case analyzer.Distribution != nil:
+		return &AnalyzeDistribution{analyzer: analyzer.Distribution}
+	case analyzer.NodeResources != nil:
+		return &AnalyzeNodeResources{analyzer: analyzer.NodeResources}
+	case analyzer.TextAnalyze != nil:
+		return &AnalyzeTextAnalyze{analyzer: analyzer.TextAnalyze}
+	case analyzer.YamlCompare != nil:
+		return &AnalyzeYamlCompare{analyzer: analyzer.YamlCompare}
+	case analyzer.JsonCompare != nil:
+		return &AnalyzeJsonCompare{analyzer: analyzer.JsonCompare}
+	case analyzer.Postgres != nil:
+		return &AnalyzePostgres{analyzer: analyzer.Postgres}
+	case analyzer.Mysql != nil:
+		return &AnalyzeMysql{analyzer: analyzer.Mysql}
+	case analyzer.Redis != nil:
+		return &AnalyzeRedis{analyzer: analyzer.Redis}
+	case analyzer.CephStatus != nil:
+		return &AnalyzeCephStatus{analyzer: analyzer.CephStatus}
+	case analyzer.Longhorn != nil:
+		return &AnalyzeLonghorn{analyzer: analyzer.Longhorn}
+	case analyzer.RegistryImages != nil:
+		return &AnalyzeRegistryImages{analyzer: analyzer.RegistryImages}
+	case analyzer.WeaveReport != nil:
+		return &AnalyzeWeaveReport{analyzer: analyzer.WeaveReport}
+	case analyzer.Sysctl != nil:
+		return &AnalyzeSysctl{analyzer: analyzer.Sysctl}
+	case analyzer.ClusterResource != nil:
+		return &AnalyzeClusterResource{analyzer: analyzer.ClusterResource}
+	default:
+		return nil
+	}
+}
+
+// deduplicates a list of troubleshootv1beta2.Analyze objects
+// marshals object to json and then uses its string value to check for uniqueness
+// there is no sorting of the keys in the analyze object's spec so if the spec isn't an exact match line for line as written, no dedup will occur
+func DedupAnalyzers(allAnalyzers []*troubleshootv1beta2.Analyze) []*troubleshootv1beta2.Analyze {
+	uniqueAnalyzers := make(map[string]bool)
+	finalAnalyzers := []*troubleshootv1beta2.Analyze{}
+
+	for _, analyzer := range allAnalyzers {
+		data, err := json.Marshal(analyzer)
+		if err != nil {
+			// return analyzer as is if for whatever reason it can't be marshalled into json
+			finalAnalyzers = append(finalAnalyzers, analyzer)
+		} else {
+			stringData := string(data)
+			if _, value := uniqueAnalyzers[stringData]; !value {
+				uniqueAnalyzers[stringData] = true
+				finalAnalyzers = append(finalAnalyzers, analyzer)
+			}
+		}
+	}
+	return finalAnalyzers
+}

pkg/analyze/analyzer_test.go

@@ -1,6 +1,7 @@
 package analyzer
 
 import (
+	"context"
 	"testing"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -65,3 +66,103 @@ func Test_GetExcludeFlag(t *testing.T) {
 		})
 	}
 }
+
+func TestAnalyzeWithNilAnalyzer(t *testing.T) {
+	got, err := Analyze(context.Background(), nil, nil, nil)
+	assert.Error(t, err)
+	assert.Nil(t, got)
+}
+
+func TestCollector_DedupCollectors(t *testing.T) {
+	tests := []struct {
+		name      string
+		Analyzers []*troubleshootv1beta2.Analyze
+		want      []*troubleshootv1beta2.Analyze
+	}{
+		{
+			name: "multiple ClusterVersion",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+			},
+		},
+		{
+			name: "multiple TextAnalyze",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "multiple TextAnalyze with different CheckName",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "test",
+						},
+					},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "test",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := DedupAnalyzers(tc.Analyzers)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}

pkg/analyze/ceph.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 )
 
 type CephHealth string
@@ -99,20 +100,47 @@ type PgMap struct {
 	TotalBytes uint64 `json:"bytes_total"`
 }
 
-func cephStatus(analyzer *troubleshootv1beta2.CephStatusAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	fileName := path.Join(collect.GetCephCollectorFilepath(analyzer.CollectorName, analyzer.Namespace), "status.json")
-	collected, err := getCollectedFileContents(fileName)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read collected ceph status")
-	}
+type AnalyzeCephStatus struct {
+	analyzer *troubleshootv1beta2.CephStatusAnalyze
+}
 
-	title := analyzer.CheckName
+func (a *AnalyzeCephStatus) Title() string {
+	title := a.analyzer.CheckName
 	if title == "" {
 		title = "Ceph Status"
 	}
 
+	return title
+}
+
+func (a *AnalyzeCephStatus) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeCephStatus) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.cephStatus(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	if result != nil {
+		result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeCephStatus) cephStatus(analyzer *troubleshootv1beta2.CephStatusAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	fileName := path.Join(collect.GetCephCollectorFilepath(analyzer.CollectorName, analyzer.Namespace), "status.json")
+	collected, err := getCollectedFileContents(fileName)
+
+	if err != nil {
+		if _, ok := err.(*types.NotFoundError); ok {
+			return nil, nil
+		}
+		return nil, errors.Wrap(err, "failed to read collected ceph status")
+	}
+
 	analyzeResult := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "rook", // maybe this should be ceph?
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16",
 	}

pkg/analyze/ceph_test.go

@@ -5,21 +5,25 @@ import (
 	"testing"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type getFile func(n string) ([]byte, error)
+
 func Test_cephStatus(t *testing.T) {
 	tests := []struct {
 		name           string
 		analyzer       troubleshootv1beta2.CephStatusAnalyze
-		expectResult   AnalyzeResult
+		expectResult   *AnalyzeResult
+		getFile        getFile
 		filePath, file string
 	}{
 		{
 			name:     "pass case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  true,
 				IsWarn:  false,
 				IsFail:  false,
@@ -51,7 +55,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -84,7 +88,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "fail case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  false,
 				IsFail:  true,
@@ -120,7 +124,7 @@ func Test_cephStatus(t *testing.T) {
 				CollectorName: "custom-namespace",
 				Namespace:     "namespace",
 			},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  true,
 				IsWarn:  false,
 				IsFail:  false,
@@ -167,7 +171,7 @@ func Test_cephStatus(t *testing.T) {
 					},
 				},
 			},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  false,
 				IsFail:  true,
@@ -200,7 +204,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case with missing osd/pg data",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -221,7 +225,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case with health status message and summary",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -249,21 +253,34 @@ func Test_cephStatus(t *testing.T) {
 				}
 			}`,
 		},
+		{
+			name:         "pass case when get file returns not found error",
+			analyzer:     troubleshootv1beta2.CephStatusAnalyze{},
+			expectResult: nil,
+			getFile: func(n string) ([]byte, error) {
+				return nil, &types.NotFoundError{}
+			},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			req := require.New(t)
-
-			getFile := func(n string) ([]byte, error) {
-				assert.Equal(t, n, test.filePath)
-				return []byte(test.file), nil
+			if test.getFile == nil {
+				test.getFile = func(n string) ([]byte, error) {
+					assert.Equal(t, n, test.filePath)
+					return []byte(test.file), nil
+				}
 			}
 
-			actual, err := cephStatus(&test.analyzer, getFile)
+			a := AnalyzeCephStatus{
+				analyzer: &test.analyzer,
+			}
+
+			actual, err := a.cephStatus(&test.analyzer, test.getFile)
 			req.NoError(err)
 
-			assert.Equal(t, test.expectResult, *actual)
+			assert.Equal(t, test.expectResult, actual)
 		})
 	}
 }

pkg/analyze/cluster_pod_statuses.go

@@ -3,19 +3,48 @@ package analyzer
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 	"strings"
 	"text/template"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 )
 
-func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChildCollectedFileContents func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
-	collected, err := getChildCollectedFileContents(filepath.Join("cluster-resources", "pods", "*.json"))
+type AnalyzeClusterPodStatuses struct {
+	analyzer *troubleshootv1beta2.ClusterPodStatuses
+}
+
+func (a *AnalyzeClusterPodStatuses) Title() string {
+	if a.analyzer.CheckName != "" {
+		return a.analyzer.CheckName
+	}
+
+	return "Cluster Pod Status"
+}
+
+func (a *AnalyzeClusterPodStatuses) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeClusterPodStatuses) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	results, err := clusterPodStatuses(a.analyzer, findFiles)
+	if err != nil {
+		return nil, err
+	}
+	for i := range results {
+		results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return results, nil
+}
+
+func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChildCollectedFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	excludeFiles := []string{}
+	collected, err := getChildCollectedFileContents(filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_PODS, "*.json"), excludeFiles)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read collected pods")
 	}
@@ -71,37 +100,40 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 				r.URI = outcome.Pass.URI
 				when = outcome.Pass.When
 			} else {
-				println("error: found an empty outcome in a clusterPodStatuses analyzer") // don't stop
+				klog.Error("error: found an empty outcome in a clusterPodStatuses analyzer\n")
 				continue
 			}
 
-			parts := strings.Split(strings.TrimSpace(when), " ")
-			if len(parts) < 2 {
-				println(fmt.Sprintf("invalid 'when' format: %s\n", when)) // don't stop
-				continue
-			}
-
-			operator := parts[0]
-			reason := parts[1]
+			operator := ""
+			reason := ""
 			match := false
-
-			switch operator {
-			case "=", "==", "===":
-				if reason == "Healthy" {
-					match = !k8sutil.IsPodUnhealthy(&pod)
-				} else {
-					match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason)
+			if when != "" {
+				parts := strings.Split(strings.TrimSpace(when), " ")
+				if len(parts) < 2 {
+					klog.Errorf("invalid 'when' format: %s\n", when)
+					continue
 				}
-			case "!=", "!==":
-				if reason == "Healthy" {
-					match = k8sutil.IsPodUnhealthy(&pod)
-				} else {
-					match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason)
-				}
-			}
+				operator = parts[0]
+				reason = parts[1]
 
-			if !match {
-				continue
+				switch operator {
+				case "=", "==", "===":
+					if reason == "Healthy" {
+						match = !k8sutil.IsPodUnhealthy(&pod)
+					} else {
+						match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason)
+					}
+				case "!=", "!==":
+					if reason == "Healthy" {
+						match = k8sutil.IsPodUnhealthy(&pod)
+					} else {
+						match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason)
+					}
+				}
+
+				if !match {
+					continue
+				}
 			}
 
 			r.InvolvedObject = &corev1.ObjectReference{

pkg/analyze/cluster_pod_statuses_test.go

@@ -0,0 +1,398 @@
+package analyzer
+
+import (
+	"testing"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func Test_ClusterPodStatuses(t *testing.T) {
+	tests := []struct {
+		name         string
+		analyzer     troubleshootv1beta2.ClusterPodStatuses
+		expectResult []*AnalyzeResult
+		files        map[string][]byte
+	}{
+		{
+			name: "pass_when_all_pods_are_healthy_in_specific_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "pass_when_all_pods_are_healthy_in_specific_namespace",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Restarting the pod may fix the issue.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_all_pods_are_healthy_in_specific_namespace",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "pass_when_pods_are_healthy_in_all_namespaces",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "pass_when_pods_are_healthy_in_all_namespaces",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Restarting the pod may fix the issue.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_pods_are_healthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_pods_are_healthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json": []byte(defaultPods),
+				"cluster-resources/pods/other.json":   []byte(otherPods),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_specific_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_specific_namespace",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_specific_namespace",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!== Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_any_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_all_namespaces",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other-unhealthy",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "== CrashLoopBackOff",
+							Message: "A pod is unhealthy.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+					Message: "A pod is unhealthy.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "=== CrashLoopBackOff",
+							Message: "A pod is unhealthy.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+					Message: "A pod is unhealthy.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+
+			getFiles := func(n string, _ []string) (map[string][]byte, error) {
+				if file, ok := test.files[n]; ok {
+					return map[string][]byte{n: file}, nil
+				}
+				return test.files, nil
+			}
+
+			actual, err := clusterPodStatuses(&test.analyzer, getFiles)
+			req.NoError(err)
+			req.Equal(len(test.expectResult), len(actual))
+			for _, a := range actual {
+				assert.Contains(t, test.expectResult, a)
+			}
+		})
+	}
+}

pkg/analyze/cluster_version.go

@@ -4,12 +4,33 @@ import (
 	"encoding/json"
 	"strings"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )
 
+type AnalyzeClusterVersion struct {
+	analyzer *troubleshootv1beta2.ClusterVersion
+}
+
+func (a *AnalyzeClusterVersion) Title() string {
+	return title(a.analyzer.CheckName)
+}
+
+func (a *AnalyzeClusterVersion) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeClusterVersion) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := analyzeClusterVersion(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
 func analyzeClusterVersion(analyzer *troubleshootv1beta2.ClusterVersion, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	clusterInfo, err := getCollectedFileContents("cluster-info/cluster_version.json")
 	if err != nil {
@@ -29,19 +50,22 @@ func analyzeClusterVersion(analyzer *troubleshootv1beta2.ClusterVersion, getColl
 	return analyzeClusterVersionResult(k8sVersion, analyzer.Outcomes, analyzer.CheckName)
 }
 
+func title(checkName string) string {
+	if checkName == "" {
+		return "Required Kubernetes Version"
+	}
+
+	return checkName
+}
+
 func analyzeClusterVersionResult(k8sVersion semver.Version, outcomes []*troubleshootv1beta2.Outcome, checkName string) (*AnalyzeResult, error) {
 	for _, outcome := range outcomes {
 		when := ""
 		message := ""
 		uri := ""
 
-		title := checkName
-		if title == "" {
-			title = "Required Kubernetes Version"
-		}
-
 		result := AnalyzeResult{
-			Title:   title,
+			Title:   title(checkName),
 			IconKey: "kubernetes_cluster_version",
 			IconURI: "https://troubleshoot.sh/images/analyzer-icons/kubernetes.svg?w=16&h=16",
 		}

pkg/analyze/cluster_version_test.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 )
 

pkg/analyze/configmap.go

@@ -8,7 +8,33 @@ import (
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )
 
-func analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+type AnalyzeConfigMap struct {
+	analyzer *troubleshootv1beta2.AnalyzeConfigMap
+}
+
+func (a *AnalyzeConfigMap) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = fmt.Sprintf("ConfigMap %s", a.analyzer.ConfigMapName)
+	}
+
+	return title
+}
+
+func (a *AnalyzeConfigMap) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeConfigMap) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeConfigMap(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeConfigMap) analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	filename := collect.GetConfigMapFileName(
 		&troubleshootv1beta2.ConfigMap{
 			Namespace: analyzer.Namespace,
@@ -28,13 +54,8 @@ func analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollect
 		return nil, err
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = fmt.Sprintf("ConfigMap %s", analyzer.ConfigMapName)
-	}
-
 	result := AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_analyze_secret", // TODO: icon
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/secret.svg?w=13&h=16",
 	}

pkg/analyze/configmap_test.go

@@ -199,7 +199,12 @@ func Test_analyzeConfigMap(t *testing.T) {
 				}
 				return contents, nil
 			}
-			got, err := analyzeConfigMap(tt.analyzer, getCollectedFileContents)
+
+			a := AnalyzeConfigMap{
+				analyzer: tt.analyzer,
+			}
+
+			got, err := a.analyzeConfigMap(tt.analyzer, getCollectedFileContents)
 			if tt.wantErr {
 				assert.Error(t, err)
 			} else {

pkg/analyze/container_runtime.go

@@ -2,16 +2,44 @@ package analyzer
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 )
 
-func analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	collected, err := getCollectedFileContents("cluster-resources/nodes.json")
+type AnalyzeContainerRuntime struct {
+	analyzer *troubleshootv1beta2.ContainerRuntime
+}
+
+func (a *AnalyzeContainerRuntime) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = "Container Runtime"
+	}
+
+	return title
+}
+
+func (a *AnalyzeContainerRuntime) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeContainerRuntime) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeContainerRuntime(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeContainerRuntime) analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	collected, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_NODES))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get contents of nodes.json")
 	}
@@ -26,12 +54,8 @@ func analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, get
 		foundRuntimes = append(foundRuntimes, node.Status.NodeInfo.ContainerRuntimeVersion)
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = "Container Runtime"
-	}
 	result := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_container_runtime",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/container-runtime.svg?w=23&h=16",
 	}

pkg/analyze/container_runtime_test.go

@@ -113,7 +113,11 @@ func Test_containerRuntime(t *testing.T) {
 				return test.files[n], nil
 			}
 
-			actual, err := analyzeContainerRuntime(&test.analyzer, getFiles)
+			a := AnalyzeContainerRuntime{
+				analyzer: &test.analyzer,
+			}
+
+			actual, err := a.analyzeContainerRuntime(&test.analyzer, getFiles)
 			req.NoError(err)
 
 			assert.Equal(t, &test.expectResult, actual)

pkg/analyze/crd.go

@@ -5,11 +5,38 @@ import (
 	"fmt"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 )
 
-func analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourceDefinition, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	crdData, err := getCollectedFileContents("cluster-resources/custom-resource-definitions.json")
+type AnalyzeCustomResourceDefinition struct {
+	analyzer *troubleshootv1beta2.CustomResourceDefinition
+}
+
+func (a *AnalyzeCustomResourceDefinition) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = fmt.Sprintf("Custom resource definition %s", a.analyzer.CustomResourceDefinitionName)
+	}
+
+	return title
+}
+
+func (a *AnalyzeCustomResourceDefinition) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeCustomResourceDefinition) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeCustomResourceDefinition(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeCustomResourceDefinition) analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourceDefinition, getFile getCollectedFileContents) (*AnalyzeResult, error) {
+	crdData, err := getFile(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_CUSTOM_RESOURCE_DEFINITIONS))
 	if err != nil {
 		return nil, err
 	}
@@ -19,13 +46,8 @@ func analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourc
 		return nil, err
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = fmt.Sprintf("Custom resource definition %s", analyzer.CustomResourceDefinitionName)
-	}
-
 	result := AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_custom_resource_definition",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/custom-resource-definition.svg?w=13&h=16",
 	}

pkg/analyze/data_test.go

@@ -33,3 +33,15 @@ var defaultStatefulSets string
 
 //go:embed files/statefulsets/monitoring.json
 var monitoringStatefulSets string
+
+//go:embed files/pods/default.json
+var defaultPods string
+
+//go:embed files/pods/other.json
+var otherPods string
+
+//go:embed files/pods/default-unhealthy.json
+var defaultPodsUnhealthy string
+
+//go:embed files/pods/other-unhealthy.json
+var otherPodsUnhealthy string

pkg/analyze/database_shared.go

@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )

pkg/analyze/deployment_status.go

@@ -7,10 +7,49 @@ import (
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	appsv1 "k8s.io/api/apps/v1"
 )
 
-func analyzeDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
+type AnalyzeDeploymentStatus struct {
+	analyzer *troubleshootv1beta2.DeploymentStatus
+}
+
+func (a *AnalyzeDeploymentStatus) Title() string {
+	if a.analyzer.CheckName != "" {
+		return a.analyzer.CheckName
+	}
+
+	if a.analyzer.Name != "" && a.analyzer.Namespace != "" {
+		return fmt.Sprintf("%s/%s Deployment Status", a.analyzer.Name, a.analyzer.Name)
+	}
+
+	if a.analyzer.Name != "" {
+		return fmt.Sprintf("%s Deployment Status", a.analyzer.Name)
+	}
+	if a.analyzer.Namespace != "" {
+		return fmt.Sprintf("%s Deployment Status", a.analyzer.Namespace)
+	}
+
+	return "Deployment Status"
+}
+
+func (a *AnalyzeDeploymentStatus) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeDeploymentStatus) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	results, err := analyzeDeploymentStatus(a.analyzer, findFiles)
+	if err != nil {
+		return nil, err
+	}
+	for i := range results {
+		results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return results, nil
+}
+
+func analyzeDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
 	if analyzer.Name == "" {
 		return analyzeAllDeploymentStatuses(analyzer, getFileContents)
 	} else {
@@ -18,8 +57,9 @@ func analyzeDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, get
 	}
 }
 
-func analyzeOneDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
-	files, err := getFileContents(filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", analyzer.Namespace)))
+func analyzeOneDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	excludeFiles := []string{}
+	files, err := getFileContents(filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", analyzer.Namespace)), excludeFiles)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read collected deployments from namespace")
 	}
@@ -61,23 +101,24 @@ func analyzeOneDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus,
 	return []*AnalyzeResult{result}, nil
 }
 
-func analyzeAllDeploymentStatuses(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents func(string) (map[string][]byte, error)) ([]*AnalyzeResult, error) {
+func analyzeAllDeploymentStatuses(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	excludeFiles := []string{}
 	fileNames := make([]string, 0)
 	if analyzer.Namespace != "" {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", analyzer.Namespace)))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", analyzer.Namespace)))
 	}
 	for _, ns := range analyzer.Namespaces {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", ns)))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", ns)))
 	}
 
 	// no namespace specified, so we need to analyze all deployments
 	if len(fileNames) == 0 {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", "*.json"))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, "*.json"))
 	}
 
 	results := []*AnalyzeResult{}
 	for _, fileName := range fileNames {
-		files, err := getFileContents(fileName)
+		files, err := getFileContents(fileName, excludeFiles)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read collected deployments from file")
 		}

pkg/analyze/deployment_status_test.go

@@ -221,7 +221,7 @@ func Test_deploymentStatus(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			req := require.New(t)
 
-			getFiles := func(n string) (map[string][]byte, error) {
+			getFiles := func(n string, _ []string) (map[string][]byte, error) {
 				if file, ok := test.files[n]; ok {
 					return map[string][]byte{n: file}, nil
 				}

pkg/analyze/distribution.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -46,6 +47,32 @@ const (
 	k3s           Provider = iota
 )
 
+type AnalyzeDistribution struct {
+	analyzer *troubleshootv1beta2.Distribution
+}
+
+func (a *AnalyzeDistribution) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = "Kubernetes Distribution"
+	}
+
+	return title
+}
+
+func (a *AnalyzeDistribution) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeDistribution) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeDistribution(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
 func CheckApiResourcesForProviders(foundProviders *providers, apiResources []*metav1.APIResourceList, provider string) string {
 	for _, resource := range apiResources {
 		if strings.HasPrefix(resource.GroupVersion, "apps.openshift.io/") {
@@ -141,11 +168,11 @@ func ParseNodesForProviders(nodes []corev1.Node) (providers, string) {
 	return foundProviders, stringProvider
 }
 
-func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+func (a *AnalyzeDistribution) analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	var unknownDistribution string
-	collected, err := getCollectedFileContents("cluster-resources/nodes.json")
+	collected, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_NODES))
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get contents of nodes.json")
+		return nil, errors.Wrap(err, fmt.Sprintf("failed to get contents of %s.json", constants.CLUSTER_RESOURCES_NODES))
 	}
 
 	var nodes corev1.NodeList
@@ -155,7 +182,7 @@ func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollecte
 
 	foundProviders, _ := ParseNodesForProviders(nodes.Items)
 
-	apiResourcesBytes, err := getCollectedFileContents("cluster-resources/resources.json")
+	apiResourcesBytes, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_RESOURCES))
 	// if the file is not found, that is not a fatal error
 	// troubleshoot 0.9.15 and earlier did not collect that file
 	if err == nil {
@@ -166,12 +193,8 @@ func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollecte
 		_ = CheckApiResourcesForProviders(&foundProviders, apiResources, "")
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = "Kubernetes Distribution"
-	}
 	result := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_distribution",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/distribution.svg?w=20&h=14",
 	}

pkg/analyze/download.go

@@ -3,8 +3,9 @@ package analyzer
 import (
 	"archive/tar"
 	"compress/gzip"
+	"context"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -12,9 +13,11 @@ import (
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/docrewrite"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 )
 
 type fileContentProvider struct {
@@ -22,7 +25,12 @@ type fileContentProvider struct {
 }
 
 // Analyze local will analyze a locally available (already downloaded) bundle
-func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analyze, hostAnalyzers []*troubleshootv1beta2.HostAnalyze) ([]*AnalyzeResult, error) {
+func AnalyzeLocal(
+	ctx context.Context,
+	localBundlePath string,
+	analyzers []*troubleshootv1beta2.Analyze,
+	hostAnalyzers []*troubleshootv1beta2.HostAnalyze,
+) ([]*AnalyzeResult, error) {
 	rootDir, err := FindBundleRootDir(localBundlePath)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find root dir")
@@ -32,9 +40,9 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 
 	analyzeResults := []*AnalyzeResult{}
 	for _, analyzer := range analyzers {
-		analyzeResult, err := Analyze(analyzer, fcp.getFileContents, fcp.getChildFileContents)
+		analyzeResult, err := Analyze(ctx, analyzer, fcp.getFileContents, fcp.getChildFileContents)
 		if err != nil {
-			logger.Printf("An analyzer failed to run: %v", err)
+			klog.Errorf("An analyzer failed to run: %v", err)
 			continue
 		}
 
@@ -47,7 +55,7 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 	}
 
 	for _, hostAnalyzer := range hostAnalyzers {
-		analyzeResult := HostAnalyze(hostAnalyzer, fcp.getFileContents, fcp.getChildFileContents)
+		analyzeResult := HostAnalyze(ctx, hostAnalyzer, fcp.getFileContents, fcp.getChildFileContents)
 		analyzeResults = append(analyzeResults, analyzeResult...)
 	}
 
@@ -55,27 +63,13 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 }
 
 func DownloadAndAnalyze(bundleURL string, analyzersSpec string) ([]*AnalyzeResult, error) {
-	tmpDir, err := ioutil.TempDir("", "troubleshoot-k8s")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to create temp dir")
-	}
-	defer os.RemoveAll(tmpDir)
-
-	if err := downloadTroubleshootBundle(bundleURL, tmpDir); err != nil {
-		return nil, errors.Wrap(err, "failed to download bundle")
-	}
-
-	rootDir, err := FindBundleRootDir(tmpDir)
+	tmpDir, rootDir, err := DownloadAndExtractSupportBundle(bundleURL)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find root dir")
 	}
+	defer os.RemoveAll(tmpDir)
 
-	_, err = os.Stat(filepath.Join(rootDir, "version.yaml"))
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read version.yaml")
-	}
-
-	analyzers := []*troubleshootv1beta2.Analyze{}
+	var analyzers []*troubleshootv1beta2.Analyze
 	hostAnalyzers := []*troubleshootv1beta2.HostAnalyze{}
 
 	if analyzersSpec == "" {
@@ -93,10 +87,37 @@ func DownloadAndAnalyze(bundleURL string, analyzersSpec string) ([]*AnalyzeResul
 		hostAnalyzers = parsedHostAnalyzers
 	}
 
-	return AnalyzeLocal(rootDir, analyzers, hostAnalyzers)
+	return AnalyzeLocal(context.Background(), rootDir, analyzers, hostAnalyzers)
+}
+
+func DownloadAndExtractSupportBundle(bundleURL string) (string, string, error) {
+	tmpDir, err := os.MkdirTemp("", "troubleshoot-k8s")
+	if err != nil {
+		return "", "", errors.Wrap(err, "failed to create temp dir")
+	}
+
+	if err := downloadTroubleshootBundle(bundleURL, tmpDir); err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to download bundle")
+	}
+
+	bundleDir, err := FindBundleRootDir(tmpDir)
+	if err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to find root dir")
+	}
+
+	_, err = os.Stat(filepath.Join(bundleDir, constants.VERSION_FILENAME))
+	if err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to read "+constants.VERSION_FILENAME)
+	}
+
+	return tmpDir, bundleDir, nil
 }
 
 func downloadTroubleshootBundle(bundleURL string, destDir string) error {
+	// TODO: Move to separate package support bundle utils package
 	if bundleURL[0] == os.PathSeparator {
 		f, err := os.Open(bundleURL)
 		if err != nil {
@@ -111,7 +132,7 @@ func downloadTroubleshootBundle(bundleURL string, destDir string) error {
 		return errors.Wrap(err, "failed to get workdir")
 	}
 
-	tmpDir, err := ioutil.TempDir("", "troubleshoot")
+	tmpDir, err := os.MkdirTemp("", "troubleshoot")
 	if err != nil {
 		return errors.Wrap(err, "failed to create tmp dir")
 	}
@@ -137,6 +158,8 @@ func downloadTroubleshootBundle(bundleURL string, destDir string) error {
 }
 
 func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
+	// TODO: Move to separate package e.g support bundle package, or sbutils
+	// if there are cyclic dependencies
 	gzReader, err := gzip.NewReader(reader)
 	if err != nil {
 		return errors.Wrap(err, "failed to create gzip reader")
@@ -159,14 +182,14 @@ func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
 				return errors.Wrap(err, "failed to mkdir")
 			}
 		case tar.TypeReg:
-			name := filepath.Join(destDir, header.Name)
+			destFileName := filepath.Join(destDir, header.Name)
 
-			dirName := filepath.Dir(name)
+			dirName := filepath.Dir(destFileName)
 			if err := os.MkdirAll(dirName, 0755); err != nil {
 				return errors.Wrapf(err, "failed to mkdir for file %s", header.Name)
 			}
 
-			file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.FileMode(header.Mode))
+			file, err := os.OpenFile(destFileName, os.O_RDWR|os.O_CREATE, os.FileMode(header.Mode))
 			if err != nil {
 				return errors.Wrap(err, "failed to open tar file")
 			}
@@ -175,6 +198,21 @@ func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
 			if err != nil {
 				return errors.Wrap(err, "failed to extract file")
 			}
+		case tar.TypeSymlink:
+			destFileName := filepath.Join(destDir, header.Name)
+
+			dirName := filepath.Dir(destFileName)
+			if err := os.MkdirAll(dirName, 0755); err != nil {
+				return errors.Wrapf(err, "failed to mkdir for symlink %s", header.Name)
+			}
+
+			// Symlink targets should be absolute paths after extraction
+			// for other parts of the code to work correctly e.g redaction, CollectorResult
+			targetPath := filepath.Join(filepath.Dir(destFileName), header.Linkname)
+			err = os.Symlink(targetPath, destFileName)
+			if err != nil {
+				return errors.Wrap(err, "failed to create symlink")
+			}
 		}
 	}
 
@@ -251,7 +289,7 @@ func FindBundleRootDir(localBundlePath string) (string, error) {
 
 	isInSubDir := true
 	for _, name := range names {
-		if name == "version.yaml" {
+		if name == constants.VERSION_FILENAME {
 			isInSubDir = false
 			break
 		}
@@ -265,17 +303,55 @@ func FindBundleRootDir(localBundlePath string) (string, error) {
 }
 
 func (f fileContentProvider) getFileContents(fileName string) ([]byte, error) {
-	return ioutil.ReadFile(filepath.Join(f.rootDir, fileName))
+	contents, err := os.ReadFile(filepath.Join(f.rootDir, fileName))
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, &types.NotFoundError{Name: fileName}
+		}
+		return nil, err
+	}
+	return contents, nil
 }
 
-func (f fileContentProvider) getChildFileContents(dirName string) (map[string][]byte, error) {
+func excludeFilePaths(files, excludeFiles []string) []string {
+	mapExcludeFiles := make(map[string]struct{}, len(excludeFiles))
+
+	for _, path := range excludeFiles {
+		mapExcludeFiles[path] = struct{}{}
+	}
+
+	var nonexcludedFiles []string
+
+	for _, path := range files {
+		if _, found := mapExcludeFiles[path]; !found {
+			nonexcludedFiles = append(nonexcludedFiles, path)
+		}
+	}
+
+	return nonexcludedFiles
+}
+
+func (f fileContentProvider) getChildFileContents(dirName string, excludeFiles []string) (map[string][]byte, error) {
 	files, err := filepath.Glob(filepath.Join(f.rootDir, dirName))
 	if err != nil {
 		return nil, errors.Wrapf(err, "invalid glob %q", dirName)
 	}
+
+	if len(excludeFiles) > 0 {
+		excludeFileNames := []string{}
+		for _, excludeFile := range excludeFiles {
+			excludeFileName, err := filepath.Glob(filepath.Join(f.rootDir, excludeFile))
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid glob %q", excludeFile)
+			}
+			excludeFileNames = append(excludeFileNames, excludeFileName...)
+		}
+		files = excludeFilePaths(files, excludeFileNames)
+	}
+
 	fileArr := map[string][]byte{}
 	for _, filePath := range files {
-		bytes, err := ioutil.ReadFile(filePath)
+		bytes, err := os.ReadFile(filePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "read %q", filePath)
 		}

pkg/analyze/download_test.go

@@ -0,0 +1,49 @@
+package analyzer
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/replicatedhq/troubleshoot/internal/testutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDownloadAndExtractSupportBundle(t *testing.T) {
+	// TODO: Add tests for web url downloads
+	tests := []struct {
+		name      string
+		bundleURL string
+		wantErr   bool
+	}{
+		{
+			name:      "extract a bundle from a local file path",
+			bundleURL: filepath.Join(testutils.FileDir(), "../../testdata/supportbundle/support-bundle.tar.gz"),
+			wantErr:   false,
+		},
+		{
+			name:      "extract a bundle from a non-existent file path",
+			bundleURL: "/home/someone/gibberish",
+			wantErr:   true,
+		},
+		{
+			name:      "extract an invalid support bundle which has no version file",
+			bundleURL: filepath.Join(testutils.FileDir(), "../../testdata/supportbundle/missing-version.tar.gz"),
+			wantErr:   true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpDir, bundleDir, err := DownloadAndExtractSupportBundle(tt.bundleURL)
+			defer os.RemoveAll(tmpDir) // clean up. Ignore error
+
+			if err == nil {
+				assert.DirExists(t, bundleDir)
+				assert.FileExists(t, filepath.Join(bundleDir, "version.yaml"))
+			} else {
+				assert.Equal(t, "", tmpDir)
+				assert.Equal(t, "", bundleDir)
+			}
+		})
+	}
+}

pkg/analyze/files/pods/default-unhealthy.json

@@ -0,0 +1,263 @@
+{
+    "kind": "PodList",
+    "apiVersion": "v1",
+    "metadata": {
+      "resourceVersion": "498"
+    },
+    "items": [
+      {
+        "kind": "Pod",
+        "apiVersion": "v1",
+        "metadata": {
+          "name": "random-pod-75b66db9b9-nqhp8",
+          "generateName": "random-pod-75b66db9b9-",
+          "namespace": "default-unhealthy",
+          "uid": "e00af024-67ca-440d-9714-999cf8617d58",
+          "resourceVersion": "488",
+          "creationTimestamp": "2023-03-23T16:43:42Z",
+          "labels": {
+            "app.kubernetes.io/appname": "random-pod",
+            "pod-template-hash": "75b66db9b9"
+          },
+          "ownerReferences": [
+            {
+              "apiVersion": "apps/v1",
+              "kind": "ReplicaSet",
+              "name": "random-pod-75b66db9b9",
+              "uid": "794592d2-539c-4cca-a036-678bd2357c78",
+              "controller": true,
+              "blockOwnerDeletion": true
+            }
+          ],
+          "managedFields": [
+            {
+              "manager": "kube-controller-manager",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:42Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:metadata": {
+                  "f:generateName": {},
+                  "f:labels": {
+                    ".": {},
+                    "f:app.kubernetes.io/appname": {},
+                    "f:pod-template-hash": {}
+                  },
+                  "f:ownerReferences": {
+                    ".": {},
+                    "k:{\"uid\":\"794592d2-539c-4cca-a036-678bd2357c78\"}": {}
+                  }
+                },
+                "f:spec": {
+                  "f:containers": {
+                    "k:{\"name\":\"random-pod\"}": {
+                      ".": {},
+                      "f:image": {},
+                      "f:imagePullPolicy": {},
+                      "f:name": {},
+                      "f:resources": {},
+                      "f:terminationMessagePath": {},
+                      "f:terminationMessagePolicy": {}
+                    }
+                  },
+                  "f:dnsPolicy": {},
+                  "f:enableServiceLinks": {},
+                  "f:restartPolicy": {},
+                  "f:schedulerName": {},
+                  "f:securityContext": {},
+                  "f:terminationGracePeriodSeconds": {}
+                }
+              }
+            },
+            {
+              "manager": "kubelet",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:46Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:status": {
+                  "f:conditions": {
+                    "k:{\"type\":\"ContainersReady\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Initialized\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Ready\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    }
+                  },
+                  "f:containerStatuses": {},
+                  "f:hostIP": {},
+                  "f:phase": {},
+                  "f:podIP": {},
+                  "f:podIPs": {
+                    ".": {},
+                    "k:{\"ip\":\"10.244.0.5\"}": {
+                      ".": {},
+                      "f:ip": {}
+                    }
+                  },
+                  "f:startTime": {}
+                }
+              },
+              "subresource": "status"
+            }
+          ]
+        },
+        "spec": {
+          "volumes": [
+            {
+              "name": "kube-api-access-hz77k",
+              "projected": {
+                "sources": [
+                  {
+                    "serviceAccountToken": {
+                      "expirationSeconds": 3607,
+                      "path": "token"
+                    }
+                  },
+                  {
+                    "configMap": {
+                      "name": "kube-root-ca.crt",
+                      "items": [
+                        {
+                          "key": "ca.crt",
+                          "path": "ca.crt"
+                        }
+                      ]
+                    }
+                  },
+                  {
+                    "downwardAPI": {
+                      "items": [
+                        {
+                          "path": "namespace",
+                          "fieldRef": {
+                            "apiVersion": "v1",
+                            "fieldPath": "metadata.namespace"
+                          }
+                        }
+                      ]
+                    }
+                  }
+                ],
+                "defaultMode": 420
+              }
+            }
+          ],
+          "containers": [
+            {
+              "name": "random-pod",
+              "image": "random-pod/server:1.27.0-alpine",
+              "resources": {},
+              "volumeMounts": [
+                {
+                  "name": "kube-api-access-hz77k",
+                  "readOnly": true,
+                  "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+                }
+              ],
+              "terminationMessagePath": "/dev/termination-log",
+              "terminationMessagePolicy": "File",
+              "imagePullPolicy": "IfNotPresent"
+            }
+          ],
+          "restartPolicy": "Always",
+          "terminationGracePeriodSeconds": 30,
+          "dnsPolicy": "ClusterFirst",
+          "serviceAccountName": "default-unhealthy",
+          "serviceAccount": "default-unhealthy",
+          "nodeName": "chart-testing-control-plane",
+          "securityContext": {},
+          "schedulerName": "default-scheduler",
+          "tolerations": [
+            {
+              "key": "node.kubernetes.io/not-ready",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            },
+            {
+              "key": "node.kubernetes.io/unreachable",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            }
+          ],
+          "priority": 0,
+          "enableServiceLinks": true,
+          "preemptionPolicy": "PreemptLowerPriority"
+        },
+        "status": {
+          "phase": "CrashLoopBackOff",
+          "conditions": [
+            {
+              "type": "Initialized",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            },
+            {
+              "type": "Ready",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "ContainersReady",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "PodScheduled",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            }
+          ],
+          "hostIP": "172.18.0.2",
+          "podIP": "10.244.0.5",
+          "podIPs": [
+            {
+              "ip": "10.244.0.5"
+            }
+          ],
+          "startTime": "2023-03-23T16:43:42Z",
+          "containerStatuses": [
+            {
+              "name": "random-pod",
+              "state": {
+                "running": {
+                  "startedAt": "2023-03-23T16:43:45Z"
+                }
+              },
+              "lastState": {},
+              "ready": true,
+              "restartCount": 0,
+              "image": "docker.io/random-pod/server:1.27.0-alpine",
+              "imageID": "docker.io/random-pod/server@sha256:7cd450642c54cc8cce64dd9b4b8664c8480e7df6de880c5e134f4a2ed6446c5c",
+              "containerID": "containerd://58e668ba55950e6d08340f496f5e7f33f3d92cbfcbf366ce5e5f5c50a3926c71",
+              "started": true
+            }
+          ],
+          "qosClass": "BestEffort"
+        }
+      }
+    ]
+  }

pkg/analyze/files/pods/default.json

@@ -0,0 +1,263 @@
+{
+    "kind": "PodList",
+    "apiVersion": "v1",
+    "metadata": {
+      "resourceVersion": "498"
+    },
+    "items": [
+      {
+        "kind": "Pod",
+        "apiVersion": "v1",
+        "metadata": {
+          "name": "random-pod-75b66db9b9-nqhp8",
+          "generateName": "random-pod-75b66db9b9-",
+          "namespace": "default",
+          "uid": "e00af024-67ca-440d-9714-999cf8617d58",
+          "resourceVersion": "488",
+          "creationTimestamp": "2023-03-23T16:43:42Z",
+          "labels": {
+            "app.kubernetes.io/appname": "random-pod",
+            "pod-template-hash": "75b66db9b9"
+          },
+          "ownerReferences": [
+            {
+              "apiVersion": "apps/v1",
+              "kind": "ReplicaSet",
+              "name": "random-pod-75b66db9b9",
+              "uid": "794592d2-539c-4cca-a036-678bd2357c78",
+              "controller": true,
+              "blockOwnerDeletion": true
+            }
+          ],
+          "managedFields": [
+            {
+              "manager": "kube-controller-manager",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:42Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:metadata": {
+                  "f:generateName": {},
+                  "f:labels": {
+                    ".": {},
+                    "f:app.kubernetes.io/appname": {},
+                    "f:pod-template-hash": {}
+                  },
+                  "f:ownerReferences": {
+                    ".": {},
+                    "k:{\"uid\":\"794592d2-539c-4cca-a036-678bd2357c78\"}": {}
+                  }
+                },
+                "f:spec": {
+                  "f:containers": {
+                    "k:{\"name\":\"random-pod\"}": {
+                      ".": {},
+                      "f:image": {},
+                      "f:imagePullPolicy": {},
+                      "f:name": {},
+                      "f:resources": {},
+                      "f:terminationMessagePath": {},
+                      "f:terminationMessagePolicy": {}
+                    }
+                  },
+                  "f:dnsPolicy": {},
+                  "f:enableServiceLinks": {},
+                  "f:restartPolicy": {},
+                  "f:schedulerName": {},
+                  "f:securityContext": {},
+                  "f:terminationGracePeriodSeconds": {}
+                }
+              }
+            },
+            {
+              "manager": "kubelet",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:46Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:status": {
+                  "f:conditions": {
+                    "k:{\"type\":\"ContainersReady\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Initialized\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Ready\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    }
+                  },
+                  "f:containerStatuses": {},
+                  "f:hostIP": {},
+                  "f:phase": {},
+                  "f:podIP": {},
+                  "f:podIPs": {
+                    ".": {},
+                    "k:{\"ip\":\"10.244.0.5\"}": {
+                      ".": {},
+                      "f:ip": {}
+                    }
+                  },
+                  "f:startTime": {}
+                }
+              },
+              "subresource": "status"
+            }
+          ]
+        },
+        "spec": {
+          "volumes": [
+            {
+              "name": "kube-api-access-hz77k",
+              "projected": {
+                "sources": [
+                  {
+                    "serviceAccountToken": {
+                      "expirationSeconds": 3607,
+                      "path": "token"
+                    }
+                  },
+                  {
+                    "configMap": {
+                      "name": "kube-root-ca.crt",
+                      "items": [
+                        {
+                          "key": "ca.crt",
+                          "path": "ca.crt"
+                        }
+                      ]
+                    }
+                  },
+                  {
+                    "downwardAPI": {
+                      "items": [
+                        {
+                          "path": "namespace",
+                          "fieldRef": {
+                            "apiVersion": "v1",
+                            "fieldPath": "metadata.namespace"
+                          }
+                        }
+                      ]
+                    }
+                  }
+                ],
+                "defaultMode": 420
+              }
+            }
+          ],
+          "containers": [
+            {
+              "name": "random-pod",
+              "image": "random-pod/server:1.27.0-alpine",
+              "resources": {},
+              "volumeMounts": [
+                {
+                  "name": "kube-api-access-hz77k",
+                  "readOnly": true,
+                  "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+                }
+              ],
+              "terminationMessagePath": "/dev/termination-log",
+              "terminationMessagePolicy": "File",
+              "imagePullPolicy": "IfNotPresent"
+            }
+          ],
+          "restartPolicy": "Always",
+          "terminationGracePeriodSeconds": 30,
+          "dnsPolicy": "ClusterFirst",
+          "serviceAccountName": "default",
+          "serviceAccount": "default",
+          "nodeName": "chart-testing-control-plane",
+          "securityContext": {},
+          "schedulerName": "default-scheduler",
+          "tolerations": [
+            {
+              "key": "node.kubernetes.io/not-ready",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            },
+            {
+              "key": "node.kubernetes.io/unreachable",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            }
+          ],
+          "priority": 0,
+          "enableServiceLinks": true,
+          "preemptionPolicy": "PreemptLowerPriority"
+        },
+        "status": {
+          "phase": "Running",
+          "conditions": [
+            {
+              "type": "Initialized",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            },
+            {
+              "type": "Ready",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "ContainersReady",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "PodScheduled",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            }
+          ],
+          "hostIP": "172.18.0.2",
+          "podIP": "10.244.0.5",
+          "podIPs": [
+            {
+              "ip": "10.244.0.5"
+            }
+          ],
+          "startTime": "2023-03-23T16:43:42Z",
+          "containerStatuses": [
+            {
+              "name": "random-pod",
+              "state": {
+                "running": {
+                  "startedAt": "2023-03-23T16:43:45Z"
+                }
+              },
+              "lastState": {},
+              "ready": true,
+              "restartCount": 0,
+              "image": "docker.io/random-pod/server:1.27.0-alpine",
+              "imageID": "docker.io/random-pod/server@sha256:7cd450642c54cc8cce64dd9b4b8664c8480e7df6de880c5e134f4a2ed6446c5c",
+              "containerID": "containerd://58e668ba55950e6d08340f496f5e7f33f3d92cbfcbf366ce5e5f5c50a3926c71",
+              "started": true
+            }
+          ],
+          "qosClass": "BestEffort"
+        }
+      }
+    ]
+  }

pkg/analyze/files/pods/other-unhealthy.json

@@ -0,0 +1,263 @@
+{
+    "kind": "PodList",
+    "apiVersion": "v1",
+    "metadata": {
+      "resourceVersion": "498"
+    },
+    "items": [
+      {
+        "kind": "Pod",
+        "apiVersion": "v1",
+        "metadata": {
+          "name": "other-pod-75b66db9b9-nqhp8",
+          "generateName": "other-pod-75b66db9b9-",
+          "namespace": "other-unhealthy",
+          "uid": "e00af024-67ca-440d-9714-999cf8617d58",
+          "resourceVersion": "488",
+          "creationTimestamp": "2023-03-23T16:43:42Z",
+          "labels": {
+            "app.kubernetes.io/appname": "other-pod",
+            "pod-template-hash": "75b66db9b9"
+          },
+          "ownerReferences": [
+            {
+              "apiVersion": "apps/v1",
+              "kind": "ReplicaSet",
+              "name": "other-pod-75b66db9b9",
+              "uid": "794592d2-539c-4cca-a036-678bd2357c78",
+              "controller": true,
+              "blockOwnerDeletion": true
+            }
+          ],
+          "managedFields": [
+            {
+              "manager": "kube-controller-manager",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:42Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:metadata": {
+                  "f:generateName": {},
+                  "f:labels": {
+                    ".": {},
+                    "f:app.kubernetes.io/appname": {},
+                    "f:pod-template-hash": {}
+                  },
+                  "f:ownerReferences": {
+                    ".": {},
+                    "k:{\"uid\":\"794592d2-539c-4cca-a036-678bd2357c78\"}": {}
+                  }
+                },
+                "f:spec": {
+                  "f:containers": {
+                    "k:{\"name\":\"other-pod\"}": {
+                      ".": {},
+                      "f:image": {},
+                      "f:imagePullPolicy": {},
+                      "f:name": {},
+                      "f:resources": {},
+                      "f:terminationMessagePath": {},
+                      "f:terminationMessagePolicy": {}
+                    }
+                  },
+                  "f:dnsPolicy": {},
+                  "f:enableServiceLinks": {},
+                  "f:restartPolicy": {},
+                  "f:schedulerName": {},
+                  "f:securityContext": {},
+                  "f:terminationGracePeriodSeconds": {}
+                }
+              }
+            },
+            {
+              "manager": "kubelet",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:46Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:status": {
+                  "f:conditions": {
+                    "k:{\"type\":\"ContainersReady\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Initialized\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Ready\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    }
+                  },
+                  "f:containerStatuses": {},
+                  "f:hostIP": {},
+                  "f:phase": {},
+                  "f:podIP": {},
+                  "f:podIPs": {
+                    ".": {},
+                    "k:{\"ip\":\"10.244.0.5\"}": {
+                      ".": {},
+                      "f:ip": {}
+                    }
+                  },
+                  "f:startTime": {}
+                }
+              },
+              "subresource": "status"
+            }
+          ]
+        },
+        "spec": {
+          "volumes": [
+            {
+              "name": "kube-api-access-hz77k",
+              "projected": {
+                "sources": [
+                  {
+                    "serviceAccountToken": {
+                      "expirationSeconds": 3607,
+                      "path": "token"
+                    }
+                  },
+                  {
+                    "configMap": {
+                      "name": "kube-root-ca.crt",
+                      "items": [
+                        {
+                          "key": "ca.crt",
+                          "path": "ca.crt"
+                        }
+                      ]
+                    }
+                  },
+                  {
+                    "downwardAPI": {
+                      "items": [
+                        {
+                          "path": "namespace",
+                          "fieldRef": {
+                            "apiVersion": "v1",
+                            "fieldPath": "metadata.namespace"
+                          }
+                        }
+                      ]
+                    }
+                  }
+                ],
+                "defaultMode": 420
+              }
+            }
+          ],
+          "containers": [
+            {
+              "name": "other-pod",
+              "image": "other-pod/server:1.27.0-alpine",
+              "resources": {},
+              "volumeMounts": [
+                {
+                  "name": "kube-api-access-hz77k",
+                  "readOnly": true,
+                  "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+                }
+              ],
+              "terminationMessagePath": "/dev/termination-log",
+              "terminationMessagePolicy": "File",
+              "imagePullPolicy": "IfNotPresent"
+            }
+          ],
+          "restartPolicy": "Always",
+          "terminationGracePeriodSeconds": 30,
+          "dnsPolicy": "ClusterFirst",
+          "serviceAccountName": "default",
+          "serviceAccount": "default",
+          "nodeName": "chart-testing-control-plane",
+          "securityContext": {},
+          "schedulerName": "default-scheduler",
+          "tolerations": [
+            {
+              "key": "node.kubernetes.io/not-ready",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            },
+            {
+              "key": "node.kubernetes.io/unreachable",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            }
+          ],
+          "priority": 0,
+          "enableServiceLinks": true,
+          "preemptionPolicy": "PreemptLowerPriority"
+        },
+        "status": {
+          "phase": "CrashLoopBackOff",
+          "conditions": [
+            {
+              "type": "Initialized",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            },
+            {
+              "type": "Ready",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "ContainersReady",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "PodScheduled",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            }
+          ],
+          "hostIP": "172.18.0.2",
+          "podIP": "10.244.0.5",
+          "podIPs": [
+            {
+              "ip": "10.244.0.5"
+            }
+          ],
+          "startTime": "2023-03-23T16:43:42Z",
+          "containerStatuses": [
+            {
+              "name": "other-pod",
+              "state": {
+                "running": {
+                  "startedAt": "2023-03-23T16:43:45Z"
+                }
+              },
+              "lastState": {},
+              "ready": true,
+              "restartCount": 0,
+              "image": "docker.io/other-pod/server:1.27.0-alpine",
+              "imageID": "docker.io/other-pod/server@sha256:7cd450642c54cc8cce64dd9b4b8664c8480e7df6de880c5e134f4a2ed6446c5c",
+              "containerID": "containerd://58e668ba55950e6d08340f496f5e7f33f3d92cbfcbf366ce5e5f5c50a3926c71",
+              "started": true
+            }
+          ],
+          "qosClass": "BestEffort"
+        }
+      }
+    ]
+  }

pkg/analyze/files/pods/other.json

@@ -0,0 +1,263 @@
+{
+    "kind": "PodList",
+    "apiVersion": "v1",
+    "metadata": {
+      "resourceVersion": "498"
+    },
+    "items": [
+      {
+        "kind": "Pod",
+        "apiVersion": "v1",
+        "metadata": {
+          "name": "other-pod-75b66db9b9-nqhp8",
+          "generateName": "other-pod-75b66db9b9-",
+          "namespace": "other",
+          "uid": "e00af024-67ca-440d-9714-999cf8617d58",
+          "resourceVersion": "488",
+          "creationTimestamp": "2023-03-23T16:43:42Z",
+          "labels": {
+            "app.kubernetes.io/appname": "other-pod",
+            "pod-template-hash": "75b66db9b9"
+          },
+          "ownerReferences": [
+            {
+              "apiVersion": "apps/v1",
+              "kind": "ReplicaSet",
+              "name": "other-pod-75b66db9b9",
+              "uid": "794592d2-539c-4cca-a036-678bd2357c78",
+              "controller": true,
+              "blockOwnerDeletion": true
+            }
+          ],
+          "managedFields": [
+            {
+              "manager": "kube-controller-manager",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:42Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:metadata": {
+                  "f:generateName": {},
+                  "f:labels": {
+                    ".": {},
+                    "f:app.kubernetes.io/appname": {},
+                    "f:pod-template-hash": {}
+                  },
+                  "f:ownerReferences": {
+                    ".": {},
+                    "k:{\"uid\":\"794592d2-539c-4cca-a036-678bd2357c78\"}": {}
+                  }
+                },
+                "f:spec": {
+                  "f:containers": {
+                    "k:{\"name\":\"other-pod\"}": {
+                      ".": {},
+                      "f:image": {},
+                      "f:imagePullPolicy": {},
+                      "f:name": {},
+                      "f:resources": {},
+                      "f:terminationMessagePath": {},
+                      "f:terminationMessagePolicy": {}
+                    }
+                  },
+                  "f:dnsPolicy": {},
+                  "f:enableServiceLinks": {},
+                  "f:restartPolicy": {},
+                  "f:schedulerName": {},
+                  "f:securityContext": {},
+                  "f:terminationGracePeriodSeconds": {}
+                }
+              }
+            },
+            {
+              "manager": "kubelet",
+              "operation": "Update",
+              "apiVersion": "v1",
+              "time": "2023-03-23T16:43:46Z",
+              "fieldsType": "FieldsV1",
+              "fieldsV1": {
+                "f:status": {
+                  "f:conditions": {
+                    "k:{\"type\":\"ContainersReady\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Initialized\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    },
+                    "k:{\"type\":\"Ready\"}": {
+                      ".": {},
+                      "f:lastProbeTime": {},
+                      "f:lastTransitionTime": {},
+                      "f:status": {},
+                      "f:type": {}
+                    }
+                  },
+                  "f:containerStatuses": {},
+                  "f:hostIP": {},
+                  "f:phase": {},
+                  "f:podIP": {},
+                  "f:podIPs": {
+                    ".": {},
+                    "k:{\"ip\":\"10.244.0.5\"}": {
+                      ".": {},
+                      "f:ip": {}
+                    }
+                  },
+                  "f:startTime": {}
+                }
+              },
+              "subresource": "status"
+            }
+          ]
+        },
+        "spec": {
+          "volumes": [
+            {
+              "name": "kube-api-access-hz77k",
+              "projected": {
+                "sources": [
+                  {
+                    "serviceAccountToken": {
+                      "expirationSeconds": 3607,
+                      "path": "token"
+                    }
+                  },
+                  {
+                    "configMap": {
+                      "name": "kube-root-ca.crt",
+                      "items": [
+                        {
+                          "key": "ca.crt",
+                          "path": "ca.crt"
+                        }
+                      ]
+                    }
+                  },
+                  {
+                    "downwardAPI": {
+                      "items": [
+                        {
+                          "path": "namespace",
+                          "fieldRef": {
+                            "apiVersion": "v1",
+                            "fieldPath": "metadata.namespace"
+                          }
+                        }
+                      ]
+                    }
+                  }
+                ],
+                "defaultMode": 420
+              }
+            }
+          ],
+          "containers": [
+            {
+              "name": "other-pod",
+              "image": "other-pod/server:1.27.0-alpine",
+              "resources": {},
+              "volumeMounts": [
+                {
+                  "name": "kube-api-access-hz77k",
+                  "readOnly": true,
+                  "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+                }
+              ],
+              "terminationMessagePath": "/dev/termination-log",
+              "terminationMessagePolicy": "File",
+              "imagePullPolicy": "IfNotPresent"
+            }
+          ],
+          "restartPolicy": "Always",
+          "terminationGracePeriodSeconds": 30,
+          "dnsPolicy": "ClusterFirst",
+          "serviceAccountName": "default",
+          "serviceAccount": "default",
+          "nodeName": "chart-testing-control-plane",
+          "securityContext": {},
+          "schedulerName": "default-scheduler",
+          "tolerations": [
+            {
+              "key": "node.kubernetes.io/not-ready",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            },
+            {
+              "key": "node.kubernetes.io/unreachable",
+              "operator": "Exists",
+              "effect": "NoExecute",
+              "tolerationSeconds": 300
+            }
+          ],
+          "priority": 0,
+          "enableServiceLinks": true,
+          "preemptionPolicy": "PreemptLowerPriority"
+        },
+        "status": {
+          "phase": "Running",
+          "conditions": [
+            {
+              "type": "Initialized",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            },
+            {
+              "type": "Ready",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "ContainersReady",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:46Z"
+            },
+            {
+              "type": "PodScheduled",
+              "status": "True",
+              "lastProbeTime": null,
+              "lastTransitionTime": "2023-03-23T16:43:42Z"
+            }
+          ],
+          "hostIP": "172.18.0.2",
+          "podIP": "10.244.0.5",
+          "podIPs": [
+            {
+              "ip": "10.244.0.5"
+            }
+          ],
+          "startTime": "2023-03-23T16:43:42Z",
+          "containerStatuses": [
+            {
+              "name": "other-pod",
+              "state": {
+                "running": {
+                  "startedAt": "2023-03-23T16:43:45Z"
+                }
+              },
+              "lastState": {},
+              "ready": true,
+              "restartCount": 0,
+              "image": "docker.io/other-pod/server:1.27.0-alpine",
+              "imageID": "docker.io/other-pod/server@sha256:7cd450642c54cc8cce64dd9b4b8664c8480e7df6de880c5e134f4a2ed6446c5c",
+              "containerID": "containerd://58e668ba55950e6d08340f496f5e7f33f3d92cbfcbf366ce5e5f5c50a3926c71",
+              "started": true
+            }
+          ],
+          "qosClass": "BestEffort"
+        }
+      }
+    ]
+  }

pkg/analyze/files/support-bundle/cluster-resources/deployments/default.json

@@ -0,0 +1,459 @@
+{
+  "kind": "DeploymentList",
+  "apiVersion": "apps/v1",
+  "metadata": {
+    "resourceVersion": "27195651"
+  },
+  "items": [
+    {
+      "metadata": {
+        "name": "kotsadm-api",
+        "namespace": "default",
+        "selfLink": "/apis/apps/v1/namespaces/default/deployments/kotsadm-api",
+        "uid": "56526035-cd29-4d08-8375-291503b1a006",
+        "resourceVersion": "1583068",
+        "generation": 1,
+        "creationTimestamp": "2019-11-07T00:34:32Z",
+        "labels": {
+          "app.kubernetes.io/managed-by": "skaffold-v0.41.0"
+        },
+        "annotations": {
+          "deployment.kubernetes.io/revision": "1",
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"},\"name\":\"kotsadm-api\",\"namespace\":\"default\"},\"spec\":{\"selector\":{\"matchLabels\":{\"app\":\"kotsadm-api\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"kotsadm-api\",\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"}},\"spec\":{\"affinity\":{\"podAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"podAffinityTerm\":{\"labelSelector\":{\"matchExpressions\":[{\"key\":\"app\",\"operator\":\"In\",\"values\":[\"ship-www\"]}]},\"topologyKey\":\"kubernetes.io/hostname\"},\"weight\":1}]},\"podAntiAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"podAffinityTerm\":{\"labelSelector\":{\"matchExpressions\":[{\"key\":\"app\",\"operator\":\"In\",\"values\":[\"kotsadm-api\"]}]},\"topologyKey\":\"kubernetes.io/hostname\"},\"weight\":2}]}},\"containers\":[{\"env\":[{\"name\":\"DEV_NAMESPACE\",\"value\":\"test\"},{\"name\":\"LOG_LEVEL\",\"value\":\"debug\"},{\"name\":\"SESSION_KEY\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"key\",\"name\":\"session\"}}},{\"name\":\"POSTGRES_URI\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"uri\",\"name\":\"ship-postgres\"}}},{\"name\":\"API_ENCRYPTION_KEY\",\"value\":\"IvWItkB8+ezMisPjSMBknT1PdKjBx7Xc/txZqOP8Y2Oe7+Jy\"},{\"name\":\"INIT_SERVER_URI\",\"value\":\"http://init-server:3000\"},{\"name\":\"WATCH_SERVER_URI\",\"value\":\"http://watch-server:3000\"},{\"name\":\"PINO_LOG_PRETTY\",\"value\":\"1\"},{\"name\":\"S3_BUCKET_NAME\",\"value\":\"shipbucket\"},{\"name\":\"AIRGAP_BUNDLE_S3_BUCKET\",\"value\":\"airgap\"},{\"name\":\"S3_ENDPOINT\",\"value\":\"http://kotsadm-s3.default.svc.cluster.local:4569/\"},{\"name\":\"S3_ACCESS_KEY_ID\",\"value\":\"***HIDDEN***\"},{\"name\":\"S3_SECRET_ACCESS_KEY\",\"value\":\"***HIDDEN***\"},{\"name\":\"S3_BUCKET_ENDPOINT\",\"value\":\"true\"},{\"name\":\"GITHUB_CLIENT_ID\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"client-id\",\"name\":\"github-app\"}}},{\"name\":\"GITHUB_CLIENT_SECRET\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"client-secret\",\"name\":\"github-app\"}}},{\"name\":\"GITHUB_INTEGRATION_ID\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"integration-id\",\"name\":\"github-app\"}}},{\"name\":\"GITHUB_PRIVATE_KEY_FILE\",\"value\":\"/keys/github/private-key.pem\"},{\"name\":\"SHIP_API_ENDPOINT\",\"value\":\"http://kotsadm-api.default.svc.cluster.local:3000\"},{\"name\":\"SHIP_API_ADVERTISE_ENDPOINT\",\"value\":\"http://localhost:30065\"},{\"name\":\"GRAPHQL_PREM_ENDPOINT\",\"value\":\"http://graphql-api-prem:3000/graphql\"},{\"name\":\"AUTO_CREATE_CLUSTER\",\"value\":\"1\"},{\"name\":\"AUTO_CREATE_CLUSTER_NAME\",\"value\":\"microk8s\"},{\"name\":\"AUTO_CREATE_CLUSTER_TOKEN\",\"value\":\"***HIDDEN***\"},{\"name\":\"ENABLE_SHIP\",\"value\":\"1\"},{\"name\":\"ENABLE_KOTS\",\"value\":\"1\"},{\"name\":\"ENABLE_KURL\",\"value\":\"1\"},{\"name\":\"POD_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.namespace\"}}}],\"image\":\"localhost:32000/kotsadm-api:v1.0.1-30-g8fa13e34-dirty@sha256:4a0ca1a2eae46472bd2d454f9e763a458ddd172689e179b17054262507bb4fc8\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"kotsadm-api\",\"ports\":[{\"containerPort\":3000,\"name\":\"http\"},{\"containerPort\":9229,\"name\":\"debug\"}],\"readinessProbe\":{\"httpGet\":{\"path\":\"/healthz\",\"port\":3000},\"initialDelaySeconds\":2,\"periodSeconds\":2},\"volumeMounts\":[{\"mountPath\":\"/keys/github\",\"name\":\"github-app-private-key\",\"readOnly\":true}]}],\"restartPolicy\":\"Always\",\"securityContext\":{\"runAsUser\":0},\"serviceAccount\":\"kotsadm-api\",\"volumes\":[{\"name\":\"github-app-private-key\",\"secret\":{\"secretName\":\"github-app-private-key\"}}]}}}}\n"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "selector": {
+          "matchLabels": {
+            "app": "kotsadm-api"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "app": "kotsadm-api"
+            }
+          },
+          "spec": {
+            "containers": [
+              {
+                "name": "kotsadm-api",
+                "image": "localhost:32000/kotsadm-api:v1.0.1-30-g8fa13e34-dirty@sha256:4a0ca1a2eae46472bd2d454f9e763a458ddd172689e179b17054262507bb4fc8",
+                "ports": [
+                  {
+                    "name": "http",
+                    "containerPort": 3000,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "DEV_NAMESPACE",
+                    "value": "test"
+                  },
+                  {
+                    "name": "POD_NAMESPACE",
+                    "valueFrom": {
+                      "fieldRef": {
+                        "apiVersion": "v1",
+                        "fieldPath": "metadata.namespace"
+                      }
+                    }
+                  }
+                ],
+                "resources": {},
+                "readinessProbe": {
+                  "httpGet": {
+                    "path": "/healthz",
+                    "port": 3000,
+                    "scheme": "HTTP"
+                  },
+                  "initialDelaySeconds": 2,
+                  "timeoutSeconds": 1,
+                  "periodSeconds": 2,
+                  "successThreshold": 1,
+                  "failureThreshold": 3
+                },
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent"
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "serviceAccountName": "kotsadm-api",
+            "serviceAccount": "kotsadm-api",
+            "schedulerName": "default-scheduler"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": "25%",
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 1,
+        "replicas": 1,
+        "updatedReplicas": 1,
+        "readyReplicas": 1,
+        "availableReplicas": 1,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:35:23Z",
+            "lastTransitionTime": "2019-11-07T00:35:23Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:35:23Z",
+            "lastTransitionTime": "2019-11-07T00:34:32Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"kotsadm-api-6f4b994bd5\" has successfully progressed."
+          }
+        ]
+      }
+    },
+    {
+      "metadata": {
+        "name": "kotsadm-operator",
+        "namespace": "default",
+        "selfLink": "/apis/apps/v1/namespaces/default/deployments/kotsadm-operator",
+        "uid": "cfae9877-eef4-44c9-acac-0bf0d1aa547e",
+        "resourceVersion": "1583379",
+        "generation": 2,
+        "creationTimestamp": "2019-11-07T00:34:32Z",
+        "labels": {
+          "app.kubernetes.io/managed-by": "skaffold-v0.41.0"
+        },
+        "annotations": {
+          "deployment.kubernetes.io/revision": "2",
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"},\"name\":\"kotsadm-operator\",\"namespace\":\"default\"},\"spec\":{\"selector\":{\"matchLabels\":{\"app\":\"kotsadm-operator\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"kotsadm-operator\",\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"}},\"spec\":{\"containers\":[{\"env\":[{\"name\":\"KOTSADM_API_ENDPOINT\",\"value\":\"http://kotsadm-api:3000\"},{\"name\":\"KOTSADM_TOKEN\",\"value\":\"***HIDDEN***\"},{\"name\":\"KOTSADM_TARGET_NAMESPACE\",\"value\":\"test\"}],\"image\":\"localhost:32000/kotsadm-operator:v1.0.1-30-g8fa13e34-dirty@sha256:177c15b6399717048e9355bc8fd8b8ed213be90615c7c6ee7b7fdcee50aca6c5\",\"imagePullPolicy\":\"Always\",\"name\":\"kotsadm-operator\",\"resources\":{\"limits\":{\"cpu\":\"200m\",\"memory\":\"1000Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"500Mi\"}}}],\"restartPolicy\":\"Always\"}}}}\n"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "selector": {
+          "matchLabels": {
+            "app": "kotsadm-operator"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "app": "kotsadm-operator"
+            }
+          },
+          "spec": {
+            "containers": [
+              {
+                "name": "kotsadm-operator",
+                "image": "localhost:32000/kotsadm-operator:v1.0.1-30-g8fa13e34-dirty@sha256:177c15b6399717048e9355bc8fd8b8ed213be90615c7c6ee7b7fdcee50aca6c5",
+                "env": [
+                  {
+                    "name": "KOTSADM_API_ENDPOINT",
+                    "value": "http://kotsadm-api:3000"
+                  },
+                  {
+                    "name": "KOTSADM_TOKEN",
+                    "value": "***HIDDEN***"
+                  },
+                  {
+                    "name": "KOTSADM_TARGET_NAMESPACE",
+                    "value": "test"
+                  }
+                ],
+                "resources": {
+                  "limits": {
+                    "cpu": "200m",
+                    "memory": "1000Mi"
+                  },
+                  "requests": {
+                    "cpu": "100m",
+                    "memory": "500Mi"
+                  }
+                },
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "Always"
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "securityContext": {},
+            "schedulerName": "default-scheduler"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": "25%",
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 2,
+        "replicas": 1,
+        "updatedReplicas": 1,
+        "readyReplicas": 1,
+        "availableReplicas": 1,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:34:37Z",
+            "lastTransitionTime": "2019-11-07T00:34:37Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:36:34Z",
+            "lastTransitionTime": "2019-11-07T00:34:32Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"kotsadm-operator-5b5c977699\" has successfully progressed."
+          }
+        ]
+      }
+    },
+    {
+      "metadata": {
+        "name": "kotsadm-postgres-watch",
+        "namespace": "default",
+        "selfLink": "/apis/apps/v1/namespaces/default/deployments/kotsadm-postgres-watch",
+        "uid": "ec195b07-4fd2-4bbe-8b01-4cbdfbe0e79d",
+        "resourceVersion": "1582762",
+        "generation": 1,
+        "creationTimestamp": "2019-11-07T00:34:39Z",
+        "annotations": {
+          "deployment.kubernetes.io/revision": "1"
+        },
+        "ownerReferences": [
+          {
+            "apiVersion": "databases.schemahero.io/v1alpha2",
+            "kind": "Database",
+            "name": "kotsadm-postgres",
+            "uid": "e6d51ce6-c4c1-428f-9e8e-56f1a3a43588",
+            "controller": true,
+            "blockOwnerDeletion": true
+          }
+        ]
+      },
+      "spec": {
+        "replicas": 1,
+        "selector": {
+          "matchLabels": {
+            "deployment": "kotsadm-postgreswatch"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "deployment": "kotsadm-postgreswatch"
+            }
+          },
+          "spec": {
+            "containers": [
+              {
+                "name": "schemahero",
+                "image": "schemahero/schemahero:alpha",
+                "args": [
+                  "watch",
+                  "--driver",
+                  "postgres",
+                  "--uri",
+                  "postgres://shipcloud:password@postgres.default.svc.cluster.local:5432/shipcloud?sslmode=disable",
+                  "--namespace",
+                  "default",
+                  "--instance",
+                  "kotsadm-postgres"
+                ],
+                "resources": {},
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "Always"
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "serviceAccountName": "kotsadm-postgres",
+            "serviceAccount": "kotsadm-postgres",
+            "securityContext": {},
+            "schedulerName": "default-scheduler"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": "25%",
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 1,
+        "replicas": 1,
+        "updatedReplicas": 1,
+        "readyReplicas": 1,
+        "availableReplicas": 1,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:35:00Z",
+            "lastTransitionTime": "2019-11-07T00:35:00Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:35:00Z",
+            "lastTransitionTime": "2019-11-07T00:34:39Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"kotsadm-postgres-watch-5cf76f4c45\" has successfully progressed."
+          }
+        ]
+      }
+    },
+    {
+      "metadata": {
+        "name": "kotsadm-web",
+        "namespace": "default",
+        "selfLink": "/apis/apps/v1/namespaces/default/deployments/kotsadm-web",
+        "uid": "4e657f8a-5edb-498b-9402-1f93f51f5dda",
+        "resourceVersion": "1582354",
+        "generation": 1,
+        "creationTimestamp": "2019-11-07T00:34:32Z",
+        "labels": {
+          "app.kubernetes.io/managed-by": "skaffold-v0.41.0",
+          "skaffold.dev/builder": "local",
+          "skaffold.dev/cleanup": "true",
+          "skaffold.dev/deployer": "kustomize",
+          "skaffold.dev/docker-api-version": "1.40",
+          "skaffold.dev/run-id": "98f0a02b-9739-4d94-ba11-3e4d273c743e",
+          "skaffold.dev/tag-policy": "git-commit",
+          "skaffold.dev/tail": "true"
+        },
+        "annotations": {
+          "deployment.kubernetes.io/revision": "1",
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"},\"name\":\"kotsadm-web\",\"namespace\":\"default\"},\"spec\":{\"selector\":{\"matchLabels\":{\"app\":\"kotsadm-web\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"kotsadm-web\",\"app.kubernetes.io/managed-by\":\"skaffold-v0.41.0\",\"skaffold.dev/builder\":\"local\",\"skaffold.dev/cleanup\":\"true\",\"skaffold.dev/deployer\":\"kustomize\",\"skaffold.dev/docker-api-version\":\"1.40\",\"skaffold.dev/run-id\":\"98f0a02b-9739-4d94-ba11-3e4d273c743e\",\"skaffold.dev/tag-policy\":\"git-commit\",\"skaffold.dev/tail\":\"true\"}},\"spec\":{\"containers\":[{\"env\":[{\"name\":\"GITHUB_CLIENT_ID\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"client-id\",\"name\":\"github-app\"}}},{\"name\":\"GITHUB_INSTALL_URL\",\"valueFrom\":{\"secretKeyRef\":{\"key\":\"install-url\",\"name\":\"github-app\"}}},{\"name\":\"SHIP_CLUSTER_API_SERVER\",\"value\":\"http://localhost:30065\"},{\"name\":\"SHIP_CLUSTER_WEB_URI\",\"value\":\"http://localhost:8000\"}],\"image\":\"localhost:32000/kotsadm-web:v1.0.1-30-g8fa13e34@sha256:5b5b5b640b6e09d8b3185d4ae15ac4dc558d4e2ea034ac3e567d8cce04eadb9c\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"kotsadm-web\",\"ports\":[{\"containerPort\":8000,\"name\":\"http\"}]}]}}}}\n"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "selector": {
+          "matchLabels": {
+            "app": "kotsadm-web"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "app": "kotsadm-web",
+              "app.kubernetes.io/managed-by": "skaffold-v0.41.0",
+              "skaffold.dev/builder": "local",
+              "skaffold.dev/cleanup": "true",
+              "skaffold.dev/deployer": "kustomize",
+              "skaffold.dev/docker-api-version": "1.40",
+              "skaffold.dev/run-id": "98f0a02b-9739-4d94-ba11-3e4d273c743e",
+              "skaffold.dev/tag-policy": "git-commit",
+              "skaffold.dev/tail": "true"
+            }
+          },
+          "spec": {
+            "containers": [
+              {
+                "name": "kotsadm-web",
+                "image": "localhost:32000/kotsadm-web:v1.0.1-30-g8fa13e34@sha256:5b5b5b640b6e09d8b3185d4ae15ac4dc558d4e2ea034ac3e567d8cce04eadb9c",
+                "ports": [
+                  {
+                    "name": "http",
+                    "containerPort": 8000,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "GITHUB_CLIENT_ID",
+                    "valueFrom": {
+                      "secretKeyRef": {
+                        "name": "github-app",
+                        "key": "client-id"
+                      }
+                    }
+                  }
+                ],
+                "resources": {},
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent"
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "securityContext": {},
+            "schedulerName": "default-scheduler"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": "25%",
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 1,
+        "replicas": 2,
+        "updatedReplicas": 1,
+        "readyReplicas": 1,
+        "availableReplicas": 1,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:34:39Z",
+            "lastTransitionTime": "2019-11-07T00:34:39Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2019-11-07T00:34:39Z",
+            "lastTransitionTime": "2019-11-07T00:34:32Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"kotsadm-web-79bfb95c48\" has successfully progressed."
+          }
+        ]
+      }
+    }
+  ]
+}

pkg/analyze/files/support-bundle/cluster-resources/deployments/kube-system.json

@@ -0,0 +1,667 @@
+{
+  "kind": "DeploymentList",
+  "apiVersion": "apps/v1",
+  "metadata": {
+    "resourceVersion": "27195652"
+  },
+  "items": [
+    {
+      "metadata": {
+        "name": "coredns",
+        "namespace": "kube-system",
+        "selfLink": "/apis/apps/v1/namespaces/kube-system/deployments/coredns",
+        "uid": "4e008c0d-39fd-4c54-acb7-45ad67c91ef4",
+        "resourceVersion": "734",
+        "generation": 1,
+        "creationTimestamp": "2021-12-17T18:38:57Z",
+        "labels": {
+          "k8s-app": "kube-dns"
+        },
+        "annotations": {
+          "deployment.kubernetes.io/revision": "1"
+        },
+        "managedFields": [
+          {
+            "manager": "kubeadm",
+            "operation": "Update",
+            "apiVersion": "apps/v1",
+            "time": "2021-12-17T18:38:57Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:labels": {
+                  ".": {},
+                  "f:k8s-app": {}
+                }
+              },
+              "f:spec": {
+                "f:progressDeadlineSeconds": {},
+                "f:replicas": {},
+                "f:revisionHistoryLimit": {},
+                "f:selector": {
+                  "f:matchLabels": {
+                    ".": {},
+                    "f:k8s-app": {}
+                  }
+                },
+                "f:strategy": {
+                  "f:rollingUpdate": {
+                    ".": {},
+                    "f:maxSurge": {},
+                    "f:maxUnavailable": {}
+                  },
+                  "f:type": {}
+                },
+                "f:template": {
+                  "f:metadata": {
+                    "f:labels": {
+                      ".": {},
+                      "f:k8s-app": {}
+                    }
+                  },
+                  "f:spec": {
+                    "f:containers": {
+                      "k:{\"name\":\"coredns\"}": {
+                        ".": {},
+                        "f:args": {},
+                        "f:image": {},
+                        "f:imagePullPolicy": {},
+                        "f:livenessProbe": {
+                          ".": {},
+                          "f:failureThreshold": {},
+                          "f:httpGet": {
+                            ".": {},
+                            "f:path": {},
+                            "f:port": {},
+                            "f:scheme": {}
+                          },
+                          "f:initialDelaySeconds": {},
+                          "f:periodSeconds": {},
+                          "f:successThreshold": {},
+                          "f:timeoutSeconds": {}
+                        },
+                        "f:name": {},
+                        "f:ports": {
+                          ".": {},
+                          "k:{\"containerPort\":53,\"protocol\":\"TCP\"}": {
+                            ".": {},
+                            "f:containerPort": {},
+                            "f:name": {},
+                            "f:protocol": {}
+                          },
+                          "k:{\"containerPort\":53,\"protocol\":\"UDP\"}": {
+                            ".": {},
+                            "f:containerPort": {},
+                            "f:name": {},
+                            "f:protocol": {}
+                          },
+                          "k:{\"containerPort\":9153,\"protocol\":\"TCP\"}": {
+                            ".": {},
+                            "f:containerPort": {},
+                            "f:name": {},
+                            "f:protocol": {}
+                          }
+                        },
+                        "f:readinessProbe": {
+                          ".": {},
+                          "f:failureThreshold": {},
+                          "f:httpGet": {
+                            ".": {},
+                            "f:path": {},
+                            "f:port": {},
+                            "f:scheme": {}
+                          },
+                          "f:periodSeconds": {},
+                          "f:successThreshold": {},
+                          "f:timeoutSeconds": {}
+                        },
+                        "f:resources": {
+                          ".": {},
+                          "f:limits": {
+                            ".": {},
+                            "f:memory": {}
+                          },
+                          "f:requests": {
+                            ".": {},
+                            "f:cpu": {},
+                            "f:memory": {}
+                          }
+                        },
+                        "f:securityContext": {
+                          ".": {},
+                          "f:allowPrivilegeEscalation": {},
+                          "f:capabilities": {
+                            ".": {},
+                            "f:add": {},
+                            "f:drop": {}
+                          },
+                          "f:readOnlyRootFilesystem": {}
+                        },
+                        "f:terminationMessagePath": {},
+                        "f:terminationMessagePolicy": {},
+                        "f:volumeMounts": {
+                          ".": {},
+                          "k:{\"mountPath\":\"/etc/coredns\"}": {
+                            ".": {},
+                            "f:mountPath": {},
+                            "f:name": {},
+                            "f:readOnly": {}
+                          }
+                        }
+                      }
+                    },
+                    "f:dnsPolicy": {},
+                    "f:nodeSelector": {
+                      ".": {},
+                      "f:kubernetes.io/os": {}
+                    },
+                    "f:priorityClassName": {},
+                    "f:restartPolicy": {},
+                    "f:schedulerName": {},
+                    "f:securityContext": {},
+                    "f:serviceAccount": {},
+                    "f:serviceAccountName": {},
+                    "f:terminationGracePeriodSeconds": {},
+                    "f:tolerations": {},
+                    "f:volumes": {
+                      ".": {},
+                      "k:{\"name\":\"config-volume\"}": {
+                        ".": {},
+                        "f:configMap": {
+                          ".": {},
+                          "f:defaultMode": {},
+                          "f:items": {},
+                          "f:name": {}
+                        },
+                        "f:name": {}
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          {
+            "manager": "kube-controller-manager",
+            "operation": "Update",
+            "apiVersion": "apps/v1",
+            "time": "2021-12-17T18:39:42Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  ".": {},
+                  "f:deployment.kubernetes.io/revision": {}
+                }
+              },
+              "f:status": {
+                "f:availableReplicas": {},
+                "f:conditions": {
+                  ".": {},
+                  "k:{\"type\":\"Available\"}": {
+                    ".": {},
+                    "f:lastTransitionTime": {},
+                    "f:lastUpdateTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {},
+                    "f:type": {}
+                  },
+                  "k:{\"type\":\"Progressing\"}": {
+                    ".": {},
+                    "f:lastTransitionTime": {},
+                    "f:lastUpdateTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {},
+                    "f:type": {}
+                  }
+                },
+                "f:observedGeneration": {},
+                "f:readyReplicas": {},
+                "f:replicas": {},
+                "f:updatedReplicas": {}
+              }
+            }
+          }
+        ]
+      },
+      "spec": {
+        "replicas": 2,
+        "selector": {
+          "matchLabels": {
+            "k8s-app": "kube-dns"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "k8s-app": "kube-dns"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "config-volume",
+                "configMap": {
+                  "name": "coredns",
+                  "items": [
+                    {
+                      "key": "Corefile",
+                      "path": "Corefile"
+                    }
+                  ],
+                  "defaultMode": 420
+                }
+              }
+            ],
+            "containers": [
+              {
+                "name": "coredns",
+                "image": "k8s.gcr.io/coredns:1.7.0",
+                "args": [
+                  "-conf",
+                  "/etc/coredns/Corefile"
+                ],
+                "ports": [
+                  {
+                    "name": "dns",
+                    "containerPort": 53,
+                    "protocol": "UDP"
+                  },
+                  {
+                    "name": "dns-tcp",
+                    "containerPort": 53,
+                    "protocol": "TCP"
+                  },
+                  {
+                    "name": "metrics",
+                    "containerPort": 9153,
+                    "protocol": "TCP"
+                  }
+                ],
+                "resources": {
+                  "limits": {
+                    "memory": "170Mi"
+                  },
+                  "requests": {
+                    "cpu": "100m",
+                    "memory": "70Mi"
+                  }
+                },
+                "volumeMounts": [
+                  {
+                    "name": "config-volume",
+                    "readOnly": true,
+                    "mountPath": "/etc/coredns"
+                  }
+                ],
+                "livenessProbe": {
+                  "httpGet": {
+                    "path": "/health",
+                    "port": 8080,
+                    "scheme": "HTTP"
+                  },
+                  "initialDelaySeconds": 60,
+                  "timeoutSeconds": 5,
+                  "periodSeconds": 10,
+                  "successThreshold": 1,
+                  "failureThreshold": 5
+                },
+                "readinessProbe": {
+                  "httpGet": {
+                    "path": "/ready",
+                    "port": 8181,
+                    "scheme": "HTTP"
+                  },
+                  "timeoutSeconds": 1,
+                  "periodSeconds": 10,
+                  "successThreshold": 1,
+                  "failureThreshold": 3
+                },
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent",
+                "securityContext": {
+                  "capabilities": {
+                    "add": [
+                      "NET_BIND_SERVICE"
+                    ],
+                    "drop": [
+                      "all"
+                    ]
+                  },
+                  "readOnlyRootFilesystem": true,
+                  "allowPrivilegeEscalation": false
+                }
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "Default",
+            "nodeSelector": {
+              "kubernetes.io/os": "linux"
+            },
+            "serviceAccountName": "coredns",
+            "serviceAccount": "coredns",
+            "securityContext": {},
+            "schedulerName": "default-scheduler",
+            "tolerations": [
+              {
+                "key": "CriticalAddonsOnly",
+                "operator": "Exists"
+              },
+              {
+                "key": "node-role.kubernetes.io/master",
+                "effect": "NoSchedule"
+              }
+            ],
+            "priorityClassName": "system-cluster-critical"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": 1,
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 1,
+        "replicas": 2,
+        "updatedReplicas": 2,
+        "readyReplicas": 2,
+        "availableReplicas": 2,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2021-12-17T18:39:25Z",
+            "lastTransitionTime": "2021-12-17T18:39:25Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2021-12-17T18:39:42Z",
+            "lastTransitionTime": "2021-12-17T18:39:05Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"coredns-f9fd979d6\" has successfully progressed."
+          }
+        ]
+      }
+    },
+    {
+      "metadata": {
+        "name": "metrics-server",
+        "namespace": "kube-system",
+        "selfLink": "/apis/apps/v1/namespaces/kube-system/deployments/metrics-server",
+        "uid": "2247dee8-5d1d-4e05-a6e8-a50f15db140b",
+        "resourceVersion": "2998",
+        "generation": 1,
+        "creationTimestamp": "2021-12-17T18:42:38Z",
+        "labels": {
+          "k8s-app": "metrics-server"
+        },
+        "annotations": {
+          "deployment.kubernetes.io/revision": "1",
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"k8s-app\":\"metrics-server\"},\"name\":\"metrics-server\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"metrics-server\"}},\"template\":{\"metadata\":{\"labels\":{\"k8s-app\":\"metrics-server\"},\"name\":\"metrics-server\"},\"spec\":{\"containers\":[{\"args\":[\"--cert-dir=/tmp\",\"--secure-port=4443\",\"--kubelet-insecure-tls\"],\"image\":\"k8s.gcr.io/metrics-server/metrics-server:v0.3.7\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"metrics-server\",\"ports\":[{\"containerPort\":4443,\"name\":\"main-port\",\"protocol\":\"TCP\"}],\"securityContext\":{\"readOnlyRootFilesystem\":true,\"runAsNonRoot\":true,\"runAsUser\":1000},\"volumeMounts\":[{\"mountPath\":\"/tmp\",\"name\":\"tmp-dir\"}]}],\"nodeSelector\":{\"kubernetes.io/os\":\"linux\"},\"serviceAccountName\":\"metrics-server\",\"volumes\":[{\"emptyDir\":{},\"name\":\"tmp-dir\"}]}}}}\n"
+        },
+        "managedFields": [
+          {
+            "manager": "kubectl-client-side-apply",
+            "operation": "Update",
+            "apiVersion": "apps/v1",
+            "time": "2021-12-17T18:42:38Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  ".": {},
+                  "f:kubectl.kubernetes.io/last-applied-configuration": {}
+                },
+                "f:labels": {
+                  ".": {},
+                  "f:k8s-app": {}
+                }
+              },
+              "f:spec": {
+                "f:progressDeadlineSeconds": {},
+                "f:replicas": {},
+                "f:revisionHistoryLimit": {},
+                "f:selector": {
+                  "f:matchLabels": {
+                    ".": {},
+                    "f:k8s-app": {}
+                  }
+                },
+                "f:strategy": {
+                  "f:rollingUpdate": {
+                    ".": {},
+                    "f:maxSurge": {},
+                    "f:maxUnavailable": {}
+                  },
+                  "f:type": {}
+                },
+                "f:template": {
+                  "f:metadata": {
+                    "f:labels": {
+                      ".": {},
+                      "f:k8s-app": {}
+                    },
+                    "f:name": {}
+                  },
+                  "f:spec": {
+                    "f:containers": {
+                      "k:{\"name\":\"metrics-server\"}": {
+                        ".": {},
+                        "f:args": {},
+                        "f:image": {},
+                        "f:imagePullPolicy": {},
+                        "f:name": {},
+                        "f:ports": {
+                          ".": {},
+                          "k:{\"containerPort\":4443,\"protocol\":\"TCP\"}": {
+                            ".": {},
+                            "f:containerPort": {},
+                            "f:name": {},
+                            "f:protocol": {}
+                          }
+                        },
+                        "f:resources": {},
+                        "f:securityContext": {
+                          ".": {},
+                          "f:readOnlyRootFilesystem": {},
+                          "f:runAsNonRoot": {},
+                          "f:runAsUser": {}
+                        },
+                        "f:terminationMessagePath": {},
+                        "f:terminationMessagePolicy": {},
+                        "f:volumeMounts": {
+                          ".": {},
+                          "k:{\"mountPath\":\"/tmp\"}": {
+                            ".": {},
+                            "f:mountPath": {},
+                            "f:name": {}
+                          }
+                        }
+                      }
+                    },
+                    "f:dnsPolicy": {},
+                    "f:nodeSelector": {
+                      ".": {},
+                      "f:kubernetes.io/os": {}
+                    },
+                    "f:restartPolicy": {},
+                    "f:schedulerName": {},
+                    "f:securityContext": {},
+                    "f:serviceAccount": {},
+                    "f:serviceAccountName": {},
+                    "f:terminationGracePeriodSeconds": {},
+                    "f:volumes": {
+                      ".": {},
+                      "k:{\"name\":\"tmp-dir\"}": {
+                        ".": {},
+                        "f:emptyDir": {},
+                        "f:name": {}
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          {
+            "manager": "kube-controller-manager",
+            "operation": "Update",
+            "apiVersion": "apps/v1",
+            "time": "2021-12-17T18:42:47Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  "f:deployment.kubernetes.io/revision": {}
+                }
+              },
+              "f:status": {
+                "f:availableReplicas": {},
+                "f:conditions": {
+                  ".": {},
+                  "k:{\"type\":\"Available\"}": {
+                    ".": {},
+                    "f:lastTransitionTime": {},
+                    "f:lastUpdateTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {},
+                    "f:type": {}
+                  },
+                  "k:{\"type\":\"Progressing\"}": {
+                    ".": {},
+                    "f:lastTransitionTime": {},
+                    "f:lastUpdateTime": {},
+                    "f:message": {},
+                    "f:reason": {},
+                    "f:status": {},
+                    "f:type": {}
+                  }
+                },
+                "f:observedGeneration": {},
+                "f:readyReplicas": {},
+                "f:replicas": {},
+                "f:updatedReplicas": {}
+              }
+            }
+          }
+        ]
+      },
+      "spec": {
+        "replicas": 1,
+        "selector": {
+          "matchLabels": {
+            "k8s-app": "metrics-server"
+          }
+        },
+        "template": {
+          "metadata": {
+            "name": "metrics-server",
+            "creationTimestamp": null,
+            "labels": {
+              "k8s-app": "metrics-server"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "tmp-dir",
+                "emptyDir": {}
+              }
+            ],
+            "containers": [
+              {
+                "name": "metrics-server",
+                "image": "k8s.gcr.io/metrics-server/metrics-server:v0.3.7",
+                "args": [
+                  "--cert-dir=/tmp",
+                  "--secure-port=4443",
+                  "--kubelet-insecure-tls"
+                ],
+                "ports": [
+                  {
+                    "name": "main-port",
+                    "containerPort": 4443,
+                    "protocol": "TCP"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "tmp-dir",
+                    "mountPath": "/tmp"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent",
+                "securityContext": {
+                  "runAsUser": 1000,
+                  "runAsNonRoot": true,
+                  "readOnlyRootFilesystem": true
+                }
+              }
+            ],
+            "restartPolicy": "Always",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "nodeSelector": {
+              "kubernetes.io/os": "linux"
+            },
+            "serviceAccountName": "metrics-server",
+            "serviceAccount": "metrics-server",
+            "securityContext": {},
+            "schedulerName": "default-scheduler"
+          }
+        },
+        "strategy": {
+          "type": "RollingUpdate",
+          "rollingUpdate": {
+            "maxUnavailable": "25%",
+            "maxSurge": "25%"
+          }
+        },
+        "revisionHistoryLimit": 10,
+        "progressDeadlineSeconds": 600
+      },
+      "status": {
+        "observedGeneration": 1,
+        "replicas": 2,
+        "updatedReplicas": 1,
+        "readyReplicas": 1,
+        "availableReplicas": 1,
+        "conditions": [
+          {
+            "type": "Available",
+            "status": "True",
+            "lastUpdateTime": "2021-12-17T18:42:47Z",
+            "lastTransitionTime": "2021-12-17T18:42:47Z",
+            "reason": "MinimumReplicasAvailable",
+            "message": "Deployment has minimum availability."
+          },
+          {
+            "type": "Progressing",
+            "status": "True",
+            "lastUpdateTime": "2021-12-17T18:42:47Z",
+            "lastTransitionTime": "2021-12-17T18:42:38Z",
+            "reason": "NewReplicaSetAvailable",
+            "message": "ReplicaSet \"metrics-server-56df6d849\" has successfully progressed."
+          }
+        ]
+      }
+    }
+  ]
+}

pkg/analyze/files/support-bundle/cluster-resources/jobs/projectcontour.json

@@ -0,0 +1,204 @@
+{
+  "kind": "JobList",
+  "apiVersion": "batch/v1",
+  "metadata": {
+    "resourceVersion": "27195661"
+  },
+  "items": [
+    {
+      "metadata": {
+        "name": "contour-certgen-v1.19.1",
+        "namespace": "projectcontour",
+        "selfLink": "/apis/batch/v1/namespaces/projectcontour/jobs/contour-certgen-v1.19.1",
+        "uid": "416a2a9d-b862-47b8-93d9-160775641fbe",
+        "resourceVersion": "1386",
+        "creationTimestamp": "2021-12-17T18:41:14Z",
+        "labels": {
+          "app": "contour-certgen",
+          "controller-uid": "416a2a9d-b862-47b8-93d9-160775641fbe",
+          "job-name": "contour-certgen-v1.19.1"
+        },
+        "annotations": {
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"batch/v1\",\"kind\":\"Job\",\"metadata\":{\"annotations\":{},\"name\":\"contour-certgen-v1.19.1\",\"namespace\":\"projectcontour\"},\"spec\":{\"backoffLimit\":1,\"completions\":1,\"parallelism\":1,\"template\":{\"metadata\":{\"labels\":{\"app\":\"contour-certgen\"}},\"spec\":{\"containers\":[{\"command\":[\"contour\",\"certgen\",\"--kube\",\"--incluster\",\"--overwrite\",\"--secrets-format=compact\",\"--namespace=$(CONTOUR_NAMESPACE)\"],\"env\":[{\"name\":\"CONTOUR_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.namespace\"}}}],\"image\":\"ghcr.io/projectcontour/contour:v1.19.1\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"contour\"}],\"restartPolicy\":\"Never\",\"securityContext\":{\"runAsGroup\":65534,\"runAsNonRoot\":true,\"runAsUser\":65534},\"serviceAccountName\":\"contour-certgen\"}},\"ttlSecondsAfterFinished\":0}}\n"
+        },
+        "managedFields": [
+          {
+            "manager": "kubectl-client-side-apply",
+            "operation": "Update",
+            "apiVersion": "batch/v1",
+            "time": "2021-12-17T18:41:14Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:metadata": {
+                "f:annotations": {
+                  ".": {},
+                  "f:kubectl.kubernetes.io/last-applied-configuration": {}
+                },
+                "f:labels": {
+                  ".": {},
+                  "f:app": {}
+                }
+              },
+              "f:spec": {
+                "f:backoffLimit": {},
+                "f:completions": {},
+                "f:parallelism": {},
+                "f:template": {
+                  "f:metadata": {
+                    "f:labels": {
+                      ".": {},
+                      "f:app": {}
+                    }
+                  },
+                  "f:spec": {
+                    "f:containers": {
+                      "k:{\"name\":\"contour\"}": {
+                        ".": {},
+                        "f:command": {},
+                        "f:env": {
+                          ".": {},
+                          "k:{\"name\":\"CONTOUR_NAMESPACE\"}": {
+                            ".": {},
+                            "f:name": {},
+                            "f:valueFrom": {
+                              ".": {},
+                              "f:fieldRef": {
+                                ".": {},
+                                "f:apiVersion": {},
+                                "f:fieldPath": {}
+                              }
+                            }
+                          }
+                        },
+                        "f:image": {},
+                        "f:imagePullPolicy": {},
+                        "f:name": {},
+                        "f:resources": {},
+                        "f:terminationMessagePath": {},
+                        "f:terminationMessagePolicy": {}
+                      }
+                    },
+                    "f:dnsPolicy": {},
+                    "f:restartPolicy": {},
+                    "f:schedulerName": {},
+                    "f:securityContext": {
+                      ".": {},
+                      "f:runAsGroup": {},
+                      "f:runAsNonRoot": {},
+                      "f:runAsUser": {}
+                    },
+                    "f:serviceAccount": {},
+                    "f:serviceAccountName": {},
+                    "f:terminationGracePeriodSeconds": {}
+                  }
+                },
+                "f:ttlSecondsAfterFinished": {}
+              }
+            }
+          },
+          {
+            "manager": "kube-controller-manager",
+            "operation": "Update",
+            "apiVersion": "batch/v1",
+            "time": "2021-12-17T18:41:17Z",
+            "fieldsType": "FieldsV1",
+            "fieldsV1": {
+              "f:status": {
+                "f:completionTime": {},
+                "f:conditions": {
+                  ".": {},
+                  "k:{\"type\":\"Complete\"}": {
+                    ".": {},
+                    "f:lastProbeTime": {},
+                    "f:lastTransitionTime": {},
+                    "f:status": {},
+                    "f:type": {}
+                  }
+                },
+                "f:startTime": {},
+                "f:succeeded": {}
+              }
+            }
+          }
+        ]
+      },
+      "spec": {
+        "parallelism": 1,
+        "completions": 1,
+        "backoffLimit": 1,
+        "selector": {
+          "matchLabels": {
+            "controller-uid": "416a2a9d-b862-47b8-93d9-160775641fbe"
+          }
+        },
+        "template": {
+          "metadata": {
+            "creationTimestamp": null,
+            "labels": {
+              "app": "contour-certgen",
+              "controller-uid": "416a2a9d-b862-47b8-93d9-160775641fbe",
+              "job-name": "contour-certgen-v1.19.1"
+            }
+          },
+          "spec": {
+            "containers": [
+              {
+                "name": "contour",
+                "image": "ghcr.io/projectcontour/contour:v1.19.1",
+                "command": [
+                  "contour",
+                  "certgen",
+                  "--kube",
+                  "--incluster",
+                  "--overwrite",
+                  "--secrets-format=compact",
+                  "--namespace=$(CONTOUR_NAMESPACE)"
+                ],
+                "env": [
+                  {
+                    "name": "CONTOUR_NAMESPACE",
+                    "valueFrom": {
+                      "fieldRef": {
+                        "apiVersion": "v1",
+                        "fieldPath": "metadata.namespace"
+                      }
+                    }
+                  }
+                ],
+                "resources": {},
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent"
+              }
+            ],
+            "restartPolicy": "Never",
+            "terminationGracePeriodSeconds": 30,
+            "dnsPolicy": "ClusterFirst",
+            "serviceAccountName": "contour-certgen",
+            "serviceAccount": "contour-certgen",
+            "securityContext": {
+              "runAsUser": 65534,
+              "runAsGroup": 65534,
+              "runAsNonRoot": true
+            },
+            "schedulerName": "default-scheduler"
+          }
+        }
+      },
+      "status": {
+        "conditions": [
+          {
+            "type": "Complete",
+            "status": "True",
+            "lastProbeTime": "2021-12-17T18:41:17Z",
+            "lastTransitionTime": "2021-12-17T18:41:17Z"
+          }
+        ],
+        "startTime": "2021-12-17T18:41:14Z",
+        "completionTime": "2021-12-17T18:41:17Z",
+        "succeeded": 0,
+        "failed": 1
+      }
+    }
+  ]
+}