chore(deps): bump k8s.io/metrics from 0.27.3 to 0.27.4 (#1276)

Bumps [k8s.io/metrics](https://github.com/kubernetes/metrics) from 0.27.3 to 0.27.4.
- [Commits](https://github.com/kubernetes/metrics/compare/v0.27.3...v0.27.4)

---
updated-dependencies:
- dependency-name: k8s.io/metrics
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Evans Mungai <evans@replicated.com>

.github/workflows/build-test-deploy.yaml

@@ -1,5 +1,10 @@
 on:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
     branches:
       - main
   push:
@@ -13,10 +18,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  fail_if_pull_request_is_draft:
+    if: github.event.pull_request.draft == true
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Fails in order to indicate that pull request needs to be marked as ready to review and unit tests workflow needs to pass.
+        run: exit 1
+
   test:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
@@ -30,26 +42,49 @@ jobs:
 
       - run: make test
 
+  test-integration:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.19"
+
+      - name: setup env
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+
+      - uses: actions/checkout@v3
+      - uses: replicatedhq/action-k3s@main
+        id: k3s
+        with:
+          version: v1.23.6-k3s1
+
+      - run: make test-integration
+
   ensure-schemas-are-generated:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
           echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
         shell: bash
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - run: make check-schemas
 
   compile-preflight:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -77,10 +112,17 @@ jobs:
           path: bin/
       - run: chmod +x bin/preflight
       - run: |
+          set +e
           ./bin/preflight --interactive=false --format=json https://preflight.replicated.com > result.json
+          EXIT_CODE=$?
           cat result.json
 
           EXIT_STATUS=0
+          if [ $EXIT_CODE -ne 3 ]; then
+            echo "Expected exit code of 3 (some checks failed), got $EXIT_CODE"
+            EXIT_STATUS=1
+          fi
+
           if grep -q "was not collected" result.json; then
             echo "Some files were not collected"
             EXIT_STATUS=1
@@ -120,12 +162,18 @@ jobs:
       - run: chmod +x bin/preflight
       - run: make preflight-e2e-test
 
+  run-examples:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: make run-examples
+
   compile-supportbundle:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -177,9 +225,9 @@ jobs:
   compile-collect:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - name: setup env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -195,22 +243,31 @@ jobs:
   goreleaser-test:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v') != true
+    strategy:
+      matrix:
+        goarch: [amd64, arm64]
+        goos: [darwin, linux, windows]
+        include:
+          - goarch: arm
+            goos: linux
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-      - name: Unshallow
-        run: git fetch --prune --unshallow
-
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: "v0.183.0"
-          args: build --rm-dist --snapshot --config deploy/.goreleaser.yaml
+          args: build --rm-dist --snapshot --config deploy/.goreleaser.yaml --single-target
+        env:
+          GOARCH: ${{ matrix.goarch }}
+          GOOS: ${{ matrix.goos }}
 
   goreleaser:
     runs-on: ubuntu-latest
@@ -222,22 +279,22 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - uses: azure/docker-login@v1
         with:
-          username: repldeploy2
+          username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Unshallow
-        run: git fetch --prune --unshallow
-
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.19"
 
-      - uses: sigstore/cosign-installer@main
+      - uses: sigstore/cosign-installer@v3
         with:
-          cosign-release: "v1.2.1"
+          # DO NOT USE v2 until we decide on whether to use Rekor or not
+          cosign-release: "v1.13.1"  # Binary version to install
 
       - name: Get Cosign Key
         run: |
@@ -253,19 +310,18 @@ jobs:
           COSIGN_KEY: ${{ secrets.COSIGN_KEY }}
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
-          version: "v0.183.0"
+          version: "v1.x"  # Binary version to install
           args: release --rm-dist --config deploy/.goreleaser.yaml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-
       - name: Update new preflight version in krew-index
-        uses: rajatjindal/krew-release-bot@v0.0.43
+        uses: rajatjindal/krew-release-bot@v0.0.46
         with:
           krew_template_file: deploy/krew/preflight.yaml
       - name: Update new support-bundle version in krew-index
-        uses: rajatjindal/krew-release-bot@v0.0.43
+        uses: rajatjindal/krew-release-bot@v0.0.46
         with:
           krew_template_file: deploy/krew/support-bundle.yaml

.github/workflows/fossa.yaml

@@ -0,0 +1,49 @@
+name: FOSSA license scan
+
+on:
+  pull_request_target: # this is safe as these scans do not execute provided code
+    branches:
+      - main
+    paths:
+      - go.sum
+
+  push:
+    branches:
+      - main
+
+jobs:
+  fossa-scan-pr:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: refs/pull/${{ github.event.number }}/merge
+      - name: "Install FOSSA" 
+        uses: replicatedhq/action-fossa/install@main
+      - name: "Run FOSSA Scan"
+        uses: replicatedhq/action-fossa/scan@main
+        with:
+          api-key: ${{ secrets.FOSSA_API_KEY }}
+          diff: true
+          diff-ref: ${{ github.event.pull_request.base.sha }}
+          debug: true
+
+  fossa-scan-merge:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - id: previous
+        run: echo "sha=$(git rev-parse HEAD~1)" >> "${GITHUB_OUTPUT}"
+      - name: "Install FOSSA" 
+        uses: replicatedhq/action-fossa/install@main
+      - name: "Run FOSSA Scan"
+        uses: replicatedhq/action-fossa/scan@main
+        with:
+          api-key: ${{ secrets.FOSSA_API_KEY }}
+          diff: true
+          diff-ref: ${{ steps.previous.outputs.sha }}
+          debug: true

.github/workflows/pr-license-scan.yaml

@@ -1,18 +0,0 @@
-name: PR license scan
-
-on:
-  pull_request_target: # this is safe as these scans do not execute provided code
-
-jobs:
-  fossa-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: refs/pull/${{ github.event.number }}/merge
-      - name: "Install FOSSA" 
-        uses: replicatedhq/action-fossa/install@main
-      - name: "Run FOSSA Scan"
-        uses: replicatedhq/action-fossa/scan@main
-        with:
-          api-key: ${{ secrets.FOSSA_API_KEY }}

.github/workflows/require-pr-labels.yaml

@@ -1,25 +0,0 @@
-name: Require pull request labels
-
-permissions:
-  pull-requests: read
-
-on:
-  pull_request:
-    types:
-      - opened
-      - labeled
-      - unlabeled
-      - synchronize
-
-jobs:
-  require-pr-labels:
-    uses: replicatedhq/reusable-workflows/.github/workflows/pr-enforce-labels.yaml@main
-  require-bug-labels:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: mheap/github-action-required-labels@v2
-        if: ${{ github.event.label.name == 'type::bug' }}
-        with:
-          mode: exactly
-          count: 1
-          labels: "bug::normal, bug::regression"

.gitignore

@@ -7,6 +7,7 @@
 *.dylib
 bin
 .DS_Store
+node_modules/
 
 # Test binary, build with `go test -c`
 *.test
@@ -26,6 +27,8 @@ vendor
 *.swp
 *.swo
 *~
+.envrc
+
 
 dist
 try.sh
@@ -41,3 +44,4 @@ sbom/
 
 # Ignore generated support bundles
 *.tar.gz
+!testdata/supportbundle/*.tar.gz

CONTRIBUTING.md

@@ -15,7 +15,7 @@ There are [community meetings](https://calendar.google.com/calendar/u/0?cid=Y19m
 
 ## Design Principles
 
-When implementing a new feature please review the [design principles](./design/design-principles.md) to help guide the approach.
+When implementing a new feature please review the [design principles](./docs/design/design-principles.md) to help guide the approach.
 
 ## Development Environment
 
@@ -32,12 +32,35 @@ To get started we recommend:
 
 6. Install [golangci-lint] linter and run `make lint` to execute additional code linters.
 
+### Syncing to a test cluster with `watchrsync`
+
+1. Install `npm`
+2. Export `REMOTES=<user>@<ip>` so that `watchrsync` knows where to sync.
+3. Maybe run `export GOOS=linux` and `export GOARCH=amd64` so that you build Linux binaries.
+4. run `make watchrsync` to build and sync binaries automatically on saving.
+
+```
+ssh-add --apple-use-keychain ~/.ssh/google_compute_engine
+export REMOTES=ada@35.229.61.56
+export GOOS=linux
+export GOARCH=amd64
+make watchrsync
+# bin/watchrsync.js
+# make support-bundle
+# go build -tags "netgo containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" -installsuffix netgo -ldflags " -s -w -X github.com/replicatedhq/troubleshoot/pkg/version.version=`git describe --tags --dirty` -X github.com/replicatedhq/troubleshoot/pkg/version.gitSHA=`git rev-parse HEAD` -X github.com/replicatedhq/troubleshoot/pkg/version.buildTime=`date -u +"%Y-%m-%dT%H:%M:%SZ"` " -o bin/support-bundle github.com/replicatedhq/troubleshoot/cmd/troubleshoot
+# rsync bin/support-bundle ada@35.229.61.56:
+# date
+# Tue May 16 14:14:13 EDT 2023
+# synced
+```
+
 ### Testing
 
 To run the tests locally run the following:
 
 ```bash
 make test
+make test RUN=TestClusterResources_Merge
 ```
 
 Additionally, e2e tests can be run with:
@@ -46,7 +69,27 @@ Additionally, e2e tests can be run with:
 make support-bundle preflight e2e-test
 ```
 
-A Kubernetes cluster as well as `jq` are required to run e2e tests.
+A running Kubernetes cluster as well as `jq` are required to run e2e tests.
+
+### Profiling
+
+You are able to collect CPU & memory runtime properties and store the data for analysis in a file. To do so, pass in the file paths using `--cpuprofile` and `--memprofile` flags in the CLI. Once you have your data collected, you can analyse it using [pprof visualization tool](https://github.com/google/pprof/blob/main/doc/README.md). Here is how
+
+Run support bundle and with CPU & memory profile flags
+
+```sh
+./bin/support-bundle examples/support-bundle/sample-supportbundle.yaml --cpuprofile=cpu.prof --memprofile=mem.prof
+```
+
+Visualize using [pprof](https://github.com/google/pprof/blob/main/doc/README.md)
+
+```sh
+go tool pprof -http=":8000" cpu.prof
+
+go tool pprof -http=":8001" mem.prof
+```
+
+More on profiling please visit https://go.dev/doc/diagnostics#profiling
 
 ## Contribution workflow
 
@@ -57,16 +100,19 @@ This is a rough outline of how to prepare a contribution:
 - Make commits of logical units.
 - When your changes are ready to merge, squash your history to 1 commit.
   - For example, if you want to squash your last 3 commits and write a new commit message:
+
       ```
       git reset --soft HEAD~3 &&
       git commit
       ```
 
   - If you want to keep the previous commit messages and concatenate them all into a new commit, you can do something like this instead:
+
       ```
       git reset --soft HEAD~3 &&
       git commit --edit -m"$(git log --format=%B --reverse HEAD..HEAD@{1})"
       ```
+
 - Push your changes to a topic branch in your fork of the repository.
 - Submit a pull request to the original repository. It will be reviewed in a timely manner.
 
@@ -85,4 +131,4 @@ Commit messages should follow the general guidelines:
 - Messages should be descriptive:
   - First line, 50 chars or less, as a heading/title that people can find
   - Then a paragraph explaining things
-- Consider a footer with links to which bugs they fix etc, bearing in mind that Github does some of this magic already
+- Consider a footer with links to which bugs they fix etc, bearing in mind that Github does some of this magic already

Makefile

@@ -2,10 +2,11 @@
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 
-SHELL := /bin/bash -o pipefail
+SHELL := bash -o pipefail
 VERSION_PACKAGE = github.com/replicatedhq/troubleshoot/pkg/version
 VERSION ?=`git describe --tags --dirty`
 DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+RUN?=""
 
 GIT_TREE = $(shell git rev-parse --is-inside-work-tree 2>/dev/null)
 ifneq "$(GIT_TREE)" ""
@@ -34,8 +35,11 @@ define LDFLAGS
 endef
 
 BUILDFLAGS = -tags "netgo containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" -installsuffix netgo
+BUILDPATHS = ./pkg/... ./cmd/... ./internal/...
+TESTFLAGS ?= -v -coverprofile cover.out
 
-all: test support-bundle preflight collect
+.DEFAULT: all
+all: test build
 
 .PHONY: ffi
 ffi: fmt vet
@@ -43,16 +47,35 @@ ffi: fmt vet
 
 .PHONY: test
 test: generate fmt vet
-	go test ${BUILDFLAGS} ./pkg/... ./cmd/... -coverprofile cover.out
+	if [ -n $(RUN) ]; then \
+		go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS} -run $(RUN); \
+	else \
+		go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS}; \
+	fi
+
+# Go tests that require a K8s instance
+# TODOLATER: merge with test, so we get unified coverage reports? it'll add 21~sec to the test job though...
+.PHONY: test-integration
+test-integration:
+	go test -v --tags "integration exclude_graphdriver_devicemapper exclude_graphdriver_btrfs" ${BUILDPATHS}
 
 .PHONY: preflight-e2e-test
 preflight-e2e-test:
 	./test/validate-preflight-e2e.sh
 
+.PHONY: run-examples
+run-examples:
+	./test/run-examples.sh
+
 .PHONY: support-bundle-e2e-test
 support-bundle-e2e-test:
 	./test/validate-support-bundle-e2e.sh
 
+# Build all binaries in parallel ( -j )
+build:
+	@echo "Build cli binaries"
+	$(MAKE) -j support-bundle preflight analyze collect
+
 .PHONY: support-bundle
 support-bundle:
 	go build ${BUILDFLAGS} ${LDFLAGS} -o bin/support-bundle github.com/replicatedhq/troubleshoot/cmd/troubleshoot
@@ -71,11 +94,11 @@ collect:
 
 .PHONY: fmt
 fmt:
-	go fmt ./pkg/... ./cmd/...
+	go fmt ${BUILDPATHS}
 
 .PHONY: vet
 vet:
-	go vet ${BUILDFLAGS} ./pkg/... ./cmd/...
+	go vet ${BUILDFLAGS} ${BUILDPATHS}
 
 .PHONY: generate
 generate: controller-gen client-gen
@@ -113,13 +136,13 @@ docs: fmt vet
 	./bin/docsgen
 
 controller-gen:
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.2
 CONTROLLER_GEN=$(shell which controller-gen)
 
 .PHONY: client-gen
 client-gen:
-ifeq (, $(shell which client-gen))
-	go install k8s.io/code-generator/cmd/client-gen@v0.22.2
+ifeq (, $(shell which client-gen 2>/dev/null))
+	go install k8s.io/code-generator/cmd/client-gen@v0.26.1
 CLIENT_GEN=$(shell go env GOPATH)/bin/client-gen
 else
 CLIENT_GEN=$(shell which client-gen)
@@ -194,16 +217,26 @@ longhorn:
 .PHONY: scan
 scan:
 	trivy fs \
-		--security-checks vuln \
+		--scanners vuln \
 		--exit-code=1 \
 		--severity="HIGH,CRITICAL" \
 		--ignore-unfixed \
 		./
 
 .PHONY: lint
-lint:
-	golangci-lint run -c .golangci.yaml
+lint: fmt vet
+	golangci-lint run --new -c .golangci.yaml ${BUILDPATHS}
 
 .PHONY: lint-and-fix
-lint-and-fix:
-	golangci-lint run --fix -c .golangci.yaml
+lint-and-fix: fmt vet
+	golangci-lint run --new --fix -c .golangci.yaml ${BUILDPATHS}
+
+## Syncronize the code with a remote server. More info: CONTRIBUTING.md
+.PHONY: watchrsync
+watchrsync: npm-install
+	bin/watchrsync.js
+
+.PHONY: npm-install
+npm-install:
+	npm --version 2>&1 >/dev/null || ( echo "npm not installed; install npm to set up watchrsync" && exit 1 )
+	npm list gaze-run-interrupt || npm install install gaze-run-interrupt@~2.0.0

README.md

@@ -4,6 +4,8 @@ Replicated Troubleshoot is a framework for collecting, redacting, and analyzing
 
 Troubleshoot provides two CLI tools as kubectl plugins (using [Krew](https://krew.dev)): `kubectl preflight` and `kubectl support-bundle`. Preflight provides pre-installation cluster conformance testing and validation (preflight checks) and support-bundle provides post-installation troubleshooting and diagnostics (support bundles).
 
+To know more about troubleshoot, please visit: https://troubleshoot.sh/
+
 ## Preflight Checks
 Preflight checks are an easy-to-run set of conformance tests that can be written to verify that specific requirements in a cluster are met.
 
@@ -13,12 +15,12 @@ To run a sample preflight check from a sample application, install the preflight
 curl https://krew.sh/preflight | bash
 ```
  and run, where https://preflight.replicated.com provides an **example** preflight spec:
- 
+
 ```
 kubectl preflight https://preflight.replicated.com
 ```
 
-**NOTE** this is an example. Do **not** use to validate real scenarios. 
+**NOTE** this is an example. Do **not** use to validate real scenarios.
 
 For more details on creating the custom resource files that drive preflight checks, visit [creating preflight checks](https://troubleshoot.sh/docs/preflight/introduction/).
 
@@ -32,12 +34,12 @@ To collect a sample support bundle, install the troubleshoot kubectl plugin:
 curl https://krew.sh/support-bundle | bash
 ```
  and run, where https://support-bundle.replicated.com provides an **example** support bundle spec:
- 
+
 ```
 kubectl support-bundle https://support-bundle.replicated.com
 ```
 
-**NOTE** this is an example. Do **not** use to validate real scenarios. 
+**NOTE** this is an example. Do **not** use to validate real scenarios.
 
 For more details on creating the custom resource files that drive support-bundle collection, visit [creating collectors](https://troubleshoot.sh/docs/collect/) and [creating analyzers](https://troubleshoot.sh/docs/analyze/).
 
@@ -45,17 +47,28 @@ And see our other tool [sbctl](https://github.com/replicatedhq/sbctl) that makes
 
 # Community
 
-For questions about using Troubleshoot, there's a [Replicated Community](https://help.replicated.com/community) forum, and a [#app-troubleshoot channel in Kubernetes Slack](https://kubernetes.slack.com/channels/app-troubleshoot).
+For questions about using Troubleshoot, how to contribute and engaging with the project in any other way, please refer to the following resources and channels.
 
-# Software Bill of Materials 
-A signed SBOM  that includes Troubleshoot dependencies is included in each release. 
-- **troubleshoot-sbom.tgz** contains a software bill of materials for Troubleshoot. 
+- [Replicated Community](https://help.replicated.com/community) forum
+- [#app-troubleshoot channel in Kubernetes Slack](https://kubernetes.slack.com/channels/app-troubleshoot)
+- [#Community meetings calendar](https://calendar.google.com/calendar/u/0?cid=Y19mMGx1aGhiZGtscGllOGo5dWpicXMwNnN1a0Bncm91cC5jYWxlbmRhci5nb29nbGUuY29t). This happen monthly but dates may change and would be kept upto date in the calendar.
+
+# Software Bill of Materials
+A signed SBOM  that includes Troubleshoot dependencies is included in each release.
+- **troubleshoot-sbom.tgz** contains a software bill of materials for Troubleshoot.
 - **troubleshoot-sbom.tgz.sig** is the digital signature for troubleshoot-sbom.tgz
 - **key.pub** is the public key from the key pair used to sign troubleshoot-sbom.tgz
 
 The following example illustrates using [cosign](https://github.com/sigstore/cosign) to verify that **troubleshoot-sbom.tgz** has
 not been tampered with.
+```sh
+$ cosign verify-blob --key key.pub --signature troubleshoot-sbom.tgz.sig troubleshoot-sbom.tgz
+Verified OK
 ```
-$ cosign verify-blob -key key.pub -signature troubleshoot-sbom.tgz.sig troubleshoot-sbom.tgz
+
+If you were to get an error similar to the one below, it means you are verifying an SBOM signed using cosign `v1` using a newer `v2` of the binary. This version introduced [breaking changes](https://github.com/sigstore/cosign/blob/main/CHANGELOG.md#breaking-changes) which require an additional flag `--insecure-ignore-tlog=true` to successfully verify SBOMs like so.
+```sh
+$ cosign verify-blob --key key.pub --signature troubleshoot-sbom.tgz.sig troubleshoot-sbom.tgz --insecure-ignore-tlog=true
+WARNING: Skipping tlog verification is an insecure practice that lacks of transparency and auditability verification for the blob.
 Verified OK
 ```

bin/watchrsync.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+
+const gri = require('gaze-run-interrupt');
+
+if (!process.env.REMOTES) {
+  console.log("Usage: `REMOTES='user@h1.1.1.1,user@1.1.1.2' ./watchrsync.js`");
+  process.exit(1);
+}
+
+process.env.GOOS = 'linux';
+process.env.GOARCH = 'amd64';
+
+const binList = [
+  // 'bin/analyze',
+  // 'bin/preflight',
+  'bin/support-bundle',
+  // 'bin/collect'
+]
+const makeList = [
+  // 'analyze',
+  // 'preflight',
+  'support-bundle',
+  // 'collect'
+]
+
+const commands = [
+  // {
+  //   command: 'rm',
+  //   args: binList,
+  // },
+  {
+    command: 'make',
+    args: makeList,
+  },
+];
+
+process.env.REMOTES.split(",").forEach(function (remote) {
+  commands.push({
+    command: 'rsync',
+    args: binList.concat(`${remote}:`),
+  });
+});
+
+commands.push({
+  command: "date",
+  args: [],
+});
+
+commands.push({
+  command: "echo",
+  args: ["synced"],
+});
+
+gri([
+  'cmd/**/*.go',
+  'pkg/**/*.go',
+], commands);

cmd/analyze/cli/root.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -23,17 +23,22 @@ func RootCmd() *cobra.Command {
 			v := viper.GetViper()
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-
 			return runAnalyzers(v, args[0])
 		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
+		},
 	}
 
 	cobra.OnInitialize(initConfig)
@@ -45,8 +50,14 @@ func RootCmd() *cobra.Command {
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
 	k8sutil.AddFlags(cmd.Flags())
 
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 

cmd/analyze/cli/run.go

@@ -7,7 +7,7 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	"github.com/spf13/viper"
 )

cmd/collect/cli/root.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -23,16 +23,22 @@ func RootCmd() *cobra.Command {
 			v := viper.GetViper()
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
 			return runCollect(v, args[0])
 		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
+		},
 	}
 
 	cobra.OnInitialize(initConfig)
@@ -58,6 +64,12 @@ func RootCmd() *cobra.Command {
 
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 

cmd/collect/cli/run.go

@@ -10,10 +10,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
-	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 	"github.com/replicatedhq/troubleshoot/pkg/docrewrite"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
@@ -88,19 +87,17 @@ func runCollect(v *viper.Viper, arg string) error {
 
 	multidocs := strings.Split(string(collectorContent), "\n---\n")
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 
-	additionalRedactors := &troubleshootv1beta2.Redactor{}
-	for idx, redactor := range v.GetStringSlice("redactors") {
-		redactorObj, err := supportbundle.GetRedactorFromURI(redactor)
-		if err != nil {
-			return errors.Wrapf(err, "failed to get redactor spec %s, #%d", redactor, idx)
-		}
+	redactors, err := supportbundle.GetRedactorsFromURIs(v.GetStringSlice("redactors"))
+	if err != nil {
+		return errors.Wrap(err, "failed to get redactors")
+	}
 
-		if redactorObj != nil {
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorObj.Spec.Redactors...)
-		}
+	additionalRedactors := &troubleshootv1beta2.Redactor{
+		Spec: troubleshootv1beta2.RedactorSpec{
+			Redactors: redactors,
+		},
 	}
 
 	for i, additionalDoc := range multidocs {

cmd/preflight/cli/oci_fetch.go

@@ -0,0 +1,26 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/replicatedhq/troubleshoot/pkg/oci"
+	"github.com/spf13/cobra"
+)
+
+func OciFetchCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "oci-fetch [URI]",
+		Args:  cobra.MinimumNArgs(1),
+		Short: "Fetch a preflight from an OCI registry and print it to standard out",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uri := args[0]
+			data, err := oci.PullPreflightFromOCI(uri)
+			if err != nil {
+				return err
+			}
+			fmt.Println(string(data))
+			return nil
+		},
+	}
+	return cmd
+}

cmd/preflight/cli/root.go

@@ -1,12 +1,18 @@
 package cli
 
 import (
+	"errors"
+	"fmt"
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/traces"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/replicatedhq/troubleshoot/pkg/preflight"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
@@ -19,35 +25,79 @@ func RootCmd() *cobra.Command {
 		Short: "Run and retrieve preflight checks in a cluster",
 		Long: `A preflight check is a set of validations that can and should be run to ensure
 that a cluster meets the requirements to run an application.`,
-		SilenceUsage: true,
+		SilenceUsage:  true,
+		SilenceErrors: true,
 		PreRun: func(cmd *cobra.Command, args []string) {
 			v := viper.GetViper()
+			v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
-			return preflight.RunPreflights(v.GetBool("interactive"), v.GetString("output"), v.GetString("format"), args[0])
+			closer, err := traces.ConfigureTracing("preflight")
+			if err != nil {
+				// Do not fail running preflights if tracing fails
+				klog.Errorf("Failed to initialize open tracing provider: %v", err)
+			} else {
+				defer closer()
+			}
+
+			err = preflight.RunPreflights(v.GetBool("interactive"), v.GetString("output"), v.GetString("format"), args)
+			if v.GetBool("debug") || v.IsSet("v") {
+				fmt.Printf("\n%s", traces.GetExporterInstance().GetSummary())
+			}
+
+			return err
+		},
+		PostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
 		},
 	}
 
 	cobra.OnInitialize(initConfig)
 
 	cmd.AddCommand(VersionCmd())
+	cmd.AddCommand(OciFetchCmd())
 	preflight.AddFlags(cmd.PersistentFlags())
 
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 
 func InitAndExecute() {
-	if err := RootCmd().Execute(); err != nil {
+	cmd := RootCmd()
+	err := cmd.Execute()
+
+	if err != nil {
+		var exitErr types.ExitError
+		if errors.As(err, &exitErr) {
+			// We need to do this, there's situations where we need the non-zero exit code (which comes as part of the custom error struct)
+			// but there's no actual error, just an exit code.
+			// If there's also an error to output (eg. invalid format etc) then print it as well
+			if exitErr.ExitStatus() != constants.EXIT_CODE_FAIL && exitErr.ExitStatus() != constants.EXIT_CODE_WARN {
+				cmd.PrintErrln("Error:", err.Error())
+			}
+
+			os.Exit(exitErr.ExitStatus())
+		}
+
+		// Fallback, should almost never be used (the above Exit() should handle almost all situations
+		cmd.PrintErrln("Error:", err.Error())
 		os.Exit(1)
 	}
 }

cmd/troubleshoot/cli/analyze.go

@@ -7,10 +7,9 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
@@ -23,15 +22,11 @@ func Analyze() *cobra.Command {
 		Short: "analyze a support bundle",
 		Long:  `Analyze a support bundle using the Analyzer definitions provided`,
 		PreRun: func(cmd *cobra.Command, args []string) {
-			viper.BindPFlag("bundle", cmd.Flags().Lookup("bundle"))
-			viper.BindPFlag("output", cmd.Flags().Lookup("output"))
-			viper.BindPFlag("quiet", cmd.Flags().Lookup("quiet"))
+			viper.BindPFlags(cmd.Flags())
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-
 			specPath := args[0]
 			analyzerSpec, err := downloadAnalyzerSpec(specPath)
 			if err != nil {
@@ -77,8 +72,6 @@ func Analyze() *cobra.Command {
 	cmd.Flags().MarkHidden("compatibility")
 	cmd.Flags().Bool("quiet", false, "enable/disable error messaging and only show parseable output")
 
-	viper.BindPFlags(cmd.Flags())
-
 	return cmd
 }
 

cmd/troubleshoot/cli/interactive_results.go

@@ -7,11 +7,13 @@ import (
 	"path"
 	"time"
 
+	"github.com/mitchellh/go-wordwrap"
 	"github.com/pkg/errors"
 	ui "github.com/replicatedhq/termui/v3"
 	"github.com/replicatedhq/termui/v3/widgets"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzerunner "github.com/replicatedhq/troubleshoot/pkg/analyze"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 )
 
 var (
@@ -178,7 +180,10 @@ func drawDetails(analysisResult *analyzerunner.AnalyzeResult) {
 
 	currentTop := 4
 	title := widgets.NewParagraph()
-	title.Text = analysisResult.Title
+	// WrapText is set to false to prevent internal wordwrap which is not accounting for the padding
+	title.WrapText = false
+	// For long title that lead to wrapping text, the terminal width is divided by 2 and deducted by MESSAGE_TEXT_PADDING to account for the padding
+	title.Text = wordwrap.WrapString(analysisResult.Title, uint(termWidth/2-constants.MESSAGE_TEXT_PADDING))
 	title.Border = false
 	if analysisResult.IsPass {
 		title.TextStyle = ui.NewStyle(ui.ColorGreen, ui.ColorClear, ui.ModifierBold)
@@ -187,34 +192,26 @@ func drawDetails(analysisResult *analyzerunner.AnalyzeResult) {
 	} else if analysisResult.IsFail {
 		title.TextStyle = ui.NewStyle(ui.ColorRed, ui.ColorClear, ui.ModifierBold)
 	}
-	height := estimateNumberOfLines(title.Text, termWidth/2)
+	height := util.EstimateNumberOfLines(title.Text)
 	title.SetRect(termWidth/2, currentTop, termWidth, currentTop+height)
 	ui.Render(title)
 	currentTop = currentTop + height + 1
 
 	message := widgets.NewParagraph()
-	message.Text = analysisResult.Message
+	// WrapText is set to false to prevent internal wordwrap which is not accounting for the padding
+	message.WrapText = false
+	// For long text that lead to wrapping text, the terminal width is divided by 2 and deducted by MESSAGE_TEXT_PADDING to account for the padding
+	message.Text = wordwrap.WrapString(analysisResult.Message, uint(termWidth/2-constants.MESSAGE_TEXT_PADDING))
+	if analysisResult.URI != "" {
+		// Add URL to the message with wordwrap
+		// Add emply lines as separator
+		urlText := wordwrap.WrapString(fmt.Sprintf("For more information: %s", analysisResult.URI), uint(termWidth/2-constants.MESSAGE_TEXT_PADDING))
+		message.Text = message.Text + "\n\n" + urlText
+	}
+	height = util.EstimateNumberOfLines(message.Text) + constants.MESSAGE_TEXT_LINES_MARGIN_TO_BOTTOM
 	message.Border = false
-	height = estimateNumberOfLines(message.Text, termWidth/2) + 2
 	message.SetRect(termWidth/2, currentTop, termWidth, currentTop+height)
 	ui.Render(message)
-	currentTop = currentTop + height + 1
-
-	if analysisResult.URI != "" {
-		uri := widgets.NewParagraph()
-		uri.Text = fmt.Sprintf("For more information: %s", analysisResult.URI)
-		uri.Border = false
-		// For long urls that lead to wrapping text, make the rectangle bigger by
-		// increasing the calculated height by 2
-		height = estimateNumberOfLines(uri.Text, termWidth/2) + 2
-		uri.SetRect(termWidth/2, currentTop, termWidth, currentTop+height)
-		ui.Render(uri)
-	}
-}
-
-func estimateNumberOfLines(text string, width int) int {
-	lines := len(text)/width + 1
-	return lines
 }
 
 func showSaved(filename string) {

cmd/troubleshoot/cli/redact.go

@@ -0,0 +1,82 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/replicatedhq/troubleshoot/pkg/supportbundle"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func Redact() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "redact [urls...]",
+		Args:  cobra.MinimumNArgs(1), // TODO
+		Short: "Redact information from a generated support bundle archive",
+		Long: `Redaction is the process of masking sensitive information from collected data in a support bundle.
+This is done using rules defined in the list of redactor manifests provided in the [urls...] command line
+argument. Default built in redactors will also be run, but these would have been run when the support
+bundle was generated. After redaction, the support bundle is archived once more. The resulting file will
+be stored in the current directory in the path provided by the --output flag.
+
+The [urls...] argument is a list of either oci://.., http://.., https://.. or local paths to yaml files.
+
+For more information on redactors visit https://troubleshoot.sh/docs/redact/
+		`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return viper.BindPFlags(cmd.Flags())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := viper.GetViper()
+
+			// 1. Decode redactors from provided URLs
+			redactors, err := supportbundle.GetRedactorsFromURIs(args)
+			if err != nil {
+				return err
+			}
+
+			// 2. Download the bundle and extract it
+			tmpDir, bundleDir, err := analyzer.DownloadAndExtractSupportBundle(v.GetString("bundle"))
+			if err != nil {
+				return err
+			}
+			defer os.RemoveAll(tmpDir)
+
+			// 3. Represent bundle as a CollectorResult
+			collectorResult, err := collect.CollectorResultFromBundle(bundleDir)
+			if err != nil {
+				return err
+			}
+
+			// 4. Perform redaction on the bundle
+			err = collect.RedactResult(bundleDir, collectorResult, redactors)
+			if err != nil {
+				return errors.Wrap(err, "failed to redact support bundle")
+			}
+
+			// 5. Compress the bundle once more after redacting
+			output := v.GetString("output")
+			if output == "" {
+				output = fmt.Sprintf("redacted-support-bundle-%s.tar.gz", time.Now().Format("2006-01-02T15_04_05"))
+			}
+			err = collectorResult.ArchiveSupportBundle(bundleDir, output)
+			if err != nil {
+				return errors.Wrap(err, "failed to create support bundle archive")
+			}
+			fmt.Println("Redacted support bundle:", output)
+			return nil
+		},
+	}
+
+	cmd.Flags().String("bundle", "", "file path of the support bundle archive to redact")
+	cmd.MarkFlagRequired("bundle")
+	cmd.Flags().BoolP("quiet", "q", false, "enable/disable error messaging and only show parseable output")
+	cmd.Flags().StringP("output", "o", "", "file path of where to save the redacted support bundle archive (default \"redacted-support-bundle-YYYY-MM-DDTHH_MM_SS.tar.gz\")")
+
+	return cmd
+}

cmd/troubleshoot/cli/root.go

@@ -1,11 +1,12 @@
 package cli
 
 import (
-	"io/ioutil"
+	"fmt"
 	"os"
 	"strings"
 
-	"github.com/go-logr/logr"
+	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/traces"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/spf13/cobra"
@@ -15,43 +16,64 @@ import (
 
 func RootCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "support-bundle [url]",
+		Use:   "support-bundle [urls...]",
 		Args:  cobra.MinimumNArgs(0),
 		Short: "Generate a support bundle",
 		Long: `A support bundle is an archive of files, output, metrics and state
 from a server that can be used to assist when troubleshooting a Kubernetes cluster.`,
 		SilenceUsage: true,
-		PreRun: func(cmd *cobra.Command, args []string) {
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			v := viper.GetViper()
+			v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 			v.BindPFlags(cmd.Flags())
 
-			if !v.GetBool("debug") {
-				klog.SetLogger(logr.Discard())
+			logger.SetupLogger(v)
+
+			if err := util.StartProfiling(); err != nil {
+				klog.Errorf("Failed to start profiling: %v", err)
 			}
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.GetViper()
 
-			logger.SetQuiet(v.GetBool("quiet"))
-			return runTroubleshoot(v, args)
+			closer, err := traces.ConfigureTracing("support-bundle")
+			if err != nil {
+				// Do not fail running support-bundle if tracing fails
+				klog.Errorf("Failed to initialize open tracing provider: %v", err)
+			} else {
+				defer closer()
+			}
+
+			err = runTroubleshoot(v, args)
+			if v.GetBool("debug") || v.IsSet("v") {
+				fmt.Printf("\n%s", traces.GetExporterInstance().GetSummary())
+			}
+
+			return err
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if err := util.StopProfiling(); err != nil {
+				klog.Errorf("Failed to stop profiling: %v", err)
+			}
 		},
 	}
 
 	cobra.OnInitialize(initConfig)
 
 	cmd.AddCommand(Analyze())
+	cmd.AddCommand(Redact())
 	cmd.AddCommand(VersionCmd())
 
 	cmd.Flags().StringSlice("redactors", []string{}, "names of the additional redactors to use")
 	cmd.Flags().Bool("redact", true, "enable/disable default redactions")
 	cmd.Flags().Bool("interactive", true, "enable/disable interactive mode")
 	cmd.Flags().Bool("collect-without-permissions", true, "always generate a support bundle, even if it some require additional permissions")
-	cmd.Flags().StringSliceP("selector", "l", []string{"troubleshoot.io/kind=supportbundle-spec"}, "selector to filter on for loading additional support bundle specs found in secrets within the cluster")
+	cmd.Flags().StringSliceP("selector", "l", []string{"troubleshoot.sh/kind=support-bundle"}, "selector to filter on for loading additional support bundle specs found in secrets within the cluster")
 	cmd.Flags().Bool("load-cluster-specs", false, "enable/disable loading additional troubleshoot specs found within the cluster. required when no specs are provided on the command line")
 	cmd.Flags().String("since-time", "", "force pod logs collectors to return logs after a specific date (RFC3339)")
 	cmd.Flags().String("since", "", "force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.")
 	cmd.Flags().StringP("output", "o", "", "specify the output file path for the support bundle")
-	cmd.Flags().Bool("debug", false, "enable debug logging")
+	cmd.Flags().Bool("debug", false, "enable debug logging. This is equivalent to --v=0")
 
 	// hidden in favor of the `insecure-skip-tls-verify` flag
 	cmd.Flags().Bool("allow-insecure-connections", false, "when set, do not verify TLS certs when retrieving spec and reporting results")
@@ -61,12 +83,14 @@ from a server that can be used to assist when troubleshooting a Kubernetes clust
 	// This flag makes sure we can also disable this and fall back to the default spec.
 	cmd.Flags().Bool("no-uri", false, "When this flag is used, Troubleshoot does not attempt to retrieve the bundle referenced by the uri: field in the spec.`")
 
-	viper.BindPFlags(cmd.Flags())
-
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-
 	k8sutil.AddFlags(cmd.Flags())
 
+	// Initialize klog flags
+	logger.InitKlogFlags(cmd)
+
+	// CPU and memory profiling flags
+	util.AddProfilingFlags(cmd)
+
 	return cmd
 }
 
@@ -80,11 +104,3 @@ func initConfig() {
 	viper.SetEnvPrefix("TROUBLESHOOT")
 	viper.AutomaticEnv()
 }
-
-func writeFile(filename string, contents []byte) error {
-	if err := ioutil.WriteFile(filename, contents, 0644); err != nil {
-		return err
-	}
-
-	return nil
-}

cmd/troubleshoot/cli/run.go

@@ -1,41 +1,43 @@
 package cli
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
+	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	cursor "github.com/ahmetalpbalkan/go-cursor"
 	"github.com/fatih/color"
-	"github.com/manifoldco/promptui"
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
-	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
-	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
 	"github.com/replicatedhq/troubleshoot/pkg/httputil"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
 	"github.com/replicatedhq/troubleshoot/pkg/specs"
 	"github.com/replicatedhq/troubleshoot/pkg/supportbundle"
 	"github.com/spf13/viper"
 	spin "github.com/tj/go-spin"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
 )
 
 func runTroubleshoot(v *viper.Viper, arg []string) error {
-	if v.GetBool("load-cluster-specs") == false && len(arg) < 1 {
+	if !v.GetBool("load-cluster-specs") && len(arg) < 1 {
 		return errors.New("flag load-cluster-specs must be set if no specs are provided on the command line")
 	}
 
@@ -77,12 +79,11 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 
 	var mainBundle *troubleshootv1beta2.SupportBundle
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
 	additionalRedactors := &troubleshootv1beta2.Redactor{}
 
 	// Defining `v` below will render using `v` in reference to Viper unusable.
 	// Therefore refactoring `v` to `val` will make sure we can still use it.
-	for i, val := range arg {
+	for _, val := range arg {
 
 		collectorContent, err := supportbundle.LoadSupportBundleSpec(val)
 		if err != nil {
@@ -98,11 +99,7 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 			return errors.Wrap(err, "failed to parse support bundle spec")
 		}
 
-		if i == 0 {
-			mainBundle = supportBundle
-		} else {
-			mainBundle = supportbundle.ConcatSpec(mainBundle, supportBundle)
-		}
+		mainBundle = supportbundle.ConcatSpec(mainBundle, supportBundle)
 
 		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
 		if err != nil {
@@ -112,94 +109,15 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 	}
 
 	if v.GetBool("load-cluster-specs") {
-		labelSelector := strings.Join(v.GetStringSlice("selector"), ",")
-
-		parsedSelector, err := labels.Parse(labelSelector)
+		sbFromCluster, redactorsFromCluster, err := loadClusterSpecs()
 		if err != nil {
-			return errors.Wrap(err, "unable to parse selector")
+			return err
 		}
-
-		namespace := ""
-		if v.GetString("namespace") != "" {
-			namespace = v.GetString("namespace")
-		}
-
-		config, err := k8sutil.GetRESTConfig()
-		if err != nil {
-			return errors.Wrap(err, "failed to convert kube flags to rest config")
-		}
-
-		client, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			return errors.Wrap(err, "failed to convert create k8s client")
-		}
-
-		var bundlesFromCluster []string
-
-		// Search cluster for Troubleshoot objects in cluster
-		bundlesFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), namespace, specs.SupportBundleKey)
-		if err != nil {
-			logger.Printf("failed to load support bundle spec from secrets: %s", err)
-		}
-		bundlesFromCluster = append(bundlesFromCluster, bundlesFromSecrets...)
-
-		bundlesFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), namespace, specs.SupportBundleKey)
-		if err != nil {
-			logger.Printf("failed to load support bundle spec from secrets: %s", err)
-		}
-		bundlesFromCluster = append(bundlesFromCluster, bundlesFromConfigMaps...)
-
-		for _, bundle := range bundlesFromCluster {
-			multidocs := strings.Split(string(bundle), "\n---\n")
-			parsedBundleFromSecret, err := supportbundle.ParseSupportBundleFromDoc([]byte(multidocs[0]))
-			if err != nil {
-				logger.Printf("failed to parse support bundle spec:  %s", err)
-				continue
-			}
-
-			if mainBundle == nil {
-				mainBundle = parsedBundleFromSecret
-			} else {
-				mainBundle = supportbundle.ConcatSpec(mainBundle, parsedBundleFromSecret)
-			}
-
-			parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
-			if err != nil {
-				logger.Printf("failed to parse redactors from doc:  %s", err)
-				continue
-			}
-
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, parsedRedactors...)
-		}
-
-		var redactorsFromCluster []string
-
-		// Search cluster for Troubleshoot objects in ConfigMaps
-		redactorsFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelector.String(), namespace, specs.RedactorKey)
-		if err != nil {
-			logger.Printf("failed to load redactor specs from config maps: %s", err)
-		}
-		redactorsFromCluster = append(redactorsFromCluster, redactorsFromSecrets...)
-
-		redactorsFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelector.String(), namespace, specs.RedactorKey)
-		if err != nil {
-			logger.Printf("failed to load redactor specs from config maps: %s", err)
-		}
-		redactorsFromCluster = append(redactorsFromCluster, redactorsFromConfigMaps...)
-
-		for _, redactor := range redactorsFromCluster {
-			multidocs := strings.Split(string(redactor), "\n---\n")
-			parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
-			if err != nil {
-				logger.Printf("failed to parse redactors from doc:  %s", err)
-			}
-
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, parsedRedactors...)
-		}
-
-		if mainBundle == nil {
+		if sbFromCluster == nil {
 			return errors.New("no specs found in cluster")
 		}
+		mainBundle = supportbundle.ConcatSpec(mainBundle, sbFromCluster)
+		additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorsFromCluster.Spec.Redactors...)
 	}
 
 	if mainBundle == nil {
@@ -208,45 +126,48 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 		return errors.New("no collectors specified in support bundle")
 	}
 
-	for idx, redactor := range v.GetStringSlice("redactors") {
-		redactorObj, err := supportbundle.GetRedactorFromURI(redactor)
-		if err != nil {
-			return errors.Wrapf(err, "failed to get redactor spec %s, #%d", redactor, idx)
-		}
-
-		if redactorObj != nil {
-			additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactorObj.Spec.Redactors...)
-		}
+	redactors, err := supportbundle.GetRedactorsFromURIs(v.GetStringSlice("redactors"))
+	if err != nil {
+		return errors.Wrap(err, "failed to get redactors")
 	}
+	additionalRedactors.Spec.Redactors = append(additionalRedactors.Spec.Redactors, redactors...)
 
-	var collectorCB func(chan interface{}, string)
-	progressChan := make(chan interface{}) // non-zero buffer can result in missed messages
-	finishedCh := make(chan bool, 1)
-	isFinishedChClosed := false
+	var wg sync.WaitGroup
+	collectorCB := func(c chan interface{}, msg string) { c <- msg }
+	progressChan := make(chan interface{})
+	isProgressChanClosed := false
+	defer func() {
+		if !isProgressChanClosed {
+			close(progressChan)
+		}
+		wg.Wait()
+	}()
 
 	if !interactive {
 		// TODO (dans): custom warning handler to capture warning in `analysisOutput`
 		restConfig.WarningHandler = rest.NoWarnings{}
-		collectorCB = func(ch chan interface{}, name string) {
-			return
-		}
 
 		// TODO (dans): maybe log to file
+		wg.Add(1)
 		go func() {
-			for {
-				select {
-				case _ = <-progressChan:
-					// do nothing
-				}
+			defer wg.Done()
+			for msg := range progressChan {
+				klog.Infof("Collecting support bundle: %v", msg)
 			}
 		}()
 	} else {
 		s := spin.New()
+		wg.Add(1)
 		go func() {
+			defer wg.Done()
 			currentDir := ""
 			for {
 				select {
-				case msg := <-progressChan:
+				case msg, ok := <-progressChan:
+					if !ok {
+						fmt.Printf("\r%s\r", cursor.ClearEntireLine())
+						return
+					}
 					switch msg := msg.(type) {
 					case error:
 						c := color.New(color.FgHiRed)
@@ -254,9 +175,6 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 					case string:
 						currentDir = filepath.Base(msg)
 					}
-				case <-finishedCh:
-					fmt.Printf("\r%s\r", cursor.ClearEntireLine())
-					return
 				case <-time.After(time.Millisecond * 100):
 					if currentDir == "" {
 						fmt.Printf("\r%s \033[36mCollecting support bundle\033[m %s", cursor.ClearEntireLine(), s.Next())
@@ -266,16 +184,6 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 				}
 			}
 		}()
-		defer func() {
-			if !isFinishedChClosed {
-				close(finishedCh)
-			}
-		}()
-
-		collectorCB = func(c chan interface{}, msg string) {
-			c <- fmt.Sprintf("%s", msg)
-		}
-
 	}
 
 	createOpts := supportbundle.SupportBundleCreateOpts{
@@ -301,11 +209,12 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to run collect and analyze process")
 	}
+
+	close(progressChan) // this removes the spinner in interactive mode
+	isProgressChanClosed = true
+
 	if len(response.AnalyzerResults) > 0 {
 		if interactive {
-			close(finishedCh) // this removes the spinner
-			isFinishedChClosed = true
-
 			if err := showInteractiveResults(mainBundle.Name, response.AnalyzerResults); err != nil {
 				interactive = false
 			}
@@ -333,7 +242,7 @@ the %s Admin Console to begin analysis.`
 			return nil
 		}
 
-		fmt.Printf("%s\n", response.ArchivePath)
+		fmt.Printf("\n%s\n", response.ArchivePath)
 		return nil
 	}
 
@@ -349,12 +258,178 @@ the %s Admin Console to begin analysis.`
 	return nil
 }
 
-func getExpectedContentType(uploadURL string) string {
-	parsedURL, err := url.Parse(uploadURL)
-	if err != nil {
-		return ""
+// loadClusterSpecs loads the support bundle and redactor specs from the cluster
+// based on the provided labels. By default this will be troubleshoot.io/kind=support-bundle and troubleshoot.sh/kind=support-bundle label selectors. We search for secrets
+// and configmaps with the label selector and parse the data as a support bundle. If the
+// user does not have sufficient permissions to list & read secrets and configmaps from
+// all namespaces, we will fallback to trying each namespace individually, and eventually
+// default to the configured kubeconfig namespace.
+func loadClusterSpecs() (*troubleshootv1beta2.SupportBundle, *troubleshootv1beta2.Redactor, error) {
+	redactors := &troubleshootv1beta2.Redactor{}
+
+	v := viper.GetViper() // It's singleton, so we can use it anywhere
+
+	klog.Info("Discover troubleshoot specs from cluster")
+
+	selectors := v.GetStringSlice("selector")
+	if reflect.DeepEqual(selectors, []string{"troubleshoot.sh/kind=support-bundle"}) {
+		// Its the default selector so we append troubleshoot.io/kind=support-bundle to it due to backwards compatibility
+		selectors = append(selectors, "troubleshoot.io/kind=support-bundle")
 	}
-	return parsedURL.Query().Get("Content-Type")
+
+	labelSelector := strings.Join(selectors, ",")
+
+	parsedSelector, err := labels.Parse(labelSelector)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to parse selector")
+	}
+
+	config, err := k8sutil.GetRESTConfig()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to convert kube flags to rest config")
+	}
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to convert create k8s client")
+	}
+
+	// List of namespaces we want to search for secrets and configmaps with support bundle specs
+	namespaces := []string{}
+	ctx := context.Background()
+
+	if v.GetString("namespace") != "" {
+		// Just progress with the namespace provided
+		namespaces = []string{v.GetString("namespace")}
+	} else {
+		// Check if I can read secrets and configmaps in all namespaces
+		ican, err := k8sutil.CanIListAndGetAllSecretsAndConfigMaps(ctx, client)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "failed to check if I can read secrets and configmaps")
+		}
+		klog.V(1).Infof("Can I read any secrets and configmaps: %v", ican)
+
+		if ican {
+			// I can read secrets and configmaps in all namespaces
+			// No need to iterate over all namespaces
+			namespaces = []string{""}
+		} else {
+			// Get list of all namespaces and try to find specs from each namespace
+			nsList, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+			if err != nil {
+				if k8serrors.IsForbidden(err) {
+					kubeconfig := k8sutil.GetKubeconfig()
+					ns, _, err := kubeconfig.Namespace()
+					if err != nil {
+						return nil, nil, errors.Wrap(err, "failed to get namespace from kubeconfig")
+					}
+					// If we are not allowed to list namespaces, just use the default namespace
+					// configured in the kubeconfig
+					namespaces = []string{ns}
+				} else {
+					return nil, nil, errors.Wrap(err, "failed to list namespaces")
+				}
+			}
+
+			for _, ns := range nsList.Items {
+				namespaces = append(namespaces, ns.Name)
+			}
+		}
+	}
+
+	var bundlesFromCluster []string
+
+	parsedSelectorStrings, err := specs.SplitTroubleshootSecretLabelSelector(client, parsedSelector)
+	if err != nil {
+		klog.Errorf("failed to parse troubleshoot labels selector %s", err)
+	}
+
+	// Search cluster for support bundle specs
+	for _, parsedSelectorString := range parsedSelectorStrings {
+		klog.V(1).Infof("Search support bundle specs from [%q] namespace using %q selector", strings.Join(namespaces, ", "), parsedSelectorString)
+		for _, ns := range namespaces {
+			bundlesFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelectorString, ns, constants.SupportBundleKey)
+			if err != nil {
+				if !k8serrors.IsForbidden(err) {
+					klog.Errorf("failed to load support bundle spec from secrets: %s", err)
+				} else {
+					klog.Warningf("Reading secrets from %q namespace forbidden", ns)
+				}
+			}
+			bundlesFromCluster = append(bundlesFromCluster, bundlesFromSecrets...)
+
+			bundlesFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelectorString, ns, constants.SupportBundleKey)
+			if err != nil {
+				if !k8serrors.IsForbidden(err) {
+					klog.Errorf("failed to load support bundle spec from configmap: %s", err)
+				} else {
+					klog.Warningf("Reading configmaps from %q namespace forbidden", ns)
+				}
+			}
+			bundlesFromCluster = append(bundlesFromCluster, bundlesFromConfigMaps...)
+		}
+	}
+
+	parsedBundle := &troubleshootv1beta2.SupportBundle{}
+
+	for _, bundle := range bundlesFromCluster {
+		multidocs := strings.Split(string(bundle), "\n---\n")
+		bundleFromDoc, err := supportbundle.ParseSupportBundleFromDoc([]byte(multidocs[0]))
+		if err != nil {
+			klog.Errorf("failed to parse support bundle spec:  %s", err)
+			continue
+		}
+
+		parsedBundle = supportbundle.ConcatSpec(parsedBundle, bundleFromDoc)
+
+		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
+		if err != nil {
+			klog.Errorf("failed to parse redactors from doc:  %s", err)
+			continue
+		}
+
+		redactors.Spec.Redactors = append(redactors.Spec.Redactors, parsedRedactors...)
+	}
+
+	var redactorsFromCluster []string
+
+	// Search cluster for redactor specs
+	for _, parsedSelectorString := range parsedSelectorStrings {
+		klog.V(1).Infof("Search redactor specs from [%q] namespace using %q selector", strings.Join(namespaces, ", "), parsedSelectorString)
+		for _, ns := range namespaces {
+			redactorsFromSecrets, err := specs.LoadFromSecretMatchingLabel(client, parsedSelectorString, ns, constants.RedactorKey)
+			if err != nil {
+				if !k8serrors.IsForbidden(err) {
+					klog.Errorf("failed to load support bundle spec from secrets: %s", err)
+				} else {
+					klog.Warningf("Reading secrets from %q namespace forbidden", ns)
+				}
+			}
+			redactorsFromCluster = append(redactorsFromCluster, redactorsFromSecrets...)
+
+			redactorsFromConfigMaps, err := specs.LoadFromConfigMapMatchingLabel(client, parsedSelectorString, ns, constants.RedactorKey)
+			if err != nil {
+				if !k8serrors.IsForbidden(err) {
+					klog.Errorf("failed to load support bundle spec from configmap: %s", err)
+				} else {
+					klog.Warningf("Reading configmaps from %q namespace forbidden", ns)
+				}
+			}
+			redactorsFromCluster = append(redactorsFromCluster, redactorsFromConfigMaps...)
+		}
+	}
+
+	for _, redactor := range redactorsFromCluster {
+		multidocs := strings.Split(string(redactor), "\n---\n")
+		parsedRedactors, err := supportbundle.ParseRedactorsFromDocs(multidocs)
+		if err != nil {
+			klog.Errorf("failed to parse redactors from doc:  %s", err)
+		}
+
+		redactors.Spec.Redactors = append(redactors.Spec.Redactors, parsedRedactors...)
+	}
+
+	return parsedBundle, redactors, nil
 }
 
 func parseTimeFlags(v *viper.Viper) (*time.Time, error) {
@@ -382,29 +457,6 @@ func parseTimeFlags(v *viper.Viper) (*time.Time, error) {
 	return &sinceTime, nil
 }
 
-func shouldRetryRequest(err error) bool {
-	if strings.Contains(err.Error(), "x509") && canTryInsecure() {
-		httputil.AddTransport(&http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		})
-		return true
-	}
-	return false
-}
-
-func canTryInsecure() bool {
-	if !isatty.IsTerminal(os.Stdout.Fd()) {
-		return false
-	}
-	prompt := promptui.Prompt{
-		Label:     "Connection appears to be insecure. Would you like to attempt to create a support bundle anyway?",
-		IsConfirm: true,
-	}
-
-	_, err := prompt.Run()
-	return err == nil
-}
-
 type analysisOutput struct {
 	Analysis    []*analyzer.AnalyzeResult
 	ArchivePath string

cmd/troubleshoot/cli/util.go

@@ -1,24 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pkg/errors"
-)
-
-func findFileName(basename, extension string) (string, error) {
-	n := 1
-	name := basename
-	for {
-		filename := name + "." + extension
-		if _, err := os.Stat(filename); os.IsNotExist(err) {
-			return filename, nil
-		} else if err != nil {
-			return "", errors.Wrap(err, "check file exists")
-		}
-
-		name = fmt.Sprintf("%s (%d)", basename, n)
-		n = n + 1
-	}
-}

cmd/troubleshoot/cli/version.go

@@ -3,13 +3,8 @@ package cli
 import (
 	"fmt"
 
-	"io/ioutil"
-	"path/filepath"
-
-	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/version"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 )
 
 func VersionCmd() *cobra.Command {
@@ -25,27 +20,3 @@ func VersionCmd() *cobra.Command {
 	}
 	return cmd
 }
-
-const VersionFilename = "version.yaml"
-
-func writeVersionFile(path string) error {
-	version := troubleshootv1beta2.SupportBundleVersion{
-		ApiVersion: "troubleshoot.sh/v1beta2",
-		Kind:       "SupportBundle",
-		Spec: troubleshootv1beta2.SupportBundleVersionSpec{
-			VersionNumber: version.Version(),
-		},
-	}
-	b, err := yaml.Marshal(version)
-	if err != nil {
-		return err
-	}
-
-	filename := filepath.Join(path, VersionFilename)
-	err = ioutil.WriteFile(filename, b, 0644)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

cmd/util/profilers.go

@@ -0,0 +1,83 @@
+package util
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	cpuProfileFile *os.File
+)
+
+// StartProfiling starts profiling CPU and memory usage if either --cpuprofile or
+// --memprofile flags were set and bound to viper configurations respectively.
+func StartProfiling() error {
+	v := viper.GetViper()
+	if v.GetString("cpuprofile") != "" {
+		var err error
+		cpuProfileFile, err = os.Create(v.GetString("cpuprofile"))
+		if err != nil {
+			return fmt.Errorf("could not create CPU profile: %w", err)
+		}
+		if err := pprof.StartCPUProfile(cpuProfileFile); err != nil {
+			cpuProfileFile.Close()
+			cpuProfileFile = nil
+			return fmt.Errorf("could not start CPU profile: %w", err)
+		}
+	}
+	return nil
+}
+
+// StopProfiling stops profiling CPU and memory usage and writes the results to
+// the files specified by --cpuprofile and --memprofile flags respectively.
+func StopProfiling() error {
+	v := viper.GetViper()
+	errs := []string{}
+
+	// Stop CPU profiling if it was started
+	if cpuProfileFile != nil {
+		pprof.StopCPUProfile()
+		err := cpuProfileFile.Close()
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+		cpuProfileFile = nil
+	}
+
+	if v.GetString("memprofile") != "" {
+		f, err := os.Create(v.GetString("memprofile"))
+		if err != nil {
+			errs = append(errs, err.Error())
+			goto BREAK_FROM_MEMPROFILE_IF
+		}
+		runtime.GC() // get up-to-date statistics
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			errs = append(errs, err.Error())
+			goto BREAK_FROM_MEMPROFILE_IF
+		}
+		err = f.Close()
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+	}
+
+BREAK_FROM_MEMPROFILE_IF:
+	if len(errs) > 0 {
+		return fmt.Errorf("errors while stopping profiling: [%s]", strings.Join(errs, ", "))
+	}
+
+	return nil
+}
+
+// AddProfilingFlags adds the --cpuprofile and --memprofile flags to the given command.
+func AddProfilingFlags(cmd *cobra.Command) {
+	// Persistent flags to make available to subcommands
+	cmd.PersistentFlags().String("cpuprofile", "", "File path to write cpu profiling data")
+	cmd.PersistentFlags().String("memprofile", "", "File path to write memory profiling data")
+}

config/crds/troubleshoot.replicated.com_analyzers.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: analyzers.troubleshoot.replicated.com
 spec:
@@ -746,9 +745,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_collectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: collectors.troubleshoot.replicated.com
 spec:
@@ -329,9 +328,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_preflights.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: preflights.troubleshoot.replicated.com
 spec:
@@ -1003,9 +1002,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_redactors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: redactors.troubleshoot.replicated.com
 spec:
@@ -79,9 +78,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.replicated.com_supportbundles.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: supportbundles.troubleshoot.replicated.com
 spec:
@@ -1032,9 +1031,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_analyzers.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: analyzers.troubleshoot.sh
 spec:
@@ -91,6 +90,53 @@ spec:
                       - namespace
                       - outcomes
                       type: object
+                    certificates:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     clusterPodStatuses:
                       properties:
                         annotations:
@@ -142,6 +188,75 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    clusterResource:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        clusterScoped:
+                          type: boolean
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        expectedValue:
+                          type: string
+                        kind:
+                          type: string
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        regex:
+                          type: string
+                        regexGroups:
+                          type: string
+                        strict:
+                          type: BoolString
+                        yamlPath:
+                          type: string
+                      required:
+                      - clusterScoped
+                      - kind
+                      - name
+                      - outcomes
+                      - yamlPath
+                      type: object
                     clusterVersion:
                       properties:
                         annotations:
@@ -617,6 +732,8 @@ spec:
                           type: BoolString
                         fileName:
                           type: string
+                        jsonPath:
+                          type: string
                         outcomes:
                           items:
                             properties:
@@ -710,6 +827,58 @@ spec:
                       - namespace
                       - outcomes
                       type: object
+                    mssql:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        fileName:
+                          type: string
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - collectorName
+                      - outcomes
+                      type: object
                     mysql:
                       properties:
                         annotations:
@@ -774,6 +943,8 @@ spec:
                           type: BoolString
                         filters:
                           properties:
+                            architecture:
+                              type: string
                             cpuAllocatable:
                               type: string
                             cpuCapacity:
@@ -1500,6 +1671,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    certificatesCollection:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     cpu:
                       properties:
                         annotations:
@@ -1990,6 +2210,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -2186,6 +2455,67 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    textAnalyze:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        excludeFiles:
+                          items:
+                            type: string
+                          type: array
+                        fileName:
+                          type: string
+                        ignoreIfNoFiles:
+                          type: boolean
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        regex:
+                          type: string
+                        regexGroups:
+                          type: string
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     time:
                       properties:
                         annotations:
@@ -2235,6 +2565,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
             type: object
@@ -2244,9 +2623,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_hostcollectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: hostcollectors.troubleshoot.sh
 spec:
@@ -144,6 +143,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    certificatesCollection:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     cpu:
                       properties:
                         annotations:
@@ -634,6 +682,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -830,6 +927,67 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    textAnalyze:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        excludeFiles:
+                          items:
+                            type: string
+                          type: array
+                        fileName:
+                          type: string
+                        ignoreIfNoFiles:
+                          type: boolean
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        regex:
+                          type: string
+                        regexGroups:
+                          type: string
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     time:
                       properties:
                         annotations:
@@ -879,6 +1037,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
               collectors:
@@ -905,6 +1112,30 @@ spec:
                       - certificatePath
                       - keyPath
                       type: object
+                    certificatesCollection:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        paths:
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - paths
+                      type: object
+                    copy:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        path:
+                          type: string
+                      required:
+                      - path
+                      type: object
                     cpu:
                       properties:
                         collectorName:
@@ -1128,6 +1359,20 @@ spec:
                       - args
                       - command
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         amzn:
@@ -1150,6 +1395,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        centos9:
+                          items:
+                            type: string
+                          type: array
                         collectorName:
                           type: string
                         exclude:
@@ -1166,6 +1415,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        ol9:
+                          items:
+                            type: string
+                          type: array
                         rhel:
                           items:
                             type: string
@@ -1178,6 +1431,22 @@ spec:
                           items:
                             type: string
                           type: array
+                        rhel9:
+                          items:
+                            type: string
+                          type: array
+                        rocky:
+                          items:
+                            type: string
+                          type: array
+                        rocky8:
+                          items:
+                            type: string
+                          type: array
+                        rocky9:
+                          items:
+                            type: string
+                          type: array
                         ubuntu:
                           items:
                             type: string
@@ -1244,6 +1513,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
             type: object
@@ -1253,9 +1535,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_hostpreflights.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: hostpreflights.troubleshoot.sh
 spec:
@@ -144,6 +143,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    certificatesCollection:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     cpu:
                       properties:
                         annotations:
@@ -634,6 +682,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    subnetAvailable:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     systemPackages:
                       properties:
                         annotations:
@@ -830,6 +927,67 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    textAnalyze:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        excludeFiles:
+                          items:
+                            type: string
+                          type: array
+                        fileName:
+                          type: string
+                        ignoreIfNoFiles:
+                          type: boolean
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        regex:
+                          type: string
+                        regexGroups:
+                          type: string
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                     time:
                       properties:
                         annotations:
@@ -879,6 +1037,55 @@ spec:
                       required:
                       - outcomes
                       type: object
+                    udpPortStatus:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        checkName:
+                          type: string
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        outcomes:
+                          items:
+                            properties:
+                              fail:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              pass:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                              warn:
+                                properties:
+                                  message:
+                                    type: string
+                                  uri:
+                                    type: string
+                                  when:
+                                    type: string
+                                type: object
+                            type: object
+                          type: array
+                        strict:
+                          type: BoolString
+                      required:
+                      - outcomes
+                      type: object
                   type: object
                 type: array
               collectors:
@@ -905,6 +1112,30 @@ spec:
                       - certificatePath
                       - keyPath
                       type: object
+                    certificatesCollection:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        paths:
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - paths
+                      type: object
+                    copy:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        path:
+                          type: string
+                      required:
+                      - path
+                      type: object
                     cpu:
                       properties:
                         collectorName:
@@ -1128,6 +1359,20 @@ spec:
                       - args
                       - command
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         amzn:
@@ -1150,6 +1395,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        centos9:
+                          items:
+                            type: string
+                          type: array
                         collectorName:
                           type: string
                         exclude:
@@ -1166,6 +1415,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        ol9:
+                          items:
+                            type: string
+                          type: array
                         rhel:
                           items:
                             type: string
@@ -1178,6 +1431,22 @@ spec:
                           items:
                             type: string
                           type: array
+                        rhel9:
+                          items:
+                            type: string
+                          type: array
+                        rocky:
+                          items:
+                            type: string
+                          type: array
+                        rocky8:
+                          items:
+                            type: string
+                          type: array
+                        rocky9:
+                          items:
+                            type: string
+                          type: array
                         ubuntu:
                           items:
                             type: string
@@ -1244,6 +1513,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
               remoteCollectors:
@@ -1463,6 +1745,20 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         collectorName:
@@ -1519,6 +1815,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
             type: object
@@ -1528,9 +1837,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_redactors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: redactors.troubleshoot.sh
 spec:
@@ -79,9 +78,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crds/troubleshoot.sh_remotecollectors.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.11.2
   creationTimestamp: null
   name: remotecollectors.troubleshoot.sh
 spec:
@@ -284,6 +283,20 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    subnetAvailable:
+                      properties:
+                        CIDRRangeAlloc:
+                          type: string
+                        collectorName:
+                          type: string
+                        desiredCIDR:
+                          type: integer
+                        exclude:
+                          type: BoolString
+                      required:
+                      - CIDRRangeAlloc
+                      - desiredCIDR
+                      type: object
                     systemPackages:
                       properties:
                         collectorName:
@@ -340,6 +353,19 @@ spec:
                         exclude:
                           type: BoolString
                       type: object
+                    udpPortStatus:
+                      properties:
+                        collectorName:
+                          type: string
+                        exclude:
+                          type: BoolString
+                        interface:
+                          type: string
+                        port:
+                          type: integer
+                      required:
+                      - port
+                      type: object
                   type: object
                 type: array
               nodeSelector:
@@ -353,9 +379,3 @@ spec:
         type: object
     served: true
     storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

deploy/.goreleaser.yaml

@@ -5,13 +5,19 @@ release:
     name: troubleshoot
 builds:
   - id: preflight
+    # NOTE: if you add any additional goos/goarch values, ensure you update ../.github/workflows/build-test-deploy.yaml
+    # specifically the matrix values for goreleaser-test
     goos:
       - linux
       - darwin
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/preflight/main.go
@@ -30,7 +36,11 @@ builds:
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/troubleshoot/main.go
@@ -49,7 +59,11 @@ builds:
       - windows
     goarch:
       - amd64
+      - arm
       - arm64
+    ignore:
+      - goos: windows
+        goarch: arm
     env:
       - CGO_ENABLED=0
     main: cmd/collect/main.go

deploy/krew/preflight.yaml

@@ -16,6 +16,28 @@ spec:
     - from: LICENSE
       to: .
     bin: preflight
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm64
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/preflight_linux_arm64.tar.gz" .TagName }}
+    files:
+    - from: preflight
+      to: .
+    - from: LICENSE
+      to: .
+    bin: preflight
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/preflight_linux_arm.tar.gz" .TagName }}
+    files:
+    - from: preflight
+      to: .
+    - from: LICENSE
+      to: .
+    bin: preflight
   - selector:
       matchLabels:
         os: darwin
@@ -37,7 +59,7 @@ spec:
       to: .
     - from: LICENSE
       to: .
-    bin: preflight    
+    bin: preflight
   - selector:
       matchLabels:
         os: windows

deploy/krew/support-bundle.yaml

@@ -16,6 +16,28 @@ spec:
     - from: LICENSE
       to: .
     bin: support-bundle
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm64
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/support-bundle_linux_arm64.tar.gz" .TagName }}
+    files:
+    - from: support-bundle
+      to: .
+    - from: LICENSE
+      to: .
+    bin: support-bundle
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm
+    {{addURIAndSha "https://github.com/replicatedhq/troubleshoot/releases/download/{{ .TagName }}/support-bundle_linux_arm.tar.gz" .TagName }}
+    files:
+    - from: support-bundle
+      to: .
+    - from: LICENSE
+      to: .
+    bin: support-bundle
   - selector:
       matchLabels:
         os: darwin
@@ -37,7 +59,7 @@ spec:
       to: .
     - from: LICENSE
       to: .
-    bin: support-bundle    
+    bin: support-bundle
   - selector:
       matchLabels:
         os: windows

design/proposal-concurrent-collectors.md

@@ -0,0 +1,68 @@
+# Run collectors concurrently
+ 
+## Goals
+
+Increase the speed at which collectors complete.
+
+## Non Goals
+
+
+## Background
+
+Currently all collectors run sequentially, in the order in which they are specified in the spec.  This is implied, and has not been guaranteed.  With the addition of the ability to supply multiple specs, the order is no longer predicatble.
+
+Although sequence is not guaranteed for any collectors, #768 ensures that the `clusterResources` collector runs first, so that the list of resources does not include those that are started during the Troubleshoot collection sequence.
+
+The performance for support-bundle and preflight runs is important to the end user experience of the product.  There are examples where a support bundle takes multiple minutes to complete, and folks are left wondering if the process has hung or crashed.  Preflights that take too long will simply be skipped, which leads to problems with installation being difficult to identify.
+
+## High-Level Design
+
+Switch collectors and host collectors to use goroutines to collect (concurrently).  This should run the list of collectors of `clusterResources` type first, then the remainder concurrently when `clusterResources` is complete.
+
+## Detailed Design
+
+Tasks that can be split to separate merges (in order):
+
+* Switch all collectors to use goroutines to collect (concurrently).
+  * In `supportbundle.runCollectors()`, refactor to use a `go func()` pattern to run all the collectors that are not `clusterResources` concurrently.  Check that the first in the list is `clusterResources` and run that separately, prior to the others.
+  * This task should introduce an optional concurrency limit field, default to, say, 10, which limits the number of goroutines/collectors that can run at any given time.
+  * This task should improve the collection speed of in-cluster collectors and preflights
+
+* Switch Host Collectors to use goroutines for concurrent collection.
+  * In `supportbundle.runHostCollectors()`, refactor to use a `go func()` pattern for each collector.
+  * No need to honor the same concurrency limit as host collectors do not load the Kubernetes API
+
+* Switch Host Preflights to use goroutines - `preflight.CollectHost()` runs `go func()` for each collector.
+
+* Switch in-cluster Preflights to use goroutines - `preflight.Collect()` runs `go func()` for each collector.
+  * Ensure `clusterResources` runs in advance of the others so that pods started by collectors do not produce an error in the results.
+
+## Limitations
+
+Relies on splitting up the list of collectors into `clusterResources` and others.
+
+Collisions between specs (e.g. if `runPod` has multiple specs with the same name) are not handled.  At the current stage with collectors running sequentially, if two collectors write to the same file the second overwrites the data from the first.  If collectors run concurrently, there can be issues creating multiple pods with the same name, and issues with concurrent access to the target file.  This issue is tracked in [#895](https://github.com/replicatedhq/troubleshoot/issues/895)
+
+This proposal excludes Remote Collectors.
+## Assumptions
+
+* The Kubernetes API can handle the load of collectors running concurrently, including when the pod logs collector runs.
+
+## Testing
+
+Any new function will need unit tests.
+
+The existing Collect functions will need unit tests altered (or added).
+
+We will need to ensure that `clusterResources` is collected prior to others.
+
+## Documentation
+
+There should be a note in https://troubleshoot.sh/docs/collect/ ensuring that folks do not expect the collectors to run in any particular order.
+
+## Alternatives Considered
+
+None.
+## Security Considerations
+
+None identified.

docs/design/proposal-sbctl-integration.md

@@ -0,0 +1,120 @@
+# Integrate SBCTL with Troubleshoot
+
+## Goals
+
+Make sbctl easier to maintain by consolidating code bases.
+
+Reduce code duplication.
+
+Improve end-user experience and improve discoverability of sbctl.
+
+## Non Goals
+
+## Background
+sbctl is our command line tool for examining K8s resources inside of Support Bundles generated by our troubleshoot project. With both projects having separate code bases we see the following problems:
+
+* Most users are familiar with the binaries from troubleshoot (support-bundle, preflight, etc.), but are unaware of the existince of sbctl because it's a separate project
+* sbctl and troubleshoot are naturally coupled together due to sbctl's reliance on support bundles to operate. Them being in separate repos leads to duplication of code in both projects increasing maintenance burden.
+
+## High-Level Design
+Because the functionality of sbctl and Troubleshoot are so tightly coupled together, they should live in the same repository. Moreover, as opposed to maintaing a separate binary for sbctl, all sbctl commands should become subcommands of `support-bundle`
+
+## Detailed Design
+The `serve` and `shell` commands become subcommands of the `support-bundle` CLI.
+
+```
+./bin/support-bundle --help
+A support bundle is an archive of files, output, metrics and state
+from a server that can be used to assist when troubleshooting a Kubernetes cluster.
+
+Usage:
+  support-bundle [url] [flags]
+  support-bundle [command]
+
+Available Commands:
+  analyze     analyze a support bundle
+  serve       Start API server
+  shell       Start interractive shell
+  completion  Generate the autocompletion script for the specified shell
+  ...
+```
+
+Similiar to the [Analyze](https://github.com/replicatedhq/troubleshoot/blob/main/cmd/troubleshoot/cli/analyze.go) subcommand, we add `serve.go` and `shell.go` to `cmd/troubleshoot/cli`
+
+### Standardize Cluster Resources Collector
+* In `pkg/collect` standardize the root directory location of the Cluster Resources collector with a variable instead of repeatedly specifying it as a string - https://github.com/replicatedhq/troubleshoot/blob/main/pkg/collect/cluster_resources.go#L121.
+* Import the directory the Cluster Resources collector uses from `pkg/collect` for usage in the APIs - https://github.com/replicatedhq/sbctl/blob/main/pkg/api/server.go#L220. This is an example of the way we can start making the sbctl code more efficient with this change.
+
+cluster_resources.go
+```go
+const ClusterResourcesPath = `cluster-resources`
+...
+
+path := filepath.Join(ClusterResourcesPath, "namespaces.json")
+output.SaveResult(c.BundlePath, path, bytes.NewBuffer(namespaces))
+```
+
+sbctl-server.go
+```go
+import (
+    ...
+    "github.com/replicatedhq/troubleshoot/pkg/collect"
+    ...
+)
+...
+
+dirName := filepath.Join(collect.ClusterResourcesPath, fmt.Sprintf("%s", resource))
+```
+
+### Package Structure
+```
+cmd
+    serve/
+    shell/
+pkg
+    serve/
+    shell/
+tests           # API tests, integration tests etc reside here.
+    serve/      # sbctl BDD tests.
+        # all tests and test fixtures moved here
+```
+
+### Makefile Changes
+
+We'll need to add make targets specific to these commands similar to the current ones
+```
+make serve run-serve
+make shell run-shell
+make ginkgo
+```
+
+### Docs 
+In troubleshoot.sh docs, add a new sub section under "Support Bundle" describing then new serve & shell subcommands.
+
+### Other Changes
+We would need to merge some of the support bundle functionality we see in both projects like so
+
+- Create a new sbutil module to store reusable support bundle functionality. At least `analyze` & `serve` modules will depend on this module
+
+```
+pkg
+    sbutil/
+```
+
+- Merge `troubleshoot/pkg/analyze/download.ExtractTroubleshootBundle` & `sbctl/pkg/sbctl/support-bundle.ExtractBundle` that duplicate support bundle archive extraction
+- Build on `sbctl/pkg/sbctl/support-bundle.ClusterData` to have it expose well known paths (ClusterResourcesDir, Version files, analysis results etc) and unmarshalled resources (this bit can be built on a need to have basis) in a support bundle especially once [Add generic kubernetes resource analyzer #780 PR](https://github.com/replicatedhq/troubleshoot/pull/780) is merged.
+- Build on `sbctl/pkg/sbctl/support-bundle.FindClusterData` to perform pre-validation of a support bundle to ensure it is valid before running any applications.
+
+## Limitations
+
+## Assumptions
+
+## Testing
+We will need to have a separate directory for sbctl (ginkgo BDD) tests. The test harness used is ginkgo cli instead of go test and there is no straight forward way of separating these tests. Also, from a directory structure point of view, such integration tests usually reside in modules in a root level tests directory. Changes to the test code need to be minimal, ideally no change, cause they will be used to ensure any refactored code does not break. We could now easily introduce tests which ensure that changes to collectors don't break APIs in sbctl.
+
+## Alternatives Considered
+* Have troubleshoot as a library dependency in sbctl keeping it as a separate binary and repo
+* Move sbctl to the Troubleshoot library "as is" and continue maintaing a separate binary
+* Leave things as they are
+
+## Security Considerations

docs/preflight.md

@@ -17,7 +17,7 @@ preflight [url] [flags]
       --as string                      Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
       --as-group stringArray           Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
       --as-uid string                  UID to impersonate for the operation.
-      --cache-dir string               Default cache directory (default "/Users/xavpaice/.kube/cache")
+      --cache-dir string               Default cache directory (default "$HOME/.kube/cache")
       --certificate-authority string   Path to a cert file for the certificate authority
       --client-certificate string      Path to a client certificate file for TLS
       --client-key string              Path to a client key file for TLS
@@ -26,12 +26,15 @@ preflight [url] [flags]
       --collector-image string         the full name of the collector image to use
       --collector-pullpolicy string    the pull policy of the collector image
       --context string                 The name of the kubeconfig context to use
+      --cpuprofile string              File path to write cpu profiling data
       --debug                          enable debug logging
+      --disable-compression            If true, opt-out of response compression for all requests to the server
       --format string                  output format, one of human, json, yaml. only used when interactive is set to false (default "human")
   -h, --help                           help for preflight
       --insecure-skip-tls-verify       If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
       --interactive                    interactive preflights (default true)
       --kubeconfig string              Path to the kubeconfig file to use for CLI requests.
+      --memprofile string              File path to write memory profiling data
   -n, --namespace string               If present, the namespace scope for this CLI request
   -o, --output string                  specify the output file path for the preflight checks
       --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
@@ -42,10 +45,12 @@ preflight [url] [flags]
       --tls-server-name string         Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used
       --token string                   Bearer token for authentication to the API server
       --user string                    The name of the kubeconfig user to use
+  -v, --v Level                        number for the log level verbosity
 ```
 
 ### SEE ALSO
 
+* [preflight oci-fetch](preflight_oci-fetch.md)	 - Fetch a preflight from an OCI registry and print it to standard out
 * [preflight version](preflight_version.md)	 - Print the current version and exit
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 8-Jun-2023

docs/preflight_oci-fetch.md

@@ -0,0 +1,36 @@
+## preflight oci-fetch
+
+Fetch a preflight from an OCI registry and print it to standard out
+
+```
+preflight oci-fetch [URI] [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for oci-fetch
+```
+
+### Options inherited from parent commands
+
+```
+      --collect-without-permissions   always run preflight checks even if some require permissions that preflight does not have (default true)
+      --collector-image string        the full name of the collector image to use
+      --collector-pullpolicy string   the pull policy of the collector image
+      --cpuprofile string             File path to write cpu profiling data
+      --debug                         enable debug logging
+      --format string                 output format, one of human, json, yaml. only used when interactive is set to false (default "human")
+      --interactive                   interactive preflights (default true)
+      --memprofile string             File path to write memory profiling data
+  -o, --output string                 specify the output file path for the preflight checks
+      --selector string               selector (label query) to filter remote collection nodes on.
+      --since string                  force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.
+      --since-time string             force pod logs collectors to return logs after a specific date (RFC3339)
+```
+
+### SEE ALSO
+
+* [preflight](preflight.md)	 - Run and retrieve preflight checks in a cluster
+
+###### Auto generated by spf13/cobra on 8-Jun-2023

docs/preflight_version.md

@@ -22,9 +22,11 @@ preflight version [flags]
       --collect-without-permissions   always run preflight checks even if some require permissions that preflight does not have (default true)
       --collector-image string        the full name of the collector image to use
       --collector-pullpolicy string   the pull policy of the collector image
+      --cpuprofile string             File path to write cpu profiling data
       --debug                         enable debug logging
       --format string                 output format, one of human, json, yaml. only used when interactive is set to false (default "human")
       --interactive                   interactive preflights (default true)
+      --memprofile string             File path to write memory profiling data
   -o, --output string                 specify the output file path for the preflight checks
       --selector string               selector (label query) to filter remote collection nodes on.
       --since string                  force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.
@@ -35,4 +37,4 @@ preflight version [flags]
 
 * [preflight](preflight.md)	 - Run and retrieve preflight checks in a cluster
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle.md

@@ -8,7 +8,7 @@ A support bundle is an archive of files, output, metrics and state
 from a server that can be used to assist when troubleshooting a Kubernetes cluster.
 
 ```
-support-bundle [url] [flags]
+support-bundle [urls...] [flags]
 ```
 
 ### Options
@@ -17,37 +17,42 @@ support-bundle [url] [flags]
       --as string                      Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
       --as-group stringArray           Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
       --as-uid string                  UID to impersonate for the operation.
-      --cache-dir string               Default cache directory (default "/Users/xavpaice/.kube/cache")
+      --cache-dir string               Default cache directory (default "$HOME/.kube/cache")
       --certificate-authority string   Path to a cert file for the certificate authority
       --client-certificate string      Path to a client certificate file for TLS
       --client-key string              Path to a client key file for TLS
       --cluster string                 The name of the kubeconfig cluster to use
       --collect-without-permissions    always generate a support bundle, even if it some require additional permissions (default true)
       --context string                 The name of the kubeconfig context to use
-      --debug                          enable debug logging
+      --cpuprofile string              File path to write cpu profiling data
+      --debug                          enable debug logging. This is equivalent to --v=0
+      --disable-compression            If true, opt-out of response compression for all requests to the server
   -h, --help                           help for support-bundle
       --insecure-skip-tls-verify       If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
       --interactive                    enable/disable interactive mode (default true)
       --kubeconfig string              Path to the kubeconfig file to use for CLI requests.
       --load-cluster-specs             enable/disable loading additional troubleshoot specs found within the cluster. required when no specs are provided on the command line
+      --memprofile string              File path to write memory profiling data
   -n, --namespace string               If present, the namespace scope for this CLI request
       --no-uri                         When this flag is used, Troubleshoot does not attempt to retrieve the bundle referenced by the uri: field in the spec.`
   -o, --output string                  specify the output file path for the support bundle
       --redact                         enable/disable default redactions (default true)
       --redactors strings              names of the additional redactors to use
       --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
-  -l, --selector strings               selector to filter on for loading additional support bundle specs found in secrets within the cluster (default [troubleshoot.io/kind=supportbundle-spec])
+  -l, --selector strings               selector to filter on for loading additional support bundle specs found in secrets within the cluster (default [troubleshoot.sh/kind=support-bundle])
   -s, --server string                  The address and port of the Kubernetes API server
       --since string                   force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h.
       --since-time string              force pod logs collectors to return logs after a specific date (RFC3339)
       --tls-server-name string         Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used
       --token string                   Bearer token for authentication to the API server
       --user string                    The name of the kubeconfig user to use
+  -v, --v Level                        number for the log level verbosity
 ```
 
 ### SEE ALSO
 
 * [support-bundle analyze](support-bundle_analyze.md)	 - analyze a support bundle
+* [support-bundle redact](support-bundle_redact.md)	 - Redact information from a generated support bundle archive
 * [support-bundle version](support-bundle_version.md)	 - Print the current version and exit
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 8-Jun-2023

docs/support-bundle_analyze.md

@@ -19,8 +19,15 @@ support-bundle analyze [url] [flags]
       --quiet           enable/disable error messaging and only show parseable output
 ```
 
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
 ### SEE ALSO
 
 * [support-bundle](support-bundle.md)	 - Generate a support bundle
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle_redact.md

@@ -0,0 +1,42 @@
+## support-bundle redact
+
+Redact information from a generated support bundle archive
+
+### Synopsis
+
+Redaction is the process of masking sensitive information from collected data in a support bundle.
+This is done using rules defined in the list of redactor manifests provided in the [urls...] command line
+argument. Default built in redactors will also be run, but these would have been run when the support
+bundle was generated. After redaction, the support bundle is archived once more. The resulting file will
+be stored in the current directory in the path provided by the --output flag.
+
+The [urls...] argument is a list of either oci://.., http://.., https://.. or local paths to yaml files.
+
+For more information on redactors visit https://troubleshoot.sh/docs/redact/
+
+
+```
+support-bundle redact [urls...] [flags]
+```
+
+### Options
+
+```
+      --bundle string   file path of the support bundle archive to redact
+  -h, --help            help for redact
+  -o, --output string   file path of where to save the redacted support bundle archive (default "redacted-support-bundle-YYYY-MM-DDTHH_MM_SS.tar.gz")
+  -q, --quiet           enable/disable error messaging and only show parseable output
+```
+
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
+### SEE ALSO
+
+* [support-bundle](support-bundle.md)	 - Generate a support bundle
+
+###### Auto generated by spf13/cobra on 3-Jan-2023

docs/support-bundle_version.md

@@ -16,8 +16,15 @@ support-bundle version [flags]
   -h, --help   help for version
 ```
 
+### Options inherited from parent commands
+
+```
+      --cpuprofile string   File path to write cpu profiling data
+      --memprofile string   File path to write memory profiling data
+```
+
 ### SEE ALSO
 
 * [support-bundle](support-bundle.md)	 - Generate a support bundle
 
-###### Auto generated by spf13/cobra on 21-Nov-2022
+###### Auto generated by spf13/cobra on 3-Jan-2023

examples/collect/host/udp-port.yaml

@@ -0,0 +1,9 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostCollector
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472

examples/collect/remote/udp-port.yaml

@@ -0,0 +1,9 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostCollector
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472

examples/preflight/e2e.yaml

@@ -7,10 +7,21 @@ spec:
     - data:
         name: config/replicas.txt
         data: "5"
-    - run:
-        collectorName: "static-hi"
-        image: 'alpine:3'
-        command: ["echo", "hi static!"]
+    - runPod:
+        collectorName: "static-hi-1"
+        podSpec:
+          containers:
+          - name: static-hi
+            image: alpine:3
+            command: ["echo", "hi static!"]
+    ## This collector is intentionally duplicated to test same pod should be terminated after success
+    - runPod:
+        collectorName: "static-hi-2"
+        podSpec:
+          containers:
+          - name: static-hi
+            image: alpine:3
+            command: ["echo", "hi static!"]
   analyzers:
     - clusterVersion:
         outcomes:
@@ -44,7 +55,16 @@ spec:
               message: You have at least 5 replicas
     - textAnalyze:
         checkName: Said hi!
-        fileName: /static-hi.log
+        fileName: /static-hi-1.log
+        regex: 'hi static'
+        outcomes:
+          - fail:
+              message: Didn't say hi.
+          - pass:
+              message: Said hi!
+    - textAnalyze:
+        checkName: Said hi!
+        fileName: /static-hi-2.log
         regex: 'hi static'
         outcomes:
           - fail:

examples/preflight/host/subnet-available.yaml

@@ -0,0 +1,19 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: subnet-available
+spec:
+  collectors:
+    # would output yes/no depending if there is a /22 available in 10.0.0.0/8
+    - subnetAvailable:
+        CIDRRangeAlloc: "10.0.0.0/8"
+        desiredCIDR: 22
+  analyzers:
+    - subnetAvailable:
+        outcomes:
+          - fail:
+            when: "no-subnet-available"
+            message: failed to find available subnet
+          - pass:
+            when: "a-subnet-is-available"
+            message: available /22 subnet found

examples/preflight/host/system-packages.yaml

@@ -21,6 +21,15 @@ spec:
         rhel8:
           - nfs-utils
           - openssl
+        rhel9:
+          - nfs-utils
+          - openssl
+        rocky8:
+          - nfs-utils
+          - openssl
+        rocky9:
+          - nfs-utils
+          - openssl
         centos:
           - iscsi-initiator-utils
         centos7:
@@ -28,6 +37,9 @@ spec:
         centos8:
           - nfs-utils
           - openssl
+        centos9:
+          - nfs-utils
+          - openssl
         ol:
           - lvm2
         ol7:
@@ -35,6 +47,8 @@ spec:
           - openssl
         ol8:
           - nfs-utils
+        ol9:
+          - nfs-utils
         amzn:
           - libzstd
         amzn2:

examples/preflight/host/udp-port.yaml

@@ -0,0 +1,24 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472
+  analyzers:
+    - udpPortStatus:
+        collectorName: flannel
+        outcomes:
+          - fail:
+              when: "address-in-use"
+              message: Another process was already listening on port 8472.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 8472 is open
+          - warn:
+              message: Unexpected port status

examples/preflight/mssql.yaml

@@ -0,0 +1,70 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: Preflight
+metadata:
+  name: sample
+spec:
+  collectors:
+    - mssql:
+        collectorName: mssql
+        uri: sqlserver://SA:nac3lle-ch@stise-bOMBay-wag@db.lab.shortrib.net:1433/master
+  analyzers:
+    - mssql:
+        checkName: Must be SQLServer 15.x or later
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version < 15.x"
+              message: The SQLServer must be at least version 15
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must be SQLServer 14 or earlier
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version > 14.x"
+              message: The SQLServer must be at most version 14
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must be SQLServer 15.0.2000.1565 
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version == 15.0.2000.1565"
+              message: The SQLServer must be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: Must not be SQLServer 15.0.2000.1565 
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == false"
+              message: Cannot connect to SQLServer
+          - fail:
+              when: "version != 15.0.2000.1565"
+              message: The SQLServer must not be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out
+    - mssql:
+        checkName: I don't want to connect to your SQL Server
+        collectorName: mssql
+        outcomes:
+          - fail:
+              when: "connected == true"
+              message: Can connect to your SQL Server
+          - fail:
+              when: "version != 15.0.2000.1565"
+              message: The SQLServer must be version 15.0.2000.1565 
+          - pass:
+              message: The SQLServer connection checks out

examples/preflight/node-resources.yaml

@@ -16,20 +16,20 @@ spec:
           - pass:
               message: This cluster has enough nodes.
     - nodeResources:
-        checkName: Must have 3 nodes with at least 6 cores
+        checkName: Must have 3 nodes with at least 6 cores each
         filters:
           cpuCapacity: "6"
         outcomes:
           - fail:
-              when: "< 3"
+              when: "count() < 3"
               message: This application requires at least 3 nodes with 6 cores each
           - pass:
-              message: This cluster has enough nodes with enough codes
+              message: This cluster has enough nodes with enough cores
     - nodeResources:
-        checkName: Must have 1 node with 16 GB (available) memory and 5 cores (on a single node)
+        checkName: Must have 1 node with 16 GB (available) memory and 10 cores (on a single node)
         filters:
           allocatableMemory: 16Gi
-          cpuCapacity: "5"
+          cpuCapacity: "10"
         outcomes:
           - fail:
               when: "< 1"
@@ -37,13 +37,32 @@ spec:
           - pass:
               message: This cluster has a node with enough memory.
     - nodeResources:
-        checkName: There must be at least 3 nodes in the cluster
+        checkName: Must have 1 node with 16 GB (available) memory and 4 cores of amd64 arch (on a single node)
+        filters:
+          allocatableMemory: 16Gi
+          cpuArchitecture: amd64
+          cpuCapacity: "4"
         outcomes:
           - fail:
-              when: "count() < 3"
-              message: This application requires at least 3 nodes in the cluster
+              when: "< 1"
+              message: This application requires at least 1 node with amd64 arch and 16GB available memory
           - pass:
-              message: This cluster haas sufficient nodes
+              message: This cluster has a node with the correct architecture and enough memory.
+    - nodeResources:
+        checkName: Must have 1 manager node with 16 GB (available) memory and 4 cores of amd64 arch
+        filters:
+          selector:
+            matchLabel:
+              node-role.kubernetes.io/master: ""
+          allocatableMemory: 16Gi
+          cpuArchitecture: amd64
+          cpuCapacity: "6"
+        outcomes:
+          - fail:
+              when: "< 1"
+              message: This application requires at least 1 manager node with amd64 arch, 16GB available memory, and the label `node-role.kubernetes.io/master=""`
+          - pass:
+              message: This cluster has a node with the correct architecture and enough memory.
     - nodeResources:
         checkName: There must be a total of at least 32Gi of memory on all nodes
         outcomes:
@@ -51,4 +70,14 @@ spec:
               when: "sum(memoryCapacity) < 32Gi"
               message: This application requires that 32Gi or more memory be available to the cluster
           - pass:
-              message: This cluster haas sufficient memory
+              message: This cluster has sufficient memory
+    - nodeResources:
+        checkName: There must be a total of at least 32Gi of memory on amd64 nodes
+        filters:
+          cpuArchitecture: amd64
+        outcomes:
+          - fail:
+              when: "sum(memoryCapacity) < 32Gi"
+              message: This application requires that 32Gi or more memory on amd64 nodes be available to the cluster
+          - pass:
+              message: This cluster has sufficient memory

examples/preflight/remote/udp-port.yaml

@@ -0,0 +1,24 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: port
+spec:
+  collectors:
+    - udpPortStatus:
+        collectorName: flannel
+        port: 8472
+  analyzers:
+    - udpPortStatus:
+        collectorName: flannel
+        outcomes:
+          - fail:
+              when: "address-in-use"
+              message: Another process was already listening on port 8472.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 8472 is open
+          - warn:
+              message: Unexpected port status

examples/redact/e2e.yaml

@@ -0,0 +1,10 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: Redactor
+metadata:
+  name: e2e-redactor
+spec:
+  redactors:
+    - name: redact-static-text
+      removals:
+        values:
+        - static

examples/sdk/helm-template/README.md

@@ -0,0 +1,14 @@
+## Parse troubleshoot specs from a helm chart
+
+This is an example of using troubleshoot's `loader.LoadSpecs` API to load specs from rendered manifests in a helm chart. The chart in this example contains a kubernetes `Secret` & `ConfigMap` with troubleshoot specs, as well as a troubleshoot custom resource. The custom resource is behind a values flag and does not get rendered by default. The code adds this flag to ensure that the manifest is rendered so as to load it.
+
+The manifests are rendered with the equivalent of `helm template --values values.yaml` where the output is a YAML multidoc. `loader.LoadSpecs` will take the YAML multidoc as an input and extract troubleshoot specs inside kuberenetes `Secrets` and `ConfigMap`s, and any troubleshoot custom resources found.
+
+This application always uses the local version of troubleshoot so as to build using the latest version of the library. This ensures that the example is kept in sync with new features.
+
+### Running
+
+```go
+go mod tidy     # sync go modules
+go run main.go  # run the application. This should print out a YAML multidoc of the loaded troubleshoot specs
+```

examples/sdk/helm-template/go.mod

@@ -0,0 +1,73 @@
+module helm-template
+
+go 1.20
+
+// Always use the local version of troubleshoot so as to build using
+// the latest version of the library. This will ensure the example
+// is kept in sync with new features
+replace github.com/replicatedhq/troubleshoot v0.0.0 => ../../../
+
+require (
+	github.com/replicatedhq/troubleshoot v0.0.0
+	helm.sh/helm/v3 v3.12.1
+	sigs.k8s.io/yaml v1.3.0
+)
+
+require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.27.3 // indirect
+	k8s.io/apiextensions-apiserver v0.27.2 // indirect
+	k8s.io/apimachinery v0.27.3 // indirect
+	k8s.io/client-go v0.27.3 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	sigs.k8s.io/controller-runtime v0.15.0 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+)

examples/sdk/helm-template/go.sum

@@ -0,0 +1,323 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
+github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
+github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
+github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
+github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+helm.sh/helm/v3 v3.12.1 h1:lzU7etZX24A6BTMXYQF3bFq0ECfD8s+fKlNBBL8AbEc=
+helm.sh/helm/v3 v3.12.1/go.mod h1:qhmSY9kcX7yH1xebe+FDMZa7E5NAeZ+LvK5j1gSln48=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y=
+k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg=
+k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo=
+k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ=
+k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
+k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
+k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8=
+k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
+k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU=
+sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

examples/sdk/helm-template/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+
+	tsloader "github.com/replicatedhq/troubleshoot/pkg/loader"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+	"sigs.k8s.io/yaml"
+)
+
+func main() {
+	// Load chart from tarball
+	// The helm loader package has a few other ways to load a chart, such as from a directory `loader.LoadDir()`
+	// or from an archive in memory `loader.LoadArchive()` etc.
+	chart, err := loader.LoadFile("mychart-0.1.0.tgz")
+	if err != nil {
+		log.Fatalf("error loading at chart: %v", err)
+	}
+
+	// Render chart to manifests. Equivalent to `helm template --values values.yaml`
+	renderedManifests, err := renderChartToManifests(chart, chartutil.Values{
+		"preflight": true,
+		"includeTroubleshootCRs": true,
+	})
+	if err != nil {
+		log.Fatalf("error rendering chart: %v", err)
+	}
+
+	fmt.Println("############# Raw helm manifests #############")
+	fmt.Println(renderedManifests)
+
+	// Load rendered manifests into a slice of troubleshoot specs
+	// 'tsKinds' will contain all the specs. This object will the the
+	// input of other troubleshoot APIs such as ones used to collect
+	// bundles, analyze bundles, redact etc.
+	ctx := context.Background()
+	tsKinds, err := tsloader.LoadSpecs(ctx, tsloader.LoadOptions{
+		RawSpec: renderedManifests,
+	})
+	if err != nil {
+		log.Fatalf("error loading troubleshoot specs: %v", err)
+	}
+
+	// Ensure we have the correct number of specs
+	if len(tsKinds.PreflightsV1Beta2) != 1 {
+		log.Fatalf("expected 1 preflight, got %d", len(tsKinds.PreflightsV1Beta2))
+	}
+	if len(tsKinds.SupportBundlesV1Beta2) != 1 {
+		log.Fatalf("expected 1 supportbundle, got %d", len(tsKinds.SupportBundlesV1Beta2))
+	}
+	if len(tsKinds.RedactorsV1Beta2) != 1 {
+		log.Fatalf("expected 1 redactor, got %d", len(tsKinds.RedactorsV1Beta2))
+	}
+
+	// Print the specs as yaml
+	fmt.Println("---\n############# Support bundle #############")
+	printSpecAsYaml(tsKinds.SupportBundlesV1Beta2[0])
+	fmt.Println("---\n############# Preflight #############")
+	printSpecAsYaml(tsKinds.PreflightsV1Beta2[0])
+	fmt.Println("---\n############# Redactor #############")
+	printSpecAsYaml(tsKinds.RedactorsV1Beta2[0])
+}
+
+func renderChartToManifests(chart *chart.Chart, inputValues chartutil.Values) (string, error) {
+	options := chartutil.ReleaseOptions{
+		Name: "my-release",	// Not mandatory, but this is how helm sets the release name
+	}
+
+	// Pull in imported values from dependencies to the parent chart
+	// https://helm.sh/docs/topics/charts/#importing-child-values-via-dependencies
+	if err := chartutil.ProcessDependencies(chart, chartutil.Values{}); err != nil {
+		return "", fmt.Errorf("failed to process chart %q dependencies: %w", chart.Name(), err)
+	}
+
+	// Gather all values necessary to render the templates
+	// imported values, values from the chart and user provided values
+	rValues, err := chartutil.ToRenderValues(chart, inputValues, options, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to render template values: %w", err)
+	}
+
+	// Render the chart templates. This will include non YAML files as well
+	renderedTemplates, err := engine.Render(chart, rValues)
+	if err != nil {
+		return "", fmt.Errorf("failed to render template: %w", err)
+	}
+
+	// Combine all rendered templates into a single yaml multidoc string
+	// Only pick up YAML files, ignore any others. Troubleshoot loader
+	// expects only valid YAML input
+	var out strings.Builder
+	for k, v := range renderedTemplates {
+		if strings.HasSuffix(k, ".yaml") || strings.HasSuffix(k, ".yml") {
+			// Add the multidoc split marker
+			out.WriteString(fmt.Sprintf("---\n# Source: %s\n%s\n", k, v))
+		}
+	}
+
+	return out.String(), nil
+}
+
+func printSpecAsYaml(v any) {
+	b, err := yaml.Marshal(v)
+	if err != nil {
+		log.Fatalf("error marshalling troubleshoot specs: %v", err)
+	}
+
+	fmt.Println(string(b))
+}

examples/support-bundle/sample-supportbundle.yaml

@@ -3,6 +3,8 @@ kind: SupportBundle
 metadata:
   name: example
 spec:
+  hostCollectors:
+    - hostOS: {}
   collectors:
     - logs:
         selector:
@@ -12,6 +14,7 @@ spec:
         limits:
           maxAge: 720h # 30*24
           maxLines: 10000
+          maxBytes: 5000000
     - logs:
         collectorName: all-logs
         name: all-logs

go.mod

@@ -4,152 +4,161 @@ go 1.19
 
 require (
 	github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412
-	github.com/blang/semver v3.5.1+incompatible
-	github.com/containers/image/v5 v5.23.1
-	github.com/docker/distribution v2.8.1+incompatible
-	github.com/fatih/color v1.13.0
-	github.com/go-logr/logr v1.2.3
+	github.com/apparentlymart/go-cidr v1.1.0
+	github.com/blang/semver/v4 v4.0.0
+	github.com/containers/image/v5 v5.25.0
+	github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
+	github.com/fatih/color v1.15.0
+	github.com/go-logr/logr v1.2.4
 	github.com/go-redis/redis/v7 v7.4.1
-	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-sql-driver/mysql v1.7.1
 	github.com/gobwas/glob v0.2.3
-	github.com/godbus/dbus v4.1.0+incompatible
+	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/handlers v1.5.1
-	github.com/hashicorp/go-getter v1.6.2
+	github.com/hashicorp/go-getter v1.7.1
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/jackc/pgx/v5 v5.1.1
+	github.com/jackc/pgx/v5 v5.4.2
 	github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e
 	github.com/manifoldco/promptui v0.9.0
-	github.com/mattn/go-isatty v0.0.16
+	github.com/mattn/go-isatty v0.0.19
 	github.com/mholt/archiver/v3 v3.5.1
-	github.com/opencontainers/image-spec v1.1.0-rc2
+	github.com/microsoft/go-mssqldb v1.3.0
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/pkg/errors v0.9.1
 	github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.6.1
+	github.com/shirou/gopsutil/v3 v3.23.6
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.14.0
-	github.com/stretchr/testify v1.8.1
+	github.com/spf13/viper v1.16.0
+	github.com/stretchr/testify v1.8.4
 	github.com/tj/go-spin v1.1.0
-	golang.org/x/sync v0.1.0
+	go.opentelemetry.io/otel v1.16.0
+	go.opentelemetry.io/otel/sdk v1.16.0
+	golang.org/x/sync v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.25.4
-	k8s.io/apiextensions-apiserver v0.25.0
-	k8s.io/apimachinery v0.25.4
-	k8s.io/apiserver v0.25.4
-	k8s.io/cli-runtime v0.25.4
-	k8s.io/client-go v0.25.4
-	k8s.io/klog/v2 v2.80.1
-	oras.land/oras-go v1.2.1
-	sigs.k8s.io/controller-runtime v0.13.1
+	k8s.io/api v0.27.4
+	k8s.io/apiextensions-apiserver v0.27.2
+	k8s.io/apimachinery v0.27.4
+	k8s.io/apiserver v0.27.3
+	k8s.io/cli-runtime v0.27.4
+	k8s.io/client-go v0.27.4
+	k8s.io/klog/v2 v2.100.1
+	oras.land/oras-go v1.2.3
+	sigs.k8s.io/controller-runtime v0.15.0
 )
 
 require (
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
-	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
+	github.com/golang-sql/sqlexp v0.1.0 // indirect
+	github.com/google/s2a-go v0.1.3 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sylabs/sif/v2 v2.8.1 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sylabs/sif/v2 v2.11.1 // indirect
+	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 )
 
 require (
-	cloud.google.com/go v0.104.0 // indirect
-	cloud.google.com/go/compute v1.12.1 // indirect
-	cloud.google.com/go/iam v0.3.0 // indirect
-	cloud.google.com/go/storage v1.23.0 // indirect
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go/iam v0.13.0 // indirect
+	cloud.google.com/go/storage v1.28.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/BurntSushi/toml v1.2.0 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.4 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
 	github.com/andybalholm/brotli v1.0.1 // indirect
-	github.com/aws/aws-sdk-go v1.43.16 // indirect
+	github.com/aws/aws-sdk-go v1.44.122 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
-	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
-	github.com/containers/ocicrypt v1.1.5 // indirect
-	github.com/containers/storage v1.43.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/containerd/containerd v1.7.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
+	github.com/containers/ocicrypt v1.1.7 // indirect
+	github.com/containers/storage v1.46.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/cli v20.10.17+incompatible // indirect
-	github.com/docker/docker v20.10.18+incompatible // indirect
+	github.com/docker/cli v23.0.1+incompatible // indirect
+	github.com/docker/docker v23.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-	github.com/felixge/httpsnoop v1.0.1 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-intervals v0.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
-	github.com/hashicorp/go-version v1.3.0 // indirect
+	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -158,57 +167,56 @@ require (
 	github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d // indirect
 	github.com/nwaples/rardecode v1.1.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
-	github.com/opencontainers/selinux v1.10.2 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
-	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.opencensus.io v0.23.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 // indirect
-	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
-	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
-	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
-	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
-	golang.org/x/text v0.4.0 // indirect
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.11.0
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.102.0 // indirect
+	google.golang.org/api v0.122.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
-	google.golang.org/grpc v1.50.1 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
-	periph.io/x/host/v3 v3.8.0
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/metrics v0.27.4
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
+	periph.io/x/host/v3 v3.8.2
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.2 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

internal/testutils/utils.go

@@ -0,0 +1,62 @@
+package testutils
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func GetTestFixture(t *testing.T, path string) string {
+	t.Helper()
+	p := filepath.Join("../../testdata", path)
+	b, err := os.ReadFile(p)
+	require.NoError(t, err)
+	return string(b)
+}
+
+// FileDir returns the directory of the current source file.
+func FileDir() string {
+	_, filename, _, _ := runtime.Caller(0)
+	return filepath.Dir(filename)
+}
+
+// Generates a temporary filename
+func TempFilename(prefix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(os.TempDir(), fmt.Sprintf("%s_%s", prefix, hex.EncodeToString(randBytes)))
+}
+
+func CreateTestFile(t *testing.T, path string) {
+	t.Helper()
+
+	CreateTestFileWithData(t, path, "Garbage for "+path)
+}
+
+func CreateTestFileWithData(t *testing.T, path, data string) {
+	t.Helper()
+
+	dir := filepath.Dir(path)
+	err := os.MkdirAll(dir, 0755)
+	require.NoError(t, err)
+	err = os.WriteFile(path, []byte(data), 0644)
+	require.NoError(t, err)
+}
+
+func LogJSON(t *testing.T, v interface{}) {
+	t.Helper()
+
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		t.Log(v)
+	} else {
+		t.Log(string(b))
+	}
+}

internal/traces/exporter.go

@@ -0,0 +1,273 @@
+package traces
+
+import (
+	"context"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+)
+
+var (
+	_        trace.SpanExporter = (*Exporter)(nil)
+	once     sync.Once
+	exporter *Exporter
+	printer  = message.NewPrinter(language.English)
+)
+
+const legend = "Succeeded (S), eXcluded (X), Failed (F)\n"
+
+// FUTURE WORK: This exporter should only be used by troubleshoot CLIs
+// until the following issue is addressed:
+// 1. The cache of spans grows infinitely at the moment. This is OK for short lived
+//    invocations such as CLI applications running one-shot commands. For long running
+//    applications, this will be a problem.
+//		* We can drop non-troubleshoot spans from the cache. There is an OTEP to add
+//        "inheritable attributes" to spans which is unsupported as of today. Once/if
+//		  it gets accepted and implemented, it will allow us to add a "troubleshoot"
+//		  attribute to the "root" span which gets propagated to downstream spans which
+//		  we can use to filter out non-troubleshoot spans.
+//		  https://github.com/dynatrace-oss-contrib/oteps/blob/feature/context-scoped-attributes/text/0207-context-scoped-attributes.md
+//		  https://github.com/open-telemetry/opentelemetry-specification/issues/1337
+//		* We can "search" for the "root" span by traversing up the span tree and drop
+//		  all spans that are not descendants of the "root" span. This can be slow, but
+//		  we can do it in a background goroutine (adds a bit of complexity).
+// 2. At the moment, the summary of an execution is the only case this exporter
+// 	  is being used for.
+
+// GetExporterInstance creates a singleton exporter instance
+func GetExporterInstance() *Exporter {
+	once.Do(func() {
+		exporter = &Exporter{
+			allSpans: make([]trace.ReadOnlySpan, 1024),
+		}
+	})
+	return exporter
+}
+
+// Exporter is an implementation of trace.SpanExporter that writes to a destination.
+type Exporter struct {
+	spansMu  sync.Mutex
+	allSpans []trace.ReadOnlySpan
+
+	stoppedMu sync.RWMutex
+	stopped   bool
+}
+
+// ExportSpans writes spans to an in-memory cache
+// This function can/will be called on every span.End() at worst.
+func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error {
+	// This is a no-op if the context is canceled.
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	e.stoppedMu.RLock()
+	stopped := e.stopped
+	e.stoppedMu.RUnlock()
+	if stopped {
+		return nil
+	}
+
+	if len(spans) == 0 {
+		return nil
+	}
+
+	e.spansMu.Lock()
+	defer e.spansMu.Unlock()
+
+	// Cache received span updates allSpans
+	e.allSpans = append(e.allSpans, spans...)
+
+	return nil
+}
+
+func isType(stub *tracetest.SpanStub, t string) bool {
+	if stub == nil {
+		return false
+	}
+
+	for _, attr := range stub.Attributes {
+		if string(attr.Key) == "type" && strings.Contains(attr.Value.AsString(), t) {
+			return true
+		}
+	}
+	return false
+}
+
+func formatStubName(stub *tracetest.SpanStub) string {
+	if stub == nil {
+		return ""
+	}
+
+	for _, attr := range stub.Attributes {
+		if stub.Status.Code == codes.Error {
+			return stub.Name + " (F)"
+		}
+		if string(attr.Key) == constants.EXCLUDED && attr.Value.AsBool() {
+			return stub.Name + " (X)"
+		}
+	}
+	return stub.Name + " (S)" // Assume success
+}
+
+// maxInt returns the larger of x or y.
+func maxInt(x, y int) int {
+	if x < y {
+		return y
+	}
+	return x
+}
+
+// GetSummary returns the runtime summary of the execution
+// so far. Call this function after your "root" span has ended
+// and the program operations needing tracing have completed.
+func (e *Exporter) GetSummary() string {
+	e.spansMu.Lock()
+	stubs := tracetest.SpanStubsFromReadOnlySpans(e.allSpans)
+	e.spansMu.Unlock()
+
+	// No spans to log
+	if len(stubs) == 0 {
+		return ""
+	}
+
+	// TODO: We may want to collect more information about the
+	// execution of the program. For example, we can collect
+	// the number of times a collector was executed, whether
+	// it was successful or not, if it was skipped, etc.
+	collectors := make(map[string]time.Duration)
+	redactors := make(map[string]time.Duration)
+	analysers := make(map[string]time.Duration)
+
+	totalDuration := time.Duration(0)
+
+	for i := range stubs {
+		stub := &stubs[i]
+
+		// Summary of span stubs
+		duration := stub.EndTime.Sub(stub.StartTime)
+		stubName := formatStubName(stub)
+		switch {
+		case stub.Name == constants.TROUBLESHOOT_ROOT_SPAN_NAME:
+			totalDuration = duration
+		case isType(stub, "Collect"):
+			collectors[stubName] = duration
+		case isType(stub, "Redactors"):
+			redactors[stub.Name] = duration
+		case isType(stub, "Analyze"):
+			analysers[stubName] = duration
+		default:
+			continue
+		}
+	}
+
+	sb := strings.Builder{}
+
+	collectorsSummary(collectors, &sb)
+	redactorsSummary(redactors, &sb)
+	analysersSummary(analysers, &sb)
+	sb.WriteString(printer.Sprintf("\nDuration: %dms\n", totalDuration/time.Millisecond))
+
+	return sb.String()
+}
+
+// summary of collector runtimes
+func collectorsSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("============ Collectors summary =============\n")
+	sb.WriteString(legend)
+	sb.WriteString("=============================================\n")
+	if len(summary) == 0 {
+		sb.WriteString("No collectors executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+// summary of redactor runtime
+func redactorsSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("\n============ Redactors summary =============\n")
+	if len(summary) == 0 {
+		sb.WriteString("No redactors executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+// summary of analyser runtime
+func analysersSummary(summary map[string]time.Duration, sb *strings.Builder) {
+	padding, keys := sortedKeysAndPadding(summary)
+
+	sb.WriteString("\n============= Analyzers summary =============\n")
+	sb.WriteString(legend)
+	sb.WriteString("=============================================\n")
+	if len(summary) == 0 {
+		sb.WriteString("No analyzers executed\n")
+		return
+	}
+
+	for _, name := range keys {
+		sb.WriteString(printer.Sprintf("%-*s : %dms\n", padding, name, summary[name]/time.Millisecond))
+	}
+}
+
+func sortedKeysAndPadding(summary map[string]time.Duration) (int, []string) {
+	keys := make([]string, 0, len(summary))
+	padding := 0
+	for k := range summary {
+		padding = maxInt(padding, len(k))
+		keys = append(keys, k)
+	}
+	sort.SliceStable(keys, func(l, r int) bool {
+		return summary[keys[l]] > summary[keys[r]]
+	})
+	return padding, keys
+}
+
+// Shutdown is called to stop the exporter, it preforms no action.
+func (e *Exporter) Shutdown(ctx context.Context) error {
+	e.stoppedMu.Lock()
+	e.stopped = true
+	e.stoppedMu.Unlock()
+
+	e.Reset()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+	return nil
+}
+
+func (e *Exporter) Reset() {
+	e.spansMu.Lock()
+	e.allSpans = e.allSpans[:0] // clear the slice
+	e.spansMu.Unlock()
+}
+
+// MarshalLog is the marshaling function used by the logging system to represent this exporter.
+func (e *Exporter) MarshalLog() interface{} {
+	return struct {
+		Type string
+	}{
+		Type: "troubleshoot",
+	}
+}

internal/traces/exporter_test.go

@@ -0,0 +1,187 @@
+package traces
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestExporter_GetSummary(t *testing.T) {
+	logger.SetQuiet(true)
+
+	tests := []struct {
+		name  string
+		spans tracetest.SpanStubs
+		want  string
+	}{
+		{
+			name:  "with no spans",
+			spans: tracetest.SpanStubs{},
+			want:  "",
+		},
+		{
+			name: "with root span only",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name:      constants.TROUBLESHOOT_ROOT_SPAN_NAME,
+					StartTime: time.Now(),
+					EndTime:   time.Now().Add(time.Second),
+				},
+			},
+			want: "Duration: 1,000ms",
+		},
+		{
+			name: "with collectors",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "all-logs", StartTime: time.Now(), EndTime: time.Now().Add(time.Minute),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectLogs"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "host-os", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "excluded-collector", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond * 2),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+						attribute.Bool("excluded", true),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "failed-collector", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.CollectHostOS"),
+					},
+					Status: trace.Status{
+						Code:        codes.Error,
+						Description: "some error",
+					},
+				},
+			},
+			want: `
+============ Collectors summary =============
+Succeeded (S), eXcluded (X), Failed (F)
+=============================================
+all-logs (S)           : 60,000ms
+host-os (S)            : 1,000ms
+excluded-collector (X) : 2ms
+failed-collector (F)   : 1ms`,
+		},
+		{
+			name: "with analyzers",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "cluster-version", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*analyzer.AnalyzeClusterVersion"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "host-cpu", StartTime: time.Now(), EndTime: time.Now().Add(time.Minute),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*analyzer.AnalyzeHostCPU"),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "excluded-analyser", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond * 2),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.AnalyzeHostCPU"),
+						attribute.Bool("excluded", true),
+					},
+				},
+				tracetest.SpanStub{
+					Name: "failed-analyser", StartTime: time.Now(), EndTime: time.Now().Add(time.Millisecond),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "*collect.AnalyzeHostCPU"),
+					},
+					Status: trace.Status{
+						Code:        codes.Error,
+						Description: "some error",
+					},
+				},
+			},
+			want: `
+============= Analyzers summary =============
+Succeeded (S), eXcluded (X), Failed (F)
+=============================================
+host-cpu (S)          : 60,000ms
+cluster-version (S)   : 1,000ms
+excluded-analyser (X) : 2ms
+failed-analyser (F)   : 1ms`,
+		},
+		{
+			name: "with redactors",
+			spans: tracetest.SpanStubs{
+				tracetest.SpanStub{
+					Name: "cluster redactor", StartTime: time.Now(), EndTime: time.Now().Add(time.Second),
+					Attributes: []attribute.KeyValue{
+						attribute.String("type", "Redactors"),
+					},
+				},
+			},
+			want: `
+============ Redactors summary =============
+cluster redactor : 1,000ms`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e := &Exporter{}
+
+			ctx := context.Background()
+			err := e.ExportSpans(ctx, tt.spans.Snapshots())
+			require.NoError(t, err)
+
+			assert.Contains(t, e.GetSummary(), strings.TrimSpace(tt.want))
+		})
+	}
+}
+
+func TestExporter_ExportSpansWithDoneContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	e := &Exporter{}
+	spans := tracetest.SpanStubs{}
+
+	assert.EqualError(t, e.ExportSpans(ctx, spans.Snapshots()), context.Canceled.Error())
+}
+
+func TestExporter_Shutdown(t *testing.T) {
+	e := &Exporter{}
+
+	ctx := context.Background()
+	spans := tracetest.SpanStubs{}
+	for i := 0; i < 5; i++ {
+		spans = append(spans, tracetest.SpanStub{Name: fmt.Sprintf("span-%d", i)})
+	}
+
+	err := e.ExportSpans(ctx, spans.Snapshots())
+	require.NoError(t, err)
+
+	assert.Len(t, e.allSpans, 5)
+
+	require.NoError(t, e.Shutdown(ctx))
+	assert.Len(t, e.allSpans, 0)
+
+	err = e.ExportSpans(ctx, spans.Snapshots())
+	require.NoError(t, err)
+
+	assert.Len(t, e.allSpans, 0)
+}

internal/traces/otel.go

@@ -0,0 +1,60 @@
+package traces
+
+import (
+	"context"
+
+	"github.com/replicatedhq/troubleshoot/pkg/version"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+	"k8s.io/klog/v2"
+)
+
+// ConfigureTracing configures the OpenTelemetry trace provider for CLI
+// commands. Projects using troubleshoot as a library would need to register
+// troubleshoot's exporter like so.
+//
+//	var tp *trace.TracerProvider	// client application's trace provider
+//	tp.RegisterSpanProcessor(
+//		trace.NewSimpleSpanProcessor(
+//			traces.GetExporterInstance(),	// Troubleshoot's exporter
+//		),
+//	)
+//
+// The client application is responsible for constructing the trace provider
+// and registering the exporter. Multiple exporters can be registered.
+func ConfigureTracing(processName string) (func(), error) {
+	r, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			resource.Default().SchemaURL(),
+			semconv.ProcessCommandKey.String(processName),
+			semconv.ProcessRuntimeVersionKey.String(version.Version()),
+			attribute.String("environment", "cli"),
+		),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Trace provider for support bundle cli. Each application is required
+	// to have its own trace provider.
+	tp := trace.NewTracerProvider(
+		trace.WithSampler(trace.AlwaysSample()),
+		trace.WithSyncer(
+			GetExporterInstance(),
+		),
+		trace.WithResource(r),
+	)
+
+	otel.SetTracerProvider(tp)
+
+	return func() {
+		if err := tp.Shutdown(context.Background()); err != nil {
+			klog.Errorf("Failed to shutdown trace provider: %v", err)
+		}
+	}, nil
+}

internal/util/util.go

@@ -4,6 +4,9 @@ import (
 	"net/url"
 	"os"
 	"strings"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 func HomeDir() string {
@@ -23,7 +26,7 @@ func IsURL(str string) bool {
 }
 
 func AppName(name string) string {
-	words := strings.Split(strings.Title(strings.Replace(name, "-", " ", -1)), " ")
+	words := strings.Split(cases.Title(language.English).String(strings.ReplaceAll(name, "-", " ")), " ")
 	casedWords := []string{}
 	for i, word := range words {
 		if strings.ToLower(word) == "ai" {
@@ -37,3 +40,15 @@ func AppName(name string) string {
 
 	return strings.Join(casedWords, " ")
 }
+
+func SplitYAML(doc string) []string {
+	return strings.Split(doc, "\n---\n")
+}
+
+func EstimateNumberOfLines(text string) int {
+	n := strings.Count(text, "\n")
+	if len(text) > 0 && !strings.HasSuffix(text, "\n") {
+		n++
+	}
+	return n
+}

pkg/analyze/analyzer.go

@@ -1,14 +1,21 @@
 package analyzer
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"strconv"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/multitype"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 )
 
 type AnalyzeResult struct {
@@ -50,18 +57,29 @@ func isExcluded(excludeVal *multitype.BoolOrString) (bool, error) {
 	return parsed, nil
 }
 
-func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) []*AnalyzeResult {
+func HostAnalyze(
+	ctx context.Context,
+	hostAnalyzer *troubleshootv1beta2.HostAnalyze,
+	getFile getCollectedFileContents,
+	findFiles getChildCollectedFileContents,
+) []*AnalyzeResult {
 	analyzer, ok := GetHostAnalyzer(hostAnalyzer)
 	if !ok {
 		return NewAnalyzeResultError(analyzer, errors.New("invalid host analyzer"))
 	}
 
+	_, span := otel.Tracer(constants.LIB_TRACER_NAME).Start(ctx, analyzer.Title())
+	span.SetAttributes(attribute.String("type", reflect.TypeOf(analyzer).String()))
+	defer span.End()
+
 	isExcluded, _ := analyzer.IsExcluded()
 	if isExcluded {
+		klog.Infof("excluding %q analyzer", analyzer.Title())
+		span.SetAttributes(attribute.Bool(constants.EXCLUDED, true))
 		return nil
 	}
 
-	result, err := analyzer.Analyze(getFile)
+	result, err := analyzer.Analyze(getFile, findFiles)
 	if err != nil {
 		return NewAnalyzeResultError(analyzer, errors.Wrap(err, "analyze"))
 	}
@@ -83,425 +101,51 @@ func NewAnalyzeResultError(analyzer HostAnalyzer, err error) []*AnalyzeResult {
 	}}
 }
 
-func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+func Analyze(
+	ctx context.Context,
+	analyzer *troubleshootv1beta2.Analyze,
+	getFile getCollectedFileContents,
+	findFiles getChildCollectedFileContents,
+) ([]*AnalyzeResult, error) {
 	if analyzer == nil {
 		return nil, errors.New("nil analyzer")
 	}
 
-	if analyzer.ClusterVersion != nil {
-		isExcluded, err := isExcluded(analyzer.ClusterVersion.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeClusterVersion(analyzer.ClusterVersion, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ClusterVersion.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.StorageClass != nil {
-		isExcluded, err := isExcluded(analyzer.StorageClass.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeStorageClass(analyzer.StorageClass, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.StorageClass.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.CustomResourceDefinition != nil {
-		isExcluded, err := isExcluded(analyzer.CustomResourceDefinition.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeCustomResourceDefinition(analyzer.CustomResourceDefinition, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.CustomResourceDefinition.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Ingress != nil {
-		isExcluded, err := isExcluded(analyzer.Ingress.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeIngress(analyzer.Ingress, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Ingress.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Secret != nil {
-		isExcluded, err := isExcluded(analyzer.Secret.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeSecret(analyzer.Secret, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Secret.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.ConfigMap != nil {
-		isExcluded, err := isExcluded(analyzer.ConfigMap.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeConfigMap(analyzer.ConfigMap, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ConfigMap.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.ImagePullSecret != nil {
-		isExcluded, err := isExcluded(analyzer.ImagePullSecret.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeImagePullSecret(analyzer.ImagePullSecret, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ImagePullSecret.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.DeploymentStatus != nil {
-		isExcluded, err := isExcluded(analyzer.DeploymentStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeDeploymentStatus(analyzer.DeploymentStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.DeploymentStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.StatefulsetStatus != nil {
-		isExcluded, err := isExcluded(analyzer.StatefulsetStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeStatefulsetStatus(analyzer.StatefulsetStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.StatefulsetStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.JobStatus != nil {
-		isExcluded, err := isExcluded(analyzer.JobStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeJobStatus(analyzer.JobStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.JobStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ReplicaSetStatus != nil {
-		isExcluded, err := isExcluded(analyzer.ReplicaSetStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeReplicaSetStatus(analyzer.ReplicaSetStatus, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.ReplicaSetStatus.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ClusterPodStatuses != nil {
-		isExcluded, err := isExcluded(analyzer.ClusterPodStatuses.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := clusterPodStatuses(analyzer.ClusterPodStatuses, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.ClusterPodStatuses.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.ContainerRuntime != nil {
-		isExcluded, err := isExcluded(analyzer.ContainerRuntime.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeContainerRuntime(analyzer.ContainerRuntime, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.ContainerRuntime.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Distribution != nil {
-		isExcluded, err := isExcluded(analyzer.Distribution.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeDistribution(analyzer.Distribution, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Distribution.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.NodeResources != nil {
-		isExcluded, err := isExcluded(analyzer.NodeResources.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeNodeResources(analyzer.NodeResources, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.NodeResources.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.TextAnalyze != nil {
-		isExcluded, err := isExcluded(analyzer.TextAnalyze.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeTextAnalyze(analyzer.TextAnalyze, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.TextAnalyze.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
-	}
-	if analyzer.YamlCompare != nil {
-		isExcluded, err := isExcluded(analyzer.YamlCompare.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeYamlCompare(analyzer.YamlCompare, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.YamlCompare.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.JsonCompare != nil {
-		isExcluded, err := isExcluded(analyzer.JsonCompare.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeJsonCompare(analyzer.JsonCompare, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.JsonCompare.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Postgres != nil {
-		isExcluded, err := isExcluded(analyzer.Postgres.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzePostgres(analyzer.Postgres, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Postgres.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Mysql != nil {
-		isExcluded, err := isExcluded(analyzer.Mysql.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeMysql(analyzer.Mysql, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Mysql.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Redis != nil {
-		isExcluded, err := isExcluded(analyzer.Redis.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeRedis(analyzer.Redis, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.Redis.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.CephStatus != nil {
-		isExcluded, err := isExcluded(analyzer.CephStatus.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := cephStatus(analyzer.CephStatus, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.CephStatus.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
-	}
-	if analyzer.Longhorn != nil {
-		isExcluded, err := isExcluded(analyzer.Longhorn.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := longhorn(analyzer.Longhorn, getFile, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.Longhorn.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
+	analyzerInst := getAnalyzer(analyzer)
+	if analyzerInst == nil {
+		return []*AnalyzeResult{{
+			IsFail:  true,
+			Title:   "nonexistent analyzer",
+			Message: "Analyzer not found",
+		}}, nil
 	}
 
-	if analyzer.RegistryImages != nil {
-		isExcluded, err := isExcluded(analyzer.RegistryImages.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeRegistry(analyzer.RegistryImages, getFile)
-		if err != nil {
-			return nil, err
-		}
-		result.Strict = analyzer.RegistryImages.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
+	_, span := otel.Tracer(constants.LIB_TRACER_NAME).Start(ctx, analyzerInst.Title())
+	span.SetAttributes(attribute.String("type", reflect.TypeOf(analyzerInst).String()))
+	defer span.End()
+
+	isExcluded, err := analyzerInst.IsExcluded()
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
+	}
+	if isExcluded {
+		klog.Infof("excluding %q analyzer", analyzerInst.Title())
+		span.SetAttributes(attribute.Bool(constants.EXCLUDED, true))
+		return nil, nil
 	}
 
-	if analyzer.WeaveReport != nil {
-		isExcluded, err := isExcluded(analyzer.WeaveReport.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		results, err := analyzeWeaveReport(analyzer.WeaveReport, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		for i := range results {
-			results[i].Strict = analyzer.WeaveReport.Strict.BoolOrDefaultFalse()
-		}
-		return results, nil
+	results, err := analyzerInst.Analyze(getFile, findFiles)
+	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
 	}
 
-	if analyzer.Sysctl != nil {
-		isExcluded, err := isExcluded(analyzer.Sysctl.Exclude)
-		if err != nil {
-			return nil, err
-		}
-		if isExcluded {
-			return nil, nil
-		}
-		result, err := analyzeSysctl(analyzer.Sysctl, findFiles)
-		if err != nil {
-			return nil, err
-		}
-		if result == nil {
-			return []*AnalyzeResult{}, nil
-		}
-		result.Strict = analyzer.Sysctl.Strict.BoolOrDefaultFalse()
-		return []*AnalyzeResult{result}, nil
+	if results == nil {
+		results = []*AnalyzeResult{}
 	}
 
-	return nil, errors.New("invalid analyzer")
+	return results, nil
 }
 
 func GetExcludeFlag(analyzer *troubleshootv1beta2.Analyze) *multitype.BoolOrString {
@@ -525,3 +169,95 @@ func GetExcludeFlag(analyzer *troubleshootv1beta2.Analyze) *multitype.BoolOrStri
 
 	return nil
 }
+
+type Analyzer interface {
+	Title() string
+	IsExcluded() (bool, error)
+	Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error)
+}
+
+func getAnalyzer(analyzer *troubleshootv1beta2.Analyze) Analyzer {
+	switch {
+	case analyzer.ClusterVersion != nil:
+		return &AnalyzeClusterVersion{analyzer: analyzer.ClusterVersion}
+	case analyzer.StorageClass != nil:
+		return &AnalyzeStorageClass{analyzer: analyzer.StorageClass}
+	case analyzer.CustomResourceDefinition != nil:
+		return &AnalyzeCustomResourceDefinition{analyzer: analyzer.CustomResourceDefinition}
+	case analyzer.Ingress != nil:
+		return &AnalyzeIngress{analyzer: analyzer.Ingress}
+	case analyzer.Secret != nil:
+		return &AnalyzeSecret{analyzer: analyzer.Secret}
+	case analyzer.ConfigMap != nil:
+		return &AnalyzeConfigMap{analyzer: analyzer.ConfigMap}
+	case analyzer.ImagePullSecret != nil:
+		return &AnalyzeImagePullSecret{analyzer: analyzer.ImagePullSecret}
+	case analyzer.DeploymentStatus != nil:
+		return &AnalyzeDeploymentStatus{analyzer: analyzer.DeploymentStatus}
+	case analyzer.StatefulsetStatus != nil:
+		return &AnalyzeStatefulsetStatus{analyzer: analyzer.StatefulsetStatus}
+	case analyzer.JobStatus != nil:
+		return &AnalyzeJobStatus{analyzer: analyzer.JobStatus}
+	case analyzer.ReplicaSetStatus != nil:
+		return &AnalyzeReplicaSetStatus{analyzer: analyzer.ReplicaSetStatus}
+	case analyzer.ClusterPodStatuses != nil:
+		return &AnalyzeClusterPodStatuses{analyzer: analyzer.ClusterPodStatuses}
+	case analyzer.ContainerRuntime != nil:
+		return &AnalyzeContainerRuntime{analyzer: analyzer.ContainerRuntime}
+	case analyzer.Distribution != nil:
+		return &AnalyzeDistribution{analyzer: analyzer.Distribution}
+	case analyzer.NodeResources != nil:
+		return &AnalyzeNodeResources{analyzer: analyzer.NodeResources}
+	case analyzer.TextAnalyze != nil:
+		return &AnalyzeTextAnalyze{analyzer: analyzer.TextAnalyze}
+	case analyzer.YamlCompare != nil:
+		return &AnalyzeYamlCompare{analyzer: analyzer.YamlCompare}
+	case analyzer.JsonCompare != nil:
+		return &AnalyzeJsonCompare{analyzer: analyzer.JsonCompare}
+	case analyzer.Postgres != nil:
+		return &AnalyzePostgres{analyzer: analyzer.Postgres}
+	case analyzer.Mysql != nil:
+		return &AnalyzeMysql{analyzer: analyzer.Mysql}
+	case analyzer.Redis != nil:
+		return &AnalyzeRedis{analyzer: analyzer.Redis}
+	case analyzer.CephStatus != nil:
+		return &AnalyzeCephStatus{analyzer: analyzer.CephStatus}
+	case analyzer.Longhorn != nil:
+		return &AnalyzeLonghorn{analyzer: analyzer.Longhorn}
+	case analyzer.RegistryImages != nil:
+		return &AnalyzeRegistryImages{analyzer: analyzer.RegistryImages}
+	case analyzer.WeaveReport != nil:
+		return &AnalyzeWeaveReport{analyzer: analyzer.WeaveReport}
+	case analyzer.Sysctl != nil:
+		return &AnalyzeSysctl{analyzer: analyzer.Sysctl}
+	case analyzer.ClusterResource != nil:
+		return &AnalyzeClusterResource{analyzer: analyzer.ClusterResource}
+	case analyzer.Certificates != nil:
+		return &AnalyzeCertificates{analyzer: analyzer.Certificates}
+	default:
+		return nil
+	}
+}
+
+// deduplicates a list of troubleshootv1beta2.Analyze objects
+// marshals object to json and then uses its string value to check for uniqueness
+// there is no sorting of the keys in the analyze object's spec so if the spec isn't an exact match line for line as written, no dedup will occur
+func DedupAnalyzers(allAnalyzers []*troubleshootv1beta2.Analyze) []*troubleshootv1beta2.Analyze {
+	uniqueAnalyzers := make(map[string]bool)
+	finalAnalyzers := []*troubleshootv1beta2.Analyze{}
+
+	for _, analyzer := range allAnalyzers {
+		data, err := json.Marshal(analyzer)
+		if err != nil {
+			// return analyzer as is if for whatever reason it can't be marshalled into json
+			finalAnalyzers = append(finalAnalyzers, analyzer)
+		} else {
+			stringData := string(data)
+			if _, value := uniqueAnalyzers[stringData]; !value {
+				uniqueAnalyzers[stringData] = true
+				finalAnalyzers = append(finalAnalyzers, analyzer)
+			}
+		}
+	}
+	return finalAnalyzers
+}

pkg/analyze/analyzer_test.go

@@ -1,6 +1,7 @@
 package analyzer
 
 import (
+	"context"
 	"testing"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -65,3 +66,103 @@ func Test_GetExcludeFlag(t *testing.T) {
 		})
 	}
 }
+
+func TestAnalyzeWithNilAnalyzer(t *testing.T) {
+	got, err := Analyze(context.Background(), nil, nil, nil)
+	assert.Error(t, err)
+	assert.Nil(t, got)
+}
+
+func TestCollector_DedupCollectors(t *testing.T) {
+	tests := []struct {
+		name      string
+		Analyzers []*troubleshootv1beta2.Analyze
+		want      []*troubleshootv1beta2.Analyze
+	}{
+		{
+			name: "multiple ClusterVersion",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					ClusterVersion: &troubleshootv1beta2.ClusterVersion{},
+				},
+			},
+		},
+		{
+			name: "multiple TextAnalyze",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "multiple TextAnalyze with different CheckName",
+			Analyzers: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "test",
+						},
+					},
+				},
+			},
+			want: []*troubleshootv1beta2.Analyze{
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "hi",
+						},
+					},
+				},
+				{
+					TextAnalyze: &troubleshootv1beta2.TextAnalyze{
+						AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+							CheckName: "test",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := DedupAnalyzers(tc.Analyzers)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}

pkg/analyze/ceph.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 )
 
 type CephHealth string
@@ -99,20 +100,47 @@ type PgMap struct {
 	TotalBytes uint64 `json:"bytes_total"`
 }
 
-func cephStatus(analyzer *troubleshootv1beta2.CephStatusAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	fileName := path.Join(collect.GetCephCollectorFilepath(analyzer.CollectorName, analyzer.Namespace), "status.json")
-	collected, err := getCollectedFileContents(fileName)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read collected ceph status")
-	}
+type AnalyzeCephStatus struct {
+	analyzer *troubleshootv1beta2.CephStatusAnalyze
+}
 
-	title := analyzer.CheckName
+func (a *AnalyzeCephStatus) Title() string {
+	title := a.analyzer.CheckName
 	if title == "" {
 		title = "Ceph Status"
 	}
 
+	return title
+}
+
+func (a *AnalyzeCephStatus) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeCephStatus) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.cephStatus(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	if result != nil {
+		result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeCephStatus) cephStatus(analyzer *troubleshootv1beta2.CephStatusAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	fileName := path.Join(collect.GetCephCollectorFilepath(analyzer.CollectorName, analyzer.Namespace), "status.json")
+	collected, err := getCollectedFileContents(fileName)
+
+	if err != nil {
+		if _, ok := err.(*types.NotFoundError); ok {
+			return nil, nil
+		}
+		return nil, errors.Wrap(err, "failed to read collected ceph status")
+	}
+
 	analyzeResult := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "rook", // maybe this should be ceph?
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/rook.svg?w=11&h=16",
 	}

pkg/analyze/ceph_test.go

@@ -5,21 +5,25 @@ import (
 	"testing"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type getFile func(n string) ([]byte, error)
+
 func Test_cephStatus(t *testing.T) {
 	tests := []struct {
 		name           string
 		analyzer       troubleshootv1beta2.CephStatusAnalyze
-		expectResult   AnalyzeResult
+		expectResult   *AnalyzeResult
+		getFile        getFile
 		filePath, file string
 	}{
 		{
 			name:     "pass case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  true,
 				IsWarn:  false,
 				IsFail:  false,
@@ -51,7 +55,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -84,7 +88,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "fail case",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  false,
 				IsFail:  true,
@@ -120,7 +124,7 @@ func Test_cephStatus(t *testing.T) {
 				CollectorName: "custom-namespace",
 				Namespace:     "namespace",
 			},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  true,
 				IsWarn:  false,
 				IsFail:  false,
@@ -167,7 +171,7 @@ func Test_cephStatus(t *testing.T) {
 					},
 				},
 			},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  false,
 				IsFail:  true,
@@ -200,7 +204,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case with missing osd/pg data",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -221,7 +225,7 @@ func Test_cephStatus(t *testing.T) {
 		{
 			name:     "warn case with health status message and summary",
 			analyzer: troubleshootv1beta2.CephStatusAnalyze{},
-			expectResult: AnalyzeResult{
+			expectResult: &AnalyzeResult{
 				IsPass:  false,
 				IsWarn:  true,
 				IsFail:  false,
@@ -249,21 +253,34 @@ func Test_cephStatus(t *testing.T) {
 				}
 			}`,
 		},
+		{
+			name:         "pass case when get file returns not found error",
+			analyzer:     troubleshootv1beta2.CephStatusAnalyze{},
+			expectResult: nil,
+			getFile: func(n string) ([]byte, error) {
+				return nil, &types.NotFoundError{}
+			},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			req := require.New(t)
-
-			getFile := func(n string) ([]byte, error) {
-				assert.Equal(t, n, test.filePath)
-				return []byte(test.file), nil
+			if test.getFile == nil {
+				test.getFile = func(n string) ([]byte, error) {
+					assert.Equal(t, n, test.filePath)
+					return []byte(test.file), nil
+				}
 			}
 
-			actual, err := cephStatus(&test.analyzer, getFile)
+			a := AnalyzeCephStatus{
+				analyzer: &test.analyzer,
+			}
+
+			actual, err := a.cephStatus(&test.analyzer, test.getFile)
 			req.NoError(err)
 
-			assert.Equal(t, test.expectResult, *actual)
+			assert.Equal(t, test.expectResult, actual)
 		})
 	}
 }

pkg/analyze/certificates.go

@@ -0,0 +1,132 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+)
+
+type AnalyzeCertificates struct {
+	analyzer *troubleshootv1beta2.CertificatesAnalyze
+}
+
+func (a *AnalyzeCertificates) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		return "Certificates Verification"
+	}
+
+	return title
+}
+
+func (a *AnalyzeCertificates) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeCertificates) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.AnalyzeCertificates(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (a *AnalyzeCertificates) AnalyzeCertificates(analyzer *troubleshootv1beta2.CertificatesAnalyze, getCollectedFileContents func(string) ([]byte, error)) ([]*AnalyzeResult, error) {
+	certificatesInfo, err := getCollectedFileContents("certificates/certificates.json")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get contents of certificates.json")
+	}
+
+	collectorCertificates := []collect.CertCollection{}
+	if err := json.Unmarshal(certificatesInfo, &collectorCertificates); err != nil {
+		return nil, errors.Wrap(err, "failed to parse certificates.json")
+	}
+
+	return a.analyzeAnalyzeCertificatesResult(collectorCertificates, analyzer.Outcomes)
+}
+
+func (a *AnalyzeCertificates) analyzeAnalyzeCertificatesResult(certificates []collect.CertCollection, outcomes []*troubleshootv1beta2.Outcome) ([]*AnalyzeResult, error) {
+	var results []*AnalyzeResult
+
+	for _, cert := range certificates {
+		var passResults []*AnalyzeResult
+		for _, certChain := range cert.CertificateChain {
+
+			when := ""
+			message := ""
+			source := ""
+
+			if cert.Source.ConfigMapName != "" {
+				source = fmt.Sprintf("obtained from %s configmap within %s namespace", cert.Source.ConfigMapName, cert.Source.Namespace)
+			}
+
+			if cert.Source.SecretName != "" {
+				source = fmt.Sprintf("obtained from %s secret within %s namespace", cert.Source.SecretName, cert.Source.Namespace)
+			}
+
+			for _, outcome := range outcomes {
+				result := AnalyzeResult{
+					Title: a.Title(),
+				}
+
+				if outcome.Fail != nil {
+					result.IsFail = true
+					when = outcome.Fail.When
+					message = outcome.Fail.Message
+				} else if outcome.Warn != nil {
+					result.IsWarn = true
+					when = outcome.Warn.When
+					message = outcome.Warn.Message
+				} else if outcome.Pass != nil {
+					result.IsPass = true
+					when = outcome.Pass.When
+					message = outcome.Pass.Message
+				} else {
+					return nil, errors.New("empty outcome")
+				}
+
+				if result.IsPass && certChain.IsValid {
+					result.Message = fmt.Sprintf("%s %s, %s", certChain.CertName, message, source)
+					// if the certificate is valid, we need to wait for the warning check whether the certificate is going to expire
+					passResults = append(passResults, &result)
+				}
+
+				if result.IsFail && !certChain.IsValid {
+					result.Message = fmt.Sprintf("%s %s, %s", certChain.CertName, message, source)
+					// return the result immediately if the certificate is invalid
+					results = append(results, &result)
+				}
+
+				if result.IsWarn && certChain.IsValid {
+					warnDate, _ := regexp.Compile(`notAfter \< Today \+ (\d+) days`)
+					warnMatch := warnDate.FindStringSubmatch(when)
+					if warnMatch != nil {
+						warnMatchDays, err := strconv.Atoi(warnMatch[1])
+						if err != nil {
+							return nil, errors.Wrap(err, "failed to convert string to integer")
+						}
+
+						targetTime := time.Now().AddDate(0, 0, warnMatchDays)
+
+						if targetTime.After(certChain.NotAfter) {
+							result.Message = fmt.Sprintf("%s %s in %d days, %s", certChain.CertName, message, warnMatchDays, source)
+							// discard passResults if the certificate is going to expire in certain days
+							passResults = []*AnalyzeResult{}
+							results = append(results, &result)
+						}
+					}
+				}
+			}
+			// append passResults if the certificate is valid and not going to expire in certain days
+			results = append(results, passResults...)
+		}
+	}
+
+	return results, nil
+}

pkg/analyze/certificates_test.go

@@ -0,0 +1,172 @@
+package analyzer
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_certificates(t *testing.T) {
+	tests := []struct {
+		name           string
+		analyzer       troubleshootv1beta2.CertificatesAnalyze
+		expectResult   []*AnalyzeResult
+		getFile        getFile
+		filePath, file string
+	}{
+		{
+			name: "pass case",
+			analyzer: troubleshootv1beta2.CertificatesAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							Message: "certificate is valid",
+						},
+					},
+				},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "Certificates Verification",
+					Message: "ca.crt certificate is valid, obtained from kube-root-ca.crt configmap within kurl namespace",
+				},
+			},
+			filePath: "certificates/certificates.json",
+			file: `[{
+				"source": {
+					"configMap": "kube-root-ca.crt",
+					"namespace": "kurl"
+				},
+				"certificateChain": [
+					{
+						"certificate": "ca.crt",
+						"subject": "CN=kubernetes",
+						"subjectAlternativeNames": [
+							"kubernetes"
+						],
+						"issuer": "CN=kubernetes",
+						"notAfter": "2033-04-16T00:30:20Z",
+						"notBefore": "2023-04-19T00:30:20Z",
+						"isValid": true,
+						"isCA": true
+					}
+				]
+			}]`,
+		},
+		{
+			name: "failed case",
+			analyzer: troubleshootv1beta2.CertificatesAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "notAfter < Today",
+							Message: "certificate has expired",
+						},
+					},
+				},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "Certificates Verification",
+					Message: "ca.crt certificate has expired, obtained from kube-root-ca.crt configmap within kurl namespace",
+				},
+			},
+			filePath: "certificates/certificates.json",
+			file: `[{
+				"source": {
+					"configMap": "kube-root-ca.crt",
+					"namespace": "kurl"
+				},
+				"certificateChain": [
+					{
+						"certificate": "ca.crt",
+						"subject": "CN=kubernetes",
+						"subjectAlternativeNames": [
+							"kubernetes"
+						],
+						"issuer": "CN=kubernetes",
+						"notAfter": "2023-04-16T00:30:20Z",
+						"notBefore": "2021-04-19T00:30:20Z",
+						"isValid": false,
+						"isCA": true
+					}
+				]
+			}]`,
+		},
+		{
+			name: "warning case",
+			analyzer: troubleshootv1beta2.CertificatesAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "notAfter < Today + 15 days",
+							Message: "certificate is about to expire",
+						},
+					},
+				},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "Certificates Verification",
+					Message: "ca.crt certificate is about to expire in 15 days, obtained from kube-root-ca.crt configmap within kurl namespace",
+				},
+			},
+			filePath: "certificates/certificates.json",
+			file: fmt.Sprintf(`[{
+				"source": {
+					"configMap": "kube-root-ca.crt",
+					"namespace": "kurl"
+				},
+				"certificateChain": [
+					{
+						"certificate": "ca.crt",
+						"subject": "CN=kubernetes",
+						"subjectAlternativeNames": [
+							"kubernetes"
+						],
+						"issuer": "CN=kubernetes",
+						"notAfter": "%s",
+						"notBefore": "2021-04-16T00:30:20Z",
+						"isValid": true,
+						"isCA": true
+					}
+				]
+			}]`, time.Now().AddDate(0, 0, 5).Format("2006-01-02T15:04:05Z")),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+			if test.getFile == nil {
+				test.getFile = func(n string) ([]byte, error) {
+					assert.Equal(t, n, test.filePath)
+					return []byte(test.file), nil
+				}
+			}
+
+			a := AnalyzeCertificates{
+				analyzer: &test.analyzer,
+			}
+
+			actual, err := a.AnalyzeCertificates(&test.analyzer, test.getFile)
+			t.Log(actual)
+			req.NoError(err)
+
+			assert.Equal(t, test.expectResult, actual)
+		})
+	}
+}

pkg/analyze/cluster_pod_statuses.go

@@ -10,13 +10,43 @@ import (
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 )
 
-func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChildCollectedFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+type AnalyzeClusterPodStatuses struct {
+	analyzer *troubleshootv1beta2.ClusterPodStatuses
+}
+
+func (a *AnalyzeClusterPodStatuses) Title() string {
+	if a.analyzer.CheckName != "" {
+		return a.analyzer.CheckName
+	}
+
+	return "Cluster Pod Status"
+}
+
+func (a *AnalyzeClusterPodStatuses) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeClusterPodStatuses) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	// findFiles is used to get the pod status and events files
+	results, err := clusterPodStatuses(a.analyzer, findFiles, findFiles)
+	if err != nil {
+		return nil, err
+	}
+	for i := range results {
+		results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return results, nil
+}
+
+func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChildCollectedFileContents getChildCollectedFileContents, getChildCollectedFileContentsEvents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
 	excludeFiles := []string{}
-	collected, err := getChildCollectedFileContents(filepath.Join("cluster-resources", "pods", "*.json"), excludeFiles)
+	collected, err := getChildCollectedFileContents(filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_PODS, "*.json"), excludeFiles)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read collected pods")
 	}
@@ -49,7 +79,38 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 
 	for _, pod := range pods {
 		if pod.Status.Reason == "" {
-			pod.Status.Reason = k8sutil.GetPodStatusReason(&pod)
+			// get pod status reason and message from the pod
+			pod.Status.Reason, pod.Status.Message = k8sutil.GetPodStatusReason(&pod)
+		}
+
+		// if the pod has no last termination message like pending or container creating, then check the pod events and get the warning messages. Errors will be logged and return empty message.
+		if pod.Status.Message == "" {
+			messages := []string{}
+			collectedEvents, err := getChildCollectedFileContentsEvents(filepath.Join(constants.CLUSTER_RESOURCES_DIR, "events", fmt.Sprintf("%s.json", pod.Namespace)), excludeFiles)
+			if err != nil {
+				klog.V(2).Infof("failed to read collected events for namespace %s: %v", pod.Namespace, err)
+			}
+
+			for _, fileContent := range collectedEvents {
+				var nsEvents []corev1.Event
+				if err := json.Unmarshal(fileContent, &nsEvents); err != nil {
+					// try new format
+					var nsEventsList corev1.EventList
+					if err := json.Unmarshal(fileContent, &nsEventsList); err != nil {
+						klog.V(2).Infof("failed to unmarshal events for namespace %s: %v", pod.Namespace, err)
+					}
+					nsEvents = nsEventsList.Items
+				}
+
+				for _, event := range nsEvents {
+					if event.InvolvedObject.Kind == "Pod" && event.InvolvedObject.Name == pod.Name && event.InvolvedObject.Namespace == pod.Namespace {
+						if event.Type == "Warning" && event.Message != "" {
+							messages = append(messages, event.Message)
+						}
+					}
+				}
+			}
+			pod.Status.Message = strings.Join(messages, ". ")
 		}
 
 		for _, outcome := range analyzer.Outcomes {
@@ -72,37 +133,40 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 				r.URI = outcome.Pass.URI
 				when = outcome.Pass.When
 			} else {
-				println("error: found an empty outcome in a clusterPodStatuses analyzer") // don't stop
+				klog.Error("error: found an empty outcome in a clusterPodStatuses analyzer\n")
 				continue
 			}
 
-			parts := strings.Split(strings.TrimSpace(when), " ")
-			if len(parts) < 2 {
-				println(fmt.Sprintf("invalid 'when' format: %s\n", when)) // don't stop
-				continue
-			}
-
-			operator := parts[0]
-			reason := parts[1]
+			operator := ""
+			reason := ""
 			match := false
-
-			switch operator {
-			case "=", "==", "===":
-				if reason == "Healthy" {
-					match = !k8sutil.IsPodUnhealthy(&pod)
-				} else {
-					match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason)
+			if when != "" {
+				parts := strings.Split(strings.TrimSpace(when), " ")
+				if len(parts) < 2 {
+					klog.Errorf("invalid 'when' format: %s\n", when)
+					continue
 				}
-			case "!=", "!==":
-				if reason == "Healthy" {
-					match = k8sutil.IsPodUnhealthy(&pod)
-				} else {
-					match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason)
-				}
-			}
+				operator = parts[0]
+				reason = parts[1]
 
-			if !match {
-				continue
+				switch operator {
+				case "=", "==", "===":
+					if reason == "Healthy" {
+						match = !k8sutil.IsPodUnhealthy(&pod)
+					} else {
+						match = reason == string(pod.Status.Phase) || reason == string(pod.Status.Reason)
+					}
+				case "!=", "!==":
+					if reason == "Healthy" {
+						match = k8sutil.IsPodUnhealthy(&pod)
+					} else {
+						match = reason != string(pod.Status.Phase) && reason != string(pod.Status.Reason)
+					}
+				}
+
+				if !match {
+					continue
+				}
 			}
 
 			r.InvolvedObject = &corev1.ObjectReference{
@@ -118,7 +182,12 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 			}
 
 			if r.Message == "" {
-				r.Message = "Pod {{ .Namespace }}/{{ .Name }} status is {{ .Status.Reason }}"
+				r.Message = "Pod {{ .Namespace }}/{{ .Name }} status is {{ .Status.Reason }}. Message is: {{ .Status.Message }}"
+			}
+
+			// if the pod has no status message, set it to None
+			if pod.Status.Message == "" {
+				pod.Status.Message = "None"
 			}
 
 			tmpl := template.New("pod")
@@ -145,7 +214,7 @@ func clusterPodStatuses(analyzer *troubleshootv1beta2.ClusterPodStatuses, getChi
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to execute template")
 			}
-			r.Message = m.String()
+			r.Message = strings.TrimSpace(m.String())
 
 			// add to results, break and check the next pod
 			allResults = append(allResults, &r)

pkg/analyze/cluster_pod_statuses_test.go

@@ -0,0 +1,695 @@
+package analyzer
+
+import (
+	"testing"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func Test_ClusterPodStatuses(t *testing.T) {
+	tests := []struct {
+		name         string
+		analyzer     troubleshootv1beta2.ClusterPodStatuses
+		expectResult []*AnalyzeResult
+		files        map[string][]byte
+		eventFiles   map[string][]byte
+	}{
+		{
+			name: "pass_when_all_pods_are_healthy_in_specific_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "pass_when_all_pods_are_healthy_in_specific_namespace",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Restarting the pod may fix the issue.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_all_pods_are_healthy_in_specific_namespace",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "pass_when_pods_are_healthy_in_all_namespaces",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "pass_when_pods_are_healthy_in_all_namespaces",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Restarting the pod may fix the issue.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_pods_are_healthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "pass_when_pods_are_healthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json": []byte(defaultPods),
+				"cluster-resources/pods/other.json":   []byte(otherPods),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_specific_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_specific_namespace",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_specific_namespace",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!== Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_specific_namespace_using_double_ne",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "fail_when_pods_are_unhealthy_in_any_namespace",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "fail_when_pods_are_unhealthy_in_all_namespaces",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A pod is unhealthy",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  false,
+					IsWarn:  false,
+					IsFail:  true,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "A pod is unhealthy",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other-unhealthy",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+				{
+					IsPass:  true,
+					IsWarn:  false,
+					IsFail:  false,
+					Title:   "fail_when_pods_are_unhealthy_in_all_namespaces",
+					Message: "All Pods are OK.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "other",
+						Name:       "other-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "== CrashLoopBackOff",
+							Message: "A pod is unhealthy.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "warn_when_pods_are_unhealthy_in_specific_namespace_using_double_equals",
+					Message: "A pod is unhealthy.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "=== CrashLoopBackOff",
+							Message: "A pod is unhealthy.",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "All Pods are OK.",
+						},
+					},
+				},
+				Namespaces: []string{"default-unhealthy"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "warn_when_pods_are_unhealthy_in_specific_namespace_using_triple_equals",
+					Message: "A pod is unhealthy.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "default-unhealthy",
+						Name:       "random-pod-75b66db9b9-nqhp8",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/default.json":           []byte(defaultPods),
+				"cluster-resources/pods/other.json":             []byte(otherPods),
+				"cluster-resources/pods/default-unhealthy.json": []byte(defaultPodsUnhealthy),
+				"cluster-resources/pods/other-unhealthy.json":   []byte(otherPodsUnhealthy),
+			},
+		},
+		{
+			name: "show_message_of_pending_pods_with_wrong_node_affinity",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_pending_pods_with_wrong_node_affinity",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-pending-node-affinity"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_pending_pods_with_wrong_node_affinity",
+					Message: "A Pod, kotsadm-b6cb54c8f-zgzrn, is unhealthy with a status of: Pending. Message is: 0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector. preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-pending-node-affinity",
+						Name:       "kotsadm-b6cb54c8f-zgzrn",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-pending-node-affinity.json": []byte(messagePendingNodeAffinity),
+			},
+		},
+		{
+			name: "show_message_of_container_creating_pod_with_failed_mount",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_container_creating_pod_with_failed_mount",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-container-creating-failed-mount"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_container_creating_pod_with_failed_mount",
+					Message: "A Pod, troubleshoot-copyfromhost-4m79m-psdjm, is unhealthy with a status of: ContainerCreating. Message is: MountVolume.SetUp failed for volume \"host\" : hostPath type check failed: /var/lib/collectd is not a directory. Unable to attach or mount volumes: unmounted volumes=[host], unattached volumes=[host kube-api-access-xddvj]: timed out waiting for the condition",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-container-creating-failed-mount",
+						Name:       "troubleshoot-copyfromhost-4m79m-psdjm",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-container-creating-failed-mount.json": []byte(messageContainerCreatingFailedMount),
+			},
+			eventFiles: map[string][]byte{
+				"cluster-resources/events/message-container-creating-failed-mount.json": []byte(messageContainerCreatingFailedMountEvents),
+			},
+		},
+		{
+			name: "show_message_of_pod_crashloop_backoff",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_pod_crashloop_backoff",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-pod-crashloop-backoff"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_pod_crashloop_backoff",
+					Message: "A Pod, init-demo, is unhealthy with a status of: CrashLoopBackOff. Message is: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: \"wge\": executable file not found in $PATH: unknown",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-pod-crashloop-backoff",
+						Name:       "init-demo",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-pod-crashloop-backoff.json": []byte(messagePodCrashLoopBackOff),
+			},
+		},
+		{
+			name: "show_message_of_init_pod_crashloop_backoff",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_init_pod_crashloop_backoff",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-pod-init-crashloop-backoff"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_init_pod_crashloop_backoff",
+					Message: "A Pod, init-demo2, is unhealthy with a status of: Init:RunContainerError. Message is: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: \"wge\": executable file not found in $PATH: unknown",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-pod-init-crashloop-backoff",
+						Name:       "init-demo2",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-pod-init-crashloop-backoff.json": []byte(messagePodInitCrashLoopBackOff),
+			},
+			eventFiles: map[string][]byte{
+				"cluster-resources/events/message-pod-init-crashloop-backoff.json": []byte(messagePodInitCrashLoopBackOffEvents),
+			},
+		},
+		{
+			name: "show_message_of_pending_pod_resource",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_pending_pod_resource",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-pending-pod-resources"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_pending_pod_resource",
+					Message: "A Pod, pending-pod-resources-5fddcf7688-djjfc, is unhealthy with a status of: Pending. Message is: 0/1 nodes are available: 1 Insufficient nvidia.com/gpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-pending-pod-resources",
+						Name:       "pending-pod-resources-5fddcf7688-djjfc",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-pending-pod-resources.json": []byte(messagePendingPodResources),
+			},
+		},
+		{
+			name: "show_message_of_oom_killed_pod",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_oom_killed_pod",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-oomkill-pod"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_oom_killed_pod",
+					Message: "A Pod, oom-kill-job3-gbb89, is unhealthy with a status of: OOMKilled. Message is: ExitCode:137",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-oomkill-pod",
+						Name:       "oom-kill-job3-gbb89",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-oomkill-pod.json": []byte(messageOOMKillPod),
+			},
+		},
+		{
+			name: "show_message_of_image_pull_fail",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_message_of_image_pull_fail",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-image-pull-fail"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_message_of_image_pull_fail",
+					Message: "A Pod, no-image-deployment-849c4c4958-rxqmt, is unhealthy with a status of: ErrImagePull. Message is: Failed to pull image \"noimage.com/no-such-image\": rpc error: code = Unknown desc = Error response from daemon: Get \"https://noimage.com/v2/\": x509: certificate is not valid for any names, but wanted to match noimage.com. Error: ErrImagePull. Error: ImagePullBackOff",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-image-pull-fail",
+						Name:       "no-image-deployment-849c4c4958-rxqmt",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-image-pull-fail.json": []byte(messageImagePullFail),
+			},
+			eventFiles: map[string][]byte{
+				"cluster-resources/events/message-image-pull-fail.json": []byte(messageImagePullFailEvents),
+			},
+		},
+		{
+			name: "show_none_message_of_no_events_pod",
+			analyzer: troubleshootv1beta2.ClusterPodStatuses{
+				AnalyzeMeta: troubleshootv1beta2.AnalyzeMeta{
+					CheckName: "show_none_message_of_no_events_pod",
+				},
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "!= Healthy",
+							Message: "A Pod, {{ .Name }}, is unhealthy with a status of: {{ .Status.Reason }}. Message is: {{ .Status.Message }}",
+						},
+					},
+				},
+				Namespaces: []string{"message-image-pull-fail"},
+			},
+			expectResult: []*AnalyzeResult{
+				{
+					IsPass:  false,
+					IsWarn:  true,
+					IsFail:  false,
+					Title:   "show_none_message_of_no_events_pod",
+					Message: "A Pod, no-image-deployment-849c4c4958-rxqmt, is unhealthy with a status of: ErrImagePull. Message is: None",
+					InvolvedObject: &corev1.ObjectReference{
+						APIVersion: "v1",
+						Kind:       "Pod",
+						Namespace:  "message-image-pull-fail",
+						Name:       "no-image-deployment-849c4c4958-rxqmt",
+					},
+				},
+			},
+			files: map[string][]byte{
+				"cluster-resources/pods/message-image-pull-fail.json": []byte(messageImagePullFail),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+
+			getFiles := func(n string, _ []string) (map[string][]byte, error) {
+				if file, ok := test.files[n]; ok {
+					return map[string][]byte{n: file}, nil
+				}
+				return test.files, nil
+			}
+
+			getEventFiles := func(n string, _ []string) (map[string][]byte, error) {
+				if file, ok := test.eventFiles[n]; ok {
+					return map[string][]byte{n: file}, nil
+				}
+				return test.files, nil
+			}
+
+			actual, err := clusterPodStatuses(&test.analyzer, getFiles, getEventFiles)
+			req.NoError(err)
+			req.Equal(len(test.expectResult), len(actual))
+			for _, a := range actual {
+				assert.Contains(t, test.expectResult, a)
+			}
+		})
+	}
+}

pkg/analyze/cluster_version.go

@@ -4,12 +4,33 @@ import (
 	"encoding/json"
 	"strings"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )
 
+type AnalyzeClusterVersion struct {
+	analyzer *troubleshootv1beta2.ClusterVersion
+}
+
+func (a *AnalyzeClusterVersion) Title() string {
+	return title(a.analyzer.CheckName)
+}
+
+func (a *AnalyzeClusterVersion) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeClusterVersion) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := analyzeClusterVersion(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
 func analyzeClusterVersion(analyzer *troubleshootv1beta2.ClusterVersion, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	clusterInfo, err := getCollectedFileContents("cluster-info/cluster_version.json")
 	if err != nil {
@@ -29,19 +50,22 @@ func analyzeClusterVersion(analyzer *troubleshootv1beta2.ClusterVersion, getColl
 	return analyzeClusterVersionResult(k8sVersion, analyzer.Outcomes, analyzer.CheckName)
 }
 
+func title(checkName string) string {
+	if checkName == "" {
+		return "Required Kubernetes Version"
+	}
+
+	return checkName
+}
+
 func analyzeClusterVersionResult(k8sVersion semver.Version, outcomes []*troubleshootv1beta2.Outcome, checkName string) (*AnalyzeResult, error) {
 	for _, outcome := range outcomes {
 		when := ""
 		message := ""
 		uri := ""
 
-		title := checkName
-		if title == "" {
-			title = "Required Kubernetes Version"
-		}
-
 		result := AnalyzeResult{
-			Title:   title,
+			Title:   title(checkName),
 			IconKey: "kubernetes_cluster_version",
 			IconURI: "https://troubleshoot.sh/images/analyzer-icons/kubernetes.svg?w=16&h=16",
 		}

pkg/analyze/cluster_version_test.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 )
 

pkg/analyze/common_status.go

@@ -47,7 +47,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 				}
 			}
 
-			match, err := compareActualToWhen(outcome.Fail.When, readyReplicas, exists)
+			match, err := compareActualToWhen(outcome.Fail.When, readyReplicas)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to parse fail range")
 			}
@@ -87,7 +87,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 				}
 			}
 
-			match, err := compareActualToWhen(outcome.Warn.When, readyReplicas, exists)
+			match, err := compareActualToWhen(outcome.Warn.When, readyReplicas)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to parse warn range")
 			}
@@ -127,7 +127,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 				}
 			}
 
-			match, err := compareActualToWhen(outcome.Pass.When, readyReplicas, exists)
+			match, err := compareActualToWhen(outcome.Pass.When, readyReplicas)
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to parse pass range")
 			}
@@ -145,9 +145,8 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 	return result, nil
 }
 
-func compareActualToWhen(when string, actual int, exists bool) (bool, error) {
+func compareActualToWhen(when string, actual int) (bool, error) {
 	parts := strings.Split(strings.TrimSpace(when), " ")
-
 	// we can make this a lot more flexible
 	if len(parts) != 2 {
 		return false, errors.New("unable to parse when range")

pkg/analyze/configmap.go

@@ -8,7 +8,33 @@ import (
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )
 
-func analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+type AnalyzeConfigMap struct {
+	analyzer *troubleshootv1beta2.AnalyzeConfigMap
+}
+
+func (a *AnalyzeConfigMap) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = fmt.Sprintf("ConfigMap %s", a.analyzer.ConfigMapName)
+	}
+
+	return title
+}
+
+func (a *AnalyzeConfigMap) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeConfigMap) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeConfigMap(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeConfigMap) analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	filename := collect.GetConfigMapFileName(
 		&troubleshootv1beta2.ConfigMap{
 			Namespace: analyzer.Namespace,
@@ -28,13 +54,8 @@ func analyzeConfigMap(analyzer *troubleshootv1beta2.AnalyzeConfigMap, getCollect
 		return nil, err
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = fmt.Sprintf("ConfigMap %s", analyzer.ConfigMapName)
-	}
-
 	result := AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_analyze_secret", // TODO: icon
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/secret.svg?w=13&h=16",
 	}

pkg/analyze/configmap_test.go

@@ -199,7 +199,12 @@ func Test_analyzeConfigMap(t *testing.T) {
 				}
 				return contents, nil
 			}
-			got, err := analyzeConfigMap(tt.analyzer, getCollectedFileContents)
+
+			a := AnalyzeConfigMap{
+				analyzer: tt.analyzer,
+			}
+
+			got, err := a.analyzeConfigMap(tt.analyzer, getCollectedFileContents)
 			if tt.wantErr {
 				assert.Error(t, err)
 			} else {

pkg/analyze/container_runtime.go

@@ -2,16 +2,44 @@ package analyzer
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 )
 
-func analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	collected, err := getCollectedFileContents("cluster-resources/nodes.json")
+type AnalyzeContainerRuntime struct {
+	analyzer *troubleshootv1beta2.ContainerRuntime
+}
+
+func (a *AnalyzeContainerRuntime) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = "Container Runtime"
+	}
+
+	return title
+}
+
+func (a *AnalyzeContainerRuntime) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeContainerRuntime) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeContainerRuntime(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeContainerRuntime) analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	collected, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_NODES))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get contents of nodes.json")
 	}
@@ -26,12 +54,8 @@ func analyzeContainerRuntime(analyzer *troubleshootv1beta2.ContainerRuntime, get
 		foundRuntimes = append(foundRuntimes, node.Status.NodeInfo.ContainerRuntimeVersion)
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = "Container Runtime"
-	}
 	result := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_container_runtime",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/container-runtime.svg?w=23&h=16",
 	}

pkg/analyze/container_runtime_test.go

@@ -113,7 +113,11 @@ func Test_containerRuntime(t *testing.T) {
 				return test.files[n], nil
 			}
 
-			actual, err := analyzeContainerRuntime(&test.analyzer, getFiles)
+			a := AnalyzeContainerRuntime{
+				analyzer: &test.analyzer,
+			}
+
+			actual, err := a.analyzeContainerRuntime(&test.analyzer, getFiles)
 			req.NoError(err)
 
 			assert.Equal(t, &test.expectResult, actual)

pkg/analyze/crd.go

@@ -5,11 +5,38 @@ import (
 	"fmt"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 )
 
-func analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourceDefinition, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	crdData, err := getCollectedFileContents("cluster-resources/custom-resource-definitions.json")
+type AnalyzeCustomResourceDefinition struct {
+	analyzer *troubleshootv1beta2.CustomResourceDefinition
+}
+
+func (a *AnalyzeCustomResourceDefinition) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = fmt.Sprintf("Custom resource definition %s", a.analyzer.CustomResourceDefinitionName)
+	}
+
+	return title
+}
+
+func (a *AnalyzeCustomResourceDefinition) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeCustomResourceDefinition) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeCustomResourceDefinition(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
+func (a *AnalyzeCustomResourceDefinition) analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourceDefinition, getFile getCollectedFileContents) (*AnalyzeResult, error) {
+	crdData, err := getFile(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_CUSTOM_RESOURCE_DEFINITIONS))
 	if err != nil {
 		return nil, err
 	}
@@ -19,13 +46,8 @@ func analyzeCustomResourceDefinition(analyzer *troubleshootv1beta2.CustomResourc
 		return nil, err
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = fmt.Sprintf("Custom resource definition %s", analyzer.CustomResourceDefinitionName)
-	}
-
 	result := AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_custom_resource_definition",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/custom-resource-definition.svg?w=13&h=16",
 	}

pkg/analyze/data_test.go

@@ -33,3 +33,45 @@ var defaultStatefulSets string
 
 //go:embed files/statefulsets/monitoring.json
 var monitoringStatefulSets string
+
+//go:embed files/pods/default.json
+var defaultPods string
+
+//go:embed files/pods/other.json
+var otherPods string
+
+//go:embed files/pods/default-unhealthy.json
+var defaultPodsUnhealthy string
+
+//go:embed files/pods/other-unhealthy.json
+var otherPodsUnhealthy string
+
+//go:embed files/pods/message-pending-node-affinity.json
+var messagePendingNodeAffinity string
+
+//go:embed files/pods/message-container-creating-failed-mount.json
+var messageContainerCreatingFailedMount string
+
+//go:embed files/events/message-container-creating-failed-mount.json
+var messageContainerCreatingFailedMountEvents string
+
+//go:embed files/pods/message-pod-crashloop-backoff.json
+var messagePodCrashLoopBackOff string
+
+//go:embed files/pods/message-pod-init-crashloop-backoff.json
+var messagePodInitCrashLoopBackOff string
+
+//go:embed files/events/message-pod-init-crashloop-backoff.json
+var messagePodInitCrashLoopBackOffEvents string
+
+//go:embed files/pods/message-pending-pod-resources.json
+var messagePendingPodResources string
+
+//go:embed files/pods/message-oomkill-pod.json
+var messageOOMKillPod string
+
+//go:embed files/pods/message-image-pull-fail.json
+var messageImagePullFail string
+
+//go:embed files/events/message-image-pull-fail.json
+var messageImagePullFailEvents string

pkg/analyze/database_shared.go

@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 )

pkg/analyze/deployment_status.go

@@ -7,9 +7,48 @@ import (
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	appsv1 "k8s.io/api/apps/v1"
 )
 
+type AnalyzeDeploymentStatus struct {
+	analyzer *troubleshootv1beta2.DeploymentStatus
+}
+
+func (a *AnalyzeDeploymentStatus) Title() string {
+	if a.analyzer.CheckName != "" {
+		return a.analyzer.CheckName
+	}
+
+	if a.analyzer.Name != "" && a.analyzer.Namespace != "" {
+		return fmt.Sprintf("%s/%s Deployment Status", a.analyzer.Name, a.analyzer.Name)
+	}
+
+	if a.analyzer.Name != "" {
+		return fmt.Sprintf("%s Deployment Status", a.analyzer.Name)
+	}
+	if a.analyzer.Namespace != "" {
+		return fmt.Sprintf("%s Deployment Status", a.analyzer.Namespace)
+	}
+
+	return "Deployment Status"
+}
+
+func (a *AnalyzeDeploymentStatus) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeDeploymentStatus) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	results, err := analyzeDeploymentStatus(a.analyzer, findFiles)
+	if err != nil {
+		return nil, err
+	}
+	for i := range results {
+		results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	}
+	return results, nil
+}
+
 func analyzeDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
 	if analyzer.Name == "" {
 		return analyzeAllDeploymentStatuses(analyzer, getFileContents)
@@ -20,7 +59,7 @@ func analyzeDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, get
 
 func analyzeOneDeploymentStatus(analyzer *troubleshootv1beta2.DeploymentStatus, getFileContents getChildCollectedFileContents) ([]*AnalyzeResult, error) {
 	excludeFiles := []string{}
-	files, err := getFileContents(filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", analyzer.Namespace)), excludeFiles)
+	files, err := getFileContents(filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", analyzer.Namespace)), excludeFiles)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read collected deployments from namespace")
 	}
@@ -66,15 +105,15 @@ func analyzeAllDeploymentStatuses(analyzer *troubleshootv1beta2.DeploymentStatus
 	excludeFiles := []string{}
 	fileNames := make([]string, 0)
 	if analyzer.Namespace != "" {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", analyzer.Namespace)))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", analyzer.Namespace)))
 	}
 	for _, ns := range analyzer.Namespaces {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", fmt.Sprintf("%s.json", ns)))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, fmt.Sprintf("%s.json", ns)))
 	}
 
 	// no namespace specified, so we need to analyze all deployments
 	if len(fileNames) == 0 {
-		fileNames = append(fileNames, filepath.Join("cluster-resources", "deployments", "*.json"))
+		fileNames = append(fileNames, filepath.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_DEPLOYMENTS, "*.json"))
 	}
 
 	results := []*AnalyzeResult{}

pkg/analyze/distribution.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -46,6 +47,32 @@ const (
 	k3s           Provider = iota
 )
 
+type AnalyzeDistribution struct {
+	analyzer *troubleshootv1beta2.Distribution
+}
+
+func (a *AnalyzeDistribution) Title() string {
+	title := a.analyzer.CheckName
+	if title == "" {
+		title = "Kubernetes Distribution"
+	}
+
+	return title
+}
+
+func (a *AnalyzeDistribution) IsExcluded() (bool, error) {
+	return isExcluded(a.analyzer.Exclude)
+}
+
+func (a *AnalyzeDistribution) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) {
+	result, err := a.analyzeDistribution(a.analyzer, getFile)
+	if err != nil {
+		return nil, err
+	}
+	result.Strict = a.analyzer.Strict.BoolOrDefaultFalse()
+	return []*AnalyzeResult{result}, nil
+}
+
 func CheckApiResourcesForProviders(foundProviders *providers, apiResources []*metav1.APIResourceList, provider string) string {
 	for _, resource := range apiResources {
 		if strings.HasPrefix(resource.GroupVersion, "apps.openshift.io/") {
@@ -141,11 +168,11 @@ func ParseNodesForProviders(nodes []corev1.Node) (providers, string) {
 	return foundProviders, stringProvider
 }
 
-func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+func (a *AnalyzeDistribution) analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
 	var unknownDistribution string
-	collected, err := getCollectedFileContents("cluster-resources/nodes.json")
+	collected, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_NODES))
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get contents of nodes.json")
+		return nil, errors.Wrap(err, fmt.Sprintf("failed to get contents of %s.json", constants.CLUSTER_RESOURCES_NODES))
 	}
 
 	var nodes corev1.NodeList
@@ -155,7 +182,7 @@ func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollecte
 
 	foundProviders, _ := ParseNodesForProviders(nodes.Items)
 
-	apiResourcesBytes, err := getCollectedFileContents("cluster-resources/resources.json")
+	apiResourcesBytes, err := getCollectedFileContents(fmt.Sprintf("%s/%s.json", constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_RESOURCES))
 	// if the file is not found, that is not a fatal error
 	// troubleshoot 0.9.15 and earlier did not collect that file
 	if err == nil {
@@ -166,12 +193,8 @@ func analyzeDistribution(analyzer *troubleshootv1beta2.Distribution, getCollecte
 		_ = CheckApiResourcesForProviders(&foundProviders, apiResources, "")
 	}
 
-	title := analyzer.CheckName
-	if title == "" {
-		title = "Kubernetes Distribution"
-	}
 	result := &AnalyzeResult{
-		Title:   title,
+		Title:   a.Title(),
 		IconKey: "kubernetes_distribution",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/distribution.svg?w=20&h=14",
 	}

pkg/analyze/download.go

@@ -3,8 +3,9 @@ package analyzer
 import (
 	"archive/tar"
 	"compress/gzip"
+	"context"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -12,9 +13,11 @@ import (
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 	"github.com/replicatedhq/troubleshoot/pkg/docrewrite"
-	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	"github.com/replicatedhq/troubleshoot/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 )
 
 type fileContentProvider struct {
@@ -22,7 +25,12 @@ type fileContentProvider struct {
 }
 
 // Analyze local will analyze a locally available (already downloaded) bundle
-func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analyze, hostAnalyzers []*troubleshootv1beta2.HostAnalyze) ([]*AnalyzeResult, error) {
+func AnalyzeLocal(
+	ctx context.Context,
+	localBundlePath string,
+	analyzers []*troubleshootv1beta2.Analyze,
+	hostAnalyzers []*troubleshootv1beta2.HostAnalyze,
+) ([]*AnalyzeResult, error) {
 	rootDir, err := FindBundleRootDir(localBundlePath)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find root dir")
@@ -32,9 +40,9 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 
 	analyzeResults := []*AnalyzeResult{}
 	for _, analyzer := range analyzers {
-		analyzeResult, err := Analyze(analyzer, fcp.getFileContents, fcp.getChildFileContents)
+		analyzeResult, err := Analyze(ctx, analyzer, fcp.getFileContents, fcp.getChildFileContents)
 		if err != nil {
-			logger.Printf("An analyzer failed to run: %v", err)
+			klog.Errorf("An analyzer failed to run: %v", err)
 			continue
 		}
 
@@ -47,7 +55,7 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 	}
 
 	for _, hostAnalyzer := range hostAnalyzers {
-		analyzeResult := HostAnalyze(hostAnalyzer, fcp.getFileContents, fcp.getChildFileContents)
+		analyzeResult := HostAnalyze(ctx, hostAnalyzer, fcp.getFileContents, fcp.getChildFileContents)
 		analyzeResults = append(analyzeResults, analyzeResult...)
 	}
 
@@ -55,27 +63,13 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
 }
 
 func DownloadAndAnalyze(bundleURL string, analyzersSpec string) ([]*AnalyzeResult, error) {
-	tmpDir, err := ioutil.TempDir("", "troubleshoot-k8s")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to create temp dir")
-	}
-	defer os.RemoveAll(tmpDir)
-
-	if err := downloadTroubleshootBundle(bundleURL, tmpDir); err != nil {
-		return nil, errors.Wrap(err, "failed to download bundle")
-	}
-
-	rootDir, err := FindBundleRootDir(tmpDir)
+	tmpDir, rootDir, err := DownloadAndExtractSupportBundle(bundleURL)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find root dir")
 	}
+	defer os.RemoveAll(tmpDir)
 
-	_, err = os.Stat(filepath.Join(rootDir, "version.yaml"))
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read version.yaml")
-	}
-
-	analyzers := []*troubleshootv1beta2.Analyze{}
+	var analyzers []*troubleshootv1beta2.Analyze
 	hostAnalyzers := []*troubleshootv1beta2.HostAnalyze{}
 
 	if analyzersSpec == "" {
@@ -93,10 +87,37 @@ func DownloadAndAnalyze(bundleURL string, analyzersSpec string) ([]*AnalyzeResul
 		hostAnalyzers = parsedHostAnalyzers
 	}
 
-	return AnalyzeLocal(rootDir, analyzers, hostAnalyzers)
+	return AnalyzeLocal(context.Background(), rootDir, analyzers, hostAnalyzers)
+}
+
+func DownloadAndExtractSupportBundle(bundleURL string) (string, string, error) {
+	tmpDir, err := os.MkdirTemp("", "troubleshoot-k8s")
+	if err != nil {
+		return "", "", errors.Wrap(err, "failed to create temp dir")
+	}
+
+	if err := downloadTroubleshootBundle(bundleURL, tmpDir); err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to download bundle")
+	}
+
+	bundleDir, err := FindBundleRootDir(tmpDir)
+	if err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to find root dir")
+	}
+
+	_, err = os.Stat(filepath.Join(bundleDir, constants.VERSION_FILENAME))
+	if err != nil {
+		os.RemoveAll(tmpDir)
+		return "", "", errors.Wrap(err, "failed to read "+constants.VERSION_FILENAME)
+	}
+
+	return tmpDir, bundleDir, nil
 }
 
 func downloadTroubleshootBundle(bundleURL string, destDir string) error {
+	// TODO: Move to separate package support bundle utils package
 	if bundleURL[0] == os.PathSeparator {
 		f, err := os.Open(bundleURL)
 		if err != nil {
@@ -111,7 +132,7 @@ func downloadTroubleshootBundle(bundleURL string, destDir string) error {
 		return errors.Wrap(err, "failed to get workdir")
 	}
 
-	tmpDir, err := ioutil.TempDir("", "troubleshoot")
+	tmpDir, err := os.MkdirTemp("", "troubleshoot")
 	if err != nil {
 		return errors.Wrap(err, "failed to create tmp dir")
 	}
@@ -137,6 +158,8 @@ func downloadTroubleshootBundle(bundleURL string, destDir string) error {
 }
 
 func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
+	// TODO: Move to separate package e.g support bundle package, or sbutils
+	// if there are cyclic dependencies
 	gzReader, err := gzip.NewReader(reader)
 	if err != nil {
 		return errors.Wrap(err, "failed to create gzip reader")
@@ -159,14 +182,14 @@ func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
 				return errors.Wrap(err, "failed to mkdir")
 			}
 		case tar.TypeReg:
-			name := filepath.Join(destDir, header.Name)
+			destFileName := filepath.Join(destDir, header.Name)
 
-			dirName := filepath.Dir(name)
+			dirName := filepath.Dir(destFileName)
 			if err := os.MkdirAll(dirName, 0755); err != nil {
 				return errors.Wrapf(err, "failed to mkdir for file %s", header.Name)
 			}
 
-			file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.FileMode(header.Mode))
+			file, err := os.OpenFile(destFileName, os.O_RDWR|os.O_CREATE, os.FileMode(header.Mode))
 			if err != nil {
 				return errors.Wrap(err, "failed to open tar file")
 			}
@@ -175,6 +198,21 @@ func ExtractTroubleshootBundle(reader io.Reader, destDir string) error {
 			if err != nil {
 				return errors.Wrap(err, "failed to extract file")
 			}
+		case tar.TypeSymlink:
+			destFileName := filepath.Join(destDir, header.Name)
+
+			dirName := filepath.Dir(destFileName)
+			if err := os.MkdirAll(dirName, 0755); err != nil {
+				return errors.Wrapf(err, "failed to mkdir for symlink %s", header.Name)
+			}
+
+			// Symlink targets should be absolute paths after extraction
+			// for other parts of the code to work correctly e.g redaction, CollectorResult
+			targetPath := filepath.Join(filepath.Dir(destFileName), header.Linkname)
+			err = os.Symlink(targetPath, destFileName)
+			if err != nil {
+				return errors.Wrap(err, "failed to create symlink")
+			}
 		}
 	}
 
@@ -251,7 +289,7 @@ func FindBundleRootDir(localBundlePath string) (string, error) {
 
 	isInSubDir := true
 	for _, name := range names {
-		if name == "version.yaml" {
+		if name == constants.VERSION_FILENAME {
 			isInSubDir = false
 			break
 		}
@@ -265,7 +303,14 @@ func FindBundleRootDir(localBundlePath string) (string, error) {
 }
 
 func (f fileContentProvider) getFileContents(fileName string) ([]byte, error) {
-	return ioutil.ReadFile(filepath.Join(f.rootDir, fileName))
+	contents, err := os.ReadFile(filepath.Join(f.rootDir, fileName))
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, &types.NotFoundError{Name: fileName}
+		}
+		return nil, err
+	}
+	return contents, nil
 }
 
 func excludeFilePaths(files, excludeFiles []string) []string {
@@ -306,7 +351,7 @@ func (f fileContentProvider) getChildFileContents(dirName string, excludeFiles [
 
 	fileArr := map[string][]byte{}
 	for _, filePath := range files {
-		bytes, err := ioutil.ReadFile(filePath)
+		bytes, err := os.ReadFile(filePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "read %q", filePath)
 		}

pkg/analyze/download_test.go

@@ -0,0 +1,49 @@
+package analyzer
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/replicatedhq/troubleshoot/internal/testutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDownloadAndExtractSupportBundle(t *testing.T) {
+	// TODO: Add tests for web url downloads
+	tests := []struct {
+		name      string
+		bundleURL string
+		wantErr   bool
+	}{
+		{
+			name:      "extract a bundle from a local file path",
+			bundleURL: filepath.Join(testutils.FileDir(), "../../testdata/supportbundle/support-bundle.tar.gz"),
+			wantErr:   false,
+		},
+		{
+			name:      "extract a bundle from a non-existent file path",
+			bundleURL: "/home/someone/gibberish",
+			wantErr:   true,
+		},
+		{
+			name:      "extract an invalid support bundle which has no version file",
+			bundleURL: filepath.Join(testutils.FileDir(), "../../testdata/supportbundle/missing-version.tar.gz"),
+			wantErr:   true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpDir, bundleDir, err := DownloadAndExtractSupportBundle(tt.bundleURL)
+			defer os.RemoveAll(tmpDir) // clean up. Ignore error
+
+			if err == nil {
+				assert.DirExists(t, bundleDir)
+				assert.FileExists(t, filepath.Join(bundleDir, "version.yaml"))
+			} else {
+				assert.Equal(t, "", tmpDir)
+				assert.Equal(t, "", bundleDir)
+			}
+		})
+	}
+}