E2E preflight tests

.github/workflows/build-test-deploy.yaml

@@ -7,7 +7,7 @@ on:
       - "main"
     tags:
       - "v*.*.*"
-  
+
 jobs:
   build:
     runs-on: ubuntu-18.04
@@ -24,8 +24,8 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - run: make 
-  
+      - run: make
+
   compile-preflight:
     runs-on: ubuntu-latest
     needs: build
@@ -39,12 +39,12 @@ jobs:
           echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
         shell: bash
       - uses: actions/checkout@master
-      - run: make preflight
+      - run: make generate preflight
       - uses: actions/upload-artifact@v1
         with:
           name: preflight
           path: bin/preflight
-        
+
   validate-preflight:
     runs-on: ubuntu-latest
     needs: compile-preflight
@@ -83,6 +83,45 @@ jobs:
 
           exit $EXIT_STATUS
 
+  validate-preflight-e2e:
+    runs-on: ubuntu-latest
+    needs: compile-preflight
+    steps:
+      - uses: actions/checkout@master
+      - name: Download preflight binary
+        uses: actions/download-artifact@v1
+        with:
+          name: preflight
+          path: bin/
+      - uses: engineerd/setup-kind@v0.5.0
+      - run: chmod +x bin/preflight
+      - run: |
+          ./bin/preflight --interactive=false --format=json examples/preflight/e2e.yaml > result.json
+          cat result.json
+
+          EXIT_STATUS=0
+          if grep -q "was not collected" result.json; then
+            echo "Some files were not collected"
+            EXIT_STATUS=1
+          fi
+
+          if (( `jq '.pass | length' result.json` < 1 )); then
+            echo "No passing preflights found"
+            EXIT_STATUS=1
+          fi
+
+          if (( `jq '.warn | length' result.json` > 0 )); then
+            echo "Warnings found"
+            EXIT_STATUS=1
+          fi
+
+          if (( `jq '.fail | length' result.json` > 0 )); then
+            echo "Failed preflights found"
+            EXIT_STATUS=1
+          fi
+
+          exit $EXIT_STATUS
+
   compile-supportbundle:
     runs-on: ubuntu-latest
     needs: build
@@ -96,7 +135,7 @@ jobs:
           echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
         shell: bash
       - uses: actions/checkout@master
-      - run: make support-bundle
+      - run: make generate support-bundle
       - uses: actions/upload-artifact@v1
         with:
           name: support-bundle
@@ -118,16 +157,35 @@ jobs:
       - run: ./bin/support-bundle ./examples/support-bundle/sample-supportbundle.yaml
       - run: ./bin/support-bundle https://kots.io
 
+  compile-collect:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/setup-go@v1
+        with:
+          go-version: '1.17'
+      - name: setup env
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+        shell: bash
+      - uses: actions/checkout@master
+      - run: make generate collect
+      - uses: actions/upload-artifact@v1
+        with:
+          name: collect
+          path: bin/collect
+
   goreleaser-test:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v') != true
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-  
+
       - name: Unshallow
         run: git fetch --prune --unshallow
-  
+
       - uses: actions/setup-go@v1
         with:
           go-version: "1.17"
@@ -142,12 +200,13 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - validate-preflight
+      - validate-preflight-e2e
       - validate-supportbundle
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-  
+
       - uses: azure/docker-login@v1
         with:
           username: repldeploy2
@@ -155,7 +214,7 @@ jobs:
 
       - name: Unshallow
         run: git fetch --prune --unshallow
-  
+
       - uses: actions/setup-go@v1
         with:
           go-version: "1.17"
@@ -169,14 +228,14 @@ jobs:
           echo $COSIGN_KEY | base64 -d > ./cosign.key
         env:
           COSIGN_KEY: ${{secrets.COSIGN_KEY}}
-          
+
       - name: Generate SBOM
         run: |
           make sbom
         env:
           COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
           COSIGN_KEY: ${{ secrets.COSIGN_KEY }}
-   
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:

Makefile

@@ -46,19 +46,19 @@ test: generate fmt vet
 	go test ${BUILDFLAGS} ./pkg/... ./cmd/... -coverprofile cover.out
 
 .PHONY: support-bundle
-support-bundle: generate fmt vet
+support-bundle:
 	go build ${BUILDFLAGS} ${LDFLAGS} -o bin/support-bundle github.com/replicatedhq/troubleshoot/cmd/troubleshoot
 
 .PHONY: preflight
-preflight: generate fmt vet
+preflight:
 	go build ${BUILDFLAGS} ${LDFLAGS} -o bin/preflight github.com/replicatedhq/troubleshoot/cmd/preflight
 
 .PHONY: analyze
-analyze: generate fmt vet
+analyze:
 	go build ${BUILDFLAGS} ${LDFLAGS} -o bin/analyze github.com/replicatedhq/troubleshoot/cmd/analyze
 	
 .PHONY: collect
-collect: generate fmt vet
+collect:
 	go build ${BUILDFLAGS} ${LDFLAGS} -o bin/collect github.com/replicatedhq/troubleshoot/cmd/collect	
 
 .PHONY: fmt

examples/preflight/e2e.yaml

@@ -0,0 +1,98 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: Preflight
+metadata:
+  name: example
+spec:
+  collectors:
+    - data:
+        name: config/replicas.txt
+        data: "5"
+    - run:
+        collectorName: "static-hi"
+        image: 'alpine:3.5'
+        command: ["echo", "hi static!"]
+  analyzers:
+    - clusterVersion:
+        outcomes:
+          - fail:
+              when: "< 1.13.0"
+              message: This application requires at least Kubernetes 1.13.0 or later, and recommends 1.15.0.
+              uri: https://www.kubernetes.io
+          - warn:
+              when: "< 1.15.0"
+              message: Your cluster meets the minimum version of Kubernetes, but we recommend you update to 1.15.0 or later.
+              uri: https://kubernetes.io
+          - pass:
+              when: ">= 1.15.0"
+              message: Your cluster meets the recommended and required versions of Kubernetes.
+    - distribution:
+        outcomes:
+          - pass:
+              when: "== k3s"
+              message: K3S is a supported distribution
+          - warn:
+              message: Unable to determine the distribution of Kubernetes
+    - textAnalyze:
+        checkName: Replica Count
+        fileName: config/replicas.txt
+        regexGroups: '(?P<Replicas>\d+)'
+        outcomes:
+          - fail:
+              when: "Replicas < 5"
+              message: That's not enough replicas!
+          - pass:
+              message: You have at least 5 replicas
+    - textAnalyze:
+        checkName: Said hi!
+        fileName: /static-hi.log
+        regex: 'hi static'
+        outcomes:
+          - fail:
+              message: Didn't say hi.
+          - pass:
+              message: Said hi!
+    - nodeResources:
+        checkName: Must be exactly 1 node in the cluster
+        outcomes:
+          - pass:
+              when: "= 1"
+              message: This cluster has exactly 1 node
+          - fail:
+              message: This application requires exactly 1 node
+    - nodeResources:
+        checkName: Must have 1 node with at least 2 cores
+        filters:
+          cpuCapacity: "2"
+        outcomes:
+          - pass:
+              when: "= 1"
+              message: This cluster has exactly 1 node with at least 2 cores
+          - fail:
+              message: This application requires exactly 1 node with at least 2 cores
+    - nodeResources:
+        checkName: Must have 1 node with 2Gi (available) memory and at least 2 cores (on a single node)
+        filters:
+          allocatableMemory: 2Gi
+          cpuCapacity: "2"
+        outcomes:
+          - pass:
+              when: "= 1"
+              message: This cluster has exactly 1 node with at least 2Gi available memory and 2 cores
+          - fail:
+              message: This application requires exactly 1 node with at least 2Gi available memory and 2 cores
+    - nodeResources:
+        checkName: There must be exactly 1 node in the cluster
+        outcomes:
+          - pass:
+              when: "count() = 1"
+              message: This has exactly 1 node in the cluster
+          - fail:
+              message: This application requires exactly 1 node in the cluster
+    - nodeResources:
+        checkName: There must be a total of at least 2Gi of memory on all nodes
+        outcomes:
+          - fail:
+              when: "sum(memoryCapacity) < 2Gi"
+              message: This application requires that 2Gi or more memory be available to the cluster
+          - pass:
+              message: This cluster has sufficient memory