fix mssql and mysql analyzers

* stashing changes

* split filesystem collector into fio and legacy functions

* read fio results into analyzer

* remove test script

* update go.mod

* remove old notes

* go mod tidy

* fix up go.mod

* fix up go.mod

* refactor tests for fio

* make schemas

* remove local scripts

* local watch script for building troubleshoot

* document watch script

* fix var names

* handle errors if run as non-root

* go mod tidy

* use String interface

* collector happy path test

* invalid filesize

* invalid filesize

* tests

* remove old code

* remove old init function

* let actions tests run this

* clean up tests

* go mod tidy

* remove duplicated type declaration

* remove old file create code

CONTRIBUTING.md

@@ -94,6 +94,12 @@ go tool pprof -http=":8000" cpu.prof
 go tool pprof -http=":8001" mem.prof
 ```
 
+**Additional flags for memory profiling**
+- `inuse_space`: Amount of memory allocated and not released yet (default).
+- `inuse_objects`: Amount of objects allocated and not released yet.
+- `alloc_space`: Total amount of memory allocated (regardless of released).
+- `alloc_objects`: Total amount of objects allocated (regardless of released).
+
 More on profiling please visit https://go.dev/doc/diagnostics#profiling
 
 ## Contribution workflow

Makefile

@@ -252,4 +252,4 @@ watchrsync: npm-install
 .PHONY: npm-install
 npm-install:
 	npm --version 2>&1 >/dev/null || ( echo "npm not installed; install npm to set up watchrsync" && exit 1 )
-	npm list gaze-run-interrupt || npm install install gaze-run-interrupt@~2.0.0
+	npm list gaze-run-interrupt || npm install install --no-save gaze-run-interrupt@~2.0.0

examples/preflight/host/cpu.yaml

@@ -19,4 +19,3 @@ spec:
               message: At least 16 CPU cores preferred
           - pass:
               message: This server has sufficient CPU cores
-

go.mod

@@ -24,31 +24,31 @@ require (
 	github.com/manifoldco/promptui v0.9.0
 	github.com/mattn/go-isatty v0.0.19
 	github.com/mholt/archiver/v3 v3.5.1
-	github.com/microsoft/go-mssqldb v1.5.0
+	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/pkg/errors v0.9.1
 	github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil/v3 v3.23.8
+	github.com/shirou/gopsutil/v3 v3.23.9
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tj/go-spin v1.1.0
-	go.opentelemetry.io/otel v1.17.0
-	go.opentelemetry.io/otel/sdk v1.17.0
+	go.opentelemetry.io/otel v1.18.0
+	go.opentelemetry.io/otel/sdk v1.18.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/sync v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.28.1
+	k8s.io/api v0.28.2
 	k8s.io/apiextensions-apiserver v0.28.1
-	k8s.io/apimachinery v0.28.1
+	k8s.io/apimachinery v0.28.2
 	k8s.io/apiserver v0.28.1
-	k8s.io/cli-runtime v0.28.1
-	k8s.io/client-go v0.28.1
+	k8s.io/cli-runtime v0.28.2
+	k8s.io/client-go v0.28.2
 	k8s.io/klog/v2 v2.100.1
 	oras.land/oras-go v1.2.3
-	sigs.k8s.io/controller-runtime v0.15.1
+	sigs.k8s.io/controller-runtime v0.16.2
 	sigs.k8s.io/e2e-framework v0.3.0
 )
 
@@ -86,6 +86,7 @@ require (
 	github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/onsi/ginkgo v1.14.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/rubenv/sql-migrate v1.3.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -98,10 +99,10 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	go.opentelemetry.io/otel/metric v1.17.0 // indirect
-	go.opentelemetry.io/otel/trace v1.17.0 // indirect
+	go.opentelemetry.io/otel/metric v1.18.0 // indirect
+	go.opentelemetry.io/otel/trace v1.18.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 	k8s.io/component-base v0.28.1 // indirect
@@ -118,7 +119,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
 	github.com/andybalholm/brotli v1.0.1 // indirect
-	github.com/aws/aws-sdk-go v1.44.122 // indirect
+	github.com/aws/aws-sdk-go v1.44.198 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect
@@ -220,11 +221,11 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/net v0.13.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.13.0
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -238,7 +239,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.12.3
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/metrics v0.28.1
+	k8s.io/metrics v0.28.2
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 	periph.io/x/host/v3 v3.8.2
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -193,12 +193,15 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
+github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 h1:yfJe15aSwEQ6Oo6J+gdfdulPNoZ3TEhmbhLIoxZcA+U=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 h1:HCc0+LpPfpCKs6LGGLAhwBARt9632unrVcI6i8s/8os=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -241,8 +244,9 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo=
 github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.198 h1:kgnvxQv4/kP5M0nbxBx0Ac0so9ndr9f8Ti0g+NmPQF8=
+github.com/aws/aws-sdk-go v1.44.198/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -328,8 +332,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
 github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI=
-github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM=
 github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
@@ -435,8 +437,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -565,8 +566,6 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
@@ -599,8 +598,6 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@@ -613,7 +610,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -635,12 +631,6 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
 github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
-github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -687,7 +677,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
@@ -748,8 +738,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
 github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
-github.com/microsoft/go-mssqldb v1.5.0 h1:CgENxkwtOBNj3Jg6T1X209y2blCfTTcwuOlznd2k9fk=
-github.com/microsoft/go-mssqldb v1.5.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho=
+github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc=
+github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.25 h1:dFwPR6SfLtrSwgDcIq2bcU/gVutB4sNApq2HBdqcakg=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
@@ -795,10 +785,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
-github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
@@ -812,14 +800,20 @@ github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE
 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/nwaples/rardecode v1.1.2 h1:Cj0yZY6T1Zx1R7AhTbyGSALm44/Mmq+BAPc4B/p/d3M=
 github.com/nwaples/rardecode v1.1.2/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
@@ -845,7 +839,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
 github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM=
 github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -910,8 +904,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod
 github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
-github.com/shirou/gopsutil/v3 v3.23.8 h1:xnATPiybo6GgdRoC4YoGnxXZFRc3dqQTGi73oLvvBrE=
-github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1id3C6aM8EDqQ=
+github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E=
+github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -1041,26 +1035,25 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM=
-go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0=
-go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc=
-go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o=
-go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE=
-go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ=
-go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ=
-go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY=
+go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs=
+go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
+go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ=
+go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
+go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY=
+go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M=
+go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10=
+go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -1079,11 +1072,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1122,7 +1112,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1155,6 +1144,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -1185,12 +1175,8 @@ golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfS
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
-golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1233,7 +1219,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1256,12 +1241,14 @@ golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1276,6 +1263,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1299,7 +1287,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1334,22 +1321,19 @@ golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1362,9 +1346,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1432,9 +1413,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1444,7 +1424,7 @@ golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNq
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1678,7 +1658,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
@@ -1714,18 +1693,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108=
-k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg=
+k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
 k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw=
 k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs=
-k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY=
-k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw=
+k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
 k8s.io/apiserver v0.28.1 h1:dw2/NKauDZCnOUAzIo2hFhtBRUo6gQK832NV8kuDbGM=
 k8s.io/apiserver v0.28.1/go.mod h1:d8aizlSRB6yRgJ6PKfDkdwCy2DXt/d1FDR6iJN9kY1w=
-k8s.io/cli-runtime v0.28.1 h1:7Njc4eD5kaO4tYdSYVJJEs54koYD/vT6gxOq8dEVf9g=
-k8s.io/cli-runtime v0.28.1/go.mod h1:yIThSWkAVLqeRs74CMkq6lNFW42GyJmvMtcNn01SZho=
-k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8=
-k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE=
+k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk=
+k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA=
+k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
+k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
 k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg=
 k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
@@ -1734,8 +1713,8 @@ k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5Ohx
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
 k8s.io/kubectl v0.28.1 h1:jAq4yKEqQL+fwkWcEsUWxhJ7uIRcOYQraJxx4SyAMTY=
 k8s.io/kubectl v0.28.1/go.mod h1:a0nk/lMMeKBulp0lMTJAKbkjZg1ykqfLfz/d6dnv1ak=
-k8s.io/metrics v0.28.1 h1:Q0AsAEZKlAzhqrvfoGyHjz2qAFlef0SqfGJ1YWJ+ITU=
-k8s.io/metrics v0.28.1/go.mod h1:8lKkAajigcZWu0o9XCEBr++YVCzT48q1ck+f9CEBhZY=
+k8s.io/metrics v0.28.2 h1:Z/oMk5SmiT/Ji1SaWOPfW2l9W831BLO9/XxDq9iS3ak=
+k8s.io/metrics v0.28.2/go.mod h1:QTIIdjMrq+KodO+rmp6R9Pr1LZO8kTArNtkWoQXw0sw=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=
@@ -1745,8 +1724,8 @@ periph.io/x/host/v3 v3.8.2/go.mod h1:yFL76AesNHR68PboofSWYaQTKmvPXsQH2Apvp/ls/K4
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c=
-sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk=
+sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU=
+sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU=
 sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U=
 sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/analyze/common_status.go

@@ -2,10 +2,11 @@ package analyzer
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
-	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"strconv"
 	"strings"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 )
 
 func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey string, iconURI string, readyReplicas int, exists bool, resourceType string) (*AnalyzeResult, error) {
@@ -64,7 +65,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 			if exists == false && outcome.Warn.When != "absent" {
 				result.IsFail = true
 				result.Message = fmt.Sprintf("The %s %q was not found", resourceType, name)
-				result.URI = outcome.Fail.URI
+				result.URI = outcome.Warn.URI
 				return result, nil
 			}
 
@@ -104,7 +105,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 			if exists == false && outcome.Pass.When != "absent" {
 				result.IsFail = true
 				result.Message = fmt.Sprintf("The %s %q was not found", resourceType, name)
-				result.URI = outcome.Fail.URI
+				result.URI = outcome.Pass.URI
 				return result, nil
 			}
 

pkg/analyze/host_filesystem_performance.go

@@ -42,7 +42,18 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(
 		return nil, errors.Wrapf(err, "failed to get collected file %s", name)
 	}
 
-	fsPerf := collect.FSPerfResults{}
+	fioResult := collect.FioResult{}
+	if err := json.Unmarshal(contents, &fioResult); err != nil {
+		return nil, errors.Wrapf(err, "failed to unmarshal fio results from %s", name)
+	}
+
+	if len(fioResult.Jobs) == 0 {
+		return nil, errors.Errorf("no jobs found in fio results from %s", name)
+	}
+
+	fioWriteLatency := fioResult.Jobs[0].Sync
+
+	fsPerf := fioWriteLatency.FSPerfResults()
 	if err := json.Unmarshal(contents, &fsPerf); err != nil {
 		return nil, errors.Wrapf(err, "failed to unmarshal filesystem performance results from %s", name)
 	}
@@ -179,7 +190,7 @@ func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsP
 		return doCompareHostFilesystemPerformance(comparator, fsPerf.P9999, desiredDuration)
 	}
 
-	return false, fmt.Errorf("Unknown filesystem performance keyword %q", keyword)
+	return false, fmt.Errorf("unknown filesystem performance keyword %q", keyword)
 }
 
 func doCompareHostFilesystemPerformance(operator string, actual time.Duration, desired time.Duration) (bool, error) {
@@ -196,7 +207,7 @@ func doCompareHostFilesystemPerformance(operator string, actual time.Duration, d
 		return actual == desired, nil
 	}
 
-	return false, fmt.Errorf("Unknown filesystem performance operator %q", operator)
+	return false, fmt.Errorf("unknown filesystem performance operator %q", operator)
 }
 
 func renderFSPerfOutcome(outcome string, fsPerf collect.FSPerfResults) string {

pkg/analyze/host_filesystem_performance_test.go

@@ -1,12 +1,9 @@
 package analyzer
 
 import (
-	"encoding/json"
 	"testing"
-	"time"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
-	"github.com/replicatedhq/troubleshoot/pkg/collect"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -14,35 +11,305 @@ import (
 func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 	tests := []struct {
 		name         string
-		fsPerf       *collect.FSPerfResults
+		fioResult    string
 		hostAnalyzer *troubleshootv1beta2.FilesystemPerformanceAnalyze
 		result       []*AnalyzeResult
 		expectErr    bool
 	}{
 		{
 			name: "Cover",
-			fsPerf: &collect.FSPerfResults{
-				Min:     200 * time.Nanosecond,
-				Max:     time.Second,
-				Average: 55 * time.Microsecond,
-				P1:      1 * time.Microsecond,
-				P5:      5 * time.Microsecond,
-				P10:     10 * time.Microsecond,
-				P20:     20 * time.Microsecond,
-				P30:     30 * time.Microsecond,
-				P40:     40 * time.Microsecond,
-				P50:     50 * time.Microsecond,
-				P60:     60 * time.Microsecond,
-				P70:     70 * time.Microsecond,
-				P80:     80 * time.Microsecond,
-				P90:     90 * time.Microsecond,
-				P95:     95 * time.Microsecond,
-				P99:     99 * time.Microsecond,
-				P995:    995 * time.Microsecond,
-				P999:    999 * time.Microsecond,
-				P9995:   5 * time.Millisecond,
-				P9999:   9 * time.Millisecond,
-			},
+			fioResult: `{
+				"fio version" : "fio-3.28",
+				"timestamp" : 1691679955,
+				"timestamp_ms" : 1691679955590,
+				"time" : "Thu Aug 10 15:05:55 2023",
+				"global options" : {
+					"rw" : "write",
+					"ioengine" : "sync",
+					"fdatasync" : "1",
+					"directory" : "/var/lib/etcd",
+					"size" : "23068672",
+					"bs" : "1024"
+				},
+				"jobs" : [
+					{
+						"jobname" : "fsperf",
+						"groupid" : 0,
+						"error" : 0,
+						"eta" : 0,
+						"elapsed" : 15,
+						"job options" : {
+							"name" : "fsperf",
+							"runtime" : "120"
+						},
+						"read" : {
+							"io_bytes" : 0,
+							"io_kbytes" : 0,
+							"bw_bytes" : 0,
+							"bw" : 0,
+							"iops" : 0.000000,
+							"runtime" : 0,
+							"total_ios" : 0,
+							"short_ios" : 22527,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"lat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"bw_min" : 0,
+							"bw_max" : 0,
+							"bw_agg" : 0.000000,
+							"bw_mean" : 0.000000,
+							"bw_dev" : 0.000000,
+							"bw_samples" : 0,
+							"iops_min" : 0,
+							"iops_max" : 0,
+							"iops_mean" : 0.000000,
+							"iops_stddev" : 0.000000,
+							"iops_samples" : 0
+						},
+						"write" : {
+							"io_bytes" : 23068672,
+							"io_kbytes" : 22528,
+							"bw_bytes" : 1651182,
+							"bw" : 1612,
+							"iops" : 1612.483001,
+							"runtime" : 13971,
+							"total_ios" : 22528,
+							"short_ios" : 0,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 200,
+								"max" : 1000000000,
+								"mean" : 55000,
+								"stddev" : 12345.6789,
+								"N" : 32400,
+								"percentile" : {
+									"1.000000" : 1000,
+									"5.000000" : 5000,
+									"10.000000" : 10000,
+									"20.000000" : 20000,
+									"30.000000" : 30000,
+									"40.000000" : 40000,
+									"50.000000" : 50000,
+									"60.000000" : 60000,
+									"70.000000" : 70000,
+									"80.000000" : 80000,
+									"90.000000" : 90000,
+									"95.000000" : 95000,
+									"99.000000" : 99000,
+									"99.500000" : 995000,
+									"99.900000" : 999000,
+									"99.950000" : 5000000,
+									"99.990000" : 9000000
+								}
+							},
+							"lat_ns" : {
+								"min" : 2684,
+								"max" : 8710446,
+								"mean" : 95169.335405,
+								"stddev" : 172145.383902,
+								"N" : 22528
+							},
+							"bw_min" : 1516,
+							"bw_max" : 1706,
+							"bw_agg" : 100.000000,
+							"bw_mean" : 1613.629630,
+							"bw_dev" : 35.708379,
+							"bw_samples" : 27,
+							"iops_min" : 1516,
+							"iops_max" : 1706,
+							"iops_mean" : 1613.629630,
+							"iops_stddev" : 35.708379,
+							"iops_samples" : 27
+						},
+						"trim" : {
+							"io_bytes" : 0,
+							"io_kbytes" : 0,
+							"bw_bytes" : 0,
+							"bw" : 0,
+							"iops" : 0.000000,
+							"runtime" : 0,
+							"total_ios" : 0,
+							"short_ios" : 0,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"lat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"bw_min" : 0,
+							"bw_max" : 0,
+							"bw_agg" : 0.000000,
+							"bw_mean" : 0.000000,
+							"bw_dev" : 0.000000,
+							"bw_samples" : 0,
+							"iops_min" : 0,
+							"iops_max" : 0,
+							"iops_mean" : 0.000000,
+							"iops_stddev" : 0.000000,
+							"iops_samples" : 0
+						},
+						"sync" : {
+							"total_ios" : 0,
+							"lat_ns" : {
+								"min" : 200,
+								"max" : 1000000000,
+								"mean" : 55000,
+								"stddev" : 12345.6789,
+								"N" : 32400,
+								"percentile" : {
+									"1.000000" : 1000,
+									"5.000000" : 5000,
+									"10.000000" : 10000,
+									"20.000000" : 20000,
+									"30.000000" : 30000,
+									"40.000000" : 40000,
+									"50.000000" : 50000,
+									"60.000000" : 60000,
+									"70.000000" : 70000,
+									"80.000000" : 80000,
+									"90.000000" : 90000,
+									"95.000000" : 95000,
+									"99.000000" : 99000,
+									"99.500000" : 995000,
+									"99.900000" : 999000,
+									"99.950000" : 5000000,
+									"99.990000" : 9000000
+								}
+							}
+						},
+						"job_runtime" : 13970,
+						"usr_cpu" : 1.410165,
+						"sys_cpu" : 5.454545,
+						"ctx" : 72137,
+						"majf" : 0,
+						"minf" : 16,
+						"iodepth_level" : {
+							"1" : 199.995561,
+							"2" : 0.000000,
+							"4" : 0.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							">=64" : 0.000000
+						},
+						"iodepth_submit" : {
+							"0" : 0.000000,
+							"4" : 100.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							"64" : 0.000000,
+							">=64" : 0.000000
+						},
+						"iodepth_complete" : {
+							"0" : 0.000000,
+							"4" : 100.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							"64" : 0.000000,
+							">=64" : 0.000000
+						},
+						"latency_ns" : {
+							"2" : 0.000000,
+							"4" : 0.000000,
+							"10" : 0.000000,
+							"20" : 0.000000,
+							"50" : 0.000000,
+							"100" : 0.000000,
+							"250" : 0.000000,
+							"500" : 0.000000,
+							"750" : 0.000000,
+							"1000" : 0.000000
+						},
+						"latency_us" : {
+							"2" : 0.000000,
+							"4" : 27.077415,
+							"10" : 42.032138,
+							"20" : 5.450994,
+							"50" : 0.306286,
+							"100" : 0.026634,
+							"250" : 0.461648,
+							"500" : 23.291016,
+							"750" : 1.269531,
+							"1000" : 0.035511
+						},
+						"latency_ms" : {
+							"2" : 0.026634,
+							"4" : 0.017756,
+							"10" : 0.010000,
+							"20" : 0.000000,
+							"50" : 0.000000,
+							"100" : 0.000000,
+							"250" : 0.000000,
+							"500" : 0.000000,
+							"750" : 0.000000,
+							"1000" : 0.000000,
+							"2000" : 0.000000,
+							">=2000" : 0.000000
+						},
+						"latency_depth" : 1,
+						"latency_target" : 0,
+						"latency_percentile" : 100.000000,
+						"latency_window" : 0
+					}
+				],
+				"disk_util" : [
+					{
+						"name" : "sda",
+						"read_ios" : 5610,
+						"write_ios" : 45550,
+						"read_merges" : 0,
+						"write_merges" : 568,
+						"read_ticks" : 1863,
+						"write_ticks" : 11605,
+						"in_queue" : 14353,
+						"util" : 99.435028
+					}
+				]
+			}`,
 			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
 				CollectorName: "etcd",
 				Outcomes: []*troubleshootv1beta2.Outcome{
@@ -298,9 +565,298 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 		},
 		{
 			name: "skip warn if pass first",
-			fsPerf: &collect.FSPerfResults{
-				P99: 9 * time.Millisecond,
-			},
+			fioResult: `{
+				"fio version" : "fio-3.28",
+				"timestamp" : 1691679955,
+				"timestamp_ms" : 1691679955590,
+				"time" : "Thu Aug 10 15:05:55 2023",
+				"global options" : {
+					"rw" : "write",
+					"ioengine" : "sync",
+					"fdatasync" : "1",
+					"directory" : "/var/lib/etcd",
+					"size" : "23068672",
+					"bs" : "1024"
+				},
+				"jobs" : [
+					{
+						"jobname" : "fsperf",
+						"groupid" : 0,
+						"error" : 0,
+						"eta" : 0,
+						"elapsed" : 15,
+						"job options" : {
+							"name" : "fsperf",
+							"runtime" : "120"
+						},
+						"read" : {
+							"io_bytes" : 0,
+							"io_kbytes" : 0,
+							"bw_bytes" : 0,
+							"bw" : 0,
+							"iops" : 0.000000,
+							"runtime" : 0,
+							"total_ios" : 0,
+							"short_ios" : 22527,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"lat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"bw_min" : 0,
+							"bw_max" : 0,
+							"bw_agg" : 0.000000,
+							"bw_mean" : 0.000000,
+							"bw_dev" : 0.000000,
+							"bw_samples" : 0,
+							"iops_min" : 0,
+							"iops_max" : 0,
+							"iops_mean" : 0.000000,
+							"iops_stddev" : 0.000000,
+							"iops_samples" : 0
+						},
+						"write" : {
+							"io_bytes" : 23068672,
+							"io_kbytes" : 22528,
+							"bw_bytes" : 1651182,
+							"bw" : 1612,
+							"iops" : 1612.483001,
+							"runtime" : 13971,
+							"total_ios" : 22528,
+							"short_ios" : 0,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 200,
+								"max" : 1000000000,
+								"mean" : 55000,
+								"stddev" : 12345.6789,
+								"N" : 32400,
+								"percentile" : {
+									"1.000000" : 1000,
+									"5.000000" : 5000,
+									"10.000000" : 10000,
+									"20.000000" : 20000,
+									"30.000000" : 30000,
+									"40.000000" : 40000,
+									"50.000000" : 50000,
+									"60.000000" : 60000,
+									"70.000000" : 70000,
+									"80.000000" : 80000,
+									"90.000000" : 90000,
+									"95.000000" : 95000,
+									"99.000000" : 99000,
+									"99.500000" : 995000,
+									"99.900000" : 999000,
+									"99.950000" : 5000000,
+									"99.990000" : 9000000
+								}
+							},
+							"lat_ns" : {
+								"min" : 2684,
+								"max" : 8710446,
+								"mean" : 95169.335405,
+								"stddev" : 172145.383902,
+								"N" : 22528
+							},
+							"bw_min" : 1516,
+							"bw_max" : 1706,
+							"bw_agg" : 100.000000,
+							"bw_mean" : 1613.629630,
+							"bw_dev" : 35.708379,
+							"bw_samples" : 27,
+							"iops_min" : 1516,
+							"iops_max" : 1706,
+							"iops_mean" : 1613.629630,
+							"iops_stddev" : 35.708379,
+							"iops_samples" : 27
+						},
+						"trim" : {
+							"io_bytes" : 0,
+							"io_kbytes" : 0,
+							"bw_bytes" : 0,
+							"bw" : 0,
+							"iops" : 0.000000,
+							"runtime" : 0,
+							"total_ios" : 0,
+							"short_ios" : 0,
+							"drop_ios" : 0,
+							"slat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"clat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"lat_ns" : {
+								"min" : 0,
+								"max" : 0,
+								"mean" : 0.000000,
+								"stddev" : 0.000000,
+								"N" : 0
+							},
+							"bw_min" : 0,
+							"bw_max" : 0,
+							"bw_agg" : 0.000000,
+							"bw_mean" : 0.000000,
+							"bw_dev" : 0.000000,
+							"bw_samples" : 0,
+							"iops_min" : 0,
+							"iops_max" : 0,
+							"iops_mean" : 0.000000,
+							"iops_stddev" : 0.000000,
+							"iops_samples" : 0
+						},
+						"sync" : {
+							"total_ios" : 0,
+							"lat_ns" : {
+								"min" : 200,
+								"max" : 1000000000,
+								"mean" : 55000,
+								"stddev" : 12345.6789,
+								"N" : 32400,
+								"percentile" : {
+									"1.000000" : 1000,
+									"5.000000" : 5000,
+									"10.000000" : 10000,
+									"20.000000" : 20000,
+									"30.000000" : 30000,
+									"40.000000" : 40000,
+									"50.000000" : 50000,
+									"60.000000" : 60000,
+									"70.000000" : 70000,
+									"80.000000" : 80000,
+									"90.000000" : 90000,
+									"95.000000" : 95000,
+									"99.000000" : 9000000,
+									"99.500000" : 995000,
+									"99.900000" : 999000,
+									"99.950000" : 5000000,
+									"99.990000" : 9000000
+								}
+							}
+						},
+						"job_runtime" : 13970,
+						"usr_cpu" : 1.410165,
+						"sys_cpu" : 5.454545,
+						"ctx" : 72137,
+						"majf" : 0,
+						"minf" : 16,
+						"iodepth_level" : {
+							"1" : 199.995561,
+							"2" : 0.000000,
+							"4" : 0.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							">=64" : 0.000000
+						},
+						"iodepth_submit" : {
+							"0" : 0.000000,
+							"4" : 100.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							"64" : 0.000000,
+							">=64" : 0.000000
+						},
+						"iodepth_complete" : {
+							"0" : 0.000000,
+							"4" : 100.000000,
+							"8" : 0.000000,
+							"16" : 0.000000,
+							"32" : 0.000000,
+							"64" : 0.000000,
+							">=64" : 0.000000
+						},
+						"latency_ns" : {
+							"2" : 0.000000,
+							"4" : 0.000000,
+							"10" : 0.000000,
+							"20" : 0.000000,
+							"50" : 0.000000,
+							"100" : 0.000000,
+							"250" : 0.000000,
+							"500" : 0.000000,
+							"750" : 0.000000,
+							"1000" : 0.000000
+						},
+						"latency_us" : {
+							"2" : 0.000000,
+							"4" : 27.077415,
+							"10" : 42.032138,
+							"20" : 5.450994,
+							"50" : 0.306286,
+							"100" : 0.026634,
+							"250" : 0.461648,
+							"500" : 23.291016,
+							"750" : 1.269531,
+							"1000" : 0.035511
+						},
+						"latency_ms" : {
+							"2" : 0.026634,
+							"4" : 0.017756,
+							"10" : 0.010000,
+							"20" : 0.000000,
+							"50" : 0.000000,
+							"100" : 0.000000,
+							"250" : 0.000000,
+							"500" : 0.000000,
+							"750" : 0.000000,
+							"1000" : 0.000000,
+							"2000" : 0.000000,
+							">=2000" : 0.000000
+						},
+						"latency_depth" : 1,
+						"latency_target" : 0,
+						"latency_percentile" : 100.000000,
+						"latency_window" : 0
+					}
+				],
+				"disk_util" : [
+					{
+						"name" : "sda",
+						"read_ios" : 5610,
+						"write_ios" : 45550,
+						"read_merges" : 0,
+						"write_merges" : 568,
+						"read_ticks" : 1863,
+						"write_ticks" : 11605,
+						"in_queue" : 14353,
+						"util" : 99.435028
+					}
+				]
+			}`,
 			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
 				CollectorName: "file system performance",
 				Outcomes: []*troubleshootv1beta2.Outcome{
@@ -332,20 +888,66 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "bail if malformed JSON",
+			fioResult: `{
+				bad JSON
+			}`,
+			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
+				CollectorName: "file system performance",
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							Message: "bad JSON should not be analyzed",
+						},
+					},
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name: "bail if fio ran no jobs",
+			fioResult: `{
+				"fio version" : "fio-3.28",
+				"timestamp" : 1691679955,
+				"timestamp_ms" : 1691679955590,
+				"time" : "Thu Aug 10 15:05:55 2023",
+				"global options" : {
+					"rw" : "write",
+					"ioengine" : "sync",
+					"fdatasync" : "1",
+					"directory" : "/var/lib/etcd",
+					"size" : "23068672",
+					"bs" : "1024"
+				},
+				"jobs" : [
+				]
+			}`,
+			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
+				CollectorName: "file system performance",
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							Message: "an empty Jobs array should not be analyzed",
+						},
+					},
+				},
+			},
+			expectErr: true,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			req := require.New(t)
-			b, err := json.Marshal(test.fsPerf)
-			if err != nil {
-				t.Fatal(err)
-			}
+
+			b := []byte(test.fioResult)
 
 			getCollectedFileContents := func(filename string) ([]byte, error) {
 				return b, nil
 			}
 
-			result, err := (&AnalyzeHostFilesystemPerformance{test.hostAnalyzer}).Analyze(getCollectedFileContents, nil)
+			a := AnalyzeHostFilesystemPerformance{test.hostAnalyzer}
+			result, err := a.Analyze(getCollectedFileContents, nil)
 			if test.expectErr {
 				req.Error(err)
 			} else {

pkg/analyze/mssql.go

@@ -43,7 +43,7 @@ func (a *AnalyzeMssql) collectorName() string {
 	if a.analyzer.CollectorName != "" {
 		return a.analyzer.CollectorName
 	}
-	return "mysql"
+	return "mssql"
 }
 
 func compareMssqlConditionalToActual(conditional string, result *collect.DatabaseConnection) (bool, error) {

pkg/analyze/mysql.go

@@ -40,11 +40,11 @@ func (a *AnalyzeMysql) collectorName() string {
 	if a.analyzer.CollectorName != "" {
 		return a.analyzer.CollectorName
 	}
-	return "mssql"
+	return "mysql"
 }
 
 func (a *AnalyzeMysql) analyzeMysql(analyzer *troubleshootv1beta2.DatabaseAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
-	fullPath := path.Join("mssql", fmt.Sprintf("%s.json", a.collectorName()))
+	fullPath := path.Join("mysql", fmt.Sprintf("%s.json", a.collectorName()))
 
 	collected, err := getCollectedFileContents(fullPath)
 	if err != nil {
@@ -58,7 +58,7 @@ func (a *AnalyzeMysql) analyzeMysql(analyzer *troubleshootv1beta2.DatabaseAnalyz
 
 	result := &AnalyzeResult{
 		Title:   a.Title(),
-		IconKey: "kubernetes_mssql_analyze",
+		IconKey: "kubernetes_mysql_analyze",
 		IconURI: "https://troubleshoot.sh/images/analyzer-icons/mysql-analyze.svg",
 	}
 

pkg/collect/ceph.go

@@ -98,6 +98,13 @@ var CephCommands = []CephCommand{
 		Format:         "txt",
 		DefaultTimeout: "30s",
 	},
+	{
+		ID:             "df", // the disk usage of each pool
+		Command:        []string{"ceph", "df"},
+		Args:           []string{"-f", "json-pretty"},
+		Format:         "json",
+		DefaultTimeout: "30s",
+	},
 	{
 		ID:             "osd-df",
 		Command:        []string{"ceph", "osd", "df"},

pkg/collect/host_filesystem_performance.go

@@ -2,16 +2,35 @@ package collect
 
 import (
 	"bytes"
+	"encoding/json"
+	"fmt"
 	"math"
-	"math/rand"
+	"os/exec"
+	"reflect"
+	"strconv"
+	"strings"
 	"text/template"
 	"time"
 
+	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/klog/v2"
 )
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
+type Durations []time.Duration
+
+func (d Durations) Len() int {
+	return len(d)
+}
+
+func (d Durations) Less(i, j int) bool {
+	return d[i] < d[j]
+}
+
+func (d Durations) Swap(i, j int) {
+	d[i], d[j] = d[j], d[i]
 }
 
 type CollectHostFilesystemPerformance struct {
@@ -90,3 +109,324 @@ func (f FSPerfResults) String() string {
 
 	return buf.String()
 }
+
+type FioResult struct {
+	FioVersion    string           `json:"fio version,omitempty"`
+	Timestamp     int64            `json:"timestamp,omitempty"`
+	TimestampMS   int64            `json:"timestamp_ms,omitempty"`
+	Time          string           `json:"time,omitempty"`
+	GlobalOptions FioGlobalOptions `json:"global options,omitempty"`
+	Jobs          []FioJobs        `json:"jobs,omitempty"`
+	DiskUtil      []FioDiskUtil    `json:"disk_util,omitempty"`
+}
+
+func (f FioResult) String() string {
+	var res string
+	res += fmt.Sprintf("FIO version - %s\n", f.FioVersion)
+	res += fmt.Sprintf("Global options - %s\n\n", f.GlobalOptions)
+	for _, job := range f.Jobs {
+		res += fmt.Sprintf("%s\n", job)
+	}
+	res += "Disk stats (read/write):\n"
+	for _, du := range f.DiskUtil {
+		res += fmt.Sprintf("%s\n", du)
+	}
+
+	return res
+}
+
+type FioGlobalOptions struct {
+	Directory  string `json:"directory,omitempty"`
+	RandRepeat string `json:"randrepeat,omitempty"`
+	Verify     string `json:"verify,omitempty"`
+	IOEngine   string `json:"ioengine,omitempty"`
+	Direct     string `json:"direct,omitempty"`
+	GtodReduce string `json:"gtod_reduce,omitempty"`
+}
+
+func (g FioGlobalOptions) String() string {
+	return fmt.Sprintf("ioengine=%s verify=%s direct=%s gtod_reduce=%s", g.IOEngine, g.Verify, g.Direct, g.GtodReduce)
+}
+
+type FioJobs struct {
+	JobName           string        `json:"jobname,omitempty"`
+	GroupID           int           `json:"groupid,omitempty"`
+	Error             int           `json:"error,omitempty"`
+	Eta               int           `json:"eta,omitempty"`
+	Elapsed           int           `json:"elapsed,omitempty"`
+	JobOptions        FioJobOptions `json:"job options,omitempty"`
+	Read              FioStats      `json:"read,omitempty"`
+	Write             FioStats      `json:"write,omitempty"`
+	Trim              FioStats      `json:"trim,omitempty"`
+	Sync              FioStats      `json:"sync,omitempty"`
+	JobRuntime        int32         `json:"job_runtime,omitempty"`
+	UsrCpu            float32       `json:"usr_cpu,omitempty"`
+	SysCpu            float32       `json:"sys_cpu,omitempty"`
+	Ctx               int32         `json:"ctx,omitempty"`
+	MajF              int32         `json:"majf,omitempty"`
+	MinF              int32         `json:"minf,omitempty"`
+	IoDepthLevel      FioDepth      `json:"iodepth_level,omitempty"`
+	IoDepthSubmit     FioDepth      `json:"iodepth_submit,omitempty"`
+	IoDepthComplete   FioDepth      `json:"iodepth_complete,omitempty"`
+	LatencyNs         FioLatency    `json:"latency_ns,omitempty"`
+	LatencyUs         FioLatency    `json:"latency_us,omitempty"`
+	LatencyMs         FioLatency    `json:"latency_ms,omitempty"`
+	LatencyDepth      int32         `json:"latency_depth,omitempty"`
+	LatencyTarget     int32         `json:"latency_target,omitempty"`
+	LatencyPercentile float32       `json:"latency_percentile,omitempty"`
+	LatencyWindow     int32         `json:"latency_window,omitempty"`
+}
+
+func (j FioJobs) String() string {
+	var job string
+	job += fmt.Sprintf("%s\n", j.JobOptions)
+	if j.Read.Iops != 0 || j.Read.BW != 0 {
+		job += fmt.Sprintf("read:\n%s\n", j.Read)
+	}
+	if j.Write.Iops != 0 || j.Write.BW != 0 {
+		job += fmt.Sprintf("write:\n%s\n", j.Write)
+	}
+	return job
+}
+
+type FioJobOptions struct {
+	Name      string `json:"name,omitempty"`
+	BS        string `json:"bs,omitempty"`
+	Directory string `json:"directory,omitempty"`
+	RW        string `json:"rw,omitempty"`
+	IOEngine  string `json:"ioengine,omitempty"`
+	FDataSync string `json:"fdatasync,omitempty"`
+	Size      string `json:"size,omitempty"`
+	RunTime   string `json:"runtime,omitempty"`
+}
+
+func (o FioJobOptions) String() string {
+	return fmt.Sprintf("JobName: %s\n  blocksize=%s filesize=%s rw=%s", o.Name, o.BS, o.Size, o.RW)
+}
+
+type FioStats struct {
+	IOBytes     int64         `json:"io_bytes,omitempty"`
+	IOKBytes    int64         `json:"io_kbytes,omitempty"`
+	BWBytes     int64         `json:"bw_bytes,omitempty"`
+	BW          int64         `json:"bw,omitempty"`
+	Iops        float32       `json:"iops,omitempty"`
+	Runtime     int64         `json:"runtime,omitempty"`
+	TotalIos    int64         `json:"total_ios,omitempty"`
+	ShortIos    int64         `json:"short_ios,omitempty"`
+	DropIos     int64         `json:"drop_ios,omitempty"`
+	SlatNs      FioNS         `json:"slat_ns,omitempty"`
+	ClatNs      FioNS         `json:"clat_ns,omitempty"`
+	LatNs       FioNS         `json:"lat_ns,omitempty"`
+	Percentile  FioPercentile `json:"percentile,omitempty"`
+	BwMin       int64         `json:"bw_min,omitempty"`
+	BwMax       int64         `json:"bw_max,omitempty"`
+	BwAgg       float32       `json:"bw_agg,omitempty"`
+	BwMean      float32       `json:"bw_mean,omitempty"`
+	BwDev       float32       `json:"bw_dev,omitempty"`
+	BwSamples   int32         `json:"bw_samples,omitempty"`
+	IopsMin     int32         `json:"iops_min,omitempty"`
+	IopsMax     int32         `json:"iops_max,omitempty"`
+	IopsMean    float32       `json:"iops_mean,omitempty"`
+	IopsStdDev  float32       `json:"iops_stddev,omitempty"`
+	IopsSamples int32         `json:"iops_samples,omitempty"`
+}
+
+func (s FioStats) String() string {
+	var stats string
+	stats += fmt.Sprintf("  IOPS=%f BW(KiB/s)=%d\n", s.Iops, s.BW)
+	stats += fmt.Sprintf("  iops: min=%d max=%d avg=%f\n", s.IopsMin, s.IopsMax, s.IopsMean)
+	stats += fmt.Sprintf("  bw(KiB/s): min=%d max=%d avg=%f", s.BwMin, s.BwMax, s.BwMean)
+	return stats
+}
+
+func (s FioStats) FSPerfResults() FSPerfResults {
+	return FSPerfResults{
+		Min:     time.Duration(s.LatNs.Min),
+		Max:     time.Duration(s.LatNs.Max),
+		Average: time.Duration(s.LatNs.Mean),
+		P1:      time.Duration(s.LatNs.Percentile.P1),
+		P5:      time.Duration(s.LatNs.Percentile.P5),
+		P10:     time.Duration(s.LatNs.Percentile.P10),
+		P20:     time.Duration(s.LatNs.Percentile.P20),
+		P30:     time.Duration(s.LatNs.Percentile.P30),
+		P40:     time.Duration(s.LatNs.Percentile.P40),
+		P50:     time.Duration(s.LatNs.Percentile.P50),
+		P60:     time.Duration(s.LatNs.Percentile.P60),
+		P70:     time.Duration(s.LatNs.Percentile.P70),
+		P80:     time.Duration(s.LatNs.Percentile.P80),
+		P90:     time.Duration(s.LatNs.Percentile.P90),
+		P95:     time.Duration(s.LatNs.Percentile.P95),
+		P99:     time.Duration(s.LatNs.Percentile.P99),
+		P995:    time.Duration(s.LatNs.Percentile.P995),
+		P999:    time.Duration(s.LatNs.Percentile.P999),
+		P9995:   time.Duration(s.LatNs.Percentile.P9995),
+		P9999:   time.Duration(s.LatNs.Percentile.P9999),
+	}
+}
+
+type FioNS struct {
+	Min        int64         `json:"min,omitempty"`
+	Max        int64         `json:"max,omitempty"`
+	Mean       float32       `json:"mean,omitempty"`
+	StdDev     float32       `json:"stddev,omitempty"`
+	N          int64         `json:"N,omitempty"`
+	Percentile FioPercentile `json:"percentile,omitempty"`
+}
+
+type FioDepth struct {
+	FioDepth0    float32 `json:"0,omitempty"`
+	FioDepth1    float32 `json:"1,omitempty"`
+	FioDepth2    float32 `json:"2,omitempty"`
+	FioDepth4    float32 `json:"4,omitempty"`
+	FioDepth8    float32 `json:"8,omitempty"`
+	FioDepth16   float32 `json:"16,omitempty"`
+	FioDepth32   float32 `json:"32,omitempty"`
+	FioDepth64   float32 `json:"64,omitempty"`
+	FioDepthGE64 float32 `json:">=64,omitempty"`
+}
+
+type FioLatency struct {
+	FioLat2      float32 `json:"2,omitempty"`
+	FioLat4      float32 `json:"4,omitempty"`
+	FioLat10     float32 `json:"10,omitempty"`
+	FioLat20     float32 `json:"20,omitempty"`
+	FioLat50     float32 `json:"50,omitempty"`
+	FioLat100    float32 `json:"100,omitempty"`
+	FioLat250    float32 `json:"250,omitempty"`
+	FioLat500    float32 `json:"500,omitempty"`
+	FioLat750    float32 `json:"750,omitempty"`
+	FioLat1000   float32 `json:"1000,omitempty"`
+	FioLat2000   float32 `json:"2000,omitempty"`
+	FioLatGE2000 float32 `json:">=2000,omitempty"`
+}
+
+type FioDiskUtil struct {
+	Name        string  `json:"name,omitempty"`
+	ReadIos     int64   `json:"read_ios,omitempty"`
+	WriteIos    int64   `json:"write_ios,omitempty"`
+	ReadMerges  int64   `json:"read_merges,omitempty"`
+	WriteMerges int64   `json:"write_merges,omitempty"`
+	ReadTicks   int64   `json:"read_ticks,omitempty"`
+	WriteTicks  int64   `json:"write_ticks,omitempty"`
+	InQueue     int64   `json:"in_queue,omitempty"`
+	Util        float32 `json:"util,omitempty"`
+}
+
+type FioPercentile struct {
+	P1    int `json:"1.000000,omitempty"`
+	P5    int `json:"5.000000,omitempty"`
+	P10   int `json:"10.000000,omitempty"`
+	P20   int `json:"20.000000,omitempty"`
+	P30   int `json:"30.000000,omitempty"`
+	P40   int `json:"40.000000,omitempty"`
+	P50   int `json:"50.000000,omitempty"`
+	P60   int `json:"60.000000,omitempty"`
+	P70   int `json:"70.000000,omitempty"`
+	P80   int `json:"80.000000,omitempty"`
+	P90   int `json:"90.000000,omitempty"`
+	P95   int `json:"95.000000,omitempty"`
+	P99   int `json:"99.000000,omitempty"`
+	P995  int `json:"99.500000,omitempty"`
+	P999  int `json:"99.900000,omitempty"`
+	P9995 int `json:"99.950000,omitempty"`
+	P9999 int `json:"99.990000,omitempty"`
+}
+
+func (d FioDiskUtil) String() string {
+	//Disk stats (read/write):
+	//rbd4: ios=30022/11982, merge=0/313, ticks=1028675/1022768, in_queue=2063740, util=99.67%
+	var du string
+	du += fmt.Sprintf("  %s: ios=%d/%d merge=%d/%d ticks=%d/%d in_queue=%d, util=%f%%", d.Name, d.ReadIos,
+		d.WriteIos, d.ReadMerges, d.WriteMerges, d.ReadTicks, d.WriteTicks, d.InQueue, d.Util)
+	return du
+}
+
+func parseCollectorOptions(hostCollector *troubleshootv1beta2.FilesystemPerformance) ([]string, *FioJobOptions, error) {
+
+	var operationSize uint64 = 1024
+	if hostCollector.OperationSizeBytes > 0 {
+		operationSize = hostCollector.OperationSizeBytes
+	}
+	var fileSize uint64 = 10 * 1024 * 1024
+	if hostCollector.FileSize != "" {
+		quantity, err := resource.ParseQuantity(hostCollector.FileSize)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "failed to parse fileSize %q", hostCollector.FileSize)
+		}
+		fileSizeInt64, ok := quantity.AsInt64()
+		if !ok {
+			return nil, nil, errors.Wrapf(err, "failed to parse fileSize %q", hostCollector.FileSize)
+		}
+		if fileSizeInt64 <= 0 {
+			return nil, nil, errors.Wrapf(err, "fileSize %q must be greater than 0", hostCollector.FileSize)
+		}
+		fileSize = uint64(fileSizeInt64)
+	}
+
+	if hostCollector.Directory == "" {
+		return nil, nil, errors.New("Directory is required to collect filesystem performance info")
+	}
+
+	latencyBenchmarkOptions := FioJobOptions{
+		RW:        "write",
+		IOEngine:  "sync",
+		FDataSync: "1",
+		Directory: hostCollector.Directory,
+		Size:      strconv.FormatUint(fileSize, 10),
+		BS:        strconv.FormatUint(operationSize, 10),
+		Name:      "fsperf",
+		RunTime:   "120",
+	}
+
+	command := buildFioCommand(latencyBenchmarkOptions)
+
+	return command, &latencyBenchmarkOptions, nil
+}
+
+func buildFioCommand(opts FioJobOptions) []string {
+	command := []string{"fio"}
+	v := reflect.ValueOf(opts)
+	t := reflect.TypeOf(opts)
+	for i := 0; i < v.NumField(); i++ {
+		field := t.Field(i)
+		value := v.Field(i)
+		if !value.IsZero() {
+			command = append(command, fmt.Sprintf("--%s=%v", strings.ToLower(field.Name), value.Interface()))
+		}
+	}
+	command = append(command, "--output-format=json")
+	return command
+}
+
+func collectFioResults(ctx context.Context, hostCollector *troubleshootv1beta2.FilesystemPerformance) (*FioResult, error) {
+
+	command, opts, err := parseCollectorOptions(hostCollector)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse collector options")
+	}
+
+	klog.V(2).Infof("collecting fio results: %s", strings.Join(command, " "))
+	output, err := exec.CommandContext(ctx, command[0], command[1:]...).Output()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if exitErr.ExitCode() == 1 {
+				return nil, errors.Wrapf(err, "fio failed; permission denied opening %s.  ensure this collector runs as root", opts.Directory)
+			} else {
+				return nil, errors.Wrapf(err, "fio failed with exit status %d", exitErr.ExitCode())
+			}
+		} else if e, ok := err.(*exec.Error); ok && e.Err == exec.ErrNotFound {
+			return nil, errors.Wrapf(err, "command not found: %v. ensure fio is installed", command)
+		} else {
+			return nil, errors.Wrapf(err, "failed to run command: %v", command)
+		}
+	}
+
+	var result FioResult
+	err = json.Unmarshal([]byte(output), &result)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal fio result")
+	}
+
+	return &result, nil
+}

pkg/collect/host_filesystem_performance_linux.go

@@ -10,36 +10,22 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"sort"
 	"sync"
 	"syscall"
 	"time"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-type Durations []time.Duration
-
-func (d Durations) Len() int {
-	return len(d)
-}
-
-func (d Durations) Less(i, j int) bool {
-	return d[i] < d[j]
-}
-
-func (d Durations) Swap(i, j int) {
-	d[i], d[j] = d[j], d[i]
-}
+// Today we only care about checking for write latency so the options struct
+// only has what we need for that.  we'll collect all the results from a single run of fio
+// and filter out the fsync results for analysis.  TODO: update the analyzer so any/all results
+// from fio can be analyzed.
 
 func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.FilesystemPerformance, bundlePath string) (map[string][]byte, error) {
 	timeout := time.Minute
+
 	if hostCollector.Timeout != "" {
 		d, err := time.ParseDuration(hostCollector.Timeout)
 		if err != nil {
@@ -50,46 +36,15 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	var operationSize uint64 = 1024
-	if hostCollector.OperationSizeBytes != 0 {
-		operationSize = hostCollector.OperationSizeBytes
+	collectorName := hostCollector.CollectorName
+	if collectorName == "" {
+		collectorName = "filesystemPerformance"
 	}
+	name := filepath.Join("host-collectors/filesystemPerformance", collectorName+".json")
 
-	var fileSize uint64 = 10 * 1024 * 1024
-	if hostCollector.FileSize != "" {
-		quantity, err := resource.ParseQuantity(hostCollector.FileSize)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse fileSize %q", hostCollector.FileSize)
-		}
-		fileSizeInt64, ok := quantity.AsInt64()
-		if !ok {
-			return nil, errors.Wrapf(err, "failed to parse fileSize %q", hostCollector.FileSize)
-		}
-		fileSize = uint64(fileSizeInt64)
-	}
-
-	if hostCollector.Directory == "" {
-		return nil, errors.New("Directory is required to collect filesystem performance info")
-	}
-	// TODO: clean up this directory if its created
 	if err := os.MkdirAll(hostCollector.Directory, 0700); err != nil {
 		return nil, errors.Wrapf(err, "failed to mkdir %q", hostCollector.Directory)
 	}
-	filename := filepath.Join(hostCollector.Directory, "fsperf")
-
-	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
-	if err != nil {
-		log.Panic(err)
-		return nil, errors.Wrapf(err, "open %s", filename)
-	}
-	defer func() {
-		if err := f.Close(); err != nil {
-			log.Println(err.Error())
-		}
-		if err := os.Remove(filename); err != nil {
-			log.Println(err.Error())
-		}
-	}()
 
 	// Start the background IOPS task and wait for warmup
 	if hostCollector.EnableBackgroundIOPS {
@@ -123,86 +78,16 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 		time.Sleep(time.Second * time.Duration(hostCollector.BackgroundIOPSWarmupSeconds))
 	}
 
-	// Sequential writes benchmark
-	var written uint64 = 0
-	var results Durations
+	var fioResult *FioResult
 
-	for {
-		if written >= fileSize {
-			break
-		}
-
-		data := make([]byte, int(operationSize))
-		rand.Read(data)
-
-		start := time.Now()
-
-		n, err := f.Write(data)
-		if err != nil {
-			return nil, errors.Wrapf(err, "write to %s", filename)
-		}
-		if hostCollector.Sync {
-			if err := f.Sync(); err != nil {
-				return nil, errors.Wrapf(err, "sync %s", filename)
-			}
-		} else if hostCollector.Datasync {
-			if err := syscall.Fdatasync(int(f.Fd())); err != nil {
-				return nil, errors.Wrapf(err, "datasync %s", filename)
-			}
-		}
-
-		d := time.Now().Sub(start)
-		results = append(results, d)
-
-		written += uint64(n)
-
-		if ctx.Err() != nil {
-			break
-		}
-	}
-
-	if len(results) == 0 {
-		return nil, errors.New("No filesystem performance results collected")
-	}
-
-	sort.Sort(results)
-
-	var sum time.Duration
-	for _, d := range results {
-		sum += d
-	}
-
-	fsPerf := &FSPerfResults{
-		Min:     results[0],
-		Max:     results[len(results)-1],
-		Average: sum / time.Duration(len(results)),
-		P1:      results[getPercentileIndex(.01, len(results))],
-		P5:      results[getPercentileIndex(.05, len(results))],
-		P10:     results[getPercentileIndex(.1, len(results))],
-		P20:     results[getPercentileIndex(.2, len(results))],
-		P30:     results[getPercentileIndex(.3, len(results))],
-		P40:     results[getPercentileIndex(.4, len(results))],
-		P50:     results[getPercentileIndex(.5, len(results))],
-		P60:     results[getPercentileIndex(.6, len(results))],
-		P70:     results[getPercentileIndex(.7, len(results))],
-		P80:     results[getPercentileIndex(.8, len(results))],
-		P90:     results[getPercentileIndex(.9, len(results))],
-		P95:     results[getPercentileIndex(.95, len(results))],
-		P99:     results[getPercentileIndex(.99, len(results))],
-		P995:    results[getPercentileIndex(.995, len(results))],
-		P999:    results[getPercentileIndex(.999, len(results))],
-		P9995:   results[getPercentileIndex(.9995, len(results))],
-		P9999:   results[getPercentileIndex(.9999, len(results))],
-	}
-
-	collectorName := hostCollector.CollectorName
-	if collectorName == "" {
-		collectorName = "filesystemPerformance"
-	}
-	name := filepath.Join("host-collectors/filesystemPerformance", collectorName+".json")
-	b, err := json.Marshal(fsPerf)
+	fioResult, err := collectFioResults(ctx, hostCollector)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal fs perf results")
+		return nil, errors.Wrap(err, "failed to collect fio results")
+	}
+
+	b, err := json.Marshal(fioResult)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal fio results")
 	}
 
 	output := NewResult()

pkg/collect/host_filesystem_performance_test.go

@@ -2,7 +2,10 @@ package collect
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 )
 
 func TestGetPercentileIndex(t *testing.T) {
@@ -57,3 +60,128 @@ func TestGetPercentileIndex(t *testing.T) {
 		})
 	}
 }
+
+func Test_parseCollectorOptions(t *testing.T) {
+	type args struct {
+		hostCollector *troubleshootv1beta2.FilesystemPerformance
+	}
+	tests := []struct {
+		name        string
+		args        args
+		wantCommand []string
+		wantOptions *FioJobOptions
+		wantErr     bool
+	}{
+		{
+			name: "Happy spec",
+			args: args{
+				hostCollector: &troubleshootv1beta2.FilesystemPerformance{
+					HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{
+						CollectorName: "fsperf",
+					},
+					OperationSizeBytes:          1024,
+					Directory:                   "/var/lib/etcd",
+					FileSize:                    "22Mi",
+					Sync:                        true,
+					Datasync:                    true,
+					Timeout:                     "120",
+					EnableBackgroundIOPS:        true,
+					BackgroundIOPSWarmupSeconds: 10,
+					BackgroundWriteIOPS:         100,
+					BackgroundReadIOPS:          100,
+					BackgroundWriteIOPSJobs:     1,
+					BackgroundReadIOPSJobs:      1,
+				},
+			},
+			wantCommand: []string{
+				"fio",
+				"--name=fsperf",
+				"--bs=1024",
+				"--directory=/var/lib/etcd",
+				"--rw=write",
+				"--ioengine=sync",
+				"--fdatasync=1",
+				"--size=23068672",
+				"--runtime=120",
+				"--output-format=json",
+			},
+			wantOptions: &FioJobOptions{
+				RW:        "write",
+				IOEngine:  "sync",
+				FDataSync: "1",
+				Directory: "/var/lib/etcd",
+				Size:      "23068672",
+				BS:        "1024",
+				Name:      "fsperf",
+				RunTime:   "120",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Empty spec fails",
+			args: args{
+				hostCollector: &troubleshootv1beta2.FilesystemPerformance{
+					HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{
+						CollectorName: "fsperf",
+					},
+				},
+			},
+			wantCommand: nil,
+			wantOptions: nil,
+			wantErr:     true,
+		},
+		{
+			name: "Invalid filesize",
+			args: args{
+				hostCollector: &troubleshootv1beta2.FilesystemPerformance{
+					HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{
+						CollectorName: "fsperf",
+					},
+					OperationSizeBytes: 1024,
+					Directory:          "/var/lib/etcd",
+					FileSize:           "abcd",
+					Sync:               true,
+					Datasync:           true,
+					Timeout:            "120",
+				},
+			},
+			wantCommand: nil,
+			wantOptions: nil,
+			wantErr:     true,
+		},
+		{
+			name: "invalid path parameter",
+			args: args{
+				hostCollector: &troubleshootv1beta2.FilesystemPerformance{
+					HostCollectorMeta: troubleshootv1beta2.HostCollectorMeta{
+						CollectorName: "fsperf",
+					},
+					OperationSizeBytes: 1024,
+					Directory:          "",
+					FileSize:           "22Mi",
+					Sync:               true,
+					Datasync:           true,
+					Timeout:            "120",
+				},
+			},
+			wantCommand: nil,
+			wantOptions: nil,
+			wantErr:     true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCommand, gotOptions, err := parseCollectorOptions(tt.args.hostCollector)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseCollectorOptions() error = %v, wantErr %v", err, tt.wantErr)
+			} else {
+				if !reflect.DeepEqual(gotCommand, tt.wantCommand) {
+					t.Errorf("parseCollectorOptions() got command = %v, want %v", gotCommand, tt.wantCommand)
+				}
+				if !reflect.DeepEqual(gotOptions, tt.wantOptions) {
+					t.Errorf("parseCollectorOptions() got options = %v, want %v", gotOptions, tt.wantOptions)
+				}
+			}
+		})
+	}
+}

pkg/collect/redact.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -17,18 +16,29 @@ import (
 	"k8s.io/klog/v2"
 )
 
+// Max number of concurrent redactors to run
+// Ensure the number is low enough since each of the redactors
+// also spawns goroutines to redact files in tar archives and
+// other goroutines for each redactor spec.
+const MAX_CONCURRENT_REDACTORS = 10
+
 func RedactResult(bundlePath string, input CollectorResult, additionalRedactors []*troubleshootv1beta2.Redact) error {
 	wg := &sync.WaitGroup{}
 
 	// Error channel to capture errors from goroutines
 	errorCh := make(chan error, len(input))
+	limitCh := make(chan struct{}, MAX_CONCURRENT_REDACTORS)
+	defer close(limitCh)
 
 	for k, v := range input {
+		limitCh <- struct{}{}
 
 		wg.Add(1)
 
 		go func(file string, data []byte) {
 			defer wg.Done()
+			defer func() { <-limitCh }() // free up after the function execution has run
+
 			var reader io.Reader
 			if data == nil {
 
@@ -84,7 +94,7 @@ func RedactResult(bundlePath string, input CollectorResult, additionalRedactors
 			// If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is
 			// decompressed and each file inside the tar redacted and compressed back into the archive.
 			if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") {
-				tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-")
+				tmpDir, err := os.MkdirTemp("", "troubleshoot-subresult-")
 				if err != nil {
 					errorCh <- errors.Wrap(err, "failed to create temp dir")
 					return

pkg/collect/result.go

@@ -132,7 +132,7 @@ func (r CollectorResult) SaveResult(bundlePath string, relativePath string, read
 		return errors.Wrap(err, "failed to stat file")
 	}
 
-	klog.V(2).Infof("Added %q (%d MB) to bundle output", relativePath, fileInfo.Size()/(1024*1024))
+	klog.V(2).Infof("Added %q (%d KB) to bundle output", relativePath, fileInfo.Size()/(1024))
 	return nil
 }
 

pkg/constants/constants.go

@@ -83,6 +83,11 @@ const (
 	MESSAGE_TEXT_PADDING                = 4
 	MESSAGE_TEXT_LINES_MARGIN_TO_BOTTOM = 4
 
-	// Bufio Reader Constants
-	MAX_BUFFER_CAPACITY = 1024 * 1024
+	// This is the initial size of the buffer allocated.
+	// Under the hood, an array of size N is allocated in memory
+	BUF_INIT_SIZE = 4096 // 4KB
+
+	// This is the muximum size the buffer can grow to
+	// Its not what the buffer will be allocated to initially
+	SCANNER_MAX_SIZE = 10 * 1024 * 1024 // 10MB
 )

pkg/redact/literal.go

@@ -2,23 +2,26 @@ package redact
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
-	"strings"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"k8s.io/klog/v2"
 )
 
 type literalRedactor struct {
-	matchString string
-	filePath    string
-	redactName  string
-	isDefault   bool
+	match      []byte
+	filePath   string
+	redactName string
+	isDefault  bool
 }
 
-func literalString(matchString, path, name string) Redactor {
+func literalString(match []byte, path, name string) Redactor {
 	return literalRedactor{
-		matchString: matchString,
-		filePath:    path,
-		redactName:  name,
+		match:      match,
+		filePath:   path,
+		redactName: name,
 	}
 }
 
@@ -28,32 +31,37 @@ func (r literalRedactor) Redact(input io.Reader, path string) io.Reader {
 	go func() {
 		var err error
 		defer func() {
-			if err == io.EOF {
+			if err == nil || err == io.EOF {
 				writer.Close()
 			} else {
+				if err == bufio.ErrTooLong {
+					s := fmt.Sprintf("Error redacting %q. A line in the file exceeded %d MB max length", path, constants.SCANNER_MAX_SIZE/1024/1024)
+					klog.V(2).Info(s)
+				} else {
+					klog.V(2).Info(fmt.Sprintf("Error redacting %q: %v", path, err))
+				}
 				writer.CloseWithError(err)
 			}
 		}()
 
-		reader := bufio.NewReader(input)
+		buf := make([]byte, constants.BUF_INIT_SIZE)
+		scanner := bufio.NewScanner(input)
+		scanner.Buffer(buf, constants.SCANNER_MAX_SIZE)
+
 		lineNum := 0
-		for {
+		for scanner.Scan() {
 			lineNum++
-			var line string
-			line, err = readLine(reader)
+			line := scanner.Bytes()
+
+			clean := bytes.ReplaceAll(line, r.match, maskTextBytes)
+
+			// Append newline since scanner strips it
+			err = writeBytes(writer, clean, NEW_LINE)
 			if err != nil {
 				return
 			}
 
-			clean := strings.ReplaceAll(line, r.matchString, MASK_TEXT)
-
-			// io.WriteString would be nicer, but scanner strips new lines
-			fmt.Fprintf(writer, "%s\n", clean)
-			if err != nil {
-				return
-			}
-
-			if clean != line {
+			if !bytes.Equal(clean, line) {
 				addRedaction(Redaction{
 					RedactorName:      r.redactName,
 					CharactersRemoved: len(line) - len(clean),
@@ -63,6 +71,9 @@ func (r literalRedactor) Redact(input io.Reader, path string) io.Reader {
 				})
 			}
 		}
+		if scanErr := scanner.Err(); scanErr != nil {
+			err = scanErr
+		}
 	}()
 	return out
 }

pkg/redact/multi_line.go

@@ -2,10 +2,9 @@ package redact
 
 import (
 	"bufio"
-	"fmt"
+	"bytes"
 	"io"
 	"regexp"
-	"strings"
 )
 
 type MultiLineRedactor struct {
@@ -20,19 +19,19 @@ type MultiLineRedactor struct {
 
 func NewMultiLineRedactor(re1 LineRedactor, re2 string, maskText, path, name string, isDefault bool) (*MultiLineRedactor, error) {
 	var scanCompiled *regexp.Regexp
-	compiled1, err := regexp.Compile(re1.regex)
+	compiled1, err := compileRegex(re1.regex)
 	if err != nil {
 		return nil, err
 	}
 
 	if re1.scan != "" {
-		scanCompiled, err = regexp.Compile(re1.scan)
+		scanCompiled, err = compileRegex(re1.scan)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	compiled2, err := regexp.Compile(re2)
+	compiled2, err := compileRegex(re2)
 	if err != nil {
 		return nil, err
 	}
@@ -48,14 +47,18 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
 			writer.CloseWithError(err)
 		}()
 
-		substStr := getReplacementPattern(r.re2, r.maskText)
+		substStr := []byte(getReplacementPattern(r.re2, r.maskText))
 
 		reader := bufio.NewReader(input)
 		line1, line2, err := getNextTwoLines(reader, nil)
 		if err != nil {
 			// this will print 2 blank lines for empty input...
-			fmt.Fprintf(writer, "%s\n", line1)
-			fmt.Fprintf(writer, "%s\n", line2)
+			// Append newlines since scanner strips them
+			err = writeBytes(writer, line1, NEW_LINE, line2, NEW_LINE)
+			if err != nil {
+				return
+			}
+
 			return
 		}
 
@@ -66,33 +69,41 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
 
 			// is scan is not nil, then check if line1 matches scan by lowercasing it
 			if r.scan != nil {
-				lowerLine1 := strings.ToLower(line1)
-				if !r.scan.MatchString(lowerLine1) {
-					fmt.Fprintf(writer, "%s\n", line1)
-					line1, line2, err = getNextTwoLines(reader, &line2)
+				lowerLine1 := bytes.ToLower(line1)
+				if !r.scan.Match(lowerLine1) {
+					// Append newline since scanner strips it
+					err = writeBytes(writer, line1, NEW_LINE)
+					if err != nil {
+						return
+					}
+					line1, line2, err = getNextTwoLines(reader, line2)
 					flushLastLine = true
 					continue
 				}
 			}
 
 			// If line1 matches re1, then transform line2 using re2
-			if !r.re1.MatchString(line1) {
-				fmt.Fprintf(writer, "%s\n", line1)
-				line1, line2, err = getNextTwoLines(reader, &line2)
+			if !r.re1.Match(line1) {
+				// Append newline since scanner strips it
+				err = writeBytes(writer, line1, NEW_LINE)
+				if err != nil {
+					return
+				}
+				line1, line2, err = getNextTwoLines(reader, line2)
 				flushLastLine = true
 				continue
 			}
 			flushLastLine = false
-			clean := r.re2.ReplaceAllString(line2, substStr)
+			clean := r.re2.ReplaceAll(line2, substStr)
 
-			// io.WriteString would be nicer, but reader strips new lines
-			fmt.Fprintf(writer, "%s\n%s\n", line1, clean)
+			// Append newlines since scanner strips them
+			err = writeBytes(writer, line1, NEW_LINE, clean, NEW_LINE)
 			if err != nil {
 				return
 			}
 
 			// if clean is not equal to line2, a redaction was performed
-			if clean != line2 {
+			if !bytes.Equal(clean, line2) {
 				addRedaction(Redaction{
 					RedactorName:      r.redactName,
 					CharactersRemoved: len(line2) - len(clean),
@@ -106,15 +117,18 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
 		}
 
 		if flushLastLine {
-			fmt.Fprintf(writer, "%s\n", line1)
+			// Append newline since scanner strip it
+			err = writeBytes(writer, line1, NEW_LINE)
+			if err != nil {
+				return
+			}
 		}
 	}()
 	return out
 }
 
-func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) {
-	line1 = ""
-	line2 = ""
+func getNextTwoLines(reader *bufio.Reader, curLine2 []byte) (line1 []byte, line2 []byte, err error) {
+	line2 = []byte{}
 
 	if curLine2 == nil {
 		line1, err = readLine(reader)
@@ -126,7 +140,7 @@ func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line
 		return
 	}
 
-	line1 = *curLine2
+	line1 = curLine2
 	line2, err = readLine(reader)
 	if err != nil {
 		return
@@ -134,3 +148,15 @@ func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line
 
 	return
 }
+
+// writeBytes writes all byte slices to the writer
+// in the order they are passed in the variadic argument
+func writeBytes(w io.Writer, bs ...[]byte) error {
+	for _, b := range bs {
+		_, err := w.Write(b)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/redact/multi_line_test.go

@@ -2,13 +2,15 @@ package redact
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
+	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func Test_NewMultiLineRedactorr(t *testing.T) {
+func Test_NewMultiLineRedactor(t *testing.T) {
 	tests := []struct {
 		name        string
 		selector    LineRedactor
@@ -101,7 +103,7 @@ func Test_NewMultiLineRedactorr(t *testing.T) {
 			req.NoError(err)
 			outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "")
 
-			gotBytes, err := ioutil.ReadAll(outReader)
+			gotBytes, err := io.ReadAll(outReader)
 			req.NoError(err)
 			req.Equal(tt.wantString, string(gotBytes))
 			GetRedactionList()
@@ -109,3 +111,40 @@ func Test_NewMultiLineRedactorr(t *testing.T) {
 		})
 	}
 }
+
+func Test_writeBytes(t *testing.T) {
+	tests := []struct {
+		name       string
+		inputBytes [][]byte
+		want       string
+	}{
+		{
+			name:       "No newline",
+			inputBytes: [][]byte{[]byte("hello"), []byte("world")},
+			want:       "helloworld",
+		},
+		{
+			name:       "With newline",
+			inputBytes: [][]byte{[]byte("hello"), NEW_LINE, []byte("world"), NEW_LINE},
+			want:       "hello\nworld\n",
+		},
+		{
+			name:       "Empty line",
+			inputBytes: [][]byte{NEW_LINE},
+			want:       "\n",
+		},
+		{
+			name: "Nothing",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var w strings.Builder
+			err := writeBytes(&w, tt.inputBytes...)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.want, w.String())
+		})
+	}
+}

pkg/redact/redact.go

@@ -17,9 +17,16 @@ const (
 	MASK_TEXT = "***HIDDEN***"
 )
 
-var allRedactions RedactionList
-var redactionListMut sync.Mutex
-var pendingRedactions sync.WaitGroup
+var (
+	allRedactions     RedactionList
+	redactionListMut  sync.Mutex
+	pendingRedactions sync.WaitGroup
+
+	// A regex cache to avoid recompiling the same regexes over and over
+	regexCache     = map[string]*regexp.Regexp{}
+	regexCacheLock sync.Mutex
+	maskTextBytes  = []byte(MASK_TEXT)
+)
 
 func init() {
 	allRedactions = RedactionList{
@@ -28,6 +35,24 @@ func init() {
 	}
 }
 
+// A regex cache to avoid recompiling the same regexes over and over
+func compileRegex(pattern string) (*regexp.Regexp, error) {
+	regexCacheLock.Lock()
+	defer regexCacheLock.Unlock()
+
+	if cached, ok := regexCache[pattern]; ok {
+		return cached, nil
+	}
+
+	compiled, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	regexCache[pattern] = compiled
+	return compiled, nil
+}
+
 type Redactor interface {
 	Redact(input io.Reader, path string) io.Reader
 }
@@ -85,10 +110,19 @@ func ResetRedactionList() {
 		ByRedactor: map[string][]Redaction{},
 		ByFile:     map[string][]Redaction{},
 	}
+
+	// Clear the regex cache as well. We do not want
+	// to keep this around in long running processes
+	// that continually redact files
+	regexCacheLock.Lock()
+	defer regexCacheLock.Unlock()
+
+	regexCache = map[string]*regexp.Regexp{}
 }
 
 func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta2.Redact) ([]Redactor, error) {
 	additionalRedactors := []Redactor{}
+
 	for i, redact := range redacts {
 		if redact == nil {
 			continue
@@ -104,7 +138,7 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta2.Redact
 		}
 
 		for j, literal := range redact.Removals.Values {
-			additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, j, redact.Name, "literal")))
+			additionalRedactors = append(additionalRedactors, literalString([]byte(literal), path, redactorName(i, j, redact.Name, "literal")))
 		}
 
 		for j, re := range redact.Removals.Regex {
@@ -458,13 +492,13 @@ func getReplacementPattern(re *regexp.Regexp, maskText string) string {
 	return substStr
 }
 
-func readLine(r *bufio.Reader) (string, error) {
+func readLine(r *bufio.Reader) ([]byte, error) {
 	var completeLine []byte
 	for {
 		var line []byte
 		line, isPrefix, err := r.ReadLine()
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 
 		completeLine = append(completeLine, line...)
@@ -472,7 +506,7 @@ func readLine(r *bufio.Reader) (string, error) {
 			break
 		}
 	}
-	return string(completeLine), nil
+	return completeLine, nil
 }
 
 func addRedaction(redaction Redaction) {

pkg/redact/single_line.go

@@ -2,12 +2,13 @@ package redact
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"regexp"
-	"strings"
 
 	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"k8s.io/klog/v2"
 )
 
 type SingleLineRedactor struct {
@@ -19,15 +20,17 @@ type SingleLineRedactor struct {
 	isDefault  bool
 }
 
+var NEW_LINE = []byte{'\n'}
+
 func NewSingleLineRedactor(re LineRedactor, maskText, path, name string, isDefault bool) (*SingleLineRedactor, error) {
 	var scanCompiled *regexp.Regexp
-	compiled, err := regexp.Compile(re.regex)
+	compiled, err := compileRegex(re.regex)
 	if err != nil {
 		return nil, err
 	}
 
 	if re.scan != "" {
-		scanCompiled, err = regexp.Compile(re.scan)
+		scanCompiled, err = compileRegex(re.scan)
 		if err != nil {
 			return nil, err
 		}
@@ -42,50 +45,62 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader {
 	go func() {
 		var err error
 		defer func() {
-			if err == io.EOF {
+			if err == nil || err == io.EOF {
 				writer.Close()
 			} else {
+				if err == bufio.ErrTooLong {
+					s := fmt.Sprintf("Error redacting %q. A line in the file exceeded %d MB max length", path, constants.SCANNER_MAX_SIZE/1024/1024)
+					klog.V(2).Info(s)
+				} else {
+					klog.V(2).Info(fmt.Sprintf("Error redacting %q: %v", path, err))
+				}
 				writer.CloseWithError(err)
 			}
 		}()
 
-		substStr := getReplacementPattern(r.re, r.maskText)
+		substStr := []byte(getReplacementPattern(r.re, r.maskText))
 
-		buf := make([]byte, constants.MAX_BUFFER_CAPACITY)
+		buf := make([]byte, constants.BUF_INIT_SIZE)
 		scanner := bufio.NewScanner(input)
-		scanner.Buffer(buf, constants.MAX_BUFFER_CAPACITY)
+		scanner.Buffer(buf, constants.SCANNER_MAX_SIZE)
 
 		lineNum := 0
 		for scanner.Scan() {
 			lineNum++
-			line := scanner.Text()
+			line := scanner.Bytes()
 
 			// is scan is not nil, then check if line matches scan by lowercasing it
 			if r.scan != nil {
-				lowerLine := strings.ToLower(line)
-				if !r.scan.MatchString(lowerLine) {
-					fmt.Fprintf(writer, "%s\n", line)
+				lowerLine := bytes.ToLower(line)
+				if !r.scan.Match(lowerLine) {
+					// Append newline since scanner strips it
+					err = writeBytes(writer, line, NEW_LINE)
+					if err != nil {
+						return
+					}
 					continue
 				}
 			}
 
 			// if scan matches, but re does not, do not redact
-			if !r.re.MatchString(line) {
-				fmt.Fprintf(writer, "%s\n", line)
+			if !r.re.Match(line) {
+				// Append newline since scanner strips it
+				err = writeBytes(writer, line, NEW_LINE)
+				if err != nil {
+					return
+				}
 				continue
 			}
 
-			clean := r.re.ReplaceAllString(line, substStr)
-
-			// io.WriteString would be nicer, but scanner strips new lines
-			fmt.Fprintf(writer, "%s\n", clean)
-
+			clean := r.re.ReplaceAll(line, substStr)
+			// Append newline since scanner strips it
+			err = writeBytes(writer, clean, NEW_LINE)
 			if err != nil {
 				return
 			}
 
 			// if clean is not equal to line, a redaction was performed
-			if clean != line {
+			if !bytes.Equal(clean, line) {
 				addRedaction(Redaction{
 					RedactorName:      r.redactName,
 					CharactersRemoved: len(line) - len(clean),

pkg/redact/single_line_test.go

@@ -2,7 +2,7 @@ package redact
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -320,7 +320,7 @@ func TestNewSingleLineRedactor(t *testing.T) {
 			req.NoError(err)
 
 			outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "")
-			gotBytes, err := ioutil.ReadAll(outReader)
+			gotBytes, err := io.ReadAll(outReader)
 			req.NoError(err)
 			req.Equal(tt.wantString, string(gotBytes))
 

testdata/filesystem_performance_preflight.yaml

@@ -0,0 +1,19 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: sample
+spec:
+  collectors:
+    - filesystemPerformance:
+        collectorName: Filesystem Latency Two Minute Benchmark
+        timeout: 2m
+        directory: /var/lib/etcd
+        fileSize: 22Mi
+        operationSizeBytes: 2300
+        datasync: true
+        enableBackgroundIOPS: true
+        backgroundIOPSWarmupSeconds: 10
+        backgroundWriteIOPS: 300
+        backgroundWriteIOPSJobs: 6
+        backgroundReadIOPS: 50
+        backgroundReadIOPSJobs: 1

testdata/kurl_preflights.yaml

@@ -0,0 +1,552 @@
+# https://kurl.sh/docs/install-with-kurl/system-requirements
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: kurl-builtin
+spec:
+  collectors:
+    - time: {}
+    - cpu: {}
+    - memory: {}
+    - hostServices: {}
+    - hostOS: {}
+    - diskUsage:
+        collectorName: "Ephemeral Disk Usage /var/lib/kubelet"
+        path: /var/lib/kubelet
+    - diskUsage:
+        collectorName: "Ephemeral Disk Usage /var/lib/docker"
+        path: /var/lib/docker
+        exclude: '{{kurl not .Installer.Spec.Docker.Version}}'
+    - diskUsage:
+        collectorName: "Ephemeral Disk Usage /var/lib/containerd"
+        path: /var/lib/containerd
+        exclude: '{{kurl not .Installer.Spec.Containerd.Version}}'
+    - diskUsage:
+        collectorName: "Ephemeral Disk Usage /var/lib/rook"
+        path: /var/lib/rook
+        exclude: '{{kurl not .Installer.Spec.Rook.Version}}'
+    - diskUsage:
+        collectorName: "Ephemeral Disk Usage /var/openebs"
+        path: /var/openebs
+        exclude: '{{kurl not .Installer.Spec.OpenEBS.Version}}'
+    - tcpLoadBalancer:
+        collectorName: "Kubernetes API Server Load Balancer"
+        port: 6443
+        address: {{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }}
+        timeout: 3m
+        # ha and is first master (primary and not join) and not is upgrade
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.LoadBalancerAddress .IsPrimary (not .IsJoin) (not .IsUpgrade) | not }}'
+    - http:
+        collectorName: "Kubernetes API Server Load Balancer Upgrade"
+        get:
+          url: https://{{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress | trimSuffix "/" }}/healthz
+          insecureSkipVerify: true
+        # ha and is first master (primary and not join) and is upgrade (the load balancer backend should already be available)
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.LoadBalancerAddress .IsPrimary .IsUpgrade (not .IsJoin) | not }}'
+    - tcpPortStatus:
+        collectorName: "Kubernetes API TCP Port Status"
+        port: 6443
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+    - tcpPortStatus:
+        collectorName: "ETCD Client API TCP Port Status"
+        port: 2379
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+    - tcpPortStatus:
+        collectorName: "ETCD Server API TCP Port Status"
+        port: 2380
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+    - tcpPortStatus:
+        collectorName: "ETCD Health Server TCP Port Status"
+        port: 2381
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        interface: lo
+    - tcpPortStatus:
+        collectorName: "Kubelet Health Server TCP Port Status"
+        port: 10248
+        exclude: '{{kurl and (not .IsUpgrade) | not }}'
+        interface: lo
+    - tcpPortStatus:
+        collectorName: "Kubelet API TCP Port Status"
+        port: 10250
+        exclude: '{{kurl and (not .IsUpgrade) | not }}'
+    - tcpPortStatus:
+        collectorName: "Kube Controller Manager Health Server TCP Port Status"
+        port: 10257
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        interface: lo
+    - tcpPortStatus:
+        collectorName: "Kube Scheduler Health Server TCP Port Status"
+        port: 10259
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        interface: lo
+    - tcpConnect:
+        collectorName: "Kubernetes API TCP Connection Status"
+        address: '{{kurl .Installer.Spec.Kubernetes.MasterAddress }}'
+        # run the collector if 1. there is a master address set AND this is a node joining the cluster AND this is not an EKCO internalLB install
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.MasterAddress .IsJoin (and .Installer.Spec.Ekco.Version .Installer.Spec.Ekco.EnableInternalLoadBalancer | not) | not }}'
+    - filesystemPerformance:
+        collectorName: Filesystem Latency Two Minute Benchmark
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        timeout: 2m
+        directory: /var/lib/etcd
+        fileSize: 22Mi
+        operationSizeBytes: 2300
+        datasync: true
+        enableBackgroundIOPS: true
+        backgroundIOPSWarmupSeconds: 10
+        backgroundWriteIOPS: 300
+        backgroundWriteIOPSJobs: 6
+        backgroundReadIOPS: 50
+        backgroundReadIOPSJobs: 1
+    - certificate:
+        collectorName: "Kubernetes API key pair certificate"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        certificatePath: /etc/kubernetes/pki/apiserver.crt
+        keyPath: /etc/kubernetes/pki/apiserver.key
+    - certificate:
+        collectorName: "Kubernetes ETCD key pair certificate"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        certificatePath: /etc/kubernetes/pki/etcd/server.crt
+        keyPath: /etc/kubernetes/pki/etcd/server.key
+    - http:
+        collectorName: "Kubernetes API Health"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        get:
+          url: https://localhost:6443/healthz
+          insecureSkipVerify: true
+  analyzers:
+    - certificate:
+        collectorName: "Kubernetes API key pair certificate"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        outcomes:
+          - fail:
+              when: "key-pair-missing"
+              message: Kubernetes API key pair certificate not found in /etc/kubernetes/pki/apiserver.*
+          - fail:
+              when: "key-pair-switched"
+              message: Kubernetes API key pair certificate and key pair are switched
+          - fail:
+              when: "key-pair-encrypted"
+              message: Kubernetes API key pair certificate private key is encrypted
+          - fail:
+              when: "key-pair-mismatch"
+              message: Kubernetes API key pair certificate and key do not match
+          - fail:
+              when: "key-pair-invalid"
+              message: Kubernetes API key pair certificate is invalid
+          - pass:
+              when: "key-pair-valid"
+              message: Kubernetes API key pair certificate is valid
+    - certificate:
+        collectorName: "Kubernetes ETCD key pair certificate"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        outcomes:
+          - fail:
+              when: "key-pair-missing"
+              message: Kubernetes ETCD key pair certificate not found in /etc/kubernetes/pki/etcd/server.*
+          - fail:
+              when: "key-pair-switched"
+              message: Kubernetes ETCD certificate and key pair are switched
+          - fail:
+              when: "key-pair-encrypted"
+              message: Kubernetes ETCD certificate private key is encrypted
+          - fail:
+              when: "key-pair-mismatch"
+              message: Kubernetes ETCD certificate and key do not match
+          - fail:
+              when: "key-pair-invalid"
+              message: Kubernetes ETCD key pair certificate is invalid
+          - pass:
+              when: "key-pair-valid"
+              message: Kubernetes ETCD key pair certificate is valid
+    - http:
+        checkName: "Kubernetes API Health"
+        exclude: '{{kurl or (not .IsPrimary) (not .IsUpgrade) }}'
+        collectorName: "Kubernetes API Health"
+        outcomes:
+          - warn:
+              when: "error"
+              message: Error connecting to Kubernetes API at https://localhost:6443/healthz
+          - pass:
+              when: "statusCode == 200"
+              message: OK HTTP status response from Kubernetes API at https://localhost:6443/healthz
+          - warn:
+              message: Unexpected status code response from Kubernetes API at https://localhost:6443/healthz
+    - cpu:
+        checkName: "Number of CPUs"
+        outcomes:
+          - fail:
+              when: "count < 2"
+              message: At least 2 CPU cores are required, and 4 CPU cores are recommended
+          - warn:
+              when: "count < 4"
+              message: At least 4 CPU cores are recommended
+          - pass:
+              message: This server has at least 4 CPU cores
+    - memory:
+        checkName: "Amount of Memory"
+        outcomes:
+          - fail:
+              when: "< 4G"
+              message: At least 4G of memory is required, and 8G of memory is recommended
+          - warn:
+              when: "< 8G"
+              message: At least 8G of memory is recommended
+          - pass:
+              message: The system has at least 8G of memory
+    - diskUsage:
+        checkName: "Ephemeral Disk Usage /var/lib/kubelet"
+        collectorName: "Ephemeral Disk Usage /var/lib/kubelet"
+        outcomes:
+          - fail:
+              when: "total < 30Gi"
+              message: The disk containing directory /var/lib/kubelet has less than 30Gi of total space
+          - fail:
+              when: "used/total > 80%"
+              message: The disk containing directory /var/lib/kubelet is more than 80% full
+          - warn:
+              when: "used/total > 60%"
+              message: The disk containing directory /var/lib/kubelet is more than 60% full
+          - warn:
+              when: "available < 10Gi"
+              message: The disk containing directory /var/lib/kubelet has less than 10Gi of disk space available
+          - pass:
+              message: The disk containing directory /var/lib/kubelet has at least 30Gi of total space, has at least 10Gi of disk space available, and is less than 60% full
+    - diskUsage:
+        checkName: "Ephemeral Disk Usage /var/lib/docker"
+        collectorName: "Ephemeral Disk Usage /var/lib/docker"
+        exclude: '{{kurl not .Installer.Spec.Docker.Version}}'
+        outcomes:
+          - fail:
+              when: "total < 30Gi"
+              message: The disk containing directory /var/lib/docker has less than 30Gi of total space
+          - fail:
+              when: "used/total > 80%"
+              message: The disk containing directory /var/lib/docker is more than 80% full
+          - warn:
+              when: "used/total > 60%"
+              message: The disk containing directory /var/lib/docker is more than 60% full
+          - warn:
+              when: "available < 10Gi"
+              message: The disk containing directory /var/lib/docker has less than 10Gi of disk space available
+          - pass:
+              message: The disk containing directory /var/lib/docker has at least 30Gi of total space, has at least 10Gi of disk space available, and is less than 60% full.
+    - diskUsage:
+        checkName: "Ephemeral Disk Usage /var/lib/containerd"
+        collectorName: "Ephemeral Disk Usage /var/lib/containerd"
+        exclude: '{{kurl not .Installer.Spec.Containerd.Version}}'
+        outcomes:
+          - fail:
+              when: "total < 30Gi"
+              message: The disk containing directory /var/lib/containerd has less than 30Gi of total space
+          - fail:
+              when: "used/total > 80%"
+              message: The disk containing directory /var/lib/containerd is more than 80% full
+          - warn:
+              when: "used/total > 60%"
+              message: The disk containing directory /var/lib/containerd is more than 60% full
+          - warn:
+              when: "available < 10Gi"
+              message: The disk containing directory /var/lib/containerd has less than 10Gi of disk space available
+          - pass:
+              message: The disk containing directory /var/lib/containerd has at least 30Gi of total space, has at least 10Gi of disk space available, and is less than 60% full.
+    - diskUsage:
+        checkName: "Ephemeral Disk Usage /var/lib/rook"
+        collectorName: "Ephemeral Disk Usage /var/lib/rook"
+        exclude: '{{kurl not .Installer.Spec.Rook.Version}}'
+        outcomes:
+          - fail:
+              when: "used/total > 80%"
+              message: The disk containing directory /var/lib/rook is more than 80% full
+          - fail:
+              when: "available < 10Gi"
+              message: The disk containing directory /var/lib/rook has less than 10Gi of disk space available
+          - pass:
+              message: The disk containing directory /var/lib/rook has sufficient space
+    - diskUsage:
+        checkName: "Ephemeral Disk Usage /var/openebs"
+        collectorName: "Ephemeral Disk Usage /var/openebs"
+        exclude: '{{kurl not .Installer.Spec.OpenEBS.Version}}'
+        outcomes:
+          - warn:
+              when: "used/total > 80%"
+              message: The disk containing directory /var/openebs is more than 80% full
+          - warn:
+              when: "available < 10Gi"
+              message: The disk containing directory /var/openebs has less than 10Gi of disk space available
+          - pass:
+              message: The disk containing directory /var/openebs has sufficient space
+    - tcpLoadBalancer:
+        checkName: "Kubernetes API Server Load Balancer"
+        collectorName: "Kubernetes API Server Load Balancer"
+        # ha and is first master (primary and not join) and not is upgrade
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.LoadBalancerAddress .IsPrimary (not .IsJoin) (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "invalid-address"
+              message: The load balancer address {{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }} is not valid.
+          - warn:
+              when: "connection-refused"
+              message: Connection to {{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }} via load balancer was refused.
+          - warn:
+              when: "connection-timeout"
+              message: Timed out connecting to {{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }} via load balancer. Check your firewall.
+          - warn:
+              when: "error"
+              message: Unexpected port status
+          - warn:
+              when: "address-in-use"
+              message: Port 6443 is unavailable
+          - pass:
+              when: "connected"
+              message: Successfully connected to {{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }} via load balancer
+          - warn:
+              message: Unexpected port status
+    - http:
+        checkName: "Kubernetes API Server Load Balancer Upgrade"
+        collectorName: "Kubernetes API Server Load Balancer Upgrade"
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.LoadBalancerAddress .IsPrimary .IsUpgrade (not .IsJoin) | not }}'
+        outcomes:
+          - fail:
+              when: "error"
+              message: Error connecting to load balancer at https://{{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }}/healthz
+          - pass:
+              when: "statusCode == 200"
+              message: OK HTTP status response from load balancer at https://{{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }}/healthz
+          - fail:
+              message: Unexpected status code response from load balancer at https://{{kurl .Installer.Spec.Kubernetes.LoadBalancerAddress }}/healthz
+    - tcpPortStatus:
+        checkName: "Kubernetes API TCP Port Status"
+        collectorName: "Kubernetes API TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 6443 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 6443.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 6443. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 6443 is open
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "ETCD Client API TCP Port Status"
+        collectorName: "ETCD Client API TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 2379 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 2379.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 2379. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 2379 is open
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "ETCD Server API TCP Port Status"
+        collectorName: "ETCD Server API TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 2380 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 2380.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 2380. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 2380 is open
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "ETCD Health Server TCP Port Status"
+        collectorName: "ETCD Health Server TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 2381 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 2381.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 2381. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 2381 is available
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "Kubelet Health Server TCP Port Status"
+        collectorName: "Kubelet Health Server TCP Port Status"
+        exclude: '{{kurl and (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 10248 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 10248.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 10248. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 10248 is available
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "Kubelet API TCP Port Status"
+        collectorName: "Kubelet API TCP Port Status"
+        exclude: '{{kurl and (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 10250 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 10250.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 10250. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 10250 is open
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "Kube Controller Manager Health Server TCP Port Status"
+        collectorName: "Kube Controller Manager Health Server TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 10257 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 10257.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 10257. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 10257 is available
+          - warn:
+              message: Unexpected port status
+    - tcpPortStatus:
+        checkName: "Kube Scheduler Health Server TCP Port Status"
+        collectorName: "Kube Scheduler Health Server TCP Port Status"
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to port 10259 was refused. This is likely to be a routing problem since this preflight configures a test server to listen on this port.
+          - warn:
+              when: "address-in-use"
+              message: Another process was already listening on port 10259.
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to port 10259. Check your firewall.
+          - fail:
+              when: "error"
+              message: Unexpected port status
+          - pass:
+              when: "connected"
+              message: Port 10259 is available
+          - warn:
+              message: Unexpected port status
+    - tcpConnect:
+        checkName: "Kubernetes API TCP Connection Status"
+        collectorName: "Kubernetes API TCP Connection Status"
+        # run the analyzer if 1. there is a master address set AND this is a node joining the cluster AND this is not an EKCO internalLB install
+        exclude: '{{kurl and .Installer.Spec.Kubernetes.Version .Installer.Spec.Kubernetes.MasterAddress .IsJoin (and .Installer.Spec.Ekco.Version .Installer.Spec.Ekco.EnableInternalLoadBalancer | not) | not }}'
+        outcomes:
+          - fail:
+              when: "connection-refused"
+              message: Connection to the Kubernetes API at address {{kurl .Installer.Spec.Kubernetes.MasterAddress }} was refused
+          - fail:
+              when: "connection-timeout"
+              message: Timed out connecting to the Kubernetes API at address {{kurl .Installer.Spec.Kubernetes.MasterAddress }}
+          - fail:
+              when: "error"
+              message: Unexpected error connecting to the Kubernetes API at address {{kurl .Installer.Spec.Kubernetes.MasterAddress }}
+          - pass:
+              when: "connected"
+              message: Successfully connected to the Kubernetes API at address {{kurl .Installer.Spec.Kubernetes.MasterAddress }}
+    - filesystemPerformance:
+        collectorName: Filesystem Latency Two Minute Benchmark
+        exclude: '{{kurl and .IsPrimary (not .IsUpgrade) | not }}'
+        outcomes:
+          - pass:
+              when: "p99 < 10ms"
+              message: "Write latency is ok (p99 target < 10ms, actual: {{ .P99 }})"
+          - warn:
+              message: "Write latency is high. p99 target < 10ms, actual:{{ .String }}"
+    - time:
+        checkName: "NTP Status"
+        outcomes:
+          - fail:
+              when: "ntp == unsynchronized+inactive"
+              message: "System clock is not synchronized"
+          - warn:
+              when: "ntp == unsynchronized+active"
+              message: System clock not yet synchronized
+          - pass:
+              when: "ntp == synchronized+active"
+              message: "System clock is synchronized"
+          - warn:
+              when: "timezone != UTC"
+              message: "Non UTC timezone can interfere with system function"
+          - pass:
+              when: "timezone == UTC"
+              message: "Timezone is set to UTC"
+    - hostOS:
+        checkName: "Docker Support"
+        exclude: '{{kurl or (not .Installer.Spec.Docker.Version) (semverCompare ">= 20.10.17" .Installer.Spec.Docker.Version) }}'
+        outcomes:
+          - fail:
+              when: "ubuntu = 22.04"
+              message: "Docker versions < 20.10.17 not supported on ubuntu 22.04"
+    # hijack hostOS analyzer in order to analyze the kURL Installer spec
+    - hostOS:
+        checkName: "Containerd and Weave Compatibility"
+        exclude: '{{kurl or (not .Installer.Spec.Weave.Version) (not .Installer.Spec.Containerd.Version) (semverCompare "1.6.0 - 1.6.4" .Installer.Spec.Containerd.Version | not) }}'
+        outcomes:
+          - fail:
+              message: "Weave is not compatible with containerd versions 1.6.0 - 1.6.4"