From 4b68be509fb38292fd051c10424075a940eef8d3 Mon Sep 17 00:00:00 2001 From: Marc Campbell Date: Tue, 16 Jul 2019 15:05:28 +0000 Subject: [PATCH] Run preflight from CLI --- cmd/preflight/cli/run.go | 109 +------ cmd/preflight/cli/run_crd.go | 103 +++++++ cmd/preflight/cli/run_nocrd.go | 267 ++++++++++++++++++ .../cli/{troubleshoot.go => util.go} | 19 ++ ...roubleshoot.replicated.com_collectors.yaml | 4 +- ...roubleshoot.replicated.com_preflights.yaml | 4 +- .../troubleshoot_v1beta1_collector.yaml | 4 +- .../troubleshoot_v1beta1_preflight.yaml | 2 +- go.mod | 1 + go.sum | 2 + pkg/analyze/analyzer.go | 24 ++ pkg/analyze/cluster_version.go | 69 +++++ .../troubleshoot/v1beta1/collector_shared.go | 4 +- .../troubleshoot/v1beta1/preflight_types.go | 10 +- pkg/collect/collector_test.go | 4 +- pkg/controller/preflightjob/collectors.go | 161 +---------- .../preflightjob/preflightjob_controller.go | 21 +- pkg/preflight/collector.go | 175 ++++++++++++ .../server.go} | 70 ++--- 19 files changed, 761 insertions(+), 292 deletions(-) create mode 100644 cmd/preflight/cli/run_crd.go create mode 100644 cmd/preflight/cli/run_nocrd.go rename cmd/preflight/cli/{troubleshoot.go => util.go} (69%) create mode 100644 pkg/analyze/analyzer.go create mode 100644 pkg/analyze/cluster_version.go create mode 100644 pkg/preflight/collector.go rename pkg/{controller/preflightjob/preflight_server.go => preflight/server.go} (56%) diff --git a/cmd/preflight/cli/run.go b/cmd/preflight/cli/run.go index eb13f1f3..cb72388e 100644 --- a/cmd/preflight/cli/run.go +++ b/cmd/preflight/cli/run.go @@ -1,18 +1,11 @@ package cli import ( - "errors" - "fmt" - "os" "path/filepath" - "time" troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" - "github.com/replicatedhq/troubleshoot/pkg/k8sutil" "github.com/spf13/cobra" "github.com/spf13/viper" - kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Run() *cobra.Command { @@ -26,93 +19,11 @@ func Run() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { v := viper.GetViper() - troubleshootClient, err := createTroubleshootK8sClient() - if err != nil { - return err + if len(args) == 0 { + return runPreflightsCRD(v) } - preflightName := v.GetString("preflight") - if preflightName == "" { - preflights, err := troubleshootClient.Preflights(v.GetString("namespace")).List(metav1.ListOptions{}) - if err != nil { - return err - } - - if len(preflights.Items) == 1 { - preflightName = preflights.Items[0].Name - } - } - - if preflightName == "" { - return errors.New("unable to fly, try using the --preflight flags") - } - - // generate a unique name - now := time.Now() - suffix := fmt.Sprintf("%d", now.Unix()) - - preflightJobName := fmt.Sprintf("%s-job-%s", preflightName, suffix[len(suffix)-4:]) - preflightJob := troubleshootv1beta1.PreflightJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: preflightJobName, - Namespace: v.GetString("namespace"), - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "preflightjob.troubleshoot.replicated.com", - }, - Spec: troubleshootv1beta1.PreflightJobSpec{ - Preflight: troubleshootv1beta1.PreflightRef{ - Name: preflightName, - Namespace: v.GetString("namespace"), - }, - Image: v.GetString("image"), - ImagePullPolicy: v.GetString("pullpolicy"), - CollectorImage: v.GetString("collector-image"), - CollectorImagePullPolicy: v.GetString("collector-pullpolicy"), - }, - } - if _, err := troubleshootClient.PreflightJobs(v.GetString("namespace")).Create(&preflightJob); err != nil { - return err - } - - // Poll the status of the Custom Resource for it to include a callback - var found *troubleshootv1beta1.PreflightJob - start := time.Now() - for { - current, err := troubleshootClient.PreflightJobs(v.GetString("namespace")).Get(preflightJobName, metav1.GetOptions{}) - if err != nil && kuberneteserrors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - - if current.Status.IsServerReady { - found = current - break - } - - if time.Now().Sub(start) > time.Duration(time.Second*10) { - return errors.New("preflightjob failed to start") - } - - time.Sleep(time.Millisecond * 200) - } - - // Connect to the callback - stopChan, err := k8sutil.PortForward(v.GetString("kubecontext"), 8000, 8000, found.Status.ServerPodNamespace, found.Status.ServerPodName) - if err != nil { - return err - } - - if err := receivePreflightResults(found.Namespace, found.Name); err != nil { - return err - } - - // Write - - close(stopChan) - return nil + return runPreflightsNoCRD(v, args[0]) }, } @@ -131,9 +42,15 @@ func Run() *cobra.Command { return cmd } -func homeDir() string { - if h := os.Getenv("HOME"); h != "" { - return h +func ensureCollectorInList(list []*troubleshootv1beta1.Collect, collector troubleshootv1beta1.Collect) []*troubleshootv1beta1.Collect { + for _, inList := range list { + if collector.ClusterResources != nil && inList.ClusterResources != nil { + return list + } + if collector.ClusterInfo != nil && inList.ClusterInfo != nil { + return list + } } - return os.Getenv("USERPROFILE") // windows + + return append(list, &collector) } diff --git a/cmd/preflight/cli/run_crd.go b/cmd/preflight/cli/run_crd.go new file mode 100644 index 00000000..002e9932 --- /dev/null +++ b/cmd/preflight/cli/run_crd.go @@ -0,0 +1,103 @@ +package cli + +import ( + "errors" + "fmt" + "time" + + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + "github.com/replicatedhq/troubleshoot/pkg/k8sutil" + "github.com/spf13/viper" + kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func runPreflightsCRD(v *viper.Viper) error { + troubleshootClient, err := createTroubleshootK8sClient() + if err != nil { + return err + } + + preflightName := v.GetString("preflight") + if preflightName == "" { + preflights, err := troubleshootClient.Preflights(v.GetString("namespace")).List(metav1.ListOptions{}) + if err != nil { + return err + } + + if len(preflights.Items) == 1 { + preflightName = preflights.Items[0].Name + } + } + + if preflightName == "" { + return errors.New("unable to preflight, try using the --preflight flags") + } + + // generate a unique name + now := time.Now() + suffix := fmt.Sprintf("%d", now.Unix()) + + preflightJobName := fmt.Sprintf("%s-job-%s", preflightName, suffix[len(suffix)-4:]) + preflightJob := troubleshootv1beta1.PreflightJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightJobName, + Namespace: v.GetString("namespace"), + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "preflightjob.troubleshoot.replicated.com", + }, + Spec: troubleshootv1beta1.PreflightJobSpec{ + Preflight: troubleshootv1beta1.PreflightRef{ + Name: preflightName, + Namespace: v.GetString("namespace"), + }, + Image: v.GetString("image"), + ImagePullPolicy: v.GetString("pullpolicy"), + CollectorImage: v.GetString("collector-image"), + CollectorImagePullPolicy: v.GetString("collector-pullpolicy"), + }, + } + if _, err := troubleshootClient.PreflightJobs(v.GetString("namespace")).Create(&preflightJob); err != nil { + return err + } + + // Poll the status of the Custom Resource for it to include a callback + var found *troubleshootv1beta1.PreflightJob + start := time.Now() + for { + current, err := troubleshootClient.PreflightJobs(v.GetString("namespace")).Get(preflightJobName, metav1.GetOptions{}) + if err != nil && kuberneteserrors.IsNotFound(err) { + continue + } else if err != nil { + return err + } + + if current.Status.IsServerReady { + found = current + break + } + + if time.Now().Sub(start) > time.Duration(time.Second*10) { + return errors.New("preflightjob failed to start") + } + + time.Sleep(time.Millisecond * 200) + } + + // Connect to the callback + stopChan, err := k8sutil.PortForward(v.GetString("kubecontext"), 8000, 8000, found.Status.ServerPodNamespace, found.Status.ServerPodName) + if err != nil { + return err + } + + if err := receivePreflightResults(found.Namespace, found.Name); err != nil { + return err + } + + // Write + + close(stopChan) + return nil +} diff --git a/cmd/preflight/cli/run_nocrd.go b/cmd/preflight/cli/run_nocrd.go new file mode 100644 index 00000000..e692ae0d --- /dev/null +++ b/cmd/preflight/cli/run_nocrd.go @@ -0,0 +1,267 @@ +package cli + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "time" + + analyzerunner "github.com/replicatedhq/troubleshoot/pkg/analyze" + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + preflightrunner "github.com/replicatedhq/troubleshoot/pkg/preflight" + "github.com/spf13/viper" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +func runPreflightsNoCRD(v *viper.Viper, arg string) error { + preflightContent := "" + if !isURL(arg) { + if _, err := os.Stat(arg); os.IsNotExist(err) { + return fmt.Errorf("%s was not found", arg) + } + + b, err := ioutil.ReadFile(arg) + if err != nil { + return err + } + + preflightContent = string(b) + } else { + resp, err := http.Get(arg) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + preflightContent = string(body) + } + + preflight := troubleshootv1beta1.Preflight{} + if err := yaml.Unmarshal([]byte(preflightContent), &preflight); err != nil { + return fmt.Errorf("unable to parse %s as a preflight", arg) + } + + allCollectedData, err := runCollectors(v, preflight) + if err != nil { + return err + } + + getCollectedFileContents := func(fileName string) ([]byte, error) { + contents, ok := allCollectedData[fileName] + if !ok { + return nil, errors.New("not found") + } + + return contents, nil + } + + for _, analyzer := range preflight.Spec.Analyzers { + analyzeResult, err := analyzerunner.Analyze(analyzer, getCollectedFileContents) + if err != nil { + return err + } + + fmt.Printf("%#v\n", analyzeResult) + } + return nil +} + +func runCollectors(v *viper.Viper, preflight troubleshootv1beta1.Preflight) (map[string][]byte, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, err + } + + client, err := client.New(cfg, client.Options{}) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + restClient := clientset.CoreV1().RESTClient() + + // deploy an object that "owns" everything to aid in cleanup + owner := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("preflight-%s-owner", preflight.Name), + Namespace: v.GetString("namespace"), + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + Data: make(map[string]string), + } + if err := client.Create(context.Background(), &owner); err != nil { + return nil, err + } + defer func() { + if err := client.Delete(context.Background(), &owner); err != nil { + fmt.Println("failed to clean up preflight.") + } + }() + + // deploy all collectors + desiredCollectors := make([]*troubleshootv1beta1.Collect, 0, 0) + for _, definedCollector := range preflight.Spec.Collectors { + desiredCollectors = append(desiredCollectors, definedCollector) + } + desiredCollectors = ensureCollectorInList(desiredCollectors, troubleshootv1beta1.Collect{ClusterInfo: &troubleshootv1beta1.ClusterInfo{}}) + desiredCollectors = ensureCollectorInList(desiredCollectors, troubleshootv1beta1.Collect{ClusterResources: &troubleshootv1beta1.ClusterResources{}}) + + podsCreated := make([]*corev1.Pod, 0, 0) + podsDeleted := make([]*corev1.Pod, 0, 0) + allCollectedData := make(map[string][]byte) + + resyncPeriod := time.Second + ctx := context.Background() + watchList := cache.NewListWatchFromClient(restClient, "pods", "", fields.Everything()) + _, controller := cache.NewInformer(watchList, &corev1.Pod{}, resyncPeriod, + cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj interface{}, newObj interface{}) { + newPod, ok := newObj.(*corev1.Pod) + if !ok { + return + } + oldPod, ok := oldObj.(*corev1.Pod) + if !ok { + return + } + labels := newPod.Labels + troubleshootRole, ok := labels["troubleshoot-role"] + if !ok || troubleshootRole != "preflight" { + return + } + preflightName, ok := labels["preflight"] + if !ok || preflightName != preflight.Name { + return + } + + if oldPod.Status.Phase == newPod.Status.Phase { + return + } + + if newPod.Status.Phase != corev1.PodSucceeded { + return + } + + podLogOpts := corev1.PodLogOptions{} + + req := clientset.CoreV1().Pods(newPod.Namespace).GetLogs(newPod.Name, &podLogOpts) + podLogs, err := req.Stream() + if err != nil { + fmt.Println("get stream") + return + } + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + fmt.Println("copy logs") + return + } + + collectedData, err := parseCollectorOutput(buf.String()) + if err != nil { + fmt.Printf("parse collected data: %v\n", err) + return + } + for k, v := range collectedData { + allCollectedData[k] = v + } + + if err := client.Delete(context.Background(), newPod); err != nil { + fmt.Println("delete pod") + } + podsDeleted = append(podsDeleted, newPod) + }, + }) + go func() { + controller.Run(ctx.Done()) + }() + + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.ConfigMap{}) + for _, collector := range desiredCollectors { + _, pod, err := preflightrunner.CreateCollector(client, s, &owner, preflight.Name, v.GetString("namespace"), collector, v.GetString("image"), v.GetString("pullpolicy")) + if err != nil { + return nil, err + } + podsCreated = append(podsCreated, pod) + } + + start := time.Now() + for { + if start.Add(time.Second * 30).Before(time.Now()) { + fmt.Println("timeout running preflight") + return nil, err + } + + if len(podsDeleted) == len(podsCreated) { + break + } + + time.Sleep(time.Millisecond * 200) + } + + ctx.Done() + + return allCollectedData, nil +} + +func parseCollectorOutput(output string) (map[string][]byte, error) { + input := make(map[string]interface{}) + files := make(map[string][]byte) + if err := json.Unmarshal([]byte(output), &input); err != nil { + return nil, err + } + + for filename, maybeContents := range input { + fileDir, fileName := filepath.Split(filename) + + switch maybeContents.(type) { + case string: + decoded, err := base64.StdEncoding.DecodeString(maybeContents.(string)) + if err != nil { + return nil, err + } + files[filepath.Join(fileDir, fileName)] = decoded + + case map[string]interface{}: + for k, v := range maybeContents.(map[string]interface{}) { + decoded, err := base64.StdEncoding.DecodeString(v.(string)) + if err != nil { + return nil, err + } + files[filepath.Join(fileDir, fileName, k)] = decoded + } + } + } + + return files, nil +} diff --git a/cmd/preflight/cli/troubleshoot.go b/cmd/preflight/cli/util.go similarity index 69% rename from cmd/preflight/cli/troubleshoot.go rename to cmd/preflight/cli/util.go index 6f25c40c..b9a7b3b4 100644 --- a/cmd/preflight/cli/troubleshoot.go +++ b/cmd/preflight/cli/util.go @@ -1,11 +1,30 @@ package cli import ( + "net/url" + "os" + troubleshootclientv1beta1 "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1" "github.com/spf13/viper" "k8s.io/client-go/tools/clientcmd" ) +func homeDir() string { + if h := os.Getenv("HOME"); h != "" { + return h + } + return os.Getenv("USERPROFILE") // windows +} + +func isURL(str string) bool { + _, err := url.ParseRequestURI(str) + if err != nil { + return false + } + + return true +} + func createTroubleshootK8sClient() (*troubleshootclientv1beta1.TroubleshootV1beta1Client, error) { v := viper.GetViper() diff --git a/config/crds/troubleshoot.replicated.com_collectors.yaml b/config/crds/troubleshoot.replicated.com_collectors.yaml index dae73ead..e84085e0 100644 --- a/config/crds/troubleshoot.replicated.com_collectors.yaml +++ b/config/crds/troubleshoot.replicated.com_collectors.yaml @@ -392,9 +392,9 @@ spec: spec: items: properties: - cluster-info: + clusterInfo: type: object - cluster-resources: + clusterResources: type: object type: object type: array diff --git a/config/crds/troubleshoot.replicated.com_preflights.yaml b/config/crds/troubleshoot.replicated.com_preflights.yaml index 470d6920..78d3db5c 100644 --- a/config/crds/troubleshoot.replicated.com_preflights.yaml +++ b/config/crds/troubleshoot.replicated.com_preflights.yaml @@ -476,9 +476,9 @@ spec: collectors: items: properties: - cluster-info: + clusterInfo: type: object - cluster-resources: + clusterResources: type: object type: object type: array diff --git a/config/samples/troubleshoot_v1beta1_collector.yaml b/config/samples/troubleshoot_v1beta1_collector.yaml index 95a0c941..d3340cfa 100644 --- a/config/samples/troubleshoot_v1beta1_collector.yaml +++ b/config/samples/troubleshoot_v1beta1_collector.yaml @@ -3,5 +3,5 @@ kind: Collector metadata: name: collector-sample spec: - - cluster-info: {} - - cluster-resources: {} + - clusterInfo: {} + - clusterResources: {} diff --git a/config/samples/troubleshoot_v1beta1_preflight.yaml b/config/samples/troubleshoot_v1beta1_preflight.yaml index 75b996a3..5ff55359 100644 --- a/config/samples/troubleshoot_v1beta1_preflight.yaml +++ b/config/samples/troubleshoot_v1beta1_preflight.yaml @@ -7,7 +7,7 @@ spec: - clusterVersion: outcomes: - fail: - when: "< 1.13.0" + when: "< 1.14.0" message: You need more kubernetes - warn: when: "< 1.15.0" diff --git a/go.mod b/go.mod index 09586f09..8064a3e4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/replicatedhq/troubleshoot go 1.12 require ( + github.com/blang/semver v3.5.1+incompatible github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/gin-gonic/gin v1.4.0 diff --git a/go.sum b/go.sum index b20eda31..8a61c5c5 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go new file mode 100644 index 00000000..9a2ea7d0 --- /dev/null +++ b/pkg/analyze/analyzer.go @@ -0,0 +1,24 @@ +package analyzer + +import ( + "errors" + + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" +) + +type AnalyzeResult struct { + IsPass bool + IsFail bool + IsWarn bool + + Message string + URI string +} + +func Analyze(analyzer *troubleshootv1beta1.Analyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) { + if analyzer.ClusterVersion != nil { + return analyzeClusterVersion(analyzer.ClusterVersion, getCollectedFileContents) + } + + return nil, errors.New("invalid analyer") +} diff --git a/pkg/analyze/cluster_version.go b/pkg/analyze/cluster_version.go new file mode 100644 index 00000000..a9bd8d46 --- /dev/null +++ b/pkg/analyze/cluster_version.go @@ -0,0 +1,69 @@ +package analyzer + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/blang/semver" + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + "github.com/replicatedhq/troubleshoot/pkg/collect" +) + +func analyzeClusterVersion(analyzer *troubleshootv1beta1.ClusterVersion, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) { + clusterInfo, err := getCollectedFileContents("cluster-info/cluster_version.json") + if err != nil { + return nil, err + } + + collectorClusterVersion := collect.ClusterVersion{} + if err := json.Unmarshal(clusterInfo, &collectorClusterVersion); err != nil { + return nil, err + } + + k8sVersion, err := semver.Make(strings.TrimLeft(collectorClusterVersion.String, "v")) + if err != nil { + return nil, err + } + + result := AnalyzeResult{} + for _, outcome := range analyzer.Outcomes { + when := "" + message := "" + uri := "" + + result = AnalyzeResult{} + if outcome.Fail != nil { + result.IsFail = true + when = outcome.Fail.When + message = outcome.Fail.Message + uri = outcome.Fail.URI + } else if outcome.Warn != nil { + result.IsWarn = true + when = outcome.Warn.When + message = outcome.Warn.Message + uri = outcome.Warn.URI + } else if outcome.Pass != nil { + result.IsPass = true + when = outcome.Pass.When + message = outcome.Pass.Message + uri = outcome.Pass.URI + } else { + return nil, errors.New("empty outcome") + } + + whenRange, err := semver.ParseRange(when) + if err != nil { + return nil, err + } + + if whenRange(k8sVersion) { + result.Message = message + result.URI = uri + + return &result, nil + } + } + + return &AnalyzeResult{}, nil +} diff --git a/pkg/apis/troubleshoot/v1beta1/collector_shared.go b/pkg/apis/troubleshoot/v1beta1/collector_shared.go index e40e086b..75b9d145 100644 --- a/pkg/apis/troubleshoot/v1beta1/collector_shared.go +++ b/pkg/apis/troubleshoot/v1beta1/collector_shared.go @@ -7,6 +7,6 @@ type ClusterResources struct { } type Collect struct { - ClusterInfo *ClusterInfo `json:"cluster-info,omitempty" yaml:"cluster-info,omitempty"` - ClusterResources *ClusterResources `json:"cluster-resources,omitempty" yaml:"cluster-resources,omitempty"` + ClusterInfo *ClusterInfo `json:"clusterInfo,omitempty" yaml:"clusterInfo,omitempty"` + ClusterResources *ClusterResources `json:"clusterResources,omitempty" yaml:"clusterResources,omitempty"` } diff --git a/pkg/apis/troubleshoot/v1beta1/preflight_types.go b/pkg/apis/troubleshoot/v1beta1/preflight_types.go index 34c20f97..c6ca5628 100644 --- a/pkg/apis/troubleshoot/v1beta1/preflight_types.go +++ b/pkg/apis/troubleshoot/v1beta1/preflight_types.go @@ -22,8 +22,8 @@ import ( // PreflightSpec defines the desired state of Preflight type PreflightSpec struct { - Collectors []*Collect `json:"collectors,omitempty"` - Analyzers []*Analyze `json:"analyzers,omitempty"` + Collectors []*Collect `json:"collectors,omitempty" yaml:"collectors,omitempty"` + Analyzers []*Analyze `json:"analyzers,omitempty" yaml:"analyzers,omitempty"` } // PreflightStatus defines the observed state of Preflight @@ -38,10 +38,10 @@ type PreflightStatus struct { // Preflight is the Schema for the preflights API // +k8s:openapi-gen=true type Preflight struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` + metav1.TypeMeta `json:",inline" yaml:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec PreflightSpec `json:"spec,omitempty"` + Spec PreflightSpec `json:"spec,omitempty" yaml:"spec,omitempty"` Status PreflightStatus `json:"status,omitempty"` } diff --git a/pkg/collect/collector_test.go b/pkg/collect/collector_test.go index dbf9f141..62e5b26f 100644 --- a/pkg/collect/collector_test.go +++ b/pkg/collect/collector_test.go @@ -15,8 +15,8 @@ func Test_ParseSpec(t *testing.T) { expectObject interface{} }{ { - name: "cluster-info", - spec: "cluster-info: {}", + name: "clusterInfo", + spec: "clusterInfo: {}", expectError: false, expectObject: &troubleshootv1beta1.Collect{ ClusterInfo: &troubleshootv1beta1.ClusterInfo{}, diff --git a/pkg/controller/preflightjob/collectors.go b/pkg/controller/preflightjob/collectors.go index b024f4db..384091a6 100644 --- a/pkg/controller/preflightjob/collectors.go +++ b/pkg/controller/preflightjob/collectors.go @@ -2,15 +2,9 @@ package preflightjob import ( "context" - "fmt" troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" - "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" - kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/replicatedhq/troubleshoot/pkg/preflight" ) func (r *ReconcilePreflightJob) reconcilePreflightCollectors(instance *troubleshootv1beta1.PreflightJob, preflight *troubleshootv1beta1.Preflight) error { @@ -63,139 +57,12 @@ func (r *ReconcilePreflightJob) reconcileOnePreflightCollector(instance *trouble return nil } - if err := r.createCollectorSpecInConfigMap(instance, collect); err != nil { - return err - } - if err := r.createCollectorPod(instance, collect); err != nil { - return err - } - - return nil -} - -func (r *ReconcilePreflightJob) createCollectorSpecInConfigMap(instance *troubleshootv1beta1.PreflightJob, collector *troubleshootv1beta1.Collect) error { - name := fmt.Sprintf("%s-%s", instance.Name, idForCollector(collector)) - - namespacedName := types.NamespacedName{ - Name: name, - Namespace: instance.Namespace, - } - - found := &corev1.ConfigMap{} - err := r.Get(context.Background(), namespacedName, found) - if err == nil || !kuberneteserrors.IsNotFound(err) { - return err - } - - specContents, err := yaml.Marshal(collector) + _, _, err := preflight.CreateCollector(r.Client, r.scheme, instance, instance.Name, instance.Namespace, collect, instance.Spec.Image, instance.Spec.ImagePullPolicy) if err != nil { return err } - specData := make(map[string]string) - specData["collector.yaml"] = string(specContents) - - configMap := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: instance.Namespace, - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, - Data: specData, - } - - if err := controllerutil.SetControllerReference(instance, &configMap, r.scheme); err != nil { - return err - } - - if err := r.Create(context.Background(), &configMap); err != nil { - return err - } - - return nil -} - -func (r *ReconcilePreflightJob) createCollectorPod(instance *troubleshootv1beta1.PreflightJob, collector *troubleshootv1beta1.Collect) error { - name := fmt.Sprintf("%s-%s", instance.Name, idForCollector(collector)) - - namespacedName := types.NamespacedName{ - Name: name, - Namespace: instance.Namespace, - } - - found := &corev1.Pod{} - err := r.Get(context.Background(), namespacedName, found) - if err == nil || !kuberneteserrors.IsNotFound(err) { - return err - } - - imageName := "replicatedhq/troubleshoot:latest" - imagePullPolicy := corev1.PullAlways - - if instance.Spec.Image != "" { - imageName = instance.Spec.Image - } - if instance.Spec.ImagePullPolicy != "" { - imagePullPolicy = corev1.PullPolicy(instance.Spec.ImagePullPolicy) - } - - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: instance.Namespace, - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - Containers: []corev1.Container{ - { - Image: imageName, - ImagePullPolicy: imagePullPolicy, - Name: idForCollector(collector), - Command: []string{"collector"}, - Args: []string{ - "run", - "--collector", - "/troubleshoot/specs/collector.yaml", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "collector", - MountPath: "/troubleshoot/specs", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "collector", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: name, - }, - }, - }, - }, - }, - }, - } - - if err := controllerutil.SetControllerReference(instance, &pod, r.scheme); err != nil { - return err - } - - if err := r.Create(context.Background(), &pod); err != nil { - return err - } - - instance.Status.CollectorsRunning = append(instance.Status.CollectorsRunning, idForCollector(collector)) + instance.Status.CollectorsRunning = append(instance.Status.CollectorsRunning, idForCollector(collect)) if err := r.Update(context.Background(), instance); err != nil { return err } @@ -203,17 +70,6 @@ func (r *ReconcilePreflightJob) createCollectorPod(instance *troubleshootv1beta1 return nil } -// Todo these will overlap with troubleshoot containers running at the same time -func idForCollector(collector *troubleshootv1beta1.Collect) string { - if collector.ClusterInfo != nil { - return "cluster-info" - } else if collector.ClusterResources != nil { - return "cluster-resources" - } - - return "" -} - func contains(s []string, e string) bool { for _, a := range s { if a == e { @@ -231,3 +87,14 @@ func remove(s []string, r string) []string { } return s } + +// Todo these will overlap with troubleshoot containers running at the same time +func idForCollector(collector *troubleshootv1beta1.Collect) string { + if collector.ClusterInfo != nil { + return "cluster-info" + } else if collector.ClusterResources != nil { + return "cluster-resources" + } + + return "" +} diff --git a/pkg/controller/preflightjob/preflightjob_controller.go b/pkg/controller/preflightjob/preflightjob_controller.go index 7aad4ad7..8bdee773 100644 --- a/pkg/controller/preflightjob/preflightjob_controller.go +++ b/pkg/controller/preflightjob/preflightjob_controller.go @@ -21,6 +21,7 @@ import ( troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" troubleshootclientv1beta1 "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1" + "github.com/replicatedhq/troubleshoot/pkg/preflight" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -96,9 +97,27 @@ func (r *ReconcilePreflightJob) Reconcile(request reconcile.Request) (reconcile. } if !instance.Status.IsServerReady { - if err := r.createPreflightServer(instance); err != nil { + preflightServerOptions := preflight.PreflightServerOptions{ + ImageName: instance.Spec.Image, + PullPolicy: instance.Spec.ImagePullPolicy, + Name: instance.Name, + Namespace: instance.Namespace, + OwnerReference: instance, + } + pod, _, err := preflight.CreatePreflightServer(r.Client, r.scheme, preflightServerOptions) + if err != nil { return reconcile.Result{}, err } + + instance.Status.ServerPodName = pod.Name + instance.Status.ServerPodNamespace = pod.Namespace + instance.Status.ServerPodPort = 8000 + instance.Status.IsServerReady = true + + if err := r.Update(context.Background(), instance); err != nil { + return reconcile.Result{}, err + } + } namespace := instance.Namespace diff --git a/pkg/preflight/collector.go b/pkg/preflight/collector.go new file mode 100644 index 00000000..ba473b19 --- /dev/null +++ b/pkg/preflight/collector.go @@ -0,0 +1,175 @@ +package preflight + +import ( + "context" + "fmt" + + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func CreateCollector(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect, image string, pullPolicy string) (*corev1.ConfigMap, *corev1.Pod, error) { + configMap, err := createCollectorSpecConfigMap(client, scheme, ownerRef, preflightJobName, preflightJobNamespace, collect) + if err != nil { + return nil, nil, err + } + + pod, err := createCollectorPod(client, scheme, ownerRef, preflightJobName, preflightJobNamespace, collect, configMap, image, pullPolicy) + if err != nil { + return nil, nil, err + } + + return configMap, pod, nil +} + +func createCollectorSpecConfigMap(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect) (*corev1.ConfigMap, error) { + name := fmt.Sprintf("%s-%s", preflightJobName, idForCollector(collect)) + + namespacedName := types.NamespacedName{ + Name: name, + Namespace: preflightJobNamespace, + } + + found := &corev1.ConfigMap{} + err := client.Get(context.Background(), namespacedName, found) + if err == nil || !kuberneteserrors.IsNotFound(err) { + return nil, err + } + + specContents, err := yaml.Marshal(collect) + if err != nil { + return nil, err + } + + specData := make(map[string]string) + specData["collector.yaml"] = string(specContents) + + configMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: preflightJobNamespace, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + Data: specData, + } + + if scheme != nil { + if err := controllerutil.SetControllerReference(ownerRef, &configMap, scheme); err != nil { + return nil, err + } + } + + if err := client.Create(context.Background(), &configMap); err != nil { + return nil, err + } + + return &configMap, nil +} + +func createCollectorPod(client client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, preflightJobName string, preflightJobNamespace string, collect *troubleshootv1beta1.Collect, configMap *corev1.ConfigMap, image string, pullPolicy string) (*corev1.Pod, error) { + name := fmt.Sprintf("%s-%s", preflightJobName, idForCollector(collect)) + + namespacedName := types.NamespacedName{ + Name: name, + Namespace: preflightJobNamespace, + } + + found := &corev1.Pod{} + err := client.Get(context.Background(), namespacedName, found) + if err == nil || !kuberneteserrors.IsNotFound(err) { + return nil, err + } + + imageName := "replicatedhq/troubleshoot:latest" + imagePullPolicy := corev1.PullAlways + + if image != "" { + imageName = image + } + if pullPolicy != "" { + imagePullPolicy = corev1.PullPolicy(pullPolicy) + } + + podLabels := make(map[string]string) + podLabels["preflight"] = preflightJobName + podLabels["troubleshoot-role"] = "preflight" + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: preflightJobNamespace, + Labels: podLabels, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Image: imageName, + ImagePullPolicy: imagePullPolicy, + Name: idForCollector(collect), + Command: []string{"collector"}, + Args: []string{ + "run", + "--collector", + "/troubleshoot/specs/collector.yaml", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "collector", + MountPath: "/troubleshoot/specs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "collector", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, + }, + }, + }, + } + + if scheme != nil { + if err := controllerutil.SetControllerReference(ownerRef, &pod, scheme); err != nil { + return nil, err + } + } + + if err := client.Create(context.Background(), &pod); err != nil { + return nil, err + } + + return &pod, nil +} + +// Todo these will overlap with troubleshoot containers running at the same time +func idForCollector(collector *troubleshootv1beta1.Collect) string { + if collector.ClusterInfo != nil { + return "cluster-info" + } else if collector.ClusterResources != nil { + return "cluster-resources" + } + + return "" +} diff --git a/pkg/controller/preflightjob/preflight_server.go b/pkg/preflight/server.go similarity index 56% rename from pkg/controller/preflightjob/preflight_server.go rename to pkg/preflight/server.go index c6152bff..91dc6d7f 100644 --- a/pkg/controller/preflightjob/preflight_server.go +++ b/pkg/preflight/server.go @@ -1,50 +1,61 @@ -package preflightjob +package preflight import ( "context" "fmt" "time" - troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -func (r *ReconcilePreflightJob) createPreflightServer(instance *troubleshootv1beta1.PreflightJob) error { - name := fmt.Sprintf("%s-%s", instance.Name, "preflight") +type PreflightServerOptions struct { + ImageName string + PullPolicy string + + Name string + Namespace string + + OwnerReference metav1.Object +} + +func CreatePreflightServer(client client.Client, scheme *runtime.Scheme, options PreflightServerOptions) (*corev1.Pod, *corev1.Service, error) { + name := fmt.Sprintf("%s-%s", options.Name, "preflight") namespacedName := types.NamespacedName{ Name: name, - Namespace: instance.Namespace, + Namespace: options.Namespace, } found := &corev1.Pod{} - err := r.Get(context.Background(), namespacedName, found) + err := client.Get(context.Background(), namespacedName, found) if err == nil || !kuberneteserrors.IsNotFound(err) { - return err + return nil, nil, err } imageName := "replicatedhq/troubleshoot:latest" imagePullPolicy := corev1.PullAlways - if instance.Spec.Image != "" { - imageName = instance.Spec.Image + if options.ImageName != "" { + imageName = options.ImageName } - if instance.Spec.ImagePullPolicy != "" { - imagePullPolicy = corev1.PullPolicy(instance.Spec.ImagePullPolicy) + if options.PullPolicy != "" { + imagePullPolicy = corev1.PullPolicy(options.PullPolicy) } podLabels := make(map[string]string) - podLabels["preflight"] = instance.Name + podLabels["preflight"] = options.Name podLabels["troubleshoot-role"] = "preflight" pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: instance.Namespace, + Namespace: options.Namespace, Labels: podLabels, }, TypeMeta: metav1.TypeMeta{ @@ -71,18 +82,20 @@ func (r *ReconcilePreflightJob) createPreflightServer(instance *troubleshootv1be }, } - if err := controllerutil.SetControllerReference(instance, &pod, r.scheme); err != nil { - return err + if scheme != nil { + if err := controllerutil.SetControllerReference(options.OwnerReference, &pod, scheme); err != nil { + return nil, nil, err + } } - if err := r.Create(context.Background(), &pod); err != nil { - return err + if err := client.Create(context.Background(), &pod); err != nil { + return nil, nil, err } service := corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: instance.Namespace, + Namespace: options.Namespace, }, TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -101,26 +114,19 @@ func (r *ReconcilePreflightJob) createPreflightServer(instance *troubleshootv1be }, } - if err := controllerutil.SetControllerReference(instance, &service, r.scheme); err != nil { - return err + if scheme != nil { + if err := controllerutil.SetControllerReference(options.OwnerReference, &service, scheme); err != nil { + return nil, nil, err + } } - if err := r.Create(context.Background(), &service); err != nil { - return err + if err := client.Create(context.Background(), &service); err != nil { + return nil, nil, err } - instance.Status.ServerPodName = name - instance.Status.ServerPodNamespace = instance.Namespace - instance.Status.ServerPodPort = 8000 - instance.Status.IsServerReady = true - // wait for the server to be ready // TODO time.Sleep(time.Second * 5) - if err := r.Update(context.Background(), instance); err != nil { - return err - } - - return nil + return &pod, &service, nil }