From 53fc9e5b71559ae3c0703bcb150db25f3d36d823 Mon Sep 17 00:00:00 2001 From: roy wang Date: Tue, 13 Oct 2020 15:43:09 +0900 Subject: [PATCH] add ready/health probes add waiting for webhook secret ready add unit tests Signed-off-by: roy wang --- Makefile | 4 +- .../templates/kubevela-controller.yaml | 18 +++- charts/vela-core/values.yaml | 3 + cmd/core/main.go | 89 +++++++++++++++++-- cmd/core/main_test.go | 66 ++++++++++++++ 5 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 cmd/core/main_test.go diff --git a/Makefile b/Makefile index 4b7e34759..ff2276473 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ all: build # Run tests test: fmt vet lint - go test ./pkg/... -coverprofile cover.out + go test ./pkg/... ./cmd/... -coverprofile cover.out # Build manager binary build: fmt vet lint @@ -78,7 +78,7 @@ docker-push: e2e-setup: ginkgo version ginkgo -v -r e2e/setup - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s + # kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s bin/vela dashboard & e2e-test: diff --git a/charts/vela-core/templates/kubevela-controller.yaml b/charts/vela-core/templates/kubevela-controller.yaml index 508f7480b..fd02f1018 100644 --- a/charts/vela-core/templates/kubevela-controller.yaml +++ b/charts/vela-core/templates/kubevela-controller.yaml @@ -111,6 +111,7 @@ spec: - "--use-webhook=true" - "--webhook-port={{ .Values.webhookService.port }}" - "--webhook-cert-dir={{ .Values.certificate.mountPath }}" + - "--health-addr=:{{ .Values.healthCheck.port }}" {{ end }} image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ quote .Values.image.pullPolicy }} @@ -121,6 +122,21 @@ spec: - containerPort: {{ .Values.webhookService.port }} name: webhook-server protocol: TCP + - containerPort: {{ .Values.healthCheck.port }} + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + initialDelaySeconds: 90 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 90 + periodSeconds: 5 volumeMounts: - mountPath: {{ .Values.certificate.mountPath }} name: tls-cert-vol @@ -142,4 +158,4 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/charts/vela-core/values.yaml b/charts/vela-core/values.yaml index 5c719ffa0..7ec1c7e2e 100644 --- a/charts/vela-core/values.yaml +++ b/charts/vela-core/values.yaml @@ -63,6 +63,9 @@ webhookService: type: ClusterIP port: 9443 +healthCheck: + port: 9440 + nodeSelector: {} tolerations: [] diff --git a/cmd/core/main.go b/cmd/core/main.go index ec63c6b2d..5ce720909 100644 --- a/cmd/core/main.go +++ b/cmd/core/main.go @@ -1,10 +1,14 @@ package main import ( + "errors" "flag" + "fmt" "io" "os" + "path/filepath" "strconv" + "time" monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" @@ -24,6 +28,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" velacore "github.com/oam-dev/kubevela/api/v1alpha1" @@ -32,7 +37,16 @@ import ( velawebhook "github.com/oam-dev/kubevela/pkg/webhook" ) -var scheme = runtime.NewScheme() +const ( + kubevelaName = "kubevela" +) + +var ( + setupLog = ctrl.Log.WithName(kubevelaName) + scheme = runtime.NewScheme() + waitSecretTimeout = 90 * time.Second + waitSecretInterval = 2 * time.Second +) func init() { _ = clientgoscheme.AddToScheme(scheme) @@ -54,6 +68,7 @@ func main() { var webhookPort int var useWebhook, useTraitInjector bool var controllerArgs oamcontroller.Args + var healthAddr string flag.BoolVar(&useWebhook, "use-webhook", false, "Enable Admission Webhook") flag.BoolVar(&useTraitInjector, "use-trait-injector", false, "Enable TraitInjector") @@ -67,6 +82,7 @@ func main() { flag.BoolVar(&logCompress, "log-compress", true, "Enable compression on the rotated logs.") flag.IntVar(&controllerArgs.RevisionLimit, "revision-limit", 50, "RevisionLimit is the maximum number of revisions that will be maintained. The default value is 50.") + flag.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") flag.Parse() // setup logging @@ -86,7 +102,6 @@ func main() { o.DestWritter = w })) - setupLog := ctrl.Log.WithName("vela-runtime") // install dependency charts first k8sClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) if err != nil { @@ -99,22 +114,32 @@ func main() { } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "vela-runtime", - Port: webhookPort, - CertDir: certDir, + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: kubevelaName, + Port: webhookPort, + CertDir: certDir, + HealthProbeBindAddress: healthAddr, }) if err != nil { setupLog.Error(err, "unable to create a controller manager") os.Exit(1) } + if err := registerHealthChecks(mgr); err != nil { + setupLog.Error(err, "unable to register ready/health checks") + os.Exit(1) + } + if useWebhook { setupLog.Info("vela webhook enabled, will serving at :" + strconv.Itoa(webhookPort)) oamwebhook.Add(mgr) velawebhook.Register(mgr) + if err := waitWebhookSecretVolume(certDir, waitSecretTimeout, waitSecretInterval); err != nil { + setupLog.Error(err, "unable to get webhook secret") + os.Exit(1) + } } if err = oamv1alpha2.Setup(mgr, controllerArgs, logging.NewLogrLogger(setupLog)); err != nil { @@ -151,3 +176,51 @@ func main() { os.Exit(1) } } + +// registerHealthChecks is used to create readiness&liveness probes +func registerHealthChecks(mgr ctrl.Manager) error { + setupLog.Info("creating readiness/health check") + if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + return err + } + + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + return err + } + return nil +} + +// waitWebhookSecretVolume waits for webhook secret ready to avoid mgr running crash +func waitWebhookSecretVolume(certDir string, timeout, interval time.Duration) error { + start := time.Now() + for { + time.Sleep(interval) + if time.Since(start) > timeout { + return fmt.Errorf("getting webhook secret timeout after %s", timeout.String()) + } + setupLog.Info(fmt.Sprintf("waiting webhook secret, time consumed: %d/%d seconds ...", + int64(time.Since(start).Seconds()), int64(timeout.Seconds()))) + if _, err := os.Stat(certDir); !os.IsNotExist(err) { + f, _ := os.Open(certDir) + defer f.Close() + // check if dir is empty + if _, err := f.Readdir(1); err == io.EOF { + continue + } + // check if secret files are empty + err := filepath.Walk(certDir, func(path string, info os.FileInfo, err error) error { + fmt.Println(info.Name(), info.Size()) + // even Cert dir is created, cert files are still empty for a while + if info.Size() == 0 { + return errors.New("secret is not ready") + } + return nil + }) + if err == nil { + setupLog.Info(fmt.Sprintf("webhook secret is ready (time consumed: %d seconds)", + int64(time.Since(start).Seconds()))) + return nil + } + } + } +} diff --git a/cmd/core/main_test.go b/cmd/core/main_test.go new file mode 100644 index 000000000..a13d926fa --- /dev/null +++ b/cmd/core/main_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "io/ioutil" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + testdir = "testdir" + testTimeout = 2 * time.Second + testInterval = 1 * time.Second +) + +func TestGinkgo(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "test main") +} + +var _ = Describe("test waitSecretVolume", func() { + BeforeEach(func() { + err := os.MkdirAll(testdir, 0755) + Expect(err).NotTo(HaveOccurred()) + }) + AfterEach(func() { + os.RemoveAll(testdir) + }) + + When("dir not exist or empty", func() { + It("return timeout error", func() { + err := waitWebhookSecretVolume(testdir, testTimeout, testInterval) + Expect(err).To(HaveOccurred()) + By("remove dir") + os.RemoveAll(testdir) + err = waitWebhookSecretVolume(testdir, testTimeout, testInterval) + Expect(err).To(HaveOccurred()) + }) + }) + + When("dir contains empty file", func() { + It("return timeout error", func() { + By("add empty file") + _, err := os.Create(testdir + "/emptyFile") + Expect(err).NotTo(HaveOccurred()) + err = waitWebhookSecretVolume(testdir, testTimeout, testInterval) + Expect(err).To(HaveOccurred()) + }) + }) + + When("files in dir are not empty", func() { + It("return nil", func() { + By("add non-empty file") + _, err := os.Create(testdir + "/file") + Expect(err).NotTo(HaveOccurred()) + err = ioutil.WriteFile(testdir+"/file", []byte("test"), os.ModeAppend) + Expect(err).NotTo(HaveOccurred()) + err = waitWebhookSecretVolume(testdir, testTimeout, testInterval) + Expect(err).NotTo(HaveOccurred()) + }) + }) + +})