Merge pull request #32 from stefanprodan/gprc-health

Implement gRPC health endpoint
This commit is contained in:
Stefan Prodan
2019-09-05 11:43:13 +03:00
committed by GitHub
11 changed files with 222 additions and 27 deletions

View File

@@ -10,7 +10,7 @@ GIT_COMMIT:=$(shell git describe --dirty --always)
VERSION:=$(shell grep 'VERSION' pkg/version/version.go | awk '{ print $$4 }' | tr -d '"')
run:
GO111MODULE=on go run -ldflags "-s -w -X github.com/stefanprodan/podinfo/pkg/version.REVISION=$(GIT_COMMIT)" cmd/podinfo/* --level=debug
GO111MODULE=on go run -ldflags "-s -w -X github.com/stefanprodan/podinfo/pkg/version.REVISION=$(GIT_COMMIT)" cmd/podinfo/* --level=debug --grpc-port=9999
test:
GO111MODULE=on go test -v -race ./...

View File

@@ -8,7 +8,9 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{- if and (not .Values.hpa.enabled) (gt .Values.replicaCount 1.0) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
strategy:
type: RollingUpdate
rollingUpdate:
@@ -23,10 +25,11 @@ spec:
app: {{ template "podinfo.name" . }}
release: {{ .Release.Name }}
annotations:
prometheus.io/scrape: 'true'
prometheus.io/scrape: "true"
prometheus.io/port: "{{ .Values.service.httpPort }}"
spec:
terminationGracePeriodSeconds: 30
{{- if .Values.serviceAccount.create }}
{{- if .Values.serviceAccount.enabled }}
serviceAccountName: {{ template "podinfo.serviceAccountName" . }}
{{- end }}
containers:
@@ -35,13 +38,20 @@ spec:
imagePullPolicy: {{ .Values.image.pullPolicy }}
command:
- ./podinfo
- --port={{ .Values.service.containerPort }}
- --port={{ .Values.service.httpPort | default 9898 }}
{{- if .Values.service.metricsPort }}
- --port-metrics={{ .Values.service.metricsPort }}
{{- end }}
{{- if .Values.service.grpcPort }}
- --grpc-port={{ .Values.service.grpcPort }}
{{- end }}
{{- if .Values.service.grpcService }}
- --grpc-service-name={{ .Values.service.grpcService }}
{{- end }}
- --level={{ .Values.logLevel }}
- --random-delay={{ .Values.faults.delay }}
- --random-error={{ .Values.faults.error }}
env:
- name: PODINFO_UI_COLOR
value: {{ .Values.color }}
{{- if .Values.message }}
- name: PODINFO_UI_MESSAGE
value: {{ .Values.message }}
@@ -52,15 +62,25 @@ spec:
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.containerPort }}
containerPort: {{ .Values.service.httpPort | default 9898 }}
protocol: TCP
{{- if .Values.service.metricsPort }}
- name: http-metrics
containerPort: {{ .Values.service.metricsPort }}
protocol: TCP
{{- end }}
{{- if .Values.service.grpcPort }}
- name: grpc
containerPort: {{ .Values.service.grpcPort }}
protocol: TCP
{{- end }}
livenessProbe:
exec:
command:
- podcli
- check
- http
- localhost:{{ .Values.service.containerPort }}/healthz
- localhost:{{ .Values.service.httpPort | default 9898 }}/healthz
initialDelaySeconds: 1
timeoutSeconds: 5
readinessProbe:
@@ -69,7 +89,7 @@ spec:
- podcli
- check
- http
- localhost:{{ .Values.service.containerPort }}/readyz
- localhost:{{ .Values.service.httpPort | default 9898 }}/readyz
initialDelaySeconds: 1
timeoutSeconds: 5
volumeMounts:

View File

@@ -18,6 +18,12 @@ spec:
{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }}
nodePort: {{ .Values.service.nodePort }}
{{- end }}
{{- if .Values.service.grpcPort }}
- port: {{ .Values.service.grpcPort }}
targetPort: grpc
protocol: TCP
name: grpc
{{- end }}
selector:
app: {{ template "podinfo.name" . }}
release: {{ .Release.Name }}

View File

@@ -0,0 +1,18 @@
apiVersion: v1
kind: Pod
metadata:
name: {{ template "podinfo.fullname" . }}-grpc-test-{{ randAlphaNum 5 | lower }}
labels:
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
app: {{ template "podinfo.name" . }}
annotations:
"helm.sh/hook": test-success
spec:
containers:
- name: grpc-health-probe
image: stefanprodan/grpc_health_probe:v0.3.0
command: ['grpc_health_probe']
args: ['-addr={{ template "podinfo.fullname" . }}:{{ .Values.service.grpcPort }}']
restartPolicy: Never

View File

@@ -2,7 +2,6 @@
replicaCount: 1
logLevel: info
color: blue
backend: #http://backend-podinfo:9898/echo
message: #UI greetings
@@ -18,8 +17,11 @@ image:
service:
enabled: true
type: ClusterIP
metricsPort: 9797
httpPort: 9898
externalPort: 9898
containerPort: 9898
grpcPort: 9999
grpcService: podinfo
nodePort: 31198
# metrics-server add-on required

View File

@@ -14,14 +14,19 @@ import (
"github.com/spf13/cobra"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/status"
)
var (
retryCount int
retryDelay time.Duration
method string
body string
timeout time.Duration
retryCount int
retryDelay time.Duration
method string
body string
timeout time.Duration
grpcServiceName string
)
var checkCmd = &cobra.Command{
@@ -51,6 +56,13 @@ var checkCertCmd = &cobra.Command{
RunE: runCheckCert,
}
var checkgRPCCmd = &cobra.Command{
Use: `grpc [address]`,
Short: "gRPC health check",
Example: ` check grpc localhost:8080 --service=podinfo --retry=1 --delay=2s --timeout=2s`,
RunE: runCheckgPRC,
}
func init() {
checkUrlCmd.Flags().StringVar(&method, "method", "GET", "HTTP method")
checkUrlCmd.Flags().StringVar(&body, "body", "", "HTTP POST/PUT content")
@@ -64,6 +76,12 @@ func init() {
checkTcpCmd.Flags().DurationVar(&timeout, "timeout", 5*time.Second, "timeout")
checkCmd.AddCommand(checkTcpCmd)
checkgRPCCmd.Flags().IntVar(&retryCount, "retry", 0, "times to retry the TCP check")
checkgRPCCmd.Flags().DurationVar(&retryDelay, "delay", 1*time.Second, "wait duration between retries")
checkgRPCCmd.Flags().DurationVar(&timeout, "timeout", 5*time.Second, "timeout")
checkgRPCCmd.Flags().StringVar(&grpcServiceName, "service", "", "gRPC service name")
checkCmd.AddCommand(checkgRPCCmd)
checkCmd.AddCommand(checkCertCmd)
rootCmd.AddCommand(checkCmd)
@@ -243,3 +261,53 @@ func fmtContentLength(b int64) string {
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
}
func runCheckgPRC(cmd *cobra.Command, args []string) error {
if retryCount < 0 {
return fmt.Errorf("--retry is required")
}
if len(args) < 1 {
return fmt.Errorf("address is required! example: check grpc localhost:8080")
}
address := args[0]
for n := 0; n <= retryCount; n++ {
if n != 1 {
time.Sleep(retryDelay)
}
conn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
logger.Info("check failed",
zap.String("address", address),
zap.Error(err))
continue
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
resp, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{
Service: grpcServiceName,
})
cancel()
if err != nil {
if stat, ok := status.FromError(err); ok && stat.Code() == codes.Unimplemented {
logger.Info("gPRC health protocol not implemented")
os.Exit(1)
} else {
logger.Info("check failed",
zap.String("address", address),
zap.Error(err))
}
continue
}
conn.Close()
logger.Info("check succeed",
zap.String("status", resp.GetStatus().String()))
os.Exit(0)
}
os.Exit(1)
return nil
}

View File

@@ -2,26 +2,31 @@ package main
import (
"fmt"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/stefanprodan/podinfo/pkg/api"
"github.com/stefanprodan/podinfo/pkg/signals"
"github.com/stefanprodan/podinfo/pkg/version"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"github.com/stefanprodan/podinfo/pkg/api"
"github.com/stefanprodan/podinfo/pkg/grpc"
"github.com/stefanprodan/podinfo/pkg/signals"
"github.com/stefanprodan/podinfo/pkg/version"
)
func main() {
// flags definition
fs := pflag.NewFlagSet("default", pflag.ContinueOnError)
fs.Int("port", 9898, "port")
fs.Int("port", 9898, "HTTP port")
fs.Int("port-metrics", 0, "metrics port")
fs.Int("grpc-port", 0, "gRPC port")
fs.String("grpc-service-name", "podinfo", "gPRC service name")
fs.String("level", "info", "log level debug, info, warn, error, flat or panic")
fs.String("backend-url", "", "backend service URL")
fs.Duration("http-client-timeout", 2*time.Minute, "client timeout duration")
@@ -90,6 +95,18 @@ func main() {
viper.Set("port", strconv.Itoa(port))
}
// load gRPC server config
var grpcCfg grpc.Config
if err := viper.Unmarshal(&grpcCfg); err != nil {
logger.Panic("config unmarshal failed", zap.Error(err))
}
// start gRPC server
if grpcCfg.Port > 0 {
grpcSrv, _ := grpc.NewServer(&grpcCfg, logger)
go grpcSrv.ListenAndServe()
}
// load HTTP server config
var srvCfg api.Config
if err := viper.Unmarshal(&srvCfg); err != nil {

View File

@@ -3,7 +3,7 @@
set -o errexit
REPO_ROOT=$(git rev-parse --show-toplevel)
KIND_VERSION=v0.4.0
KIND_VERSION=v0.5.1
if [[ "$1" ]]; then
KIND_VERSION=$1

4
go.mod
View File

@@ -3,7 +3,6 @@ module github.com/stefanprodan/podinfo
go 1.12
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/aws/aws-sdk-go v1.15.63 // indirect
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
@@ -14,7 +13,7 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/fatih/color v1.7.0
github.com/fsnotify/fsnotify v1.4.7
github.com/golang/protobuf v1.2.0 // indirect
github.com/go-chi/chi v4.0.2+incompatible // indirect
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.0
github.com/hashicorp/go-cleanhttp v0.5.0 // indirect
@@ -49,4 +48,5 @@ require (
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.9.1
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
google.golang.org/grpc v1.23.0
)

18
go.sum
View File

@@ -1,3 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
@@ -20,6 +21,7 @@ github.com/chzyer/readline v0.0.0-20160726135117-62c6fe619375 h1:JVe1zduaiPlSLOu
github.com/chzyer/readline v0.0.0-20160726135117-62c6fe619375/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -31,6 +33,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
@@ -40,8 +44,12 @@ github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k
github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
@@ -123,19 +131,29 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468 h1:fTfk6GjmihJbK0mSUFgPPgYpsdmApQ86Mcd4GuKax9U=
golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

46
pkg/grpc/server.go Normal file
View File

@@ -0,0 +1,46 @@
package grpc
import (
"fmt"
"net"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
"google.golang.org/grpc/health/grpc_health_v1"
)
type Server struct {
logger *zap.Logger
config *Config
}
type Config struct {
Port int `mapstructure:"grpc-port"`
ServiceName string `mapstructure:"grpc-service-name"`
}
func NewServer(config *Config, logger *zap.Logger) (*Server, error) {
srv := &Server{
logger: logger,
config: config,
}
return srv, nil
}
func (s *Server) ListenAndServe() {
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", s.config.Port))
if err != nil {
s.logger.Fatal("failed to listen", zap.Int("port", s.config.Port))
}
srv := grpc.NewServer()
server := health.NewServer()
grpc_health_v1.RegisterHealthServer(srv, server)
server.SetServingStatus(s.config.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING)
if err := srv.Serve(listener); err != nil {
s.logger.Fatal("failed to serve", zap.Error(err))
}
}