From 7bbb77091a7db170bd9a3be79e1b375925afd19e Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Mon, 7 Nov 2016 13:20:26 -0500 Subject: [PATCH] Add a chart for CockroachDB to the incubator repo (#167) --- incubator/cockroachdb/Chart.yaml | 9 + incubator/cockroachdb/README.md | 190 ++++++++++++++++++ incubator/cockroachdb/templates/NOTES.txt | 27 +++ .../templates/cockroachdb-petset.yaml | 188 +++++++++++++++++ incubator/cockroachdb/values.yaml | 21 ++ 5 files changed, 435 insertions(+) create mode 100755 incubator/cockroachdb/Chart.yaml create mode 100644 incubator/cockroachdb/README.md create mode 100644 incubator/cockroachdb/templates/NOTES.txt create mode 100644 incubator/cockroachdb/templates/cockroachdb-petset.yaml create mode 100644 incubator/cockroachdb/values.yaml diff --git a/incubator/cockroachdb/Chart.yaml b/incubator/cockroachdb/Chart.yaml new file mode 100755 index 0000000000..7d35adb8bd --- /dev/null +++ b/incubator/cockroachdb/Chart.yaml @@ -0,0 +1,9 @@ +name: cockroachdb +home: https://www.cockroachlabs.com +version: 0.1.0 +description: CockroachDB Helm chart for Kubernetes. +sources: + - https://github.com/cockroachdb/cockroach +maintainers: + - name: Alex Robinson + email: alex@cockroachlabs.com diff --git a/incubator/cockroachdb/README.md b/incubator/cockroachdb/README.md new file mode 100644 index 0000000000..cd54c6df06 --- /dev/null +++ b/incubator/cockroachdb/README.md @@ -0,0 +1,190 @@ +# CockroachDB Helm Chart + +## Prerequisites Details +* Kubernetes 1.4 with alpha APIs enabled +* PV support on the underlying infrastructure + +## PetSet Details +* http://kubernetes.io/docs/user-guide/petset/ + +## PetSet Caveats +* http://kubernetes.io/docs/user-guide/petset/#alpha-limitations + +## Todo +* Support setting up clusters with certificate-based authentication + +## Chart Details +This chart will do the following: + +* Set up a dynamically scalable CockroachDB cluster using a Kubernetes PetSet + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```shell +helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator +helm install --name my-release incubator/cockroachdb +``` + +## Configuration + +The following tables lists the configurable parameters of the CockroachDB chart and their default values. + +| Parameter | Description | Default | +| ----------------------- | ---------------------------------- | ---------------------------------------------------------- | +| `Name` | Chart name | `cockroachdb` | +| `Image` | Container image name | `cockroachdb/cockroach` | +| `ImageTag` | Container image tag | `latest` | +| `ImagePullPolicy` | Container pull policy | `Always` | +| `Replicas` | k8s petset replicas | `3` | +| `Component` | k8s selector key | `cockroachdb` | +| `GrpcPort` | CockroachDB primary serving port | `26257` | +| `HttpPort` | CockroachDB HTTP port | `8080` | +| `Cpu` | Container requested cpu | `100m` | +| `Memory` | Container requested memory | `512Mi` | +| `Storage` | Persistent volume size | `1Gi` | +| `StorageClass` | Persistent volume class | `anything` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```shell +helm install --name my-release -f values.yaml incubator/cockroachdb +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +# Deep dive + +## Connecting to the CockroachDB cluster + +Once you've created the cluster, you can start talking to it it by connecting +to its "public" service. CockroachDB is PostgreSQL wire protocol compatible so +there's a [wide variety of supported +clients](https://www.cockroachlabs.com/docs/install-client-drivers.html). For +the sake of example, we'll open up a SQL shell using CockroachDB's built-in +shell and play around with it a bit, like this (likely needing to replace +"my-release-cockroachdb-public" with the name of the "-public" service that +was created with your installed chart): + +```console +$ kubectl run -it --rm cockroach-client \ + --image=cockroachdb/cockroach \ + --restart=Never \ + --command -- ./cockroach sql --host my-release-cockroachdb-public +Waiting for pod default/cockroach-client to be running, status is Pending, +pod ready: false +If you don't see a command prompt, try pressing enter. +root@my-release-cockroachdb-public:26257> SHOW DATABASES; ++--------------------+ +| Database | ++--------------------+ +| information_schema | +| pg_catalog | +| system | ++--------------------+ +(3 rows) +root@my-release-cockroachdb-public:26257> CREATE DATABASE bank; +CREATE DATABASE +root@my-release-cockroachdb-public:26257> CREATE TABLE bank.accounts (id INT +PRIMARY KEY, balance DECIMAL); +CREATE TABLE +root@my-release-cockroachdb-public:26257> INSERT INTO bank.accounts VALUES +(1234, 10000.50); +INSERT 1 +root@my-release-cockroachdb-public:26257> SELECT * FROM bank.accounts; ++------+---------+ +| id | balance | ++------+---------+ +| 1234 | 10000.5 | ++------+---------+ +(1 row) +root@my-release-cockroachdb-public:26257> \q +Waiting for pod default/cockroach-client to terminate, status is Running +pod "cockroach-client" deleted +``` + +## Cluster health + +Because our pod spec includes regular health checks of the CockroachDB processes, +simply running `kubectl get pods` and looking at the `STATUS` column is sufficient +to determine the health of each instance in the cluster. + +If you want more detailed information about the cluster, the best place to look +is the admin UI. + +## Accessing the admin UI + +If you want to see information about how the cluster is doing, you can try +pulling up the CockroachDB admin UI by port-forwarding from your local machine +to one of the pods (replacing "release-cockroachdb-0" with one of your pods' names): + +```shell +kubectl port-forward release-cockroachdb-0 8080 +``` + +Once you’ve done that, you should be able to access the admin UI by visiting +http://localhost:8080/ in your web browser. + +## Failover + +If any CockroachDB member fails it gets restarted or recreated automatically by +the Kubernetes infrastructure, and will rejoin the cluster automatically when +it comes back up. You can test this scenario by killing any of the pets: + +```shell +kubectl delete pod my-release-cockroachdb-1 +``` + +```shell +$ kubectl get pods -l "component=my-release-cockroachdb" +NAME READY STATUS RESTARTS AGE +my-release-cockroachdb-0 1/1 Running 0 5m +my-release-cockroachdb-2 1/1 Running 0 5m +``` + +After a while: + +```console +$ kubectl get pods -l "component=my-release-cockroachdb" +NAME READY STATUS RESTARTS AGE +my-release-cockroachdb-0 1/1 Running 0 5m +my-release-cockroachdb-1 1/1 Running 0 20s +my-release-cockroachdb-2 1/1 Running 0 5m +``` + +You can check state of re-joining from the new pod's logs: + +```console +$ kubectl logs my-release-cockroachdb-1 +[...] +I161028 19:32:09.754026 1 server/node.go:586 [n1] node connected via gossip and +verified as part of cluster {"35ecbc27-3f67-4e7d-9b8f-27c31aae17d6"} +[...] +cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local:26257 +build: beta-20161027-55-gd2d3c7f @ 2016/10/28 19:27:25 (go1.7.3) +admin: http://0.0.0.0:8080 +sql: +postgresql://root@my-release-cockroachdb-1.my-release-cockroachdb.default.svc.cluster.local:26257?sslmode=disable +logs: cockroach-data/logs +store[0]: path=cockroach-data +status: restarted pre-existing node +clusterID: {35ecbc27-3f67-4e7d-9b8f-27c31aae17d6} +nodeID: 2 +[...] +``` + +## Scaling + +Scaling should typically be managed via the `helm upgrade` command, but PetSets +don't yet work with `helm upgrade`. In the meantime until `helm upgrade` works, +if you want to change the number of replicas, you can use the `kubectl patch` +command with the desired number of replicas, as shown below (or see the +[PetSet documentation](http://kubernetes.io/docs/user-guide/petset/#scaling-a-petset) +for more options): + +```shell +kubectl patch petset my-release-cockroachdb -p '{"spec":{"replicas":4}}' +``` diff --git a/incubator/cockroachdb/templates/NOTES.txt b/incubator/cockroachdb/templates/NOTES.txt new file mode 100644 index 0000000000..a0a2a9cba1 --- /dev/null +++ b/incubator/cockroachdb/templates/NOTES.txt @@ -0,0 +1,27 @@ +CockroachDB can be accessed via port 26257 (or whatever you set the GrpcPort +value to) at the following DNS name from within your cluster: +{{ .Release.Name }}-public.{{ .Release.Namespace }}.svc.cluster.local + +Because CockroachDB supports the PostgreSQL wire protocol, you can connect to +the cluster using any available PostgreSQL client. + +For example, you can open up a SQL shell to the cluster by running: + + kubectl run -it --rm cockroach-client \ + --image=cockroachdb/cockroach \ + --restart=Never \ + --command -- ./cockroach sql --host {{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-public.{{ .Release.Namespace }} + +From there, you can interact with the SQL shell as you would any other SQL shell, +confident that any data you write will be safe and available even if parts of +your cluster fail. + +Finally, to open up the CockroachDB admin UI, you can port-forward from your +local machine into one of the instances in the cluster: + + kubectl port-forward {{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-0 8080 + +Then you can access the admin UI at http://localhost:8080/ in your web browser. + +For more information on using CockroachDB, please see the project's docs at +https://www.cockroachlabs.com/docs/ diff --git a/incubator/cockroachdb/templates/cockroachdb-petset.yaml b/incubator/cockroachdb/templates/cockroachdb-petset.yaml new file mode 100644 index 0000000000..9f0e9255ee --- /dev/null +++ b/incubator/cockroachdb/templates/cockroachdb-petset.yaml @@ -0,0 +1,188 @@ +apiVersion: v1 +kind: Service +metadata: + # This service is meant to be used by clients of the database. It exposes a ClusterIP that will + # automatically load balance connections to the different database pods. + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-public" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + helm.sh/created: {{.Release.Time.Seconds | quote }} +spec: + ports: + # The main port, served by gRPC, serves Postgres-flavor SQL, internode + # traffic and the cli. + - port: {{.Values.GrpcPort}} + targetPort: {{.Values.GrpcPort}} + name: grpc + # The secondary port serves the UI as well as health and debug endpoints. + - port: {{.Values.HttpPort}} + targetPort: {{.Values.HttpPort}} + name: http + selector: + component: "{{.Release.Name}}-{{.Values.Component}}" +--- +apiVersion: v1 +kind: Service +metadata: + # This service only exists to create DNS entries for each pet in the petset such that they can resolve + # each other's IP addresses. It does not create a load-balanced ClusterIP and should not be used + # directly by clients in most circumstances. + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + helm.sh/created: {{.Release.Time.Seconds | quote }} + # This is needed to make the peer-finder work properly and to help avoid + # edge cases where instance 0 comes up after losing its data and needs to + # decide whether it should create a new cluster or try to join an existing + # one. If it creates a new cluster when it should have joined an existing + # one, we'd end up with two separate clusters listening at the same service + # endpoint, which would be very bad. + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + # Enable automatic monitoring of all instances when Prometheus is running in the cluster. + prometheus.io/scrape: "true" + prometheus.io/path: "_status/vars" + prometheus.io/port: "8080" +spec: + ports: + - port: {{.Values.GrpcPort}} + targetPort: {{.Values.GrpcPort}} + name: grpc + - port: {{.Values.HttpPort}} + targetPort: {{.Values.HttpPort}} + name: http + clusterIP: None + selector: + component: "{{.Release.Name}}-{{.Values.Component}}" +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + annotations: + helm.sh/created: {{.Release.Time.Seconds | quote }} +spec: + serviceName: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + replicas: {{default 3 .Values.Replicas}} + template: + metadata: + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + helm.sh/created: {{.Release.Time.Seconds | quote }} + pod.alpha.kubernetes.io/initialized: "true" + # Init containers are run only once in the lifetime of a pod, before + # it's started up for the first time. It has to exit successfully + # before the pod's main containers are allowed to start. + # This particular init container does a DNS lookup for other pods in + # the petset to help determine whether or not a cluster already exists. + # If any other pets exist, it creates a file in the cockroach-data + # directory to pass that information along to the primary container that + # has to decide what command-line flags to use when starting CockroachDB. + # This only matters when a pod's persistent volume is empty - if it has + # data from a previous execution, that data will always be used. + # The cockroachdb/cockroach-k8s-init image is defined at + # github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/init + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "bootstrap", + "image": "{{.Values.BootstrapImage}}:{{.Values.BootstrapImageTag}}", + "args": [ + "-on-start=/on-start.sh", + "-service={{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "datadir", + "mountPath": "/cockroach/cockroach-data" + } + ] + } + ]' + spec: + containers: + - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + image: "{{.Values.Image}}:{{.Values.ImageTag}}" + imagePullPolicy: "{{.Values.ImagePullPolicy}}" + ports: + - containerPort: {{.Values.GrpcPort}} + name: grpc + - containerPort: {{.Values.HttpPort}} + name: http + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: PETSET_NAME + value: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + livenessProbe: + httpGet: + path: /_admin/v1/health + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /_admin/v1/health + port: http + initialDelaySeconds: 10 + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + # The use of qualified `hostname -f` is crucial: + # Other nodes aren't able to look up the unqualified hostname. + CRARGS=("start" "--logtostderr" "--insecure" "--host" "$(hostname -f)" "--http-host" "0.0.0.0") + # We only want to initialize a new cluster (by omitting the join flag) + # if we're sure that we're the first node (i.e. index 0) and that + # there aren't any other nodes running as part of the cluster that + # this is supposed to be a part of (which indicates that a cluster + # already exists and we should make sure not to create a new one). + # It's fine to run without --join on a restart if there aren't any + # other nodes. + if [ ! "$(hostname)" == "${PETSET_NAME}-0" ] || \ + [ -e "/cockroach/cockroach-data/cluster_exists_marker" ] + then + CRARGS+=("--join" "${PETSET_NAME}-public") + fi + exec /cockroach/cockroach ${CRARGS[*]} + # No pre-stop hook is required, a SIGTERM plus some time is all that's + # needed for graceful shutdown of a node. + terminationGracePeriodSeconds: 60 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: datadir + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: "{{.Values.StorageClass}}" + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "{{.Values.Storage}}" diff --git a/incubator/cockroachdb/values.yaml b/incubator/cockroachdb/values.yaml new file mode 100644 index 0000000000..abc9fe59be --- /dev/null +++ b/incubator/cockroachdb/values.yaml @@ -0,0 +1,21 @@ +# Default values for cockroachdb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +Name: "cockroachdb" +Image: "cockroachdb/cockroach" +ImageTag: "latest" # Keep at latest until 1.0, then switch to a specific version +ImagePullPolicy: "Always" +BootstrapImage: "cockroachdb/cockroach-k8s-init" +BootstrapImageTag: "0.1" +Replicas: 3 +Component: "cockroachdb" +GrpcPort: 26257 +HttpPort: 8080 +Resources: + requests: + cpu: "100m" + memory: "512Mi" +Storage: "1Gi" +StorageClass: "anything"