From e0e7644acd0b4a91a1361d12a4567f1c9ae14a1a Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 12 Jan 2016 14:35:49 +0000 Subject: [PATCH 1/4] k8s: Use service account token when no api server url is provided --- probe/kubernetes/client.go | 16 +++++++++++++++- prog/probe.go | 3 ++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/probe/kubernetes/client.go b/probe/kubernetes/client.go index 5a4f4e1d9..02f1ff06c 100644 --- a/probe/kubernetes/client.go +++ b/probe/kubernetes/client.go @@ -33,7 +33,21 @@ type client struct { // NewClient returns a usable Client. Don't forget to Stop it. func NewClient(addr string, resyncPeriod time.Duration) (Client, error) { - c, err := unversioned.New(&unversioned.Config{Host: addr}) + var config *unversioned.Config + if addr != "" { + config = &unversioned.Config{Host: addr} + } else { + // If no API server address was provided, assume we are running + // inside a pod. Try to connect to the API server through its + // Service environment variables, using the default Service + // Account Token. + var err error + if config, err = unversioned.InClusterConfig(); err != nil { + return nil, err + } + } + + c, err := unversioned.New(config) if err != nil { return nil, err } diff --git a/prog/probe.go b/prog/probe.go index d7dba92e5..f8aa9ed1b 100644 --- a/prog/probe.go +++ b/prog/probe.go @@ -43,7 +43,7 @@ func probeMain() { dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes") dockerBridge = flag.String("docker.bridge", "docker0", "the docker bridge name") kubernetesEnabled = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node") - kubernetesAPI = flag.String("kubernetes.api", "http://localhost:8080", "Address of kubernetes master api") + kubernetesAPI = flag.String("kubernetes.api", "", "Address of kubernetes master api") kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data") weaveRouterAddr = flag.String("weave.router.addr", "", "IP address or FQDN of the Weave router") procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem") @@ -144,6 +144,7 @@ func probeMain() { p.AddReporter(kubernetes.NewReporter(client)) } else { log.Printf("Kubernetes: failed to start client: %v", err) + log.Printf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url") } } From fa43df2de7b69db21fdb0a53b7649ab35f4b676b Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 12 Jan 2016 15:26:44 +0000 Subject: [PATCH 2/4] k8s: Log errors when contacting the API server --- probe/kubernetes/client.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/probe/kubernetes/client.go b/probe/kubernetes/client.go index 02f1ff06c..a6b605a0c 100644 --- a/probe/kubernetes/client.go +++ b/probe/kubernetes/client.go @@ -1,6 +1,7 @@ package kubernetes import ( + "log" "time" "k8s.io/kubernetes/pkg/api" @@ -8,6 +9,7 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/util" ) // These constants are keys used in node metadata @@ -31,6 +33,17 @@ type client struct { serviceStore *cache.StoreToServiceLister } +// runReflectorUntil is equivalent to cache.Reflector.RunUntil, but it also logs +// errors, which cache.Reflector.RunUntil simply ignores +func runReflectorUntil(r *cache.Reflector, resyncPeriod time.Duration, stopCh <-chan struct{}) { + loggingListAndWatch := func() { + if err := r.ListAndWatch(stopCh); err != nil { + log.Printf("Kubernetes reflector error: %v", err) + } + } + go util.Until(loggingListAndWatch, resyncPeriod, stopCh) +} + // NewClient returns a usable Client. Don't forget to Stop it. func NewClient(addr string, resyncPeriod time.Duration) (Client, error) { var config *unversioned.Config @@ -61,8 +74,8 @@ func NewClient(addr string, resyncPeriod time.Duration) (Client, error) { serviceReflector := cache.NewReflector(serviceListWatch, &api.Service{}, serviceStore, resyncPeriod) quit := make(chan struct{}) - podReflector.RunUntil(quit) - serviceReflector.RunUntil(quit) + runReflectorUntil(podReflector, resyncPeriod, quit) + runReflectorUntil(serviceReflector, resyncPeriod, quit) return &client{ quit: quit, From 312b9ac370968c561dcf3be468b2658892bf3831 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 13 Jan 2016 17:51:14 +0000 Subject: [PATCH 3/4] k8s: Add documentation about deploying Scope --- README.md | 72 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 4c536c1d8..703e97b69 100644 --- a/README.md +++ b/README.md @@ -150,44 +150,54 @@ sudo scope launch --service-token= ## Using Weave Scope with Kubernetes -To use Scope's Kubernetes integration, you need to start Scope with the -`--probe.kubernetes true` flag. Scope needs to be installed on all -nodes (master and minions), but this flag should only be enabled on the -Kubernetes master node. +Scope comes with built-in Kubernetes support. We recommend to run Scope natively +in your Kubernetes cluster using +[this resource definitions](https://github.com/TheNewNormal/kube-charts/tree/master/weavescope/manifests) +which are easily deployable with [Helm](https://helm.sh/). -As per the normal requirements, you will need to run Scope on every -machine you want to monitor, as shown in [Getting -Started](#getting-started). However, when launching Scope you -need to pass different arguments to the Kubernetes master and minion -nodes. +1. Make sure your cluster supports + [DaemonSets](https://github.com/kubernetes/kubernetes/blob/master/docs/design/daemon.md) + in your cluster. DaemonSets are needed to ensure that each Kubernetes node + runs a Scope Probe: + + * To enable them in an existing cluster, make sure to add a + `--runtime-config=extensions/v1beta1/daemonsets=true` argument to the + [apiserver](https://github.com/kubernetes/kubernetes/blob/master/docs/admin/kube-apiserver.md)'s configuration + (normally found at `/etc/kubernetes/manifest/kube-apiserver.manifest`) followed by a + [restart of the apiserver and controller manager](https://github.com/kubernetes/kubernetes/issues/18656). -On the master node you need to launch Scope with Kubernetes support: + * If you are creating a new cluster, set `KUBE_ENABLE_DAEMONSETS=true` in + your cluster configuration. +2. Install [Helm](https://helm.sh/) +3. Add the kube-charts helm repo: + + ``` +helm up +helm repo add kube-charts https://github.com/TheNewNormal/kube-charts +helm up ``` -sudo scope launch --probe.kubernetes true +4. Fetch the weavescope Chart: + + ``` +helm fetch kube-charts/weavescope ``` -Depending on your setup, you may find that Kubernetes has renamed your -Docker bridge interface. In this instance you'll need to tell Scope -about the new name when launching it. For example, if your Docker bridge is -named `cbr0`: - +5. Tweak the Scope probe configuration at `$HOME/.helm/workspace/charts/weavescope/manifests/scope-probe-ds.yaml`, namely: + * If you have an account at http://scope.weave.works and want to use Scope in + service mode, uncomment the `--probe.token=foo` argument, substitute `foo` + by the token found in your account page, and comment out the + `$(WEAVE_SCOPE_APP_SERVICE_HOST):$(WEAVE_SCOPE_APP_SERVICE_PORT)` argument. +6. Install Scope in your cluster by using kubectl directly. Unfortunately `helm + install` cannot be used because the Scope App is optional (only needed in + standalone installations) and + [a specific deployment order is required](https://github.com/TheNewNormal/kube-charts/blob/915fcacb2a14f8b6a42c44ca5e2d217e21945137/weavescope/manifests/scope-probe-ds.yaml#L40-L42)): + + ``` +kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-app-rc.yaml # only if you want to run scope in standalone mode +kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-app-svc.yaml # only if you want to run scope in standalone mode +kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-probe-ds.yaml ``` -sudo DOCKER_BRIDGE=cbr0 scope launch --probe.docker.bridge cbr0 --probe.kubernetes true -``` - -On each minion node you need to launch Scope telling it -to connect to the master node. - -``` -sudo scope launch --no-app kubernetes-master.my.network -``` - -Again, if your Docker bridge interface is named differently, you'll -need to pass that to your probe when launching it. - -Once the first few reports come in, the UI should begin displaying two -Kubernetes-specific views "Pods", and "Pods by Service". ## Developing From e0dfeb18b766841c7ee40d0d4b5fe348a89db8fd Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 14 Jan 2016 11:49:31 +0000 Subject: [PATCH 4/4] Leave Helm out of the k8s install instructions for now --- README.md | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 703e97b69..146bf993c 100644 --- a/README.md +++ b/README.md @@ -152,10 +152,14 @@ sudo scope launch --service-token= Scope comes with built-in Kubernetes support. We recommend to run Scope natively in your Kubernetes cluster using -[this resource definitions](https://github.com/TheNewNormal/kube-charts/tree/master/weavescope/manifests) -which are easily deployable with [Helm](https://helm.sh/). +[this resource definitions](https://github.com/TheNewNormal/kube-charts/tree/master/weavescope/manifests). -1. Make sure your cluster supports +1. If you are running a Kubernetes version lower than 1.1, make sure your + cluster allows running pods in privileged mode (required by the Scope + probes). To allow privileged pods, your API Server and all your Kubelets must + be provided with flag `--allow_privileged` at launch time. + +2. Make sure your cluster supports [DaemonSets](https://github.com/kubernetes/kubernetes/blob/master/docs/design/daemon.md) in your cluster. DaemonSets are needed to ensure that each Kubernetes node runs a Scope Probe: @@ -169,34 +173,24 @@ which are easily deployable with [Helm](https://helm.sh/). * If you are creating a new cluster, set `KUBE_ENABLE_DAEMONSETS=true` in your cluster configuration. -2. Install [Helm](https://helm.sh/) -3. Add the kube-charts helm repo: +3. Download the resource definitions: ``` -helm up -helm repo add kube-charts https://github.com/TheNewNormal/kube-charts -helm up -``` -4. Fetch the weavescope Chart: - - ``` -helm fetch kube-charts/weavescope +for I in app-rc app-svc probe-ds; do curl -s -L https://raw.githubusercontent.com/TheNewNormal/kube-charts/master/weavescope/manifests/scope-$I.yaml -o scope-$I.yaml; done ``` -5. Tweak the Scope probe configuration at `$HOME/.helm/workspace/charts/weavescope/manifests/scope-probe-ds.yaml`, namely: +4. Tweak the Scope probe configuration at `scope-probe-ds.yaml`, namely: * If you have an account at http://scope.weave.works and want to use Scope in - service mode, uncomment the `--probe.token=foo` argument, substitute `foo` + Cloud Service Mode, uncomment the `--probe.token=foo` argument, substitute `foo` by the token found in your account page, and comment out the `$(WEAVE_SCOPE_APP_SERVICE_HOST):$(WEAVE_SCOPE_APP_SERVICE_PORT)` argument. -6. Install Scope in your cluster by using kubectl directly. Unfortunately `helm - install` cannot be used because the Scope App is optional (only needed in - standalone installations) and - [a specific deployment order is required](https://github.com/TheNewNormal/kube-charts/blob/915fcacb2a14f8b6a42c44ca5e2d217e21945137/weavescope/manifests/scope-probe-ds.yaml#L40-L42)): + +5. Install Scope in your cluster (order is important): ``` -kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-app-rc.yaml # only if you want to run scope in standalone mode -kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-app-svc.yaml # only if you want to run scope in standalone mode -kubectl create -f $HOME/.helm/workspace/charts/weavescope/manifests/scope-probe-ds.yaml +kubectl create -f scope-app-rc.yaml # Only if you want to run Scope in Standalone Mode +kubectl create -f scope-app-svc.yaml # Only if you want to run Scope in Standalone Mode +kubectl create -f scope-probe-ds.yaml ```