github/node-problem-detector
1
0
Fork 0
mirror of https://github.com/kubernetes/node-problem-detector.git synced 2026-08-19 04:06:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #39 from Random-Liu/journald-support

Browse Source 
Journald support
This commit is contained in:
Dawn Chen
2017-02-01 12:41:51 -08:00
committed by GitHub
parent ba5f5a158d 457272c4f6
commit b66c4df364
26 changed files with 829 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1 +1,2 @@
/bin/node-problem-detector
/Dockerfile
Dockerfile → Dockerfile.in
+5 -1
View File
@@ -12,8 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM alpine:3.4
FROM @BASEIMAGE@
MAINTAINER Random Liu <lantaol@google.com>
# Avoid symlink of /etc/localtime.
RUN test -h /etc/localtime && rm -f /etc/localtime && cp /usr/share/zoneinfo/UTC /etc/localtime || true
ADD ./bin/node-problem-detector /node-problem-detector
ADD config /config
ENTRYPOINT ["/node-problem-detector", "--kernel-monitor=/config/kernel-monitor.json"]
Godeps/Godeps.json
Generated
+4 -4
View File
@@ -128,13 +128,13 @@
},
{
"ImportPath": "github.com/google/cadvisor/info/v1",
"Comment": "v0.23.2",
"Rev": "7ddf6eb5d1f84363fbc181a498313a880b12ba07"
"Comment": "v0.24.0-alpha1-60-gb5b0547",
"Rev": "b5b0547e330937317d464f50dfe9e6f34460b125"
},
{
"ImportPath": "github.com/google/cadvisor/utils/tail",
"Comment": "v0.23.2",
"Rev": "7ddf6eb5d1f84363fbc181a498313a880b12ba07"
"Comment": "v0.24.0-alpha1-60-gb5b0547",
"Rev": "b5b0547e330937317d464f50dfe9e6f34460b125"
},
{
"ImportPath": "github.com/google/gofuzz",
Makefile
+67 -15
View File
@@ -1,24 +1,73 @@
.PHONY: all build-container build-tar build push-container push-tar push clean vet fmt version
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Build the node-problem-detector image.
.PHONY: all build-container build-tar build push-container push-tar push clean vet fmt version Dockerfile
all: build
VERSION := $(shell git describe --tags --dirty)
# VERSION is the version of the binary.
VERSION:=$(shell git describe --tags --dirty)
TAG ?= $(VERSION)
# TAG is the tag of the container image, default to binary version.
TAG?=$(VERSION)
UPLOAD_PATH ?= gs://kubernetes-release
# PROJ is the image project.
PROJ?=gcr.io/google_containers
# UPLOAD_PATH is the cloud storage path to upload release tar.
UPLOAD_PATH?=gs://kubernetes-release
# Trim the trailing '/' in the path
UPLOAD_PATH := $(shell echo $(UPLOAD_PATH) | sed '$$s/\/*$$//')
UPLOAD_PATH:=$(shell echo $(UPLOAD_PATH) | sed '$$s/\/*$$//')
PROJ ?= google_containers
# PKG is the package name of node problem detector repo.
PKG:=k8s.io/node-problem-detector
PKG := k8s.io/node-problem-detector
# PKG_SOURCES are all the go source code.
PKG_SOURCES:=$(shell find pkg cmd -name '*.go')
PKG_SOURCES := $(shell find pkg cmd -name '*.go')
# TARBALL is the name of release tar. Include binary version by default.
TARBALL:=node-problem-detector-$(VERSION).tar.gz
TARBALL := node-problem-detector-$(VERSION).tar.gz
# IMAGE is the image name of the node problem detector container image.
IMAGE:=$(PROJ)/node-problem-detector:$(TAG)
IMAGE := gcr.io/$(PROJ)/node-problem-detector:$(TAG)
# ENABLE_JOURNALD enables build journald support or not. Building journald support needs libsystemd-dev
# or libsystemd-journal-dev.
# TODO(random-liu): Build NPD inside container.
ENABLE_JOURNALD?=1
# TODO(random-liu): Support different architectures.
BASEIMAGE:=alpine:3.4
# Disable cgo by default to make the binary statically linked.
CGO_ENABLED:=0
# NOTE that enable journald will increase the image size.
ifeq ($(ENABLE_JOURNALD), 1)
# Enable journald build tag.
BUILD_TAGS:=-tags journald
# Use fedora because it has newer systemd version (229) and support +LZ4. +LZ4 is needed
# on some os distros such as GCI.
BASEIMAGE:=fedora
# Enable cgo because sdjournal needs cgo to compile. The binary will be dynamically
# linked if CGO_ENABLED is enabled. This is fine because fedora already has necessary
# dynamic library. We can not use `-extldflags "-static"` here, because go-systemd uses
# dlopen, and dlopen will not work properly in a statically linked application.
CGO_ENABLED:=1
endif
vet:
go list ./... | grep -v "./vendor/*" | xargs go vet
@@ -30,14 +79,17 @@ version:
@echo $(VERSION)
./bin/node-problem-detector: $(PKG_SOURCES)
GOOS=linux go build -o bin/node-problem-detector \
-ldflags '-w -extldflags "-static" -X $(PKG)/pkg/version.version=$(VERSION)' \
cmd/node_problem_detector.go
CGO_ENABLED=$(CGO_ENABLED) GOOS=linux go build -o bin/node-problem-detector \
-ldflags '-w -X $(PKG)/pkg/version.version=$(VERSION)' \
$(BUILD_TAGS) cmd/node_problem_detector.go
Dockerfile: Dockerfile.in
sed -e 's|@BASEIMAGE@|$(BASEIMAGE)|g' $< >$@
test: vet fmt
go test -timeout=1m -v -race ./pkg/...
go test -timeout=1m -v -race ./pkg/... $(BUILD_TAGS)
build-container: ./bin/node-problem-detector
build-container: ./bin/node-problem-detector Dockerfile
docker build -t $(IMAGE) .
build-tar: ./bin/node-problem-detector
README.md
+6
View File
@@ -100,11 +100,17 @@ spec:
- name: log
mountPath: /log
readOnly: true
- name: localtime
mountPath: /etc/localtime
readOnly: true
volumes:
- name: log
# Config `log` to your system log directory
hostPath:
path: /var/log/
- name: localtime
hostPath:
path: /etc/localtime
```
* Edit node-problem-detector.yaml to fit your environment: Set `log` volume to your system log diretory. (Used by KernelMonitor)
* Create the DaemonSet with `kubectl create -f node-problem-detector.yaml`
cmd/node_problem_detector.go
+6 -4
View File
@@ -18,15 +18,15 @@ package main
import (
"flag"
"fmt"
"net/url"
"os"
"github.com/golang/glog"
"k8s.io/node-problem-detector/pkg/kernelmonitor"
"k8s.io/node-problem-detector/pkg/problemdetector"
"k8s.io/node-problem-detector/pkg/version"
"github.com/golang/glog"
"fmt"
)
// TODO: Move flags to options directory.
@@ -86,5 +86,7 @@ func main() {
k := kernelmonitor.NewKernelMonitorOrDie(*kernelMonitorConfigPath)
p := problemdetector.NewProblemDetector(k, *apiServerOverride, nodeName)
p.Run()
if err := p.Run(); err != nil {
glog.Fatalf("Problem detector failed with error: %v", err)
}
}
config/kernel-monitor.json
+2 -1
View File
@@ -1,5 +1,6 @@
{
"logPath": "/log/kern.log",
"plugin": "journald",
"logPath": "/var/log/journal",
"lookback": "10m",
"startPattern": "Initializing cgroup subsys cpuset",
"bufferSize": 10,
node-problem-detector.yaml
+10 -1
View File
@@ -12,6 +12,7 @@ spec:
- name: node-problem-detector
command:
- /node-problem-detector
- --logtostderr
- --kernel-monitor=/config/kernel-monitor.json
image: gcr.io/google_containers/node-problem-detector:v0.2
imagePullPolicy: Always
@@ -24,7 +25,12 @@ spec:
fieldPath: spec.nodeName
volumeMounts:
- name: log
mountPath: /log
mountPath: /var/log
readOnly: true
# Make sure node problem detector is in the same timezone
# with the host.
- name: localtime
mountPath: /etc/localtime
readOnly: true
- name: config
mountPath: /config
@@ -34,6 +40,9 @@ spec:
# Config `log` to your system log directory
hostPath:
path: /var/log/
- name: localtime
hostPath:
path: /etc/localtime
- name: config
configMap:
name: node-problem-detector-config
pkg/kernelmonitor/README.md
+18 -13
View File
@@ -9,12 +9,8 @@ The rule list is extensible.
## Limitations
* Kernel Monitor only supports file based kernel log now. It doesn't support log tools
like journald. There is an [open issue](https://github.com/kubernetes/node-problem-detector/issues/14)
to add journald support.
* Kernel Monitor has assumption on kernel log format, now it only works on Ubuntu and
Debian. However, it is easy to extend it to [support other log format](#support-other-log-format).
* Kernel Monitor only supports syslog (rsyslog) and journald now, but it is easy
to extend it with [new log watcher](#new-log-watcher)
## Add New NodeConditions
@@ -43,14 +39,23 @@ with new rule definition:
}
```
## Change Log Path
## Log Watchers
Kernel log in different OS distros may locate in different path. The `log`
Kernel monitor supports different log management tools with different log
watchers:
* [syslog](https://github.com/kubernetes/node-problem-detector/blob/master/pkg/kernelmonitor/logwatchers/syslog)
* [journald](https://github.com/kubernetes/node-problem-detector/blob/master/pkg/kernelmonitor/logwatchers/journald)
### Change Log Path
Kernel log on different OS distros may locate in different path. The `logPath`
field in `config/kernel-monitor.json` is the log path inside the container.
You can always configure it to match your OS distro.
You can always configure `logPath` and volume mount to match your OS distro.
* syslog: `logPath` is the kernel log path, usually `/var/log/kern.log`.
* journald: `logPath` is the journal log directory, usually `/var/log/journal`.
## Support Other Log Format
### New Log Watcher
Kernel monitor uses [`Translator`](https://github.com/kubernetes/node-problem-detector/blob/master/pkg/kernelmonitor/translator/translator.go)
plugin to translate kernel log the internal data structure. It is easy to
implement a new translator for a new log format.
Kernel monitor uses [Log
Watcher](https://github.com/kubernetes/node-problem-detector/blob/master/pkg/kernelmonitor/logwatchers/types/log_watcher.go) to support different log management tools.
It is easy to implement a new log watcher.
pkg/kernelmonitor/config.go
+49
View File
@@ -0,0 +1,49 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kernelmonitor
import (
watchertypes "k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/types"
)
// MonitorConfig is the configuration of kernel monitor.
type MonitorConfig struct {
// WatcherConfig is the configuration of kernel log watcher.
watchertypes.WatcherConfig
// BufferSize is the size (in lines) of the log buffer.
BufferSize int `json:"bufferSize"`
// Source is the source name of the kernel monitor
Source string `json:"source"`
// DefaultConditions are the default states of all the conditions kernel monitor should handle.
DefaultConditions []types.Condition `json:"conditions"`
// Rules are the rules kernel monitor will follow to parse the log file.
Rules []kerntypes.Rule `json:"rules"`
// StartPattern is the pattern of the start line
StartPattern string `json:"startPattern, omitempty"`
}
// applyDefaultConfiguration applies default configurations.
func applyDefaultConfiguration(cfg *MonitorConfig) {
if cfg.BufferSize == 0 {
cfg.BufferSize = 10
}
if cfg.WatcherConfig.Lookback == "" {
cfg.WatcherConfig.Lookback = "0"
}
}
pkg/kernelmonitor/kernel_log_watcher.go
-213
View File
@@ -1,213 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kernelmonitor
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"strings"
"time"
"k8s.io/node-problem-detector/pkg/kernelmonitor/translator"
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
utilclock "code.cloudfoundry.org/clock"
"github.com/coreos/go-systemd/sdjournal"
"github.com/golang/glog"
"github.com/google/cadvisor/utils/tail"
)
const (
defaultKernelLogPath = "/var/log/kern.log"
)
// WatcherConfig is the configuration of kernel log watcher.
type WatcherConfig struct {
// KernelLogPath is the path to the kernel log
KernelLogPath string `json:"logPath, omitempty"`
// StartPattern is the pattern of the start line
StartPattern string `json:"startPattern, omitempty"`
// Lookback is the time kernel watcher looks up
Lookback string `json:"lookback, omitempty"`
}
// KernelLogWatcher watches and translates the kernel log. Once there is new log line,
// it will translate and report the log.
type KernelLogWatcher interface {
// Watch starts the kernel log watcher and returns a watch channel.
Watch() (<-chan *types.KernelLog, error)
// Stop stops the kernel log watcher.
Stop()
}
type kernelLogWatcher struct {
// trans is the translator translates the log into internal format.
trans translator.Translator
cfg WatcherConfig
reader *bufio.Reader
logCh chan *types.KernelLog
tomb *util.Tomb
clock utilclock.Clock
}
// NewKernelLogWatcher creates a new kernel log watcher.
func NewKernelLogWatcher(cfg WatcherConfig) KernelLogWatcher {
return &kernelLogWatcher{
trans: translator.NewDefaultTranslator(),
cfg: cfg,
tomb: util.NewTomb(),
// A capacity 1000 buffer should be enough
logCh: make(chan *types.KernelLog, 1000),
clock: utilclock.NewClock(),
}
}
func (k *kernelLogWatcher) Watch() (<-chan *types.KernelLog, error) {
path := defaultKernelLogPath
if k.cfg.KernelLogPath != "" {
path = k.cfg.KernelLogPath
}
// NOTE(random-liu): This is a hack. KernelMonitor doesn't support some OS distros e.g. GCI. Ideally,
// KernelMonitor should only run on nodes with supported OS distro. However, NodeProblemDetector is
// running as DaemonSet, it has to be deployed on each node (There is no node affinity support for
// DaemonSet now #22205). If some nodes have unsupported OS distro e.g. the OS distro of master node
// in gke/gce is GCI, KernelMonitor will keep throwing out error, and NodeProblemDetector will be
// restarted again and again.
// To avoid this, we decide to add this temporarily hack. When KernelMonitor can't find the kernel
// log file, it will print a log and then return nil channel and no error. Since nil channel will
// always be blocked, the NodeProblemDetector will block forever.
if _, err := os.Stat(path); os.IsNotExist(err) {
glog.Infof("kernel log %q is not found, kernel monitor doesn't support the os distro", path)
return nil, nil
}
// TODO(random-liu): Rate limit tail file.
// Notice that, kernel log watcher doesn't look back to the rolled out logs.
reader, err := getLogReader(path)
if err != nil {
return nil, err
}
k.reader = bufio.NewReader(reader)
glog.Info("Start watching kernel log")
go k.watchLoop()
return k.logCh, nil
}
func (k *kernelLogWatcher) Stop() {
k.tomb.Stop()
}
// watchLoop is the main watch loop of kernel log watcher.
func (k *kernelLogWatcher) watchLoop() {
defer func() {
close(k.logCh)
k.tomb.Done()
}()
lookback, err := parseDuration(k.cfg.Lookback)
if err != nil {
glog.Fatalf("failed to parse duration %q: %v", k.cfg.Lookback, err)
}
var buffer bytes.Buffer
for {
select {
case <-k.tomb.Stopping():
glog.Infof("Stop watching kernel log")
return
default:
}
line, err := k.reader.ReadString('\n')
if err != nil && err != io.EOF {
glog.Errorf("exiting kernel log watch with error: %v", err)
return
}
if err == io.EOF {
buffer.WriteString(line)
time.Sleep(100 * time.Millisecond)
continue
}
if line == "" {
time.Sleep(100 * time.Millisecond)
continue
}
if err == nil {
buffer.WriteString(line)
// trim `\n`
line = strings.TrimRight(buffer.String(), "\n")
buffer.Reset()
log, err := k.trans.Translate(line)
if err != nil {
glog.Infof("Unable to parse line: %q, %v", line, err)
continue
}
// If the log is older than look back duration, discard it.
if k.clock.Since(log.Timestamp) > lookback {
continue
}
k.logCh <- log
}
}
}
// getLogReader gets a kernel log reader.
func getLogReader(path string) (io.Reader, error) {
if len(path) != 0 {
return tryLogFile(path)
}
return tryJournal()
}
func tryJournal() (io.Reader, error) {
r, err := sdjournal.NewJournalReader(sdjournal.JournalReaderConfig{
NumFromTail: uint64(0),
Matches: []sdjournal.Match{
{
Field: sdjournal.SD_JOURNAL_FIELD_TRANSPORT,
Value: "kernel",
},
},
})
if err != nil {
return nil, fmt.Errorf("error opening journal: %v", err)
}
if r == nil {
return nil, fmt.Errorf("got a nil reader")
}
glog.Info("Kernel log watcher use journal")
return r, nil
}
func tryLogFile(path string) (io.Reader, error) {
tail, err := tail.NewTail(path)
if err != nil {
return nil, err
}
glog.Infof("Kernel log watcher use log file: %s", path)
time.Sleep(1000 * time.Millisecond)
return tail, nil
}
func parseDuration(s string) (time.Duration, error) {
// If the duration is not configured, just return 0 by default
if s == "" {
return 0, nil
}
return time.ParseDuration(s)
}
pkg/kernelmonitor/kernel_monitor.go
+12 -19
View File
@@ -22,6 +22,8 @@ import (
"regexp"
"time"
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers"
watchertypes "k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
"k8s.io/node-problem-detector/pkg/types"
@@ -29,20 +31,6 @@ import (
"github.com/golang/glog"
)
// MonitorConfig is the configuration of kernel monitor.
type MonitorConfig struct {
// WatcherConfig is the configuration of kernel log watcher.
WatcherConfig
// BufferSize is the size (in lines) of the log buffer.
BufferSize int `json:"bufferSize"`
// Source is the source name of the kernel monitor
Source string `json:"source"`
// DefaultConditions are the default states of all the conditions kernel monitor should handle.
DefaultConditions []types.Condition `json:"conditions"`
// Rules are the rules kernel monitor will follow to parse the log file.
Rules []kerntypes.Rule `json:"rules"`
}
// KernelMonitor monitors the kernel log and reports node problem condition and event according to
// the rules.
type KernelMonitor interface {
@@ -53,7 +41,7 @@ type KernelMonitor interface {
}
type kernelMonitor struct {
watcher KernelLogWatcher
watcher watchertypes.LogWatcher
buffer LogBuffer
config MonitorConfig
conditions []types.Condition
@@ -69,18 +57,23 @@ func NewKernelMonitorOrDie(configPath string) KernelMonitor {
}
f, err := ioutil.ReadFile(configPath)
if err != nil {
panic(err)
glog.Fatalf("Failed to read configuration file %q: %v", configPath, err)
}
err = json.Unmarshal(f, &k.config)
if err != nil {
panic(err)
glog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err)
}
// Apply default configurations
applyDefaultConfiguration(&k.config)
err = validateRules(k.config.Rules)
if err != nil {
panic(err)
glog.Fatalf("Failed to validate matching rules %#v: %v", k.config.Rules, err)
}
glog.Infof("Finish parsing log file: %+v", k.config)
k.watcher = NewKernelLogWatcher(k.config.WatcherConfig)
k.watcher, err = logwatchers.GetLogWatcher(k.config.WatcherConfig)
if err != nil {
glog.Fatalf("Failed to create log watcher with watcher config %#v: %v", k.config.WatcherConfig, err)
}
k.buffer = NewLogBuffer(k.config.BufferSize)
// A 1000 size channel should be big enough.
k.output = make(chan *types.Status, 1000)
pkg/kernelmonitor/logwatchers/journald/log_watcher.go
+163
View File
@@ -0,0 +1,163 @@
// +build journald
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package journald
import (
"fmt"
"strings"
"time"
"github.com/coreos/go-systemd/sdjournal"
"github.com/golang/glog"
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
)
// Compiling go-systemd/sdjournald needs libsystemd-dev or libsystemd-journal-dev,
// which is not always available on all os distros and versions.
// So we add the build tag in this file, so that on unsupported os distro, user can
// disable this build tag.
// journaldWatcher is the log watcher for journald.
type journaldWatcher struct {
journal *sdjournal.Journal
cfg types.WatcherConfig
logCh chan *kerntypes.KernelLog
tomb *util.Tomb
}
// NewJournaldWatcher is the create function of journald watcher.
func NewJournaldWatcher(cfg types.WatcherConfig) types.LogWatcher {
return &journaldWatcher{
cfg: cfg,
tomb: util.NewTomb(),
// A capacity 1000 buffer should be enough
logCh: make(chan *kerntypes.KernelLog, 1000),
}
}
// Make sure NewJournaldWatcher is types.WatcherCreateFunc .
var _ types.WatcherCreateFunc = NewJournaldWatcher
// Watch starts the journal watcher.
func (j *journaldWatcher) Watch() (<-chan *kerntypes.KernelLog, error) {
journal, err := getJournal(j.cfg)
if err != nil {
return nil, err
}
j.journal = journal
glog.Info("Start watching journald")
go j.watchLoop()
return j.logCh, nil
}
// Stop stops the journald watcher.
func (j *journaldWatcher) Stop() {
j.tomb.Stop()
}
// waitLogTimeout is the timeout waiting for new log.
const waitLogTimeout = 5 * time.Second
// watchLoop is the main watch loop of journald watcher.
func (j *journaldWatcher) watchLoop() {
defer func() {
if err := j.journal.Close(); err != nil {
glog.Errorf("Failed to close journal client: %v", err)
}
j.tomb.Done()
}()
for {
select {
case <-j.tomb.Stopping():
glog.Infof("Stop watching journald")
return
default:
}
// Get next log entry.
n, err := j.journal.Next()
if err != nil {
glog.Errorf("Failed to get next journal entry: %v", err)
continue
}
// If next reaches the end, wait for waitLogTimeout.
if n == 0 {
j.journal.Wait(waitLogTimeout)
continue
}
entry, err := j.journal.GetEntry()
if err != nil {
glog.Errorf("failed to get journal entry: %v", err)
continue
}
j.logCh <- translate(entry)
}
}
// defaultJournalLogPath is the default path of journal log.
const defaultJournalLogPath = "/var/log/journal"
// getJournal returns a journal client.
func getJournal(cfg types.WatcherConfig) (*sdjournal.Journal, error) {
// Get journal log path.
path := defaultJournalLogPath
if cfg.LogPath != "" {
path = cfg.LogPath
}
// Get lookback duration.
since, err := time.ParseDuration(cfg.Lookback)
if err != nil {
return nil, fmt.Errorf("failed to parse lookback duration %q: %v", cfg.Lookback, err)
}
// Get journal client from the log path.
journal, err := sdjournal.NewJournalFromDir(path)
if err != nil {
return nil, fmt.Errorf("failed to create journal client from path %q: %v", path, err)
}
// Seek journal client based on the lookback duration.
start := time.Now().Add(-since)
err = journal.SeekRealtimeUsec(uint64(start.UnixNano() / 1000))
if err != nil {
return nil, fmt.Errorf("failed to lookback %q: %v", since, err)
}
// TODO(random-liu): Make this configurable to support parsing other logs.
kernelMatch := sdjournal.Match{
Field: sdjournal.SD_JOURNAL_FIELD_TRANSPORT,
Value: "kernel",
}
err = journal.AddMatch(kernelMatch.String())
if err != nil {
return nil, fmt.Errorf("failed to add log filter %#v: %v", kernelMatch, err)
}
return journal, nil
}
// translate translates journal entry into internal type.
func translate(entry *sdjournal.JournalEntry) *kerntypes.KernelLog {
timestamp := time.Unix(0, int64(time.Duration(entry.RealtimeTimestamp)*time.Microsecond))
message := strings.TrimSpace(entry.Fields["MESSAGE"])
return &kerntypes.KernelLog{
Timestamp: timestamp,
Message: message,
}
}
pkg/kernelmonitor/logwatchers/journald/log_watcher_test.go
+64
View File
@@ -0,0 +1,64 @@
// +build journald
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package journald
import (
"testing"
"time"
"github.com/coreos/go-systemd/sdjournal"
"github.com/stretchr/testify/assert"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
)
func TestTranslate(t *testing.T) {
testCases := []struct {
entry *sdjournal.JournalEntry
log *kerntypes.KernelLog
}{
{
// has log message
entry: &sdjournal.JournalEntry{
Fields: map[string]string{"MESSAGE": "log message"},
RealtimeTimestamp: 123456789,
},
log: &kerntypes.KernelLog{
Timestamp: time.Unix(0, 123456789*1000),
Message: "log message",
},
},
{
// no log message
entry: &sdjournal.JournalEntry{
Fields: map[string]string{},
RealtimeTimestamp: 987654321,
},
log: &kerntypes.KernelLog{
Timestamp: time.Unix(0, 987654321*1000),
Message: "",
},
},
}
for c, test := range testCases {
t.Logf("TestCase #%d: %#v", c+1, test)
assert.Equal(t, test.log, translate(test.entry))
}
}
pkg/kernelmonitor/logwatchers/log_watchers.go
+43
View File
@@ -0,0 +1,43 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logwatchers
import (
"fmt"
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
"github.com/golang/glog"
)
// createFuncs is a table of createFuncs for all supported log watchers.
var createFuncs = map[string]types.WatcherCreateFunc{}
// registerLogWatcher registers a createFunc for a log watcher.
func registerLogWatcher(name string, create types.WatcherCreateFunc) {
createFuncs[name] = create
}
// GetLogWatcher get a log watcher based on the passed in configuration.
func GetLogWatcher(config types.WatcherConfig) (types.LogWatcher, error) {
create, ok := createFuncs[config.Plugin]
if !ok {
return nil, fmt.Errorf("no create function found for plugin %q", config.Plugin)
}
glog.Infof("Use log watcher of plugin %q", config.Plugin)
return create(config), nil
}
pkg/kernelmonitor/logwatchers/register_journald.go
+30
View File
@@ -0,0 +1,30 @@
// +build journald
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logwatchers
import (
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/journald"
)
const journaldPluginName = "journald"
func init() {
// Register the syslog plugin.
registerLogWatcher(journaldPluginName, journald.NewJournaldWatcher)
}
pkg/kernelmonitor/logwatchers/register_syslog.go
+28
View File
@@ -0,0 +1,28 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logwatchers
import (
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/syslog"
)
const syslogPluginName = "syslog"
func init() {
// Register the syslog plugin.
registerLogWatcher(syslogPluginName, syslog.NewSyslogWatcher)
}
pkg/kernelmonitor/translator/translator.go → pkg/kernelmonitor/logwatchers/syslog/helpers.go
+42 -25
View File
@@ -13,50 +13,41 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package translator
package syslog
import (
"fmt"
"io"
"os"
"strings"
"time"
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"github.com/google/cadvisor/utils/tail"
)
// Translator translates a log line into types.KernelLog, so that kernel monitor
// could parse it whatever the original format is.
type Translator interface {
// Translate translates one log line into types.KernelLog.
Translate(string) (*types.KernelLog, error)
}
// defaultTranslator works well for ubuntu and debian, but may not work well with
// other os distros. However it is easy to add a new translator for new os distro.
type defaultTranslator struct{}
// NewDefaultTranslator creates a default translator.
func NewDefaultTranslator() Translator {
return &defaultTranslator{}
}
func (t *defaultTranslator) Translate(line string) (*types.KernelLog, error) {
timestamp, message, err := t.parseLine(line)
// translate translates the log line into internal type.
func translate(line string) (*kerntypes.KernelLog, error) {
timestamp, message, err := parseLine(line)
if err != nil {
return nil, err
}
return &types.KernelLog{
return &kerntypes.KernelLog{
Timestamp: timestamp,
Message: message,
}, nil
}
var (
timestampLen = 15
const (
// timestampLen is the length of timestamp in syslog logging format.
timestampLen = 15
// messagePrefix is the character before real message.
messagePrefix = "]"
)
func (t *defaultTranslator) parseLine(line string) (time.Time, string, error) {
// parseLine parses one log line into timestamp and message.
func parseLine(line string) (time.Time, string, error) {
// Trim the spaces to make sure timestamp could be found
line = strings.TrimSpace(line)
if len(line) < timestampLen {
@@ -83,3 +74,29 @@ func (t *defaultTranslator) parseLine(line string) (time.Time, string, error) {
return timestamp, message, nil
}
// defaultKernelLogPath the default path of syslog kernel log.
const defaultKernelLogPath = "/var/log/kern.log"
// getLogReader returns log reader for syslog log. Note that getLogReader doesn't look back
// to the rolled out logs.
func getLogReader(path string) (io.ReadCloser, error) {
if path == "" {
path = defaultKernelLogPath
}
// To handle log rotation, tail will not report error immediately if
// the file doesn't exist. So we check file existence frist.
// This could go wrong during mid-rotation. It should recover after
// several restart when the log file is created again. The chance
// is slim but we should still fix this in the future.
// TODO(random-liu): Handle log missing during rotation.
_, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("failed to stat the file %q: %v", path, err)
}
tail, err := tail.NewTail(path)
if err != nil {
return nil, fmt.Errorf("failed to tail the file %q: %v", path, err)
}
return tail, nil
}
pkg/kernelmonitor/translator/translator_test.go → pkg/kernelmonitor/logwatchers/syslog/helpers_test.go
+24 -21
View File
@@ -14,32 +14,38 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package translator
package syslog
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
)
func TestDefaultTranslator(t *testing.T) {
tr := NewDefaultTranslator()
func TestTranslate(t *testing.T) {
year := time.Now().Year()
testCases := []struct {
input string
err bool
timestamp time.Time
message string
input string
err bool
log *kerntypes.KernelLog
}{
{
input: "May 1 12:23:45 hostname kernel: [0.000000] component: log message",
timestamp: time.Date(year, time.May, 1, 12, 23, 45, 0, time.Local),
message: "component: log message",
input: "May 1 12:23:45 hostname kernel: [0.000000] component: log message",
log: &kerntypes.KernelLog{
Timestamp: time.Date(year, time.May, 1, 12, 23, 45, 0, time.Local),
Message: "component: log message",
},
},
{
// no log message
input: "May 21 12:23:45 hostname kernel: [9.999999]",
timestamp: time.Date(year, time.May, 21, 12, 23, 45, 0, time.Local),
message: "",
input: "May 21 12:23:45 hostname kernel: [9.999999]",
log: &kerntypes.KernelLog{
Timestamp: time.Date(year, time.May, 21, 12, 23, 45, 0, time.Local),
Message: "",
},
},
{
// the right square bracket is missing
@@ -49,15 +55,12 @@ func TestDefaultTranslator(t *testing.T) {
}
for c, test := range testCases {
log, err := tr.Translate(test.input)
if test.err {
if err == nil {
t.Errorf("case %d: expect error should occur, got %+v, %v", c+1, log, err)
}
t.Logf("TestCase #%d: %#v", c+1, test)
log, err := translate(test.input)
if (err != nil) != test.err {
t.Errorf("case %d: error assertion failed, got log: %+v, error: %v", c+1, log, err)
continue
}
if test.timestamp != log.Timestamp || test.message != log.Message {
t.Errorf("case %d: expect %v, %q; got %v, %q", c+1, test.timestamp, test.message, log.Timestamp, log.Message)
}
assert.Equal(t, test.log, log)
}
}
pkg/kernelmonitor/logwatchers/syslog/log_watcher.go
+122
View File
@@ -0,0 +1,122 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package syslog
import (
"bufio"
"bytes"
"io"
"time"
utilclock "code.cloudfoundry.org/clock"
"github.com/golang/glog"
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
)
type syslogWatcher struct {
cfg types.WatcherConfig
reader *bufio.Reader
closer io.Closer
logCh chan *kerntypes.KernelLog
tomb *util.Tomb
clock utilclock.Clock
}
// NewSyslogWatcher creates a new kernel log watcher.
func NewSyslogWatcher(cfg types.WatcherConfig) types.LogWatcher {
return &syslogWatcher{
cfg: cfg,
tomb: util.NewTomb(),
// A capacity 1000 buffer should be enough
logCh: make(chan *kerntypes.KernelLog, 1000),
clock: utilclock.NewClock(),
}
}
// Make sure NewSyslogWathcer is types.WatcherCreateFunc.
var _ types.WatcherCreateFunc = NewSyslogWatcher
// Watch starts the syslog watcher.
func (s *syslogWatcher) Watch() (<-chan *kerntypes.KernelLog, error) {
r, err := getLogReader(s.cfg.LogPath)
if err != nil {
return nil, err
}
s.reader = bufio.NewReader(r)
s.closer = r
glog.Info("Start watching syslog")
go s.watchLoop()
return s.logCh, nil
}
// Stop stops the syslog watcher.
func (s *syslogWatcher) Stop() {
s.tomb.Stop()
}
// watchPollInterval is the interval syslog log watcher will
// poll for pod change after reading to the end.
const watchPollInterval = 500 * time.Millisecond
// watchLoop is the main watch loop of syslog watcher.
func (s *syslogWatcher) watchLoop() {
defer func() {
s.closer.Close()
close(s.logCh)
s.tomb.Done()
}()
lookback, err := time.ParseDuration(s.cfg.Lookback)
if err != nil {
glog.Fatalf("Failed to parse duration %q: %v", s.cfg.Lookback, err)
}
glog.Info("Lookback:", lookback)
var buffer bytes.Buffer
for {
select {
case <-s.tomb.Stopping():
glog.Infof("Stop watching syslog")
return
default:
}
line, err := s.reader.ReadString('\n')
if err != nil && err != io.EOF {
glog.Errorf("Exiting syslog watch with error: %v", err)
return
}
buffer.WriteString(line)
if err == io.EOF {
time.Sleep(watchPollInterval)
continue
}
line = buffer.String()
buffer.Reset()
log, err := translate(line)
if err != nil {
glog.Warningf("Unable to parse line: %q, %v", line, err)
continue
}
// If the log is older than look back duration, discard it.
if s.clock.Since(log.Timestamp) > lookback {
continue
}
s.logCh <- log
}
}
pkg/kernelmonitor/kernel_log_watcher_test.go → pkg/kernelmonitor/logwatchers/syslog/log_watcher_test.go
+27 -23
View File
@@ -14,18 +14,19 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package kernelmonitor
package syslog
import (
"io/ioutil"
//"os"
"reflect"
"os"
"testing"
"time"
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"k8s.io/node-problem-detector/pkg/kernelmonitor/logwatchers/types"
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
"code.cloudfoundry.org/clock/fakeclock"
"github.com/stretchr/testify/assert"
)
func TestWatch(t *testing.T) {
@@ -34,7 +35,7 @@ func TestWatch(t *testing.T) {
fakeClock := fakeclock.NewFakeClock(now)
testCases := []struct {
log string
logs []types.KernelLog
logs []kerntypes.KernelLog
lookback string
}{
{
@@ -43,7 +44,8 @@ func TestWatch(t *testing.T) {
Jan 2 03:04:06 kernel: [1.000000] 2
Jan 2 03:04:07 kernel: [2.000000] 3
`,
logs: []types.KernelLog{
lookback: "0",
logs: []kerntypes.KernelLog{
{
Timestamp: now,
Message: "1",
@@ -64,7 +66,8 @@ func TestWatch(t *testing.T) {
Jan 2 03:04:05 kernel: [1.000000] 2
Jan 2 03:04:06 kernel: [2.000000] 3
`,
logs: []types.KernelLog{
lookback: "0",
logs: []kerntypes.KernelLog{
{
Timestamp: now,
Message: "2",
@@ -82,7 +85,7 @@ func TestWatch(t *testing.T) {
Jan 2 03:04:05 kernel: [2.000000] 3
`,
lookback: "1s",
logs: []types.KernelLog{
logs: []kerntypes.KernelLog{
{
Timestamp: now.Add(-time.Second),
Message: "2",
@@ -95,31 +98,32 @@ func TestWatch(t *testing.T) {
},
}
for c, test := range testCases {
t.Logf("TestCase #%d: %#v", c+1, test)
f, err := ioutil.TempFile("", "kernel_log_watcher_test")
if err != nil {
t.Fatal(err)
}
assert.NoError(t, err)
defer func() {
f.Close()
//os.Remove(f.Name())
os.Remove(f.Name())
}()
_, err = f.Write([]byte(test.log))
if err != nil {
t.Fatal(err)
}
assert.NoError(t, err)
w := NewKernelLogWatcher(WatcherConfig{KernelLogPath: f.Name(), Lookback: test.lookback})
w := NewSyslogWatcher(types.WatcherConfig{
Plugin: "syslog",
LogPath: f.Name(),
Lookback: test.lookback,
})
// Set the fake clock.
w.(*kernelLogWatcher).clock = fakeClock
w.(*syslogWatcher).clock = fakeClock
logCh, err := w.Watch()
if err != nil {
t.Fatal(err)
}
assert.NoError(t, err)
defer w.Stop()
for _, expected := range test.logs {
got := <-logCh
if !reflect.DeepEqual(&expected, got) {
t.Errorf("case %d: expect %+v, got %+v", c+1, expected, *got)
select {
case got := <-logCh:
assert.Equal(t, &expected, got)
case <-time.After(30 * time.Second):
t.Errorf("timeout waiting for log")
}
}
// The log channel should have already been drained
pkg/kernelmonitor/logwatchers/types/log_watcher.go
+43
View File
@@ -0,0 +1,43 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package types
import (
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
)
// LogWatcher is the interface of a log watcher.
type LogWatcher interface {
// Watch starts watching logs and returns logs via a channel.
Watch() (<-chan *types.KernelLog, error)
// Stop stops the log watcher. Resources open should be closed properly.
Stop()
}
// WatcherConfig is the configuration of the log watcher.
type WatcherConfig struct {
// Plugin is the name of plugin which is currently used.
// Currently supported: syslog, journald.
Plugin string `json:"plugin, omitempty"`
// LogPath is the path to the log
LogPath string `json:"logPath, omitempty"`
// Lookback is the time kernel watcher looks up
Lookback string `json:"lookback, omitempty"`
}
// WatcherCreateFunc is the create function of a log watcher.
type WatcherCreateFunc func(WatcherConfig) LogWatcher
vendor/github.com/google/cadvisor/info/v1/container.go
Generated Vendored
+39 -15
View File
@@ -266,7 +266,7 @@ type LoadStats struct {
// CPU usage time statistics.
type CpuUsage struct {
// Total CPU usage.
// Units: nanoseconds
// Unit: nanoseconds.
Total uint64 `json:"total"`
// Per CPU/core usage of the container.
@@ -274,17 +274,31 @@ type CpuUsage struct {
PerCpu []uint64 `json:"per_cpu_usage,omitempty"`
// Time spent in user space.
// Unit: nanoseconds
// Unit: nanoseconds.
User uint64 `json:"user"`
// Time spent in kernel space.
// Unit: nanoseconds
// Unit: nanoseconds.
System uint64 `json:"system"`
}
// Cpu Completely Fair Scheduler statistics.
type CpuCFS struct {
// Total number of elapsed enforcement intervals.
Periods uint64 `json:"periods"`
// Total number of times tasks in the cgroup have been throttled.
ThrottledPeriods uint64 `json:"throttled_periods"`
// Total time duration for which tasks in the cgroup have been throttled.
// Unit: nanoseconds.
ThrottledTime uint64 `json:"throttled_time"`
}
// All CPU usage metrics are cumulative from the creation of the container
type CpuStats struct {
Usage CpuUsage `json:"usage"`
CFS CpuCFS `json:"cfs"`
// Smoothed average of number of runnable threads x 1000.
// We multiply by thousand to avoid using floats, but preserving precision.
// Load is smoothed over the last 10 seconds. Instantaneous value can be read
@@ -324,6 +338,10 @@ type MemoryStats struct {
// Units: Bytes.
RSS uint64 `json:"rss"`
// The amount of swap currently used by the processes in this cgroup
// Units: Bytes.
Swap uint64 `json:"swap"`
// The amount of working set memory, this includes recently accessed memory,
// dirty memory, and kernel memory. Working set is <= "usage".
// Units: Bytes.
@@ -371,27 +389,27 @@ type NetworkStats struct {
}
type TcpStat struct {
//Count of TCP connections in state "Established"
// Count of TCP connections in state "Established"
Established uint64
//Count of TCP connections in state "Syn_Sent"
// Count of TCP connections in state "Syn_Sent"
SynSent uint64
//Count of TCP connections in state "Syn_Recv"
// Count of TCP connections in state "Syn_Recv"
SynRecv uint64
//Count of TCP connections in state "Fin_Wait1"
// Count of TCP connections in state "Fin_Wait1"
FinWait1 uint64
//Count of TCP connections in state "Fin_Wait2"
// Count of TCP connections in state "Fin_Wait2"
FinWait2 uint64
//Count of TCP connections in state "Time_Wait
// Count of TCP connections in state "Time_Wait
TimeWait uint64
//Count of TCP connections in state "Close"
// Count of TCP connections in state "Close"
Close uint64
//Count of TCP connections in state "Close_Wait"
// Count of TCP connections in state "Close_Wait"
CloseWait uint64
//Count of TCP connections in state "Listen_Ack"
// Count of TCP connections in state "Listen_Ack"
LastAck uint64
//Count of TCP connections in state "Listen"
// Count of TCP connections in state "Listen"
Listen uint64
//Count of TCP connections in state "Closing"
// Count of TCP connections in state "Closing"
Closing uint64
}
@@ -415,6 +433,12 @@ type FsStats struct {
// Number of bytes available for non-root user.
Available uint64 `json:"available"`
// HasInodes when true, indicates that Inodes info will be available.
HasInodes bool `json:"has_inodes"`
// Number of Inodes
Inodes uint64 `json:"inodes"`
// Number of available Inodes
InodesFree uint64 `json:"inodes_free"`
@@ -487,7 +511,7 @@ type ContainerStats struct {
// Task load stats
TaskStats LoadStats `json:"task_stats,omitempty"`
//Custom metrics from all collectors
// Custom metrics from all collectors
CustomMetrics map[string][]MetricVal `json:"custom_metrics,omitempty"`
}
vendor/github.com/google/cadvisor/info/v1/machine.go
Generated Vendored
+3
View File
@@ -26,6 +26,9 @@ type FsInfo struct {
// Total number of inodes available on the filesystem.
Inodes uint64 `json:"inodes"`
// HasInodes when true, indicates that Inodes info will be available.
HasInodes bool `json:"has_inodes"`
}
type Node struct {
vendor/github.com/google/cadvisor/info/v1/metric.go
Generated Vendored
+3 -3
View File
@@ -26,10 +26,10 @@ const (
MetricGauge MetricType = "gauge"
// A counter-like value that is only expected to increase.
MetricCumulative = "cumulative"
MetricCumulative MetricType = "cumulative"
// Rate over a time period.
MetricDelta = "delta"
MetricDelta MetricType = "delta"
)
// DataType for metric being exported.
@@ -37,7 +37,7 @@ type DataType string
const (
IntType DataType = "int"
FloatType = "float"
FloatType DataType = "float"
)
// Spec for custom metric.
vendor/github.com/google/cadvisor/utils/tail/tail.go
Generated Vendored
+18 -6
View File
@@ -45,6 +45,16 @@ const (
// NewTail starts opens the given file and watches it for deletion/rotation
func NewTail(filename string) (*Tail, error) {
t, err := newTail(filename)
if err != nil {
return nil, err
}
go t.watchLoop()
return t, nil
}
// newTail creates a Tail object.
func newTail(filename string) (*Tail, error) {
t := &Tail{
filename: filename,
}
@@ -54,7 +64,9 @@ func NewTail(filename string) (*Tail, error) {
if err != nil {
return nil, fmt.Errorf("inotify init failed on %s: %v", t.filename, err)
}
go t.watchLoop()
// Initialize readerErr as io.EOF, so that the reader can work properly
// during initialization.
t.readerErr = io.EOF
return t, nil
}
@@ -62,23 +74,23 @@ func NewTail(filename string) (*Tail, error) {
func (t *Tail) Read(p []byte) (int, error) {
t.readerLock.RLock()
defer t.readerLock.RUnlock()
if t.reader == nil || t.readerErr != nil {
if t.readerErr != nil {
return 0, t.readerErr
}
return t.reader.Read(p)
}
var _ io.Reader = &Tail{}
var _ io.ReadCloser = &Tail{}
// Close stops watching and closes the file
func (t *Tail) Close() {
func (t *Tail) Close() error {
close(t.stop)
return nil
}
func (t *Tail) attemptOpen() error {
t.readerLock.Lock()
defer t.readerLock.Unlock()
t.reader = nil
t.readerErr = nil
attempt := 0
for interval := defaultRetryInterval; ; interval *= 2 {
@@ -88,7 +100,7 @@ func (t *Tail) attemptOpen() error {
t.file, err = os.Open(t.filename)
if err == nil {
// TODO: not interested in old events?
//t.file.Seek(0, os.SEEK_END)
// t.file.Seek(0, os.SEEK_END)
t.reader = bufio.NewReader(t.file)
return nil
}