From 5822164ee53ee0cfe353a31ea052d4a3161a0add Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Thu, 2 Mar 2017 14:30:09 -0800 Subject: [PATCH] Add test kernel log generator. --- demo/demo | 81 ------------------- test/kernel_log_generator/Dockerfile | 23 ++++++ test/kernel_log_generator/Makefile | 31 +++++++ test/kernel_log_generator/generator.sh | 66 +++++++++++++++ .../problems}/au_opts_verify | 0 .../problems}/aufs_umount_hung | 0 .../problems}/divide_zero | 0 .../problems}/docker_hung | 0 .../problems}/null_pointer | 0 .../kernel_log_generator/problems}/oom_kill | 0 .../problems}/unregister_netdevice | 0 11 files changed, 120 insertions(+), 81 deletions(-) delete mode 100755 demo/demo create mode 100644 test/kernel_log_generator/Dockerfile create mode 100644 test/kernel_log_generator/Makefile create mode 100755 test/kernel_log_generator/generator.sh rename {demo => test/kernel_log_generator/problems}/au_opts_verify (100%) rename {demo => test/kernel_log_generator/problems}/aufs_umount_hung (100%) rename {demo => test/kernel_log_generator/problems}/divide_zero (100%) rename {demo => test/kernel_log_generator/problems}/docker_hung (100%) rename {demo => test/kernel_log_generator/problems}/null_pointer (100%) rename {demo => test/kernel_log_generator/problems}/oom_kill (100%) rename {demo => test/kernel_log_generator/problems}/unregister_netdevice (100%) diff --git a/demo/demo b/demo/demo deleted file mode 100755 index 21d773e2..00000000 --- a/demo/demo +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -ZONE=us-central1-b -KERNLOG=/var/log/kern.log - -usage () { - echo "Usage : `basename $0` COMMAND [arg...] - Commands: - create: create node problem detector daemon. - nodes: describe all nodes or specific node. - pods: describe all pods or specific pod. - inject: inject error kernel log into specific node. - reboot: generate a fake reboot log on specific node. - delete: delete node problem detector daemon." - exit -} - -runCmd() { - echo $1 - eval $1 -} - -rebootAll() { - TEMP=`mktemp` - kubectl get nodes | awk 'NR>1{print $1}' > $TEMP - while read -r node; do - reboot $node - done < $TEMP - rm $TEMP -} - -reboot() { - LATEST=`gcloud compute ssh -n root@$1 --zone=$ZONE "tail -1 $KERNLOG"` - PREFIX=`echo $LATEST | cut -d "]" -f 1 -`"]" - runCmd "gcloud compute ssh -n root@$1 --zone=$ZONE \"echo '$PREFIX Initializing cgroup subsys cpuset' >> $KERNLOG\"" -} - -case $1 in - create ) - runCmd "kubectl create configmap node-problem-detector-config --from-file=../config/" - runCmd "kubectl create -f ../node-problem-detector.yaml --validate=false" - ;; - nodes ) - runCmd "kubectl describe nodes $2" - ;; - pods ) - runCmd "kubectl describe pods $2" - ;; - inject ) - if [ -z $3 ]; then - usage - exit 1 - fi - NODE=$3 - LATEST=`gcloud compute ssh root@$NODE --zone=$ZONE "tail -1 $KERNLOG"` - PREFIX=`echo $LATEST | cut -d "]" -f 1 -`"]" - PREFIX=`printf "%q" "$PREFIX"` - COMMAND= - while read error - do - ERROR=`printf "%q" "$error"` - COMMAND=$COMMAND"echo $PREFIX $ERROR >> $KERNLOG; " - done < $2 - runCmd "gcloud compute ssh root@$NODE --zone=$ZONE '$COMMAND'" - ;; - reboot ) - if [ -z $2 ]; then - usage - exit 1 - fi - reboot $2 - ;; - delete ) - runCmd "kubectl delete -f ../node-problem-detector.yaml" - runCmd "kubectl delete configmap node-problem-detector-config" - ;; - * ) - usage - exit 1 - ;; -esac diff --git a/test/kernel_log_generator/Dockerfile b/test/kernel_log_generator/Dockerfile new file mode 100644 index 00000000..e248c82b --- /dev/null +++ b/test/kernel_log_generator/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM bashell/alpine-bash +MAINTAINER Random Liu + +# Use oom kill problem as default +ENV PROBLEM /problems/oom_kill + +ADD problems /problems +ADD generator.sh /generator.sh +ENTRYPOINT ["bash", "-c", "/generator.sh"] diff --git a/test/kernel_log_generator/Makefile b/test/kernel_log_generator/Makefile new file mode 100644 index 00000000..5482e521 --- /dev/null +++ b/test/kernel_log_generator/Makefile @@ -0,0 +1,31 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the kernel problem generator. + +.PHONY: all build push + +PROJ ?= gcr.io/google_containers +TAG := 0.1 + +all: push + +build: + docker build -t $(PROJ)/kernel_log_generator:$(TAG) . + +push: + gcloud docker -- push $(PROJ)/kernel_log_generator:$(TAG) + +clean: + docker rmi $(PROJ)/kernel_log_generator:$(TAG) diff --git a/test/kernel_log_generator/generator.sh b/test/kernel_log_generator/generator.sh new file mode 100755 index 00000000..dd863105 --- /dev/null +++ b/test/kernel_log_generator/generator.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script generate kernel log in given rate. +# Note that this only works for journald. +# Note that this script can't be used to test multi-line patterns, because +# the test kernel log lines may be splitted by real kernel log. + +# PROBLEM is the path to the problem log. +PROBLEM=${PROBLEM:-""} +# LINES_PER_SECOND is the log generating rate. +LINES_PER_SECOND=${LINES_PER_SECOND:-1} +# LOG_PATH is the path to generate kernel logs. +LOG_PATH=${LOG_PATH:-"/dev/kmsg"} +# RUN_ONCE specifies whether to keep generating problem logs or only +# generate once. +RUN_ONCE=${RUN_ONCE:-"false"} + +if [[ -z ${PROBLEM} ]]; then + echo "$(basename $0): PROBLEM is empty" + exit 1 +fi + +# now returns the current unix time in micro seconds +now() { + echo $(($(date +'%s%N') / 1000)) +} + +prefix="kernel: " +logs=$(<${PROBLEM}) +lines=$(printf "%s\n" "${logs}" | wc -l) +current_lines=0 +start=$(now) +while true; do + while read log; do + echo "${prefix}${log}" >> ${LOG_PATH} + ((current_lines++)) + if [[ ${current_lines} == ${LINES_PER_SECOND} ]]; then + current_lines=0 + current=$(now) + duration=$((1000000 - current + start)) + if [[ ${duration} -gt 0 ]]; then + echo "generated ${LINES_PER_SECOND} lines of log, sleep for ${duration}" + usleep ${duration} + fi + start=$(now) + fi + done <<< "$logs" + if [[ "${RUN_ONCE}" == "true" ]]; then + echo "stop the generator" + exit 0 + fi +done diff --git a/demo/au_opts_verify b/test/kernel_log_generator/problems/au_opts_verify similarity index 100% rename from demo/au_opts_verify rename to test/kernel_log_generator/problems/au_opts_verify diff --git a/demo/aufs_umount_hung b/test/kernel_log_generator/problems/aufs_umount_hung similarity index 100% rename from demo/aufs_umount_hung rename to test/kernel_log_generator/problems/aufs_umount_hung diff --git a/demo/divide_zero b/test/kernel_log_generator/problems/divide_zero similarity index 100% rename from demo/divide_zero rename to test/kernel_log_generator/problems/divide_zero diff --git a/demo/docker_hung b/test/kernel_log_generator/problems/docker_hung similarity index 100% rename from demo/docker_hung rename to test/kernel_log_generator/problems/docker_hung diff --git a/demo/null_pointer b/test/kernel_log_generator/problems/null_pointer similarity index 100% rename from demo/null_pointer rename to test/kernel_log_generator/problems/null_pointer diff --git a/demo/oom_kill b/test/kernel_log_generator/problems/oom_kill similarity index 100% rename from demo/oom_kill rename to test/kernel_log_generator/problems/oom_kill diff --git a/demo/unregister_netdevice b/test/kernel_log_generator/problems/unregister_netdevice similarity index 100% rename from demo/unregister_netdevice rename to test/kernel_log_generator/problems/unregister_netdevice