untangle plugin runner a bit

add some docs and make it clearer what is actually going on
(parallel rule execution on start and then on timer)

pkg/custompluginmonitor/plugin/plugin.go

@@ -61,58 +61,60 @@ func (p *Plugin) Run() {
 	runTicker := time.NewTicker(*p.config.PluginGlobalConfig.InvokeInterval)
 	defer runTicker.Stop()
 
-	runner := func() {
-		glog.Info("Start to run custom plugins")
-
-		for _, rule := range p.config.Rules {
-			p.syncChan <- struct{}{}
-			p.Add(1)
-
-			go func(rule *cpmtypes.CustomRule) {
-				defer p.Done()
-				defer func() {
-					<-p.syncChan
-				}()
-
-				start := time.Now()
-				exitStatus, message := p.run(*rule)
-				end := time.Now()
-
-				glog.V(3).Infof("Rule: %+v. Start time: %v. End time: %v. Duration: %v", rule, start, end, end.Sub(start))
-
-				result := cpmtypes.Result{
-					Rule:       rule,
-					ExitStatus: exitStatus,
-					Message:    message,
-				}
-
-				p.resultChan <- result
-
-				glog.Infof("Add check result %+v for rule %+v", result, rule)
-			}(rule)
-		}
-
-		p.Wait()
-		glog.Info("Finish running custom plugins")
-	}
-
+	// on boot run once
 	select {
 	case <-p.tomb.Stopping():
 		return
 	default:
-		runner()
+		p.runRules()
 	}
 
+	// run every InvokeInterval
 	for {
 		select {
 		case <-runTicker.C:
-			runner()
+			p.runRules()
 		case <-p.tomb.Stopping():
 			return
 		}
 	}
 }
 
+// run each rule in parallel and wait for them to complete
+func (p *Plugin) runRules() {
+	glog.Info("Start to run custom plugins")
+
+	for _, rule := range p.config.Rules {
+		p.syncChan <- struct{}{}
+		p.Add(1)
+		go func(rule *cpmtypes.CustomRule) {
+			defer p.Done()
+			defer func() {
+				<-p.syncChan
+			}()
+
+			start := time.Now()
+			exitStatus, message := p.run(*rule)
+			end := time.Now()
+
+			glog.V(3).Infof("Rule: %+v. Start time: %v. End time: %v. Duration: %v", rule, start, end, end.Sub(start))
+
+			result := cpmtypes.Result{
+				Rule:       rule,
+				ExitStatus: exitStatus,
+				Message:    message,
+			}
+
+			p.resultChan <- result
+
+			glog.Infof("Add check result %+v for rule %+v", result, rule)
+		}(rule)
+	}
+
+	p.Wait()
+	glog.Info("Finish running custom plugins")
+}
+
 func (p *Plugin) run(rule cpmtypes.CustomRule) (exitStatus cpmtypes.Status, output string) {
 	var ctx context.Context
 	var cancel context.CancelFunc