Support multiple configs per input metric.

When generating metric values using expression evaluation, there are new
use-cases for more than one configuration for each sensor value.
Expressions are more flexible than aggregation rules in Prometheus, so
it makes sense to derive multiple metrics from the same sensor input.

pkg/metrics/extractor.go

@@ -32,19 +32,16 @@ func NewJSONObjectExtractor(p Parser) Extractor {
 				continue
 			}
 
-			// Find a valid metrics config
+			// Find all valid metric configs
-			config, found := p.findMetricConfig(path, deviceID)
+			for _, config := range p.findMetricConfigs(path, deviceID) {
-			if !found {
+				id := metricID(topic, path, deviceID, config.PrometheusName)
-				continue
+				m, err := p.parseMetric(config, id, rawValue)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err)
+				}
+				m.Topic = topic
+				mc = append(mc, m)
 			}
-
-			id := metricID(topic, path, deviceID, config.PrometheusName)
-			m, err := p.parseMetric(config, id, rawValue)
-			if err != nil {
-				return nil, fmt.Errorf("failed to parse valid metric value: %w", err)
-			}
-			m.Topic = topic
-			mc = append(mc, m)
 		}
 		return mc, nil
 	}
@@ -52,35 +49,34 @@ func NewJSONObjectExtractor(p Parser) Extractor {
 
 func NewMetricPerTopicExtractor(p Parser, metricNameRegex *config.Regexp) Extractor {
 	return func(topic string, payload []byte, deviceID string) (MetricCollection, error) {
+		var mc MetricCollection
 		metricName := metricNameRegex.GroupValue(topic, config.MetricNameRegexGroup)
 		if metricName == "" {
 			return nil, fmt.Errorf("failed to find valid metric in topic path")
 		}
 
-		// Find a valid metrics config
+		// Find all valid metric configs
-		config, found := p.findMetricConfig(metricName, deviceID)
+		for _, config := range p.findMetricConfigs(metricName, deviceID) {
-		if !found {
+			var rawValue interface{}
-			return nil, nil
+			if config.PayloadField != "" {
-		}
+				parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload))
-
+				rawValue = parsed.Find(config.PayloadField)
-		var rawValue interface{}
+				parsed.Reset()
-		if config.PayloadField != "" {
+				if rawValue == nil {
-			parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload))
+					return nil, fmt.Errorf("failed to extract field %q from payload %q for metric %q", config.PayloadField, payload, metricName)
-			rawValue = parsed.Find(config.PayloadField)
+				}
-			parsed.Reset()
+			} else {
-			if rawValue == nil {
+				rawValue = string(payload)
-				return nil, fmt.Errorf("failed to extract field %s from payload %s", config.PayloadField, payload)
 			}
-		} else {
-			rawValue = string(payload)
-		}
 
-		id := metricID(topic, metricName, deviceID, config.PrometheusName)
+			id := metricID(topic, metricName, deviceID, config.PrometheusName)
-		m, err := p.parseMetric(config, id, rawValue)
+			m, err := p.parseMetric(config, id, rawValue)
-		if err != nil {
+			if err != nil {
-			return nil, fmt.Errorf("failed to parse metric: %w", err)
+				return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err)
+			}
+			m.Topic = topic
+			mc = append(mc, m)
 		}
-		m.Topic = topic
+		return mc, nil
-		return MetricCollection{m}, nil
 	}
 }

pkg/metrics/extractor_test.go

@@ -11,7 +11,7 @@ import (
 func TestNewJSONObjectExtractor_parseMetric(t *testing.T) {
 	now = testNow
 	type fields struct {
-		metricConfigs map[string][]config.MetricConfig
+		metricConfigs map[string][]*config.MetricConfig
 	}
 	type args struct {
 		metricPath string
@@ -31,8 +31,8 @@ func TestNewJSONObjectExtractor_parseMetric(t *testing.T) {
 			name:      "string value",
 			separator: "->",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"SDS0X1->PM2->5": []config.MetricConfig{
+					"SDS0X1->PM2->5": {
 						{
 							PrometheusName: "temperature",
 							MQTTName:       "SDS0X1.PM2.5",
@@ -57,8 +57,8 @@ func TestNewJSONObjectExtractor_parseMetric(t *testing.T) {
 			name:      "string value with dots in path",
 			separator: "->",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"SDS0X1->PM2.5": []config.MetricConfig{
+					"SDS0X1->PM2.5": {
 						{
 							PrometheusName: "temperature",
 							MQTTName:       "SDS0X1->PM2.5",
@@ -83,8 +83,8 @@ func TestNewJSONObjectExtractor_parseMetric(t *testing.T) {
 			name:      "metric matching SensorNameFilter",
 			separator: ".",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName:   "temperature",
 							MQTTName:         "temperature",
@@ -110,8 +110,8 @@ func TestNewJSONObjectExtractor_parseMetric(t *testing.T) {
 			name:      "metric not matching SensorNameFilter",
 			separator: ".",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName:   "temperature",
 							MQTTName:         "temperature",

pkg/metrics/parser.go

@@ -44,7 +44,7 @@ type Parser struct {
 	separator string
 	// Maps the mqtt metric name to a list of configs
 	// The first that matches SensorNameFilter will be used
-	metricConfigs map[string][]config.MetricConfig
+	metricConfigs map[string][]*config.MetricConfig
 	// Directory holding state files
 	stateDir string
 	// Per-metric state
@@ -130,10 +130,10 @@ func defaultExprEnv() map[string]interface{} {
 }
 
 func NewParser(metrics []config.MetricConfig, separator, stateDir string) Parser {
-	cfgs := make(map[string][]config.MetricConfig)
+	cfgs := make(map[string][]*config.MetricConfig)
 	for i := range metrics {
 		key := metrics[i].MQTTName
-		cfgs[key] = append(cfgs[key], metrics[i])
+		cfgs[key] = append(cfgs[key], &metrics[i])
 	}
 	return Parser{
 		separator:     separator,
@@ -144,24 +144,24 @@ func NewParser(metrics []config.MetricConfig, separator, stateDir string) Parser
 }
 
 // Config returns the underlying metrics config
-func (p *Parser) config() map[string][]config.MetricConfig {
+func (p *Parser) config() map[string][]*config.MetricConfig {
 	return p.metricConfigs
 }
 
-// validMetric returns config matching the metric and deviceID
+// validMetric returns all configs matching the metric and deviceID.
-// Second return value indicates if config was found.
+func (p *Parser) findMetricConfigs(metric string, deviceID string) []*config.MetricConfig {
-func (p *Parser) findMetricConfig(metric string, deviceID string) (config.MetricConfig, bool) {
+	configs := []*config.MetricConfig{}
 	for _, c := range p.metricConfigs[metric] {
 		if c.SensorNameFilter.Match(deviceID) {
-			return c, true
+			configs = append(configs, c)
 		}
 	}
-	return config.MetricConfig{}, false
+	return configs
 }
 
 // parseMetric parses the given value according to the given deviceID and metricPath. The config allows to
 // parse a metric value according to the device ID.
-func (p *Parser) parseMetric(cfg config.MetricConfig, metricID string, value interface{}) (Metric, error) {
+func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value interface{}) (Metric, error) {
 	var metricValue float64
 	var err error
 

pkg/metrics/parser_test.go

@@ -32,7 +32,7 @@ func TestParser_parseMetric(t *testing.T) {
 
 	now = testNow
 	type fields struct {
-		metricConfigs map[string][]config.MetricConfig
+		metricConfigs map[string][]*config.MetricConfig
 	}
 	type args struct {
 		metricPath string
@@ -50,8 +50,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "value without timestamp",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName: "temperature",
 							ValueType:      "gauge",
@@ -76,8 +76,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "string value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName: "temperature",
 							ValueType:      "gauge",
@@ -101,8 +101,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "scaled string value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName: "temperature",
 							ValueType:      "gauge",
@@ -127,8 +127,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "string value failure",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName: "temperature",
 							ValueType:      "gauge",
@@ -146,8 +146,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "float value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"temperature": []config.MetricConfig{
+					"temperature": {
 						{
 							PrometheusName: "temperature",
 							ValueType:      "gauge",
@@ -171,8 +171,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "scaled float value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"humidity": []config.MetricConfig{
+					"humidity": {
 						{
 							PrometheusName: "humidity",
 							ValueType:      "gauge",
@@ -197,8 +197,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "negative scaled float value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"humidity": []config.MetricConfig{
+					"humidity": {
 						{
 							PrometheusName: "humidity",
 							ValueType:      "gauge",
@@ -223,8 +223,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "bool value true",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -248,8 +248,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "scaled bool value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -274,8 +274,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "bool value false",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -299,8 +299,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "string mapping value success",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -330,8 +330,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "string mapping value failure default to error value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -362,8 +362,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "string mapping value failure no error value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -387,8 +387,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "metric not configured",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -413,8 +413,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "unexpected type",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"enabled": []config.MetricConfig{
+					"enabled": {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
@@ -439,8 +439,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "monotonic gauge, step 1: initial value",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"aenergy.total": []config.MetricConfig{
+					"aenergy.total": {
 						{
 							PrometheusName:  "total_energy",
 							ValueType:       "gauge",
@@ -464,8 +464,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "monotonic gauge, step 2: monotonic increase does not add offset",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"aenergy.total": []config.MetricConfig{
+					"aenergy.total": {
 						{
 							PrometheusName:  "total_energy",
 							ValueType:       "gauge",
@@ -489,8 +489,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "monotonic gauge, step 3: raw metric is reset, last value becomes the new offset",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"aenergy.total": []config.MetricConfig{
+					"aenergy.total": {
 						{
 							PrometheusName:  "total_energy",
 							ValueType:       "gauge",
@@ -514,8 +514,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "monotonic gauge, step 4: monotonic increase with offset",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"aenergy.total": []config.MetricConfig{
+					"aenergy.total": {
 						{
 							PrometheusName:  "total_energy",
 							ValueType:       "gauge",
@@ -539,8 +539,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "integrate positive values using expressions, step 1",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"apower": []config.MetricConfig{
+					"apower": {
 						{
 							PrometheusName: "total_energy",
 							ValueType:      "gauge",
@@ -564,8 +564,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "integrate positive values using expressions, step 2",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"apower": []config.MetricConfig{
+					"apower": {
 						{
 							PrometheusName: "total_energy",
 							ValueType:      "gauge",
@@ -590,8 +590,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "integrate positive values using expressions, step 3",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"apower": []config.MetricConfig{
+					"apower": {
 						{
 							PrometheusName: "total_energy",
 							ValueType:      "gauge",
@@ -616,8 +616,8 @@ func TestParser_parseMetric(t *testing.T) {
 		{
 			name: "integrate positive values using expressions, step 4",
 			fields: fields{
-				map[string][]config.MetricConfig{
+				map[string][]*config.MetricConfig{
-					"apower": []config.MetricConfig{
+					"apower": {
 						{
 							PrometheusName: "total_energy",
 							ValueType:      "gauge",
@@ -649,13 +649,14 @@ func TestParser_parseMetric(t *testing.T) {
 			p.metricConfigs = tt.fields.metricConfigs
 
 			// Find a valid metrics config
-			config, found := p.findMetricConfig(tt.args.metricPath, tt.args.deviceID)
+			configs := p.findMetricConfigs(tt.args.metricPath, tt.args.deviceID)
-			if !found {
+			if len(configs) != 1 {
 				if !tt.wantErr {
 					t.Errorf("MetricConfig not found")
 				}
 				return
 			}
+			config := configs[0]
 
 			id := metricID("", tt.args.metricPath, tt.args.deviceID, config.PrometheusName)
 			got, err := p.parseMetric(config, id, tt.args.value)