collected lua call, started on lua metrics, added testLua call

README.md

@@ -4,7 +4,7 @@ This exporter exports some variables from an
 [AVM Fritzbox](http://avm.de/produkte/fritzbox/)
 to prometheus.
 
-This exporter is tested with a Fritzbox 7590 software version 07.12 and 07.20.
+This exporter is tested with a Fritzbox 7590 software version 07.12, 07.20 and 07.21.
 
 The goal of the fork is:
   - [x] allow passing of username / password using evironment variable
@@ -12,6 +12,7 @@ The goal of the fork is:
   - [x] move config of metrics to be exported to config file rather then code
   - [x] add config for additional metrics to collect (especially from TR-064 API)
   - [x] create a grafana dashboard consuming the additional metrics
+  - [ ] collect metrics from lua APIs not available in UPNP APIs
  
 Other changes:
   - replaced digest authentication code with own implementation
@@ -20,6 +21,7 @@ Other changes:
   - **New:** collect option to directly test collection of results
   - **New:** additional metrics to collect details about connected hosts and DECT devices
   - **New:** support to use results like hostname or MAC address as labels to metrics
+  - **New:** support for metrics from lua APIs (e.g. CPU temperature, utilization, ...)
  
 
 ## Building

fritzbox_lua/README.md

@@ -0,0 +1,24 @@
+# Client for LUA API of FRITZ!Box UI
+
+**Note:** This client only support calls that return JSON (some seem to return HTML they are not supported)
+
+There does not seem to be a complete documentation of the API, the authentication and getting a sid (Session ID) is described here:
+[https://avm.de/fileadmin/user_upload/Global/Service/Schnittstellen/AVM_Technical_Note_-_Session_ID.pdf]
+
+## Details
+Most of the calls seem to be using the data.lua url with a http FORM POST request. As parameters the page and session id are required (e.g.: sid=<SID>&page=engery). The result is JSON with the data needed to create the respective UI.
+
+Since no public documentation for the JSON format of the various pages seem to exist, you need to observe the calls made by the UI and analyse the JSON result. However the client should be generic enough to get metric and label values from all kind of nested hash and array structures contained in the JSONs.
+
+## Compatibility
+The client was developed on a Fritzbox 7590 running on 07.21, other models or versions may behave differently so just test and see what works, but again the generic part of the client should still work as long as there is a JSON result.
+
+## Translations
+Since the API is used to drive the UI, labels are translated and will be returned in the language configured in the Fritzbox. There seems to be a lang parameter but it looks like it is simply ignored. Having translated labels is annoying, therefore the clients also support renaming them based on regex.
+Currently the regex are defined for:
+  - German
+  
+If your Fritzbox is running in another language you need to adjust them or you will receive different labels, that may not work with dashboards using them for filtering!
+
+
+

fritzbox_lua/lua_client.go

@@ -59,6 +59,7 @@ type LuaPage struct {
 type LuaMetricValueDefinition struct {
 	Path        string
 	Key         string
+	OkValue     string
 	Labels      []string
 	FixedLabels map[string]string
 }
@@ -95,10 +96,22 @@ func (lua *LuaSession) doLogin(response string) error {
 		return fmt.Errorf("Error decoding SessionInfo: %s", err.Error())
 	}
 
+	if lua.SessionInfo.BlockTime > 0 {
+		return fmt.Errorf("To many failed logins, login blocked for %d seconds", lua.SessionInfo.BlockTime)
+	}
+
 	return nil
 }
 
 func (lmvDef *LuaMetricValueDefinition) createValue(name string, value string) LuaMetricValue {
+	if lmvDef.OkValue != "" {
+		if value == lmvDef.OkValue {
+			value = "1"
+		} else {
+			value = "0"
+		}
+	}
+
 	lmv := LuaMetricValue{
 		Name:   name,
 		Value:  value,
@@ -144,7 +157,17 @@ func (lua *LuaSession) Login() error {
 
 // LoadData load a lua bage and return content
 func (lua *LuaSession) LoadData(page LuaPage) ([]byte, error) {
-	dataURL := fmt.Sprintf("%s/%s", lua.BaseURL, page.Path)
+	method := "POST"
+	path := page.Path
+
+	// handle method prefix
+	pathParts := strings.SplitN(path, ":", 2)
+	if len(pathParts) > 1 {
+		method = pathParts[0]
+		path = pathParts[1]
+	}
+
+	dataURL := fmt.Sprintf("%s/%s", lua.BaseURL, path)
 
 	callDone := false
 	var resp *http.Response
@@ -167,7 +190,14 @@ func (lua *LuaSession) LoadData(page LuaPage) ([]byte, error) {
 			params += "&" + page.Params
 		}
 
-		resp, err = http.Post(dataURL, "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(params)))
+		if method == "POST" {
+			resp, err = http.Post(dataURL, "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(params)))
+		} else if method == "GET" {
+			resp, err = http.Get(dataURL + "?" + params)
+		} else {
+			err = fmt.Errorf("method %s is unsupported in path %s", method, page.Path)
+		}
+
 		if err != nil {
 			return nil, err
 		}

luaTest-many.json

@@ -0,0 +1,110 @@
+[
+    {
+        "path": "data.lua",
+        "params": "page=overview"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=ipv6"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dnsSrv"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=kidLis"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=trafapp"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=portoverview"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dslOv"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dialLi"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=bookLi"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dectSet"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dectMon"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=homeNet"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=netDev"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=netSet"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=usbOv"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=mServSet"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=wSet"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=chan"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=sh_dev"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=energy"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=ecoStat"
+    },
+    {
+        "path": "GET:internet/inetstat_monitor.lua",
+        "params": "action=get_graphic&useajax=1"
+    },
+    {
+        "path": "GET:internet/internet_settings.lua",
+        "params": "multiwan_page=dsl&useajax=1"
+    },
+    {
+        "path": "GET:internet/dsl_stats_tab.lua",
+        "params": "update=mainDiv&useajax=1"
+    },
+    {
+        "path": "GET:net/network.lua",
+        "params": "useajax=1"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=netCnt"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dslStat"
+    }
+]

luaTest.json

@@ -0,0 +1,34 @@
+[
+    {
+        "path": "data.lua",
+        "params": "page=overview"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dslOv"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=dectMon"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=netDev"
+    },
+   {
+        "path": "data.lua",
+        "params": "page=usbOv"
+    },
+     {
+        "path": "data.lua",
+        "params": "page=sh_dev"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=energy"
+    },
+    {
+        "path": "data.lua",
+        "params": "page=ecoStat"
+    }
+]

main.go

@@ -41,14 +41,15 @@ import (
 const serviceLoadRetryTime = 1 * time.Minute
 
 var (
-	flag_luacall = flag.Bool("testLua", false, "test LUA") // TODO cleanup once completed
-
 	flag_test    = flag.Bool("test", false, "print all available metrics to stdout")
+	flag_luatest = flag.Bool("testLua", false, "read luaTest.json file make all contained calls ans print results")
 	flag_collect = flag.Bool("collect", false, "print configured metrics to stdout and exit")
 	flag_jsonout = flag.String("json-out", "", "store metrics also to JSON file when running test")
 
-	flag_addr         = flag.String("listen-address", "127.0.0.1:9042", "The address to listen on for HTTP requests.")
+	flag_addr             = flag.String("listen-address", "127.0.0.1:9042", "The address to listen on for HTTP requests.")
-	flag_metrics_file = flag.String("metrics-file", "metrics.json", "The JSON file with the metric definitions.")
+	flag_metrics_file     = flag.String("metrics-file", "metrics.json", "The JSON file with the metric definitions.")
+	flag_disable_lua      = flag.Bool("nolua", false, "disable collecting lua metrics")
+	flag_lua_metrics_file = flag.String("lua-metrics-file", "metrics-lua.json", "The JSON file with the lua metric definitions.")
 
 	flag_gateway_url      = flag.String("gateway-url", "http://fritz.box:49000", "The URL of the FRITZ!Box")
 	flag_gateway_luaurl   = flag.String("gateway-luaurl", "http://fritz.box", "The URL of the FRITZ!Box UI")
@@ -91,12 +92,39 @@ type Metric struct {
 	MetricType prometheus.ValueType
 }
 
+type LuaTest struct {
+	Path   string `json:"path"`
+	Params string `json:"params"`
+}
+
+type LuaLabelRename struct {
+	MatchRegex  string `json:"matchRegex"`
+	RenameLabel string `json:"renameLabel"`
+}
+
 type LuaMetric struct {
 	// initialized loading JSON
+	Path        string            `json:"path"`
+	Params      string            `json:"params"`
+	ResultPath  string            `json:"resultPath"`
+	ResultKey   string            `json:"resultKey"`
+	OkValue     string            `json:"okValue"`
+	FixedLabels map[string]string `json:"fixedLabels"`
+	PromDesc    JSON_PromDesc     `json:"promDesc"`
+	PromType    string            `json:"promType"`
 
+	// initialized at startup
+	Desc       *prometheus.Desc
+	MetricType prometheus.ValueType
+}
+
+type LuaMetricsFile struct {
+	LabelRenames []LuaLabelRename `json:"labelRenames"`
+	Metrics      []LuaMetric      `json:"metrics"`
 }
 
 var metrics []*Metric
+var luaMetricsFile *LuaMetricsFile
 
 type FritzboxCollector struct {
 	Url      string
@@ -408,27 +436,41 @@ func test() {
 	}
 }
 
-func testLuaCall() {
+func testLua() {
-	var luaSession lua.LuaSession
-	luaSession.BaseURL = *flag_gateway_luaurl
-	luaSession.Username = *flag_gateway_username
-	luaSession.Password = *flag_gateway_password
-
-	var jsonData []byte
-	var err error
-
-	page := lua.LuaPage{Path: "data.lua", Params: "page=energy"}
-	//page := lua.LuaPage{Path: "data.lua", Params: "page=ecoStat"}
-	//page := lua.LuaPage{Path: "data.lua", Params: "page=usbOv"}
-	jsonData, err = luaSession.LoadData(page)
 
+	jsonData, err := ioutil.ReadFile("luaTest.json")
 	if err != nil {
-		fmt.Println(err.Error())
+		fmt.Println("error reading luaTest.json:", err)
 		return
 	}
 
-	fmt.Println(fmt.Sprintf("JSON: %s", string(jsonData)))
+	var luaTests []LuaTest
+	err = json.Unmarshal(jsonData, &luaTests)
+	if err != nil {
+		fmt.Println("error parsing luaTest JSON:", err)
+		return
+	}
 
+	// create session struct and init params
+	luaSession := lua.LuaSession{BaseURL: *flag_gateway_luaurl, Username: *flag_gateway_username, Password: *flag_gateway_password}
+
+	for _, test := range luaTests {
+		fmt.Printf("TESTING: %s (%s)\n", test.Path, test.Params)
+
+		page := lua.LuaPage{Path: test.Path, Params: test.Params}
+		pageData, err := luaSession.LoadData(page)
+
+		if err != nil {
+			fmt.Println(err.Error())
+		} else {
+			fmt.Println(string(pageData))
+		}
+
+		fmt.Println("\n")
+	}
+}
+
+func extraceLuaData(jsonData []byte) {
 	data, err := lua.ParseJSON(jsonData)
 	if err != nil {
 		fmt.Println(err.Error())
@@ -524,10 +566,11 @@ func main() {
 		return
 	}
 
-	if *flag_luacall {
+	if *flag_luatest {
-		testLuaCall()
+		testLua()
 		return
 	}
+
 	// read metrics
 	jsonData, err := ioutil.ReadFile(*flag_metrics_file)
 	if err != nil {
@@ -541,6 +584,20 @@ func main() {
 		return
 	}
 
+	if !*flag_disable_lua {
+		jsonData, err := ioutil.ReadFile(*flag_lua_metrics_file)
+		if err != nil {
+			fmt.Println("error reading lua metric file:", err)
+			return
+		}
+
+		err = json.Unmarshal(jsonData, &luaMetricsFile)
+		if err != nil {
+			fmt.Println("error parsing lua JSON:", err)
+			return
+		}
+	}
+
 	// init metrics
 	for _, m := range metrics {
 		pd := m.PromDesc

metrics-lua.json

@@ -0,0 +1,162 @@
+{
+    "labelRenames": [
+        {
+            "matchRegex": "(?i)prozessor",
+            "renameLabel": "CPU"
+        },
+        {
+            "matchRegex": "(?i)system",
+            "renameLabel": "System"
+        },
+        {
+            "matchRegex": "(?i)DSL",
+            "renameLabel": "DSL"
+        },        
+        {
+            "matchRegex": "(?i)FON",
+            "renameLabel": "Phone"
+        },
+        {
+            "matchRegex": "(?i)WLAN",
+            "renameLabel": "WLAN"
+        },
+        {
+            "matchRegex": "(?i)USB",
+            "renameLabe": "USB"
+        }
+    ],
+    "metrics": [
+        {
+            "path": "data.lua",
+            "params": "page=energy",
+            "resultPath": "data.drain.*",
+            "resultKey": "actPerc",
+            "promDesc": {
+                "fqName": "gateway_data_energy_consumption",
+                "help": "percentage of energy consumed from data.lua?page=energy",
+                "varLabels": [
+                    "gateway", "name"
+                ]
+            },
+            "promType": "GaugeValue"
+        },       
+        {
+            "path": "data.lua",
+            "params": "page=energy",
+            "resultPath": "data.drain.*.lan.*",
+            "resultKey": "class",
+            "okValue": "green",
+            "promDesc": {
+                "fqName": "gateway_data_energy_lan_status",
+                "help": "status of LAN connection from data.lua?page=energy (1 = up)",
+                "varLabels": [
+                    "gateway", "name"
+                ]
+            },
+            "promType": "GaugeValue"
+        },
+        {
+            "path": "data.lua",
+            "params": "page=ecoStat",
+            "resultPath": "data.cputemp.series.0",
+            "resultKey": "-1",
+            "promDesc": {
+                "fqName": "gateway_data_ecostat_cputemp",
+                "help": "cpu temperature from data.lua?page=ecoStat",
+                "varLabels": [
+                    "gateway"
+                ]
+            },
+            "promType": "GaugeValue"
+        },
+        {
+            "path": "data.lua",
+            "params": "page=ecoStat",
+            "resultPath": "data.cpuutil.series.0",
+            "resultKey": "-1",
+            "promDesc": {
+                "fqName": "gateway_data_ecostat_cpuutil",
+                "help": "percentage of cpu utilization from data.lua?page=ecoStat",
+                "varLabels": [
+                    "gateway"
+                ]
+            },
+            "promType": "GaugeValue"
+        },        
+        {
+            "path": "data.lua",
+            "params": "page=ecoStat",
+            "resultPath": "data.ramusage.series.0",
+            "resultKey": "-1",
+            "fixedLabels": { "ram_type" : "Fixed" },
+            "promDesc": {
+                "fqName": "gateway_data_energy_consumption",
+                "help": "percentage of energy consumed from data.lua?page=energy",
+                "varLabels": [
+                    "gateway", "ram_type"
+                ]
+            },
+            "promType": "GaugeValue"
+        },
+        {
+            "path": "data.lua",
+            "params": "page=ecoStat",
+            "resultPath": "data.ramusage.series.1",
+            "resultKey": "-1",
+            "fixedLabels": { "ram_type" : "Dynamic" },
+            "promDesc": {
+                "fqName": "gateway_data_energy_consumption",
+                "help": "percentage of energy consumed from data.lua?page=energy",
+                "varLabels": [
+                    "gateway", "ram_type"
+                ]
+            },
+            "promType": "GaugeValue"
+        },
+        {
+            "path": "data.lua",
+            "params": "page=ecoStat",
+            "resultPath": "data.ramusage.series.2",
+            "resultKey": "-1",
+            "fixedLabels": { "ram_type" : "Free" },
+            "promDesc": {
+                "fqName": "gateway_data_energy_consumption",
+                "help": "percentage of energy consumed from data.lua?page=energy",
+                "varLabels": [
+                    "gateway", "ram_type"
+                ]
+            },
+            "promType": "GaugeValue"
+        },        
+        {
+            "path": "data.lua",
+            "params": "page=usbOv",
+            "resultPath": "data.usbOverview.devices.*",
+            "resultKey": "partitions.0.totalStorageInBytes",
+            "promDesc": {
+                "fqName": "gateway_data_usb_storage_total",
+                "help": "total storage in bytes from data.lua?page=usbOv",
+                "varLabels": [
+                    "gateway", "deviceType", "deviceName"
+                ]
+            },
+            "promType": "GaugeValue"
+        },
+        {
+            "path": "data.lua",
+            "params": "page=usbOv",
+            "resultPath": "data.usbOverview.devices.*",
+            "resultKey": "partitions.0.usedStorageInBytes",
+            "promDesc": {
+                "fqName": "gateway_data_usb_storage_used",
+                "help": "used storage in bytes from data.lua?page=usbOv",
+                "varLabels": [
+                    "gateway", "deviceType", "deviceName"
+                ]
+            },
+            "promType": "GaugeValue"
+        }        
+    ]
+
+}
+