Merge pull request #117 from flobz/feature-client_filter

Feature client filter

web/app.py

@@ -27,7 +27,8 @@ myp: MyPSACC = None
 chc: ChargeControls = None
 
 
-def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pylint: disable=too-many-arguments
+def start_app(title, base_path, debug: bool, host, port, reloader=False,   # pylint: disable=too-many-arguments
+              unminified=False):
     global app, dash_app, dispatcher
     try:
         lang = locale.getlocale()[0].split("_")[0]
@@ -36,6 +37,8 @@ def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pyl
     except (IndexError, locale.Error):
         locale_url = None
         logger.warning("Can't get language")
+    if unminified:
+        locale_url = ["assets/plotly-with-meta.js"]
     app = Flask(__name__)
     app.config["DEBUG"] = debug
     if base_path == "/":
@@ -46,6 +49,7 @@ def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pyl
         requests_pathname_prefix = base_path + "/"
     dash_app = dash.Dash(external_stylesheets=[dbc.themes.BOOTSTRAP], external_scripts=locale_url, title=title,
                          server=app, requests_pathname_prefix=requests_pathname_prefix)
+    dash_app.enable_dev_tools(reloader)
     # keep this line
     import web.views  # pylint: disable=unused-import,import-outside-toplevel
     return run_simple(host, port, application, use_reloader=reloader, use_debugger=debug)

web/assets/clientside.js

@@ -0,0 +1,227 @@
+class Avg {
+  constructor () {
+    this.total = 0
+    this.count = 0
+  }
+
+  addValue (value) {
+    if (typeof value === 'number') {
+      this.count++
+      this.total = ((this.total * (this.count - 1)) / this.count) + (value / this.count)
+    }
+  }
+
+  average () {
+    return this.total
+  }
+
+  static getAverageFromKey (array, key) {
+    const avg = new Avg()
+    array.forEach(function (obj) { avg.addValue(obj[key]) })
+    return avg.average()
+  }
+}
+const logger = (function () {
+  let oldConsoleLog = null
+  const pub = {}
+
+  pub.enableLogger = function enableLogger () {
+    if (oldConsoleLog == null) {
+      return
+    }
+
+    window.console.log = oldConsoleLog
+  }
+
+  pub.disableLogger = function disableLogger () {
+    oldConsoleLog = console.log
+    window.console.log = function () {}
+  }
+
+  return pub
+}())
+function addLocaleDate (data, dateKey) {
+  const dateOption = [undefined, { hour: 'numeric', minute: 'numeric' }]
+  function dateToLocale (row, key) {
+    const date = new Date(row[key])
+    row[key] = date
+    row[key + '_str'] = date.toLocaleDateString(...dateOption)
+  }
+  let datasetName, dataset
+  for ([datasetName, dataset] of Object.entries(data)) {
+    dataset.forEach(function (row) {
+      dateKey[datasetName].forEach(key => dateToLocale(row, key))
+    })
+  }
+}
+
+function filterDataset (data, range) {
+  function dateFromISO (st) {
+    return new Date(st).getTime() / 1000
+  }
+  function isInRange (st) {
+    const tsDate = dateFromISO(st)
+    return tsDate >= range[0] && tsDate <= range[1]
+  }
+  const res = {}
+  res.trips = data.trips.filter(line => isInRange(line.start_at))
+  res.chargings = data.chargings.filter(line => isInRange(line.start_at))
+  console.log('filtered_dataset', res)
+  return res
+}
+
+function filterShortTrip (data) {
+  const longTrips = {
+    trips: data.trips.filter(line => line.distance > 10),
+    chargings: data.chargings
+  }
+  console.log('long trips:', longTrips)
+  return longTrips
+}
+
+function updateFigures (data, oldFigure, x, y) {
+  const trips = data.trips
+  const figures = []
+  let i = 0
+  y.forEach(function (yLabel) {
+    const xLabel = x[i]
+    const figure = Object.assign({}, oldFigure[i])
+    i++
+    // console.log(oldFigure[i]);
+    // var unique_y_label = y[i].filter((v, i, a) => a.indexOf(v) === i);
+    // var data_nonnull = trips
+    // unique_y_label.forEach(function(label) {
+    //     data_nonnull = data_nonnull.filter(line => line[label]);
+    // });
+    if ('mapbox' in figure.layout) {
+      figure.data[0].lat = []
+      figure.data[0].lon = []
+      figure.data[0].hovertext = []
+      let trip = null
+      for (trip of trips) {
+        const xPos = trip.positions[xLabel]
+        figure.data[0].lat.push(...xPos, null)
+        figure.data[0].lon.push(...trip.positions[yLabel[0]])
+        figure.data[0].hovertext.push(...Array(xPos.length).fill(trip[yLabel[1]]), null)
+      }
+      if (trip) {
+        const lastPos = trip.positions[yLabel[0]].length - 1
+        figure.layout.mapbox.center.lat = trip.positions[xLabel][lastPos]
+        figure.layout.mapbox.center.lon = trip.positions[yLabel[0]][lastPos]
+        figure.data[1].lat = [figure.layout.mapbox.center.lat]
+        figure.data[1].lon = [figure.layout.mapbox.center.lon]
+      }
+    } else {
+      const xValues = trips.map(a => a[xLabel])
+      // for each y label
+      for (let j = 0; j < yLabel.length; j++) {
+        figure.data[j].y = trips.map(a => a[yLabel[j]])
+        figure.data[j].x = xValues
+      }
+    }
+    console.log(xLabel, figure)
+    figures.push(figure)
+  })
+  return figures
+}
+
+function updateTables (data, tables) {
+  console.log('tables', tables)
+  const figures = []
+  tables.forEach(function (table) {
+    figures.push(data[table.src])
+  })
+  return figures
+}
+
+function updateCardsValue (data) {
+  const res = {}
+  let avgPriceKw
+  let avgC02 = new Avg(); let avgKw = new Avg(); const avgTime = new Avg()
+  const avgPrice = new Avg()
+  data.chargings.forEach(function (charge) {
+    const diff = ((new Date(charge.stop_at)) - (new Date(charge.start_at))) / 3600000
+    avgKw.addValue(charge.kw)
+    avgC02.addValue(charge.co2)
+    avgPrice.addValue(charge.price)
+    if (diff > 0) {
+      avgTime.addValue(diff)
+    }
+  })
+  if (data.chargings.length > 0) {
+    avgKw = avgKw.average()
+    avgC02 = avgC02.average()
+    avgPriceKw = avgPrice.average() / avgKw
+    res.avg_emission_kw = avgC02
+    res.avg_chg_speed = avgKw / avgTime.average()
+  }
+  if (data.trips.length > 0) {
+    const totalDistance = data.trips[data.trips.length - 1].mileage - data.trips[0].mileage
+    res.avg_consum_kw = Avg.getAverageFromKey(data.trips, 'consumption_km')
+    res.elec_consum_kw = totalDistance * res.avg_consum_kw / 100
+  }
+  if (data.trips.length > 0 && data.chargings.length > 0) {
+    res.avg_emission_km = res.avg_emission_kw * res.avg_consum_kw / 100
+    res.elec_consum_price = avgPriceKw * res.elec_consum_kw
+    res.avg_consum_price = avgPriceKw * res.avg_consum_kw
+  }
+  for (const [key, value] of Object.entries(res)) {
+    document.getElementById(key).innerHTML = value.toPrecision(3)
+  }
+}
+
+function sortDataset (ctx, data, tables) {
+  const tableId = ctx.prop_id.split('.')[0]
+  if (ctx.value.length > 0) {
+    const asc = ctx.value[0].direction === 'asc'
+    let columnId = ctx.value[0].column_id
+    const table = tables.filter(table => table.table_id === tableId)[0]
+    let sorted
+    if (columnId.endsWith('_str')) {
+      columnId = columnId.slice(0, -4)
+      sorted = data[table.src].sort(function (a, b) {
+        return a[columnId] - b[columnId]
+      })
+    } else if (typeof data[table.src][0][columnId] === 'number') {
+      sorted = data[table.src].sort(function (a, b) {
+        return a[columnId] - b[columnId]
+      })
+    } else {
+      sorted = data[table.src].sort((a, b) => a[columnId].localeCompare(b[columnId]))
+    }
+    if (asc === false) {
+      sorted = sorted.reverse()
+    }
+    data[table.src] = sorted
+  }
+}
+
+function filterAndSort (data, range, figures, p, log) { // eslint-disable-line no-unused-vars
+  if (log > 10) {
+    logger.disableLogger()
+  }
+  const ctx = dash_clientside.callback_context.triggered // eslint-disable-line  no-undef
+  const outFigures = []; let dataFiltered
+  console.log('figures:', figures)
+  console.log('data:', data)
+  console.log('ctx', ctx)
+  if (ctx.length > 0 && ctx[0].prop_id.endsWith('sort_by')) {
+    dataFiltered = filterDataset(data, range)
+    sortDataset(ctx[0], dataFiltered, p.table_src)
+    outFigures.push(...updateTables(dataFiltered, p.table_src))
+    outFigures.push(...figures.graph)
+    outFigures.push(...figures.maps)
+  } else {
+    addLocaleDate(data, p.date_columns)
+    dataFiltered = filterDataset(data, range)
+    outFigures.push(...updateTables(dataFiltered, p.table_src))
+    console.log(dataFiltered.trips.length)
+    const longTrips = filterShortTrip(dataFiltered)
+    console.log('trips', dataFiltered.trips.length)
+    console.log('longTrips', longTrips.trips.length)
+    outFigures.push(...updateFigures(longTrips, figures.graph, p.graph_x_label, p.graph_y_label))
+    outFigures.push(...updateFigures(dataFiltered, figures.maps, p.map_x_label, p.map_y_label))
+    updateCardsValue(longTrips)
+  }
+  return outFigures
+}

web/assets/style.json

@@ -0,0 +1,22 @@
+{
+        "version": 8,
+        "sources": {
+            "osm": {
+                "type": "raster",
+                "tiles": [
+                    "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
+                ],
+                "tileSize": 256,
+                "attribution": "Map tiles by <a target=\"_top\" rel=\"noopener\" href=\"https://tile.openstreetmap.org/\">OpenStreetMap tile servers</a>, under the <a target=\"_top\" rel=\"noopener\" href=\"https://operations.osmfoundation.org/policies/tiles/\">tile usage policy</a>. Data by <a target=\"_top\" rel=\"noopener\" href=\"http://openstreetmap.org\">OpenStreetMap</a>"
+            }
+        },
+        "sprite": "",
+        "glyphs": "https://api.maptiler.com/fonts/{fontstack}/{range}.pbf",
+        "layers": [
+            {
+                "id": "osm",
+                "type": "raster",
+                "source": "osm"
+            }
+        ]
+}

web/db.py

@@ -11,7 +11,7 @@ from geojson import Feature, Point, FeatureCollection
 from geojson import dumps as geo_dumps
 
 from mylogger import logger
-from utils import get_temp
+from libs.utils import get_temp
 
 NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]]
 NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]]
@@ -126,7 +126,10 @@ class Database:
     def clean_battery(conn):
         # delete charging longer than 17h
         conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;")
-        conn.execute("DELETE FROM battery WHERE start_level==end_level;")
+        # delete charging not finished longer than 17h
+        conn.execute("DELETE from battery where stop_at is NULL and JULIANDAY()-JULIANDAY(start_at)>0.7;")
+        #delete little charge
+        conn.execute("DELETE FROM battery WHERE start_level >= end_level-1;")
 
     @staticmethod
     def clean_position(conn):

web/figure_filter.py

@@ -0,0 +1,121 @@
+import json
+from logging import DEBUG
+
+from dash.dependencies import Output, Input
+from dash_core_components import Store
+from mylogger import logger
+
+
+class Graph:
+    def __init__(self, graph_id, x, y: [], figure):
+        self.graph_id = graph_id
+        self.x = x
+        self.y = y
+        self.figure = figure
+
+
+class Table:
+    def __init__(self, table_id, src, figure):
+        self.table_id = table_id
+        self.src = src
+        self.figure = figure
+        self.date_columns = []
+
+
+def figures_to_dict(figures):
+    el_list = []
+    for figure in figures:
+        res = {}
+        for key, value in figure.__dict__.items():
+            if key != "figure":
+                res[key] = value
+        el_list.append(res)
+    return el_list
+
+
+class Figure_Filter:
+
+    def __init__(self):
+        self.graphs = []
+        self.tables = []
+        self.maps = []
+        self.src = {}
+
+    def add_map(self, dash_Graph, latitude, longitude, figure):
+        self.maps.append(Graph(dash_Graph.id, latitude, longitude, figure))
+        return dash_Graph
+
+    def add_graph(self, dash_Graph, x, y, figure):
+        self.graphs.append(Graph(dash_Graph.id, x, y, figure))
+        return dash_Graph
+
+    def add_table(self, src, figure):
+        table = Table(figure.id, src, figure)
+        table.date_columns = [col["id"][:-4] for col in figure.columns if col["type"] == "datetime" and
+                              col["id"].endswith("_str")]
+        self.tables.append(table)
+
+    def __get_table_date_column_id(self):
+        res = {table.src: table.date_columns for table in self.tables}
+        return res
+
+    def __get_table_src(self):
+        return [table.src for table in self.tables]
+
+    def __get_figures(self):
+        return {"graph": [graph.figure for graph in self.graphs],
+                "tables": [table.figure for table in self.tables],
+                "maps": [map.figure for map in self.maps]}
+
+    def __get_output(self) -> list:
+        outputs = [Output(table.table_id, "data") for table in self.tables]
+        outputs.extend([Output(graph.graph_id, "figure") for graph in self.graphs])
+        outputs.extend([Output(graph.graph_id, "figure") for graph in self.maps])
+        return outputs
+
+    def __get_graph_x_label(self, graphs):
+        return [graph.x for graph in graphs]
+
+    def __get_graph_y_label(self, graphs):
+        return [graph.y for graph in graphs]
+
+    def __get_table_input_sort_by(self):
+        inputs = [Input(table.table_id, 'sort_by') for table in self.tables]
+        return inputs
+
+    def gen_unused_variable(self):
+        res = ", ".join([chr(i) for i in range(ord('a'), ord('a') + len(self.tables))])
+        return res
+
+    def get_params(self):
+        params = json.dumps({
+            "date_columns": self.__get_table_date_column_id(),
+            "table_src": figures_to_dict(self.tables),
+            "graph_x_label": self.__get_graph_x_label(self.graphs),
+            "graph_y_label": self.__get_graph_y_label(self.graphs),
+            "map_x_label": self.__get_graph_x_label(self.maps),
+            "map_y_label": self.__get_graph_y_label(self.maps)
+        }, indent=4)
+        return params
+
+    def get_clientside_callback(self):
+        if logger.isEnabledFor(DEBUG):
+            log_level = 10
+        else:
+            log_level = 20
+        fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{
+                        const params={self.get_params()};
+                        const logLevel={log_level};
+                        return filterAndSort(data, range, figures, params, logLevel);
+                      }}"""
+        res = [fct_def,
+               *self.__get_output(),
+               Input('clientside-data-store', 'data'),
+               Input('date-slider', 'value'),
+               Input('clientside-figure-store', 'data'),
+               *self.__get_table_input_sort_by()]
+        return res
+
+    def get_store(self):
+        return [Store(id='clientside-figure-store', data=self.__get_figures()),
+                Store(id='clientside-data-store', data=self.src)]

web/figures.py

@@ -1,52 +1,18 @@
 from copy import deepcopy
 
-from typing import List
-
 import dash_bootstrap_components as dbc
 import dash_table
-import numpy as np
 from dash_core_components import Graph
 from dash_table.Format import Format, Scheme, Symbol
-from dateutil.relativedelta import relativedelta
 import plotly.express as px
 import plotly.graph_objects as go
-from pandas import DataFrame
-from pandas import options as pandas_options
 import dash_html_components as html
 
 from libs.car import Car
 from libs.elec_price import ElecPrice
-from trip import Trips, Trip
+from trip import Trip
 from web.db import Database
-
-
-def unix_time_millis(date):
-    return int(date.timestamp())
-
-
-def get_marks_from_start_end(start, end):
-    nb_marks = 10
-    result = []
-    time_delta = int((end - start).total_seconds() / nb_marks)
-    current = start
-    if time_delta > 0:
-        while current <= end:
-            result.append(current)
-            current += relativedelta(seconds=time_delta)
-        result[-1] = end
-        if time_delta < 3600 * 24:
-            if time_delta > 3600:
-                date_f = '%x %Hh'
-            else:
-                date_f = '%x %Hh%M'
-        else:
-            date_f = '%x'
-        marks = {}
-        for date in result:
-            marks[unix_time_millis(date)] = str(date.strftime(date_f))
-        return marks
-    return None
-
+from web.utils import card_value_div, dash_date_to_datetime
 
 # pylint: disable=invalid-name
 ERROR_DIV = dbc.Alert("No data to show, there is probably no trips recorded yet", color="danger")
@@ -55,45 +21,59 @@ consumption_fig = ERROR_DIV
 consumption_df = ERROR_DIV
 trips_map = ERROR_DIV
 consumption_fig_by_speed = ERROR_DIV
-consumption_graph_by_temp = ERROR_DIV
+consumption_fig_by_temp = ERROR_DIV
 table_fig = ERROR_DIV
-pandas_options.display.float_format = '${:.2f}'.format
 info = ""
-battery_info = ERROR_DIV
-battery_table = None
+battery_table = ERROR_DIV
 
-SUMMARY_CARDS = {"Average consumption": {"text": None, "src": "static/images/consumption.svg"},
-                 "Average emission": {"text": None, "src": "static/images/pollution.svg"},
-                 "Average charge speed": {"text": None, "src": "static/images/battery-charge-line.svg"},
-                 "Electricity consumption": {"text": None, "src": "static/images/electricity bill.svg"}
+AVG_CHARGE_SPEED = "avg_chg_speed"
+AVG_EMISSION_KM = "avg_emission_km"
+AVG_EMISSION_KW = "avg_emission_kw"
+ELEC_CONSUM_KW = "elec_consum_kw"
+ELEC_CONSUM_PRICE = "elec_consum_price"
+AVG_CONSUM_KW = "avg_consum_kw"
+AVG_CONSUM_PRICE = "avg_consum_price"
+
+SUMMARY_CARDS = {"Average consumption": {"text": [card_value_div(AVG_CONSUM_KW, "kWh/100km"),
+                                                  card_value_div(AVG_CONSUM_PRICE, f"{ElecPrice.currency}/100km")],
+                                         "src": "assets/images/consumption.svg"},
+                 "Average emission": {"text": [card_value_div(AVG_EMISSION_KM, " g/km"),
+                                               card_value_div(AVG_EMISSION_KW, "g/kWh")],
+                                      "src": "assets/images/pollution.svg"},
+                 "Average charge speed": {"text": [card_value_div(AVG_CHARGE_SPEED, " kW")],
+                                          "src": "assets/images/battery-charge-line.svg"},
+                 "Electricity consumption": {"text": [card_value_div(ELEC_CONSUM_KW, "kWh"),
+                                                      card_value_div(ELEC_CONSUM_PRICE, ElecPrice.currency)],
+                                             "src": "assets/images/electricity bill.svg"}
                  }
 
 
 # pylint: disable=too-many-locals
-def get_figures(trips: Trips, charging: List[dict]):
-    global consumption_fig, consumption_df, trips_map, consumption_fig_by_speed, table_fig, info, battery_info, \
-        battery_table, consumption_graph_by_temp
-    lats = []
-    lons = []
-    names = []
-    for trip in trips:
-        for points in trip.positions:
-            lats = np.append(lats, points.latitude)
-            lons = np.append(lons, points.longitude)
-            names = np.append(names, [str(trip.start_at)])
-        lats = np.append(lats, None)
-        lons = np.append(lons, None)
-        names = np.append(names, None)
-    trips_map = px.line_mapbox(lat=lats, lon=lons, hover_name=names,
-                               mapbox_style="stamen-terrain", zoom=12)
+def get_figures(car: Car):
+    global consumption_fig, consumption_df, trips_map, consumption_fig_by_speed, table_fig, info, \
+        battery_table, consumption_fig_by_temp
+    lats = [42, 41]
+    lons = [1, 2]
+    names = ["undefined", "undefined"]
+    trips_map = px.line_mapbox(lat=lats, lon=lons, hover_name=names, zoom=12, mapbox_style="assets/style2.json")
+    trips_map.add_trace(go.Scattermapbox(
+        mode="markers",
+        marker={"symbol": "marker", "size": 20},
+        lon=[lons[0]], lat=[lats[0]],
+        showlegend=False, name="Last Position"))
     # table
     nb_format = Format(precision=2, scheme=Scheme.fixed, symbol=Symbol.yes)  # pylint: disable=no-member
+    style_cell_conditional = []
+    if car.is_electric():
+        style_cell_conditional.append({'if': {'column_id': 'consumption_fuel_km', }, 'display': 'None', })
+    if car.is_thermal():
+        style_cell_conditional.append({'if': {'column_id': 'consumption_km', }, 'display': 'None', })
     table_fig = dash_table.DataTable(
         id='trips-table',
-        sort_action='native',
+        sort_action='custom',
         sort_by=[{'column_id': 'id', 'direction': 'desc'}],
         columns=[{'id': 'id', 'name': '#', 'type': 'numeric'},
-                 {'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
+                 {'id': 'start_at_str', 'name': 'start at', 'type': 'datetime'},
                  {'id': 'duration', 'name': 'duration', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" min").precision(0)},
                  {'id': 'speed_average', 'name': 'average speed', 'type': 'numeric',
@@ -116,51 +96,31 @@ def get_figures(trips: Trips, charging: List[dict]):
                 "text-decoration": "underline"
             }
         ],
-        data=trips.get_info(),
+        style_cell_conditional=style_cell_conditional,
+        data=[],
         page_size=50
     )
     # consumption_fig
-    consumption_df = DataFrame.from_records(trips.get_long_trips())
-    consumption_fig = px.histogram(consumption_df, x="date", y="consumption_km", title='Consumption of the car',
+    consumption_fig = px.histogram(x=[0], y=[1], title='Consumption of the car',
                                    histfunc="avg")
-    consumption_fig.update_layout(yaxis_title="Consumption kWh/100Km")
+    consumption_fig.update_layout(yaxis_title="Consumption kWh/100Km", xaxis_title="date")
 
-    consumption_fig_by_speed = px.histogram(consumption_df, x="speed", y="consumption_km", histfunc="avg",
+    consumption_fig_by_speed = px.histogram(data_frame=[{"start_at": 1, "speed_average": 2}], x="start_at",
+                                            y="speed_average", histfunc="avg",
                                             title="Consumption by speed")
     consumption_fig_by_speed.update_traces(xbins_size=15)
     consumption_fig_by_speed.update_layout(bargap=0.05)
-    consumption_fig_by_speed.add_trace(
-        go.Scatter(mode="markers", x=consumption_df["speed"], y=consumption_df["consumption_km"],
-                   name="Trips"))
+    consumption_fig_by_speed.add_trace(go.Scatter(mode="markers", x=[0],
+                                                  y=[0], name="Trips"))
     consumption_fig_by_speed.update_layout(xaxis_title="average Speed km/h", yaxis_title="Consumption kWh/100Km")
-    kw_per_km = float(consumption_df["consumption_km"].mean())
-    info = "Average consumption: {:.1f} kWh/100km".format(kw_per_km)
 
-    # charging
-    charging_data = DataFrame.from_records(charging)
-    co2_per_kw = __calculate_co2_per_kw(charging_data)
-    co2_per_km = co2_per_kw * kw_per_km / 100
-    try:
-        charge_speed = 3600 * charging_data["kw"].mean() / \
-                       (charging_data["stop_at"] - charging_data["start_at"]).mean().total_seconds()
-        price_kw = (charging_data["price"] / charging_data["kw"]).mean()
-        total_elec = kw_per_km * trips.get_distance() / 100
-    except (TypeError, KeyError, ZeroDivisionError):  # when there is no data yet:
-        charge_speed = 0
-        price_kw = 0
-        total_elec = 0
-
-    SUMMARY_CARDS["Average charge speed"]["text"] = f"{charge_speed:.2f} kW"
-    SUMMARY_CARDS["Average emission"]["text"] = [html.P(f"{co2_per_km:.1f} g/km"), html.P(f"{co2_per_kw:.1f} g/kWh")]
-    SUMMARY_CARDS["Electricity consumption"]["text"] = [f"{total_elec:.0f} kWh", html.Br(), \
-                                                        f"{total_elec * price_kw:.0f} {ElecPrice.currency}"]
-    SUMMARY_CARDS["Average consumption"]["text"] = f"{consumption_df['consumption_km'].mean():.1f} kWh/100km"
+    # battery_table
     battery_table = dash_table.DataTable(
         id='battery-table',
-        sort_action='native',
-        sort_by=[{'column_id': 'start_at', 'direction': 'desc'}],
-        columns=[{'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
-                 {'id': 'stop_at', 'name': 'stop at', 'type': 'datetime'},
+        sort_action='custom',
+        sort_by=[{'column_id': 'start_at_str', 'direction': 'desc'}],
+        columns=[{'id': 'start_at_str', 'name': 'start at', 'type': 'datetime'},
+                 {'id': 'stop_at_str', 'name': 'stop at', 'type': 'datetime'},
                  {'id': 'start_level', 'name': 'start level', 'type': 'numeric'},
                  {'id': 'end_level', 'name': 'end level', 'type': 'numeric'},
                  {'id': 'co2', 'name': 'CO2', 'type': 'numeric',
@@ -170,7 +130,7 @@ def get_figures(trips: Trips, charging: List[dict]):
                  {'id': 'price', 'name': 'price', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" " + ElecPrice.currency).precision(2), 'editable': True}
                  ],
-        data=charging,
+        data=[],
         style_data_conditional=[
             {
                 'if': {'column_id': ['start_level', "end_level"]},
@@ -179,42 +139,25 @@ def get_figures(trips: Trips, charging: List[dict]):
             },
             {
                 'if': {'column_id': 'price'},
-                'backgroundColor': 'rgb(230, 246, 254)'
+                'backgroundColor': '#ABE2FB'
             }
         ],
     )
-    consumption_by_temp_df = consumption_df[consumption_df["consumption_by_temp"].notnull()]
-    if len(consumption_by_temp_df) > 0:
-        consumption_fig_by_temp = px.histogram(consumption_by_temp_df, x="consumption_by_temp", y="consumption_km",
-                                               histfunc="avg", title="Consumption by temperature")
-        consumption_fig_by_temp.update_traces(xbins_size=2)
-        consumption_fig_by_temp.update_layout(bargap=0.05)
-        consumption_fig_by_temp.add_trace(
-            go.Scatter(mode="markers", x=consumption_by_temp_df["consumption_by_temp"],
-                       y=consumption_by_temp_df["consumption_km"], name="Trips"))
-        consumption_fig_by_temp.update_layout(xaxis_title="average temperature in °C",
-                                              yaxis_title="Consumption kWh/100Km")
-        consumption_graph_by_temp = html.Div(Graph(figure=consumption_fig_by_temp), id="consumption_graph_by_temp")
-
-    else:
-        consumption_graph_by_temp = html.Div(Graph(style={'display': 'none'}), id="consumption_graph_by_temp")
+    consumption_fig_by_temp = px.histogram(x=[0], y=[0],
+                                           histfunc="avg", title="Consumption by temperature")
+    consumption_fig_by_temp.update_traces(xbins_size=2)
+    consumption_fig_by_temp.update_layout(bargap=0.05)
+    consumption_fig_by_temp.add_trace(
+        go.Scatter(mode="markers", x=[0],
+                   y=[0], name="Trips"))
+    consumption_fig_by_temp.update_layout(xaxis_title="average temperature in °C",
+                                          yaxis_title="Consumption kWh/100Km")
     return True
 
 
-def __calculate_co2_per_kw(charging_data):
-    try:
-        co2_data = charging_data[charging_data["co2"] > 0]
-        co2_kw_sum = co2_data["kw"].sum()
-        if co2_kw_sum > 0:
-            return co2_data["co2"].sum() / co2_kw_sum
-    except KeyError:
-        return 0
-    return 0
-
-
 def get_battery_curve_fig(row: dict, car: Car):
-    start_date = Database.convert_datetime_from_string(row["start_at"])
-    stop_at = Database.convert_datetime_from_string(row["stop_at"])
+    start_date = dash_date_to_datetime(row["start_at"])
+    stop_at = dash_date_to_datetime(row["stop_at"])
     conn = Database.get_db()
     res = Database.get_battery_curve(conn, start_date, car.vin)
     conn.close()

web/utils.py

@@ -0,0 +1,66 @@
+from datetime import datetime, timedelta
+
+import dash_bootstrap_components as dbc
+import dash_html_components as html
+from dash.development.base_component import Component
+
+
+def unix_time_millis(date):
+    return int(date.timestamp())
+
+
+def get_marks_from_start_end(start, end):
+    nb_marks = 10
+    result = []
+    time_delta = int((end - start).total_seconds() / nb_marks)
+    current = start
+    if time_delta > 0:
+        while current <= end:
+            result.append(current)
+            current += timedelta(seconds=time_delta)
+        result[-1] = end
+        if time_delta < 3600 * 24:
+            if time_delta > 3600:
+                date_f = '%x %Hh'
+            else:
+                date_f = '%x %Hh%M'
+        else:
+            date_f = '%x'
+        marks = {}
+        for date in result:
+            marks[unix_time_millis(date)] = str(date.strftime(date_f))
+        return marks
+    return None
+
+
+def card_value_div(card_id, unit, value="-"):
+    return html.Div([html.Div(value, id=card_id, className="mr-2"), html.Div(unit)],
+                    className="d-flex flex-row justify-content-center")
+
+
+def dash_date_to_datetime(st):
+    return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def create_card(card: dict):
+    res = []
+    for tile, value in card.items():
+        rows = value["text"]
+        # if isinstance(text, str):
+        #     text = html.H3(text)
+        html_text = []
+        for row in rows:
+            html_text.append(html.Div(row, className="d-flex flex-row justify-content-center"))
+        res.append(html.Div(
+            dbc.Card([
+                html.H4(tile, className="card-title text-center"),
+                dbc.Row([
+                    dbc.Col(dbc.CardBody(html_text, style={"whiteSpace": "nowrap", "fontSize": "160%"}),
+                            className="text-center"),
+                    dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"maxHeight": "7rem"}))
+                ],
+                    className="align-items-center flex-nowrap")
+            ], className="h-100 p-2"),
+            className="col-sm-12 col-md-6 col-lg-3 py-2"
+        ))
+    return res

web/views.py

@@ -1,17 +1,17 @@
 import json
-from datetime import datetime, timezone
 from typing import List
 
 import dash_bootstrap_components as dbc
 from dash.dependencies import Output, Input, MATCH, State
-from dash.development.base_component import Component
 from dash.exceptions import PreventUpdate
 import dash_core_components as dcc
 import dash_html_components as html
 import dash_daq as daq
-import pandas as pd
+from deepdiff import DeepDiff
 from flask import jsonify, request, Response as FlaskResponse
 
+import web.utils
+from libs.car import Cars, Car
 from mylogger import logger
 
 from trip import Trips
@@ -23,65 +23,22 @@ from web.app import app, dash_app, myp, chc
 from web.db import Database
 
 # pylint: disable=invalid-name
+from web.figure_filter import Figure_Filter
+from web.utils import dash_date_to_datetime, create_card
+
 RESPONSE = "-response"
 EMPTY_DIV = "empty-div"
 ABRP_SWITCH = 'abrp-switch'
 CALLBACK_CREATED = False
 
-trips: Trips
+trips: Trips = Trips()
 chargings: List[dict]
 min_date = max_date = min_millis = max_millis = step = marks = cached_layout = None
 
 
-def diff_dashtable(data, data_previous, row_id_name="row_id"):
-    df, df_previous = pd.DataFrame(data=data), pd.DataFrame(data_previous)
-    for _df in [df, df_previous]:
-        assert row_id_name in _df.columns
-        _df = _df.set_index(row_id_name)
-    mask = df.ne(df_previous)
-    df_diff = df[mask].dropna(how="all", axis="columns").dropna(how="all", axis="rows")
-    changes = []
-    for idx, row in df_diff.iterrows():
-        row.dropna(inplace=True)
-        for change in row.iteritems():
-            changes.append(
-                {
-                    row_id_name: data[idx][row_id_name],
-                    "column_name": change[0],
-                    "current_value": change[1],
-                    "previous_value": df_previous.at[idx, change[0]],
-                }
-            )
-    return changes
-
-
 def create_callback():  # noqa: MC0001
     global CALLBACK_CREATED
     if not CALLBACK_CREATED:
-        @dash_app.callback(Output('trips_map', 'figure'),
-                           Output('consumption_fig', 'figure'),
-                           Output('consumption_fig_by_speed', 'figure'),
-                           Output('consumption_graph_by_temp', 'children'),
-                           Output('summary-cards', 'children'),
-                           Output('tab_trips_fig', 'children'),
-                           Output('tab_charge', 'children'),
-                           Output('date-slider', 'max'),
-                           Output('date-slider', 'step'),
-                           Output('date-slider', 'marks'),
-                           Input('date-slider', 'value'))
-        def display_value(value):  # pylint: disable=unused-variable
-            mini = datetime.fromtimestamp(value[0], tz=timezone.utc)
-            maxi = datetime.fromtimestamp(value[1], tz=timezone.utc)
-            filtered_trips = Trips()
-            for trip in trips:
-                if mini <= trip.start_at <= maxi:
-                    filtered_trips.append(trip)
-            filtered_chargings = Charging.get_chargings(mini, maxi)
-            figures.get_figures(filtered_trips, filtered_chargings)
-            return figures.trips_map, figures.consumption_fig, figures.consumption_fig_by_speed, \
-                   figures.consumption_graph_by_temp, create_card(figures.SUMMARY_CARDS), \
-                   figures.table_fig, figures.battery_table, max_millis, step, marks
-
         @dash_app.callback(Output(EMPTY_DIV, "children"),
                            [Input("battery-table", "data_timestamp")],
                            [State("battery-table", "data"),
@@ -89,17 +46,22 @@ def create_callback():  # noqa: MC0001
         def capture_diffs_in_battery_table(timestamp, data, data_previous):  # pylint: disable=unused-variable
             if timestamp is None:
                 raise PreventUpdate
-            diff_data = diff_dashtable(data, data_previous, "start_at")
-            for changed_line in diff_data:
-                if changed_line['column_name'] == 'price':
+            diff_data = DeepDiff(data_previous, data, ignore_numeric_type_changes=True, ignore_order=True, view="tree",
+                                 verbose_level=1)
+            for value_changed in diff_data["values_changed"]:
+                index, column_name = value_changed.path(output_format='list')
+                new_value = value_changed.t2
+                if column_name == 'price':
                     conn = Database.get_db()
-                    if not Database.set_chargings_price( conn, changed_line['start_at'],
-                                                        changed_line['current_value']):
+                    date = dash_date_to_datetime(data[index]['start_at'])
+                    if not Database.set_chargings_price(conn, date, new_value):
                         logger.error("Can't find line to update in the database")
+                    else:
+                        logger.debug("update price %s of %s", value_changed, date)
                     conn.close()
-            return ""
+            return ""  # don't need to update dashboard
 
-        @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ],
+        @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open")],
                            [Input("battery-table", "active_cell"),
                             Input("tab_battery_popup-close", "n_clicks")],
                            [State('battery-table', 'data'),
@@ -116,7 +78,7 @@ def create_callback():  # noqa: MC0001
                            [Input("trips-table", "active_cell"),
                             Input("tab_trips_popup-close", "n_clicks")],
                            State("tab_trips_popup", "is_open"))
-        def get_altitude(active_cell, close, is_open):  # pylint: disable=unused-argument, unused-variable
+        def get_altitude_graph(active_cell, close, is_open):  # pylint: disable=unused-argument, unused-variable
             if is_open is None:
                 is_open = False
             if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open:
@@ -160,6 +122,21 @@ def get_vehicle_info(vin):
     return response
 
 
+STYLE_CACHE = None
+
+
+@app.route("/assets/style2.json")
+def get_style():
+    global STYLE_CACHE
+    if not STYLE_CACHE:
+        with open(app.root_path + "/assets/style.json", "r") as f:
+            res = json.loads(f.read())
+            STYLE_CACHE = res
+    url_root = request.url_root
+    STYLE_CACHE["sprite"] = url_root + "assets/sprites/osm-liberty@2x"
+    return jsonify(STYLE_CACHE)
+
+
 @app.route('/charge_now/<string:vin>/<int:charge>')
 def charge_now(vin, charge):
     return jsonify(myp.charge_now(vin, charge != 0))
@@ -248,14 +225,17 @@ def update_trips():
     conn.close()
     min_date = None
     max_date = None
+    car = myp.vehicles_list[0]  # todo handle multiple car
     try:
-        trips_by_vin = Trips.get_trips(myp.vehicles_list)
-        trips = next(iter(trips_by_vin.values()))  # todo handle multiple car
+        trips_by_vin = Trips.get_trips(Cars([car]))
+        trips = trips_by_vin[car.vin]
         assert len(trips) > 0
         min_date = trips[0].start_at
         max_date = trips[-1].start_at
-    except (StopIteration, AssertionError):
+        figures.get_figures(trips[0].car)
+    except (AssertionError, KeyError):
         logger.debug("No trips yet")
+        figures.get_figures(Car("vin","vid","brand"))
     try:
         chargings = Charging.get_chargings()
         assert len(chargings) > 0
@@ -272,11 +252,12 @@ def update_trips():
     # update for slider
     try:
         logger.debug("min_date:%s - max_date:%s", min_date, max_date)
-        min_millis = figures.unix_time_millis(min_date)
-        max_millis = figures.unix_time_millis(max_date)
+        min_millis = web.utils.unix_time_millis(min_date)
+        max_millis = web.utils.unix_time_millis(max_date)
         step = (max_millis - min_millis) / 100
-        marks = figures.get_marks_from_start_end(min_date, max_date)
+        marks = web.utils.get_marks_from_start_end(min_date, max_date)
         cached_layout = None  # force regenerate layout
+        figures.get_figures(car)
     except (ValueError, IndexError):
         logger.error("update_trips (slider): %s", exc_info=True)
     except AttributeError:
@@ -303,40 +284,12 @@ def __get_control_tabs():
     return tabs
 
 
-def create_card(card: dict):
-    res = []
-    for tile, value in card.items():
-        text = value["text"]
-        # if isinstance(text, str):
-        #     text = html.H3(text)
-        res.append(html.Div(
-            dbc.Card([
-                html.H4(tile, className="card-title text-center"),
-                dbc.Row([
-                    dbc.Col(dbc.CardBody(text, style={"white-space": "nowrap", "font-size": "160%"}),
-                            className="text-center"),
-                    dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"max-height": "7rem"}))
-                ],
-                    className="align-items-center flex-nowrap")
-            ], className="h-100 p-2"),
-            className="col-sm-12 col-md-6 col-lg-3 py-2"
-        ))
-    return res
-
-
 def serve_layout():
     global cached_layout
     if cached_layout is None:
         logger.debug("Create new layout")
+        fig_filter = Figure_Filter()
         try:
-            figures.get_figures(trips, chargings)
-            summary_tab = [dbc.Container(dbc.Row(id="summary-cards",
-                                                 children=create_card(figures.SUMMARY_CARDS)), fluid=True),
-                           dcc.Graph(figure=figures.consumption_fig, id="consumption_fig"),
-                           dcc.Graph(figure=figures.consumption_fig_by_speed, id="consumption_fig_by_speed"),
-                           figures.consumption_graph_by_temp]
-            maps = dcc.Graph(figure=figures.trips_map, id="trips_map", style={"height": '90vh'})
-            create_callback()
             range_slider = dcc.RangeSlider(
                 id='date-slider',
                 min=min_millis,
@@ -345,12 +298,31 @@ def serve_layout():
                 marks=marks,
                 value=[min_millis, max_millis],
             )
-        except (IndexError, TypeError, NameError):
+            summary_tab = [
+                dbc.Container(dbc.Row(id="summary-cards",
+                                      children=create_card(figures.SUMMARY_CARDS)), fluid=True),
+                fig_filter.add_graph(dcc.Graph(id="consumption_fig"), "start_at", ["consumption_km"],
+                                     figures.consumption_fig),
+                fig_filter.add_graph(dcc.Graph(id="consumption_fig_by_speed"), "speed_average",
+                                     ["consumption_km"] * 2, figures.consumption_fig_by_speed),
+                fig_filter.add_graph(dcc.Graph(id="consumption_graph_by_temp"), "consumption_by_temp",
+                                     ["consumption_km"] * 2, figures.consumption_fig_by_temp)]
+            maps = fig_filter.add_map(dcc.Graph(id="trips_map", style={"height": '90vh'}), "lat",
+                                      ["long", "start_at"], figures.trips_map)
+            fig_filter.add_table("trips", figures.table_fig)
+            fig_filter.add_table("chargings", figures.battery_table)
+            fig_filter.src = {"trips": trips.get_trips_as_dict(), "chargings": chargings}
+            dash_app.clientside_callback(*fig_filter.get_clientside_callback())
+            create_callback()
+        except (IndexError, TypeError, NameError, AssertionError, NameError):
             summary_tab = figures.ERROR_DIV
             maps = figures.ERROR_DIV
             logger.warning("Failed to generate figure, there is probably not enough data yet", exc_info_debug=True)
             range_slider = html.Div()
+            figures.battery_table = figures.ERROR_DIV
+
         data_div = html.Div([
+            *fig_filter.get_store(),
             range_slider,
             html.Div([
                 dbc.Tabs([
@@ -404,8 +376,8 @@ def serve_layout():
 
 
 try:
-    Database.set_db_callback(update_trips)
     Charging.set_default_price()
+    Database.set_db_callback(update_trips)
     update_trips()
 except (IndexError, TypeError):
     logger.debug("Failed to get trips, there is probably not enough data yet:", exc_info=True)