Merge pull request #117 from flobz/feature-client_filter

Feature client filter

.eslintrc.yml

@@ -0,0 +1,11 @@
+env:
+  browser: true
+  es2021: true
+extends:
+  - standard
+parserOptions:
+  ecmaFeatures:
+    jsx: true
+  ecmaVersion: 12
+  sourceType: module
+rules: {}

.pre-commit-config.yaml

@@ -5,4 +5,4 @@ repos:
     rev: 1.3.1 # The version of Prospector to use, at least 1.1.7
     hooks:
     -   id: prospector
-        language: system
+        language: system

.prospector.yaml

@@ -3,6 +3,7 @@ use: flask
 max-line-length: 120
 ignore-paths:
   - psa_connectedcar
+  - test
 pep8:
   disable:
     - E722

docs/Install.md

@@ -13,7 +13,7 @@ We will retrieve this information:
 - On debian based distribution you can install some requirement from repos, it's faster than installtion with pip: 
  
  ```
- sudo apt-get install python3-typing-extensions python3-pandas python3-plotly python3-paho-mqtt  python3-six python3-dateutil python3-brotli  libblas-dev  liblapack-dev gfortran python3-pycryptodome python3-numpy libatlas3-base python3-cryptography
+ sudo apt-get install python3-typing-extensions python3-plotly python3-paho-mqtt  python3-six python3-dateutil python3-brotli  libblas-dev  liblapack-dev gfortran python3-pycryptodome python3-cryptography
  ```
     
 - For everyone :

libs/car.py

@@ -38,6 +38,12 @@ class Car:
     def is_hybrid(self) -> bool:
         return self.fuel_capacity > 0 and self.battery_power > 0
 
+    def has_battery(self):
+        return self.battery_power > 0
+
+    def has_fuel(self):
+        return self.fuel_capacity > 0
+
     def get_status(self):
         if self.status is not None:
             return self.status

libs/charging.py

@@ -13,17 +13,9 @@ class Charging:
     elec_price: ElecPrice = ElecPrice(None)
 
     @staticmethod
-    def get_chargings(mini=None, maxi=None) -> List[dict]:
+    def get_chargings() -> List[dict]:
         conn = Database.get_db()
-        if mini is not None:
-            if maxi is not None:
-                res = conn.execute("select * from battery WHERE start_at>=? and start_at<=?", (mini, maxi)).fetchall()
-            else:
-                res = conn.execute("select * from battery WHERE start_at>=?", (mini,)).fetchall()
-        elif maxi is not None:
-            res = conn.execute("select * from battery WHERE start_at<=?", (maxi,)).fetchall()
-        else:
-            res = conn.execute("select * from battery").fetchall()
+        res = conn.execute("select * from battery ORDER BY start_at").fetchall()
         conn.close()
         return list(map(dict, res))
 

my_psacc.py

@@ -22,7 +22,7 @@ from otp.otp import load_otp, new_otp_session, save_otp, ConfigException, Otp
 from psa_connectedcar.rest import ApiException
 from mylogger import logger
 
-from utils import rate_limit
+from libs.utils import rate_limit
 from web.abrp import Abrp
 from web.db import Database
 

requirements-dev.txt

@@ -1,3 +1,2 @@
 prospector>=1.3.0
 pre-commit
-deepdiff

requirements.txt

@@ -4,10 +4,8 @@ dash_daq
 plotly>=4
 cryptography>=2.6
 Werkzeug>=1.0.0
-pandas
 oauth2_client
 requests
-numpy
 pytz
 typing
 argparse
@@ -17,6 +15,7 @@ geojson
 reverse_geocode
 androguard
 pycryptodomex
+deepdiff
 
 #swagger req
 certifi >= 14.05.14

server.py

@@ -16,7 +16,7 @@ from libs.elec_price import ElecPrice
 from mylogger import my_logger
 from mylogger import logger
 from my_psacc import MyPSACC
-from utils import is_port_in_use
+from libs.utils import is_port_in_use
 from web.app import start_app, save_config
 
 CONFIG_NAME = "config.json"

test/test_unit.py

@@ -0,0 +1,255 @@
+# flake8: noqa
+import json
+import os
+import unittest
+from datetime import datetime, timedelta
+from psa_connectedcar import ApiClient
+import psa_connectedcar as psacc
+import reverse_geocode
+from libs.car import Car, Cars
+from libs.charging import Charging
+from libs.elec_price import ElecPrice
+from my_psacc import MyPSACC
+from ecomix import Ecomix
+from libs.car_model import CarModel
+from mylogger import my_logger
+from otp.otp import load_otp, save_otp
+from charge_control import ChargeControls
+from trip import Trips
+from libs.utils import get_temp
+from web.db import Database
+from web.figures import get_figures, get_battery_curve_fig, get_altitude_fig
+import pytz
+from deepdiff import DeepDiff
+
+latitude = 47.2183
+longitude = -1.55362
+date3 = datetime.utcnow().replace(2021, 3, 1, 12, 00, 00, 00, tzinfo=pytz.UTC)
+date2 = date3 - timedelta(minutes=20)
+date1 = date3 - timedelta(minutes=40)
+date0 = date3 - timedelta(minutes=60)
+DATA_DIR = os.path.dirname(os.path.realpath(__file__)) + "/data/"
+
+
+def compare_dict(result, expected):
+    diff = DeepDiff(expected,  result)
+    if diff != {}:
+        raise AssertionError(str(diff))
+    return True
+
+
+dummy_value = 0
+
+
+def callback_test():
+    global dummy_value
+    dummy_value += 1
+
+
+class TestUnit(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        super().__init__(methodName)
+        self.test_online = os.environ.get("TEST_ONLINE", "0") == "1"
+        self.vehicule_list = Cars()
+        self.vehicule_list.extend(
+            [Car("VR3UHZKX", "vid", "Peugeot"), Car("VXXXXX", "XXXX", "Peugeot", label="SUV 3008")])
+
+    @staticmethod
+    def get_new_test_db():
+        try:
+            os.remove(DATA_DIR + "tmp.db")
+        except:
+            pass
+        Database.DEFAULT_DB_FILE = DATA_DIR + "tmp.db"
+        Database.db_initialized = False
+        conn = Database.get_db()
+        return conn
+
+    def test_car(self):
+        car1 = Car("VRAAAAAAA", "1sdfdksnfk222", "Peugeot", "208", 46, 0)
+        car2 = Car("VR3UHZKX", "1sdfdksnfk222", "Peugeot")
+        cars = Cars([car1, car2])
+        cars.save_cars(name=DATA_DIR + "test_car.json")
+        Cars.load_cars(name=DATA_DIR + "test_car.json")
+
+    def test_otp_config(self):
+        otp_config = load_otp(filename=DATA_DIR + "otp_test.bin")
+        assert otp_config is not None
+        save_otp(otp_config, filename=DATA_DIR + "otp_test2.bin")
+
+    def test_mypsacc(self):
+        if self.test_online:
+            myp = MyPSACC.load_config("config.json")
+            myp.refresh_token()
+            myp.get_vehicles()
+            car = myp.vehicles_list[0]
+            myp.abrp.abrp_enable_vin.add(car.vin)
+            res = myp.get_vehicle_info(myp.vehicles_list[0].vin)
+            myp.abrp.call(car, 22.1)
+            myp.save_config()
+            assert isinstance(get_temp(str(latitude), str(longitude), myp.weather_api), float)
+
+    def test_car_model(self):
+        assert CarModel.find_model_by_vin("VR3UHZKXZL").name == "e-208"
+        assert CarModel.find_model_by_vin("VR3UKZKXZM").name == "e-2008"
+        assert CarModel.find_model_by_vin("VXKUHZKXZL").name == "corsa-e"
+
+    def test_c02_signal_cache(self):
+        start = datetime.now() - timedelta(minutes=30)
+        end = datetime.now()
+        Ecomix._cache = {'FR': [[start - timedelta(days=1), 100],
+                                [start + timedelta(minutes=1), 10],
+                                [start + timedelta(minutes=2), 20],
+                                [start + timedelta(minutes=3), 30]]}
+        assert Ecomix.get_co2_from_signal_cache(start, end, "FR") == 20
+
+    def test_c02_signal(self):
+        if self.test_online:
+            key = "d186c74bfbcd1da8"
+            Ecomix.co2_signal_key = key
+            def_country = "FR"
+            Ecomix.get_data_from_co2_signal(latitude, longitude, def_country)
+            res = Ecomix.get_co2_from_signal_cache(datetime.now() - timedelta(minutes=5), datetime.now(), def_country)
+            assert isinstance(res, float)
+
+    def test_charge_control(self):
+        charge_control = ChargeControls()
+        charge_control.file_name = "test_charge_control.json"
+        charge_control.save_config(force=True)
+
+    def test_battery_curve(self):
+        from libs.car import Car
+        from libs.charging import Charging
+        try:
+            os.remove("tmp.db")
+        except:
+            pass
+        Database.DEFAULT_DB_FILE = "tmp.db"
+        conn = Database.get_db()
+        list(map(dict, conn.execute('PRAGMA database_list').fetchall()))
+        vin = "VR3UHZKXZL"
+        car = Car(vin, "id", "Peugeot")
+        Charging.record_charging(car, "InProgress", date0, 50, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "InProgress", date1, 75, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "InProgress", date2, 85, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "InProgress", date3, 90, latitude, longitude, "FR", "slow")
+
+        res = Database.get_battery_curve(Database.get_db(), date0, vin)
+        assert len(res) == 3
+
+    def test_sdk(self):
+
+        res = {
+            'lastPosition': {'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [9.65457, 49.96119, 21]},
+                             'properties': {'updatedAt': '2021-03-29T05:16:10Z', 'heading': 126,
+                                            'type': 'Estimated'}}, 'preconditionning': {
+                'airConditioning': {'updatedAt': '2021-04-01T16:17:01Z', 'status': 'Disabled', 'programs': [
+                    {'enabled': False, 'slot': 1, 'recurrence': 'Daily', 'start': 'PT21H40M',
+                     'occurence': {'day': ['Sat']}}]}},
+            'energy': [{'updatedAt': '2021-02-23T22:29:03Z', 'type': 'Fuel', 'level': 0},
+                       {'updatedAt': '2021-04-01T16:17:01Z', 'type': 'Electric', 'level': 70, 'autonomy': 192,
+                        'charging': {'plugged': False, 'status': 'Disconnected', 'remainingTime': 'PT0S',
+                                     'chargingRate': 0, 'chargingMode': 'No', 'nextDelayedTime': 'PT21H30M'}}],
+            'createdAt': '2021-04-01T16:17:01Z',
+            'battery': {'voltage': 99, 'current': 0, 'createdAt': '2021-04-01T16:17:01Z'},
+            'kinetic': {'createdAt': '2021-03-29T05:16:10Z', 'moving': False},
+            'privacy': {'createdAt': '2021-04-01T16:17:01Z', 'state': 'None'},
+            'service': {'type': 'Electric', 'updatedAt': '2021-02-23T21:10:29Z'}, '_links': {'self': {
+                'href': 'https://api.groupe-psa.com/connectedcar/v4/user/vehicles/myid/status'},
+                'vehicles': {
+                    'href': 'https://api.groupe-psa.com/connectedcar/v4/user/vehicles/myid'}},
+            'timed.odometer': {'createdAt': None, 'mileage': 1107.1}, 'updatedAt': '2021-04-01T16:17:01Z'}
+        api = ApiClient()
+        status: psacc.models.status.Status = api._ApiClient__deserialize(res, "Status")
+        geocode_res = reverse_geocode.search([(status.last_position.geometry.coordinates[:2])[::-1]])[0]
+        assert geocode_res["country_code"] == "DE"
+        TestUnit.get_new_test_db()
+        car = Car("XX", "vid", "Peugeot")
+        car.status = status
+        myp = MyPSACC.load_config(DATA_DIR + "config.json")
+        myp.record_info(car)
+        assert "features" in json.loads(Database.get_recorded_position())
+        # electric should be first
+        assert car.status.energy[0].type == 'Electric'
+
+    def test_record_position_charging(self):
+        TestUnit.get_new_test_db()
+        ElecPrice.CONFIG_FILENAME = DATA_DIR + "config.ini"
+        car = self.vehicule_list[0]
+        Database.record_position(None, car.vin, 11, latitude, longitude - 0.05, None, date0, 40, None, False)
+        Database.record_position(None, car.vin, 20, latitude, longitude, 32, date1, 35, None, False)
+        Database.record_position(None, car.vin, 30, latitude, longitude, 42, date2, 30, None, False)
+        Database.add_altitude_to_db(Database.get_db())
+        data = json.loads(Database.get_recorded_position())
+        assert data["features"][1]["geometry"]["coordinates"] == [float(longitude), float(latitude)]
+        trips = Trips.get_trips(self.vehicule_list)[car.vin]
+        trip = trips[0]
+        map(trip.add_temperature, [10, 13, 15])
+        res = trip.get_info()
+        assert compare_dict(res, {'consumption_km': 24.21052631578947,
+                                  'start_at': date0,
+                                  'consumption_by_temp': None,
+                                  'positions': {'lat': [latitude], 'long': [longitude]},
+                                  'duration': 40.0, 'speed_average': 28.5, 'distance': 19.0, 'mileage': 30.0,
+                                  'altitude_diff': 2, 'id': 1, 'consumption': 4.6})
+
+        Charging.elec_price = ElecPrice.read_config()
+        start_level = 40
+        end_level = 85
+        Charging.record_charging(car, "InProgress", date0, start_level, latitude, longitude, None, "slow")
+        Charging.record_charging(car, "InProgress", date1, 70, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "InProgress", date1, 70, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "InProgress", date2, 80, latitude, longitude, "FR", "slow")
+        Charging.record_charging(car, "Stopped", date3, end_level, latitude, longitude, "FR", "slow")
+        chargings = Charging.get_chargings()
+        co2 = chargings[0]["co2"]
+        assert isinstance(co2, float)
+        assert compare_dict(chargings, [{'start_at': date0,
+                                         'stop_at': date3,
+                                         'VIN': 'VR3UHZKX',
+                                         'start_level': 40,
+                                         'end_level': 85,
+                                         'co2': co2,
+                                         'kw': 20.7,
+                                         'price': 3.84,
+                                         'charging_mode': 'slow'}])
+        assert get_figures(car)
+        row = {"start_at": date0.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
+               "stop_at": date3.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "start_level": start_level, "end_level": end_level}
+        assert get_battery_curve_fig(row, car) is not None
+        assert get_altitude_fig(trip) is not None
+
+    def test_fuel_car(self):
+        TestUnit.get_new_test_db()
+        ElecPrice.CONFIG_FILENAME = DATA_DIR + "config.ini"
+        car = self.vehicule_list[1]
+        Database.record_position(None, car.vin, 11, latitude, longitude, 22, date0, 40, 30, False)
+        Database.record_position(None, car.vin, 20, latitude, longitude, 22, date1, 35, 29, False)
+        Database.record_position(None, car.vin, 30, latitude, longitude, 22, date2, 30, 28, False)
+        trips = Trips.get_trips(self.vehicule_list)
+        res = trips[car.vin].get_trips_as_dict()
+        assert compare_dict(res, [{'consumption_km': 5.684210526315789,
+                                   'start_at': date0,
+                                   'consumption_by_temp': None,
+                                   'positions': {'lat': [latitude],
+                                                'long': [longitude]},
+                                   'duration': 40.0,
+                                   'speed_average': 28.5,
+                                   'distance': 19.0,
+                                   'mileage': 30.0,
+                                   'altitude_diff': 0,
+                                   'id': 1,
+                                   'consumption': 1.08,
+                                   'consumption_fuel_km': 10.53}])
+
+    def test_db_callback(self):
+        old_dummy_value = dummy_value
+        TestUnit.get_new_test_db()
+        Database.set_db_callback(callback_test)
+        assert old_dummy_value == dummy_value
+        Database.record_position(None, "xx", 11, latitude, longitude - 0.05, None, date0, 40, None, False)
+        assert old_dummy_value != dummy_value
+
+if __name__ == '__main__':
+    my_logger(handler_level=os.environ.get("DEBUG_LEVEL", 20))
+    unittest.main()

trip.py

@@ -2,7 +2,6 @@ import logging
 from statistics import mean
 from typing import List, Dict
 
-from dateutil import tz
 from geojson import Feature, FeatureCollection, MultiLineString
 
 from libs.car import Cars, Car
@@ -38,6 +37,7 @@ class Trip:
         self.car: Car = None
         self.altitude_diff = None
         self.temperatures = []
+        self.id = None
 
     def add_points(self, latitude, longitude):
         self.positions.append(Points(latitude, longitude))
@@ -78,19 +78,22 @@ class Trip:
                                                                "average consumption": self.consumption_km,
                                                                "average consumption fuel": self.consumption_fuel_km})
 
-    def get_info(self, row_id=None):
-        res = {"start_at": self.start_at.astimezone(tz.tzlocal()).replace(tzinfo=None).strftime("%x %X"),
-               # convert to naive tz,
-               "duration": self.duration * 60, "speed_average": self.speed_average,
-               "consumption_km": self.consumption_km, "consumption_fuel_km": self.consumption_fuel_km,
-               "distance": self.distance, "mileage": self.mileage, "altitude_diff": self.altitude_diff}
-        if row_id is not None:
-            res["id"] = row_id
-        return res
 
-    def get_consumption(self):
-        return {"speed": self.speed_average, "consumption_km": self.consumption_km, "date": self.start_at,
-                "consumption_by_temp": self.get_temperature()}
+    def get_info(self):
+
+        res =  {"consumption_km": self.consumption_km, "start_at": self.start_at,
+                "consumption_by_temp": self.get_temperature(), "positions": self.get_positions(),
+                "duration": self.duration * 60, "speed_average": self.speed_average, "distance": self.distance,
+                "mileage": self.mileage, "altitude_diff": self.altitude_diff, "id": self.id,
+                "consumption": self.consumption
+                }
+        if self.car.has_battery():
+            res["consumption_km"] = self.consumption_km
+
+        if self.car.has_fuel():
+            res["consumption_fuel_km"] = self.consumption_fuel_km
+
+        return res
 
     def set_altitude_diff(self, start, end):
         try:
@@ -98,18 +101,26 @@ class Trip:
         except (NameError, TypeError):
             pass
 
+    def get_positions(self):
+        lat = []
+        long = []
+        for position in self.positions:
+            lat.append(position.latitude)
+            long.append(position.longitude)
+        return {"lat": lat, "long": long}
+
 
 class Trips(list):
     def __init__(self, *args):
         list.__init__(self, *args)
+        self.trip_num = 1
 
     def to_geo_json(self):
         feature_collection = FeatureCollection(self)
         return feature_collection
 
-    def get_long_trips(self):
-        res = [trip.get_consumption() for trip in self if trip.consumption > 1.8]
-        return res
+    def get_trips_as_dict(self):
+        return [trip.get_info() for trip in self]
 
     def get_distance(self):
         return self[-1].mileage - self[0].mileage
@@ -117,6 +128,8 @@ class Trips(list):
     def check_and_append(self, trip: Trip):
         if trip.consumption_km <= trip.car.max_elec_consumption and \
                 trip.consumption_fuel_km <= trip.car.max_fuel_consumption:
+            trip.id = self.trip_num
+            self.trip_num += 1
             self.append(trip)
             return True
         logger.debugv("trip discarded")
@@ -210,11 +223,3 @@ class Trips(list):
                 trips_by_vin[vin] = trips
         conn.close()
         return trips_by_vin
-
-    def get_info(self):
-        res = []
-        row_id = 1
-        for trip in self:
-            res.append(trip.get_info(row_id))
-            row_id += 1
-        return res

web/app.py

@@ -27,7 +27,8 @@ myp: MyPSACC = None
 chc: ChargeControls = None
 
 
-def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pylint: disable=too-many-arguments
+def start_app(title, base_path, debug: bool, host, port, reloader=False,   # pylint: disable=too-many-arguments
+              unminified=False):
     global app, dash_app, dispatcher
     try:
         lang = locale.getlocale()[0].split("_")[0]
@@ -36,6 +37,8 @@ def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pyl
     except (IndexError, locale.Error):
         locale_url = None
         logger.warning("Can't get language")
+    if unminified:
+        locale_url = ["assets/plotly-with-meta.js"]
     app = Flask(__name__)
     app.config["DEBUG"] = debug
     if base_path == "/":
@@ -46,6 +49,7 @@ def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pyl
         requests_pathname_prefix = base_path + "/"
     dash_app = dash.Dash(external_stylesheets=[dbc.themes.BOOTSTRAP], external_scripts=locale_url, title=title,
                          server=app, requests_pathname_prefix=requests_pathname_prefix)
+    dash_app.enable_dev_tools(reloader)
     # keep this line
     import web.views  # pylint: disable=unused-import,import-outside-toplevel
     return run_simple(host, port, application, use_reloader=reloader, use_debugger=debug)

web/assets/clientside.js

@@ -0,0 +1,227 @@
+class Avg {
+  constructor () {
+    this.total = 0
+    this.count = 0
+  }
+
+  addValue (value) {
+    if (typeof value === 'number') {
+      this.count++
+      this.total = ((this.total * (this.count - 1)) / this.count) + (value / this.count)
+    }
+  }
+
+  average () {
+    return this.total
+  }
+
+  static getAverageFromKey (array, key) {
+    const avg = new Avg()
+    array.forEach(function (obj) { avg.addValue(obj[key]) })
+    return avg.average()
+  }
+}
+const logger = (function () {
+  let oldConsoleLog = null
+  const pub = {}
+
+  pub.enableLogger = function enableLogger () {
+    if (oldConsoleLog == null) {
+      return
+    }
+
+    window.console.log = oldConsoleLog
+  }
+
+  pub.disableLogger = function disableLogger () {
+    oldConsoleLog = console.log
+    window.console.log = function () {}
+  }
+
+  return pub
+}())
+function addLocaleDate (data, dateKey) {
+  const dateOption = [undefined, { hour: 'numeric', minute: 'numeric' }]
+  function dateToLocale (row, key) {
+    const date = new Date(row[key])
+    row[key] = date
+    row[key + '_str'] = date.toLocaleDateString(...dateOption)
+  }
+  let datasetName, dataset
+  for ([datasetName, dataset] of Object.entries(data)) {
+    dataset.forEach(function (row) {
+      dateKey[datasetName].forEach(key => dateToLocale(row, key))
+    })
+  }
+}
+
+function filterDataset (data, range) {
+  function dateFromISO (st) {
+    return new Date(st).getTime() / 1000
+  }
+  function isInRange (st) {
+    const tsDate = dateFromISO(st)
+    return tsDate >= range[0] && tsDate <= range[1]
+  }
+  const res = {}
+  res.trips = data.trips.filter(line => isInRange(line.start_at))
+  res.chargings = data.chargings.filter(line => isInRange(line.start_at))
+  console.log('filtered_dataset', res)
+  return res
+}
+
+function filterShortTrip (data) {
+  const longTrips = {
+    trips: data.trips.filter(line => line.distance > 10),
+    chargings: data.chargings
+  }
+  console.log('long trips:', longTrips)
+  return longTrips
+}
+
+function updateFigures (data, oldFigure, x, y) {
+  const trips = data.trips
+  const figures = []
+  let i = 0
+  y.forEach(function (yLabel) {
+    const xLabel = x[i]
+    const figure = Object.assign({}, oldFigure[i])
+    i++
+    // console.log(oldFigure[i]);
+    // var unique_y_label = y[i].filter((v, i, a) => a.indexOf(v) === i);
+    // var data_nonnull = trips
+    // unique_y_label.forEach(function(label) {
+    //     data_nonnull = data_nonnull.filter(line => line[label]);
+    // });
+    if ('mapbox' in figure.layout) {
+      figure.data[0].lat = []
+      figure.data[0].lon = []
+      figure.data[0].hovertext = []
+      let trip = null
+      for (trip of trips) {
+        const xPos = trip.positions[xLabel]
+        figure.data[0].lat.push(...xPos, null)
+        figure.data[0].lon.push(...trip.positions[yLabel[0]])
+        figure.data[0].hovertext.push(...Array(xPos.length).fill(trip[yLabel[1]]), null)
+      }
+      if (trip) {
+        const lastPos = trip.positions[yLabel[0]].length - 1
+        figure.layout.mapbox.center.lat = trip.positions[xLabel][lastPos]
+        figure.layout.mapbox.center.lon = trip.positions[yLabel[0]][lastPos]
+        figure.data[1].lat = [figure.layout.mapbox.center.lat]
+        figure.data[1].lon = [figure.layout.mapbox.center.lon]
+      }
+    } else {
+      const xValues = trips.map(a => a[xLabel])
+      // for each y label
+      for (let j = 0; j < yLabel.length; j++) {
+        figure.data[j].y = trips.map(a => a[yLabel[j]])
+        figure.data[j].x = xValues
+      }
+    }
+    console.log(xLabel, figure)
+    figures.push(figure)
+  })
+  return figures
+}
+
+function updateTables (data, tables) {
+  console.log('tables', tables)
+  const figures = []
+  tables.forEach(function (table) {
+    figures.push(data[table.src])
+  })
+  return figures
+}
+
+function updateCardsValue (data) {
+  const res = {}
+  let avgPriceKw
+  let avgC02 = new Avg(); let avgKw = new Avg(); const avgTime = new Avg()
+  const avgPrice = new Avg()
+  data.chargings.forEach(function (charge) {
+    const diff = ((new Date(charge.stop_at)) - (new Date(charge.start_at))) / 3600000
+    avgKw.addValue(charge.kw)
+    avgC02.addValue(charge.co2)
+    avgPrice.addValue(charge.price)
+    if (diff > 0) {
+      avgTime.addValue(diff)
+    }
+  })
+  if (data.chargings.length > 0) {
+    avgKw = avgKw.average()
+    avgC02 = avgC02.average()
+    avgPriceKw = avgPrice.average() / avgKw
+    res.avg_emission_kw = avgC02
+    res.avg_chg_speed = avgKw / avgTime.average()
+  }
+  if (data.trips.length > 0) {
+    const totalDistance = data.trips[data.trips.length - 1].mileage - data.trips[0].mileage
+    res.avg_consum_kw = Avg.getAverageFromKey(data.trips, 'consumption_km')
+    res.elec_consum_kw = totalDistance * res.avg_consum_kw / 100
+  }
+  if (data.trips.length > 0 && data.chargings.length > 0) {
+    res.avg_emission_km = res.avg_emission_kw * res.avg_consum_kw / 100
+    res.elec_consum_price = avgPriceKw * res.elec_consum_kw
+    res.avg_consum_price = avgPriceKw * res.avg_consum_kw
+  }
+  for (const [key, value] of Object.entries(res)) {
+    document.getElementById(key).innerHTML = value.toPrecision(3)
+  }
+}
+
+function sortDataset (ctx, data, tables) {
+  const tableId = ctx.prop_id.split('.')[0]
+  if (ctx.value.length > 0) {
+    const asc = ctx.value[0].direction === 'asc'
+    let columnId = ctx.value[0].column_id
+    const table = tables.filter(table => table.table_id === tableId)[0]
+    let sorted
+    if (columnId.endsWith('_str')) {
+      columnId = columnId.slice(0, -4)
+      sorted = data[table.src].sort(function (a, b) {
+        return a[columnId] - b[columnId]
+      })
+    } else if (typeof data[table.src][0][columnId] === 'number') {
+      sorted = data[table.src].sort(function (a, b) {
+        return a[columnId] - b[columnId]
+      })
+    } else {
+      sorted = data[table.src].sort((a, b) => a[columnId].localeCompare(b[columnId]))
+    }
+    if (asc === false) {
+      sorted = sorted.reverse()
+    }
+    data[table.src] = sorted
+  }
+}
+
+function filterAndSort (data, range, figures, p, log) { // eslint-disable-line no-unused-vars
+  if (log > 10) {
+    logger.disableLogger()
+  }
+  const ctx = dash_clientside.callback_context.triggered // eslint-disable-line  no-undef
+  const outFigures = []; let dataFiltered
+  console.log('figures:', figures)
+  console.log('data:', data)
+  console.log('ctx', ctx)
+  if (ctx.length > 0 && ctx[0].prop_id.endsWith('sort_by')) {
+    dataFiltered = filterDataset(data, range)
+    sortDataset(ctx[0], dataFiltered, p.table_src)
+    outFigures.push(...updateTables(dataFiltered, p.table_src))
+    outFigures.push(...figures.graph)
+    outFigures.push(...figures.maps)
+  } else {
+    addLocaleDate(data, p.date_columns)
+    dataFiltered = filterDataset(data, range)
+    outFigures.push(...updateTables(dataFiltered, p.table_src))
+    console.log(dataFiltered.trips.length)
+    const longTrips = filterShortTrip(dataFiltered)
+    console.log('trips', dataFiltered.trips.length)
+    console.log('longTrips', longTrips.trips.length)
+    outFigures.push(...updateFigures(longTrips, figures.graph, p.graph_x_label, p.graph_y_label))
+    outFigures.push(...updateFigures(dataFiltered, figures.maps, p.map_x_label, p.map_y_label))
+    updateCardsValue(longTrips)
+  }
+  return outFigures
+}

web/assets/style.json

@@ -0,0 +1,22 @@
+{
+        "version": 8,
+        "sources": {
+            "osm": {
+                "type": "raster",
+                "tiles": [
+                    "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
+                ],
+                "tileSize": 256,
+                "attribution": "Map tiles by <a target=\"_top\" rel=\"noopener\" href=\"https://tile.openstreetmap.org/\">OpenStreetMap tile servers</a>, under the <a target=\"_top\" rel=\"noopener\" href=\"https://operations.osmfoundation.org/policies/tiles/\">tile usage policy</a>. Data by <a target=\"_top\" rel=\"noopener\" href=\"http://openstreetmap.org\">OpenStreetMap</a>"
+            }
+        },
+        "sprite": "",
+        "glyphs": "https://api.maptiler.com/fonts/{fontstack}/{range}.pbf",
+        "layers": [
+            {
+                "id": "osm",
+                "type": "raster",
+                "source": "osm"
+            }
+        ]
+}

web/db.py

@@ -11,7 +11,7 @@ from geojson import Feature, Point, FeatureCollection
 from geojson import dumps as geo_dumps
 
 from mylogger import logger
-from utils import get_temp
+from libs.utils import get_temp
 
 NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]]
 NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]]
@@ -126,7 +126,10 @@ class Database:
     def clean_battery(conn):
         # delete charging longer than 17h
         conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;")
-        conn.execute("DELETE FROM battery WHERE start_level==end_level;")
+        # delete charging not finished longer than 17h
+        conn.execute("DELETE from battery where stop_at is NULL and JULIANDAY()-JULIANDAY(start_at)>0.7;")
+        #delete little charge
+        conn.execute("DELETE FROM battery WHERE start_level >= end_level-1;")
 
     @staticmethod
     def clean_position(conn):

web/figure_filter.py

@@ -0,0 +1,121 @@
+import json
+from logging import DEBUG
+
+from dash.dependencies import Output, Input
+from dash_core_components import Store
+from mylogger import logger
+
+
+class Graph:
+    def __init__(self, graph_id, x, y: [], figure):
+        self.graph_id = graph_id
+        self.x = x
+        self.y = y
+        self.figure = figure
+
+
+class Table:
+    def __init__(self, table_id, src, figure):
+        self.table_id = table_id
+        self.src = src
+        self.figure = figure
+        self.date_columns = []
+
+
+def figures_to_dict(figures):
+    el_list = []
+    for figure in figures:
+        res = {}
+        for key, value in figure.__dict__.items():
+            if key != "figure":
+                res[key] = value
+        el_list.append(res)
+    return el_list
+
+
+class Figure_Filter:
+
+    def __init__(self):
+        self.graphs = []
+        self.tables = []
+        self.maps = []
+        self.src = {}
+
+    def add_map(self, dash_Graph, latitude, longitude, figure):
+        self.maps.append(Graph(dash_Graph.id, latitude, longitude, figure))
+        return dash_Graph
+
+    def add_graph(self, dash_Graph, x, y, figure):
+        self.graphs.append(Graph(dash_Graph.id, x, y, figure))
+        return dash_Graph
+
+    def add_table(self, src, figure):
+        table = Table(figure.id, src, figure)
+        table.date_columns = [col["id"][:-4] for col in figure.columns if col["type"] == "datetime" and
+                              col["id"].endswith("_str")]
+        self.tables.append(table)
+
+    def __get_table_date_column_id(self):
+        res = {table.src: table.date_columns for table in self.tables}
+        return res
+
+    def __get_table_src(self):
+        return [table.src for table in self.tables]
+
+    def __get_figures(self):
+        return {"graph": [graph.figure for graph in self.graphs],
+                "tables": [table.figure for table in self.tables],
+                "maps": [map.figure for map in self.maps]}
+
+    def __get_output(self) -> list:
+        outputs = [Output(table.table_id, "data") for table in self.tables]
+        outputs.extend([Output(graph.graph_id, "figure") for graph in self.graphs])
+        outputs.extend([Output(graph.graph_id, "figure") for graph in self.maps])
+        return outputs
+
+    def __get_graph_x_label(self, graphs):
+        return [graph.x for graph in graphs]
+
+    def __get_graph_y_label(self, graphs):
+        return [graph.y for graph in graphs]
+
+    def __get_table_input_sort_by(self):
+        inputs = [Input(table.table_id, 'sort_by') for table in self.tables]
+        return inputs
+
+    def gen_unused_variable(self):
+        res = ", ".join([chr(i) for i in range(ord('a'), ord('a') + len(self.tables))])
+        return res
+
+    def get_params(self):
+        params = json.dumps({
+            "date_columns": self.__get_table_date_column_id(),
+            "table_src": figures_to_dict(self.tables),
+            "graph_x_label": self.__get_graph_x_label(self.graphs),
+            "graph_y_label": self.__get_graph_y_label(self.graphs),
+            "map_x_label": self.__get_graph_x_label(self.maps),
+            "map_y_label": self.__get_graph_y_label(self.maps)
+        }, indent=4)
+        return params
+
+    def get_clientside_callback(self):
+        if logger.isEnabledFor(DEBUG):
+            log_level = 10
+        else:
+            log_level = 20
+        fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{
+                        const params={self.get_params()};
+                        const logLevel={log_level};
+                        return filterAndSort(data, range, figures, params, logLevel);
+                      }}"""
+        res = [fct_def,
+               *self.__get_output(),
+               Input('clientside-data-store', 'data'),
+               Input('date-slider', 'value'),
+               Input('clientside-figure-store', 'data'),
+               *self.__get_table_input_sort_by()]
+        return res
+
+    def get_store(self):
+        return [Store(id='clientside-figure-store', data=self.__get_figures()),
+                Store(id='clientside-data-store', data=self.src)]

web/figures.py

@@ -1,52 +1,18 @@
 from copy import deepcopy
 
-from typing import List
-
 import dash_bootstrap_components as dbc
 import dash_table
-import numpy as np
 from dash_core_components import Graph
 from dash_table.Format import Format, Scheme, Symbol
-from dateutil.relativedelta import relativedelta
 import plotly.express as px
 import plotly.graph_objects as go
-from pandas import DataFrame
-from pandas import options as pandas_options
 import dash_html_components as html
 
 from libs.car import Car
 from libs.elec_price import ElecPrice
-from trip import Trips, Trip
+from trip import Trip
 from web.db import Database
-
-
-def unix_time_millis(date):
-    return int(date.timestamp())
-
-
-def get_marks_from_start_end(start, end):
-    nb_marks = 10
-    result = []
-    time_delta = int((end - start).total_seconds() / nb_marks)
-    current = start
-    if time_delta > 0:
-        while current <= end:
-            result.append(current)
-            current += relativedelta(seconds=time_delta)
-        result[-1] = end
-        if time_delta < 3600 * 24:
-            if time_delta > 3600:
-                date_f = '%x %Hh'
-            else:
-                date_f = '%x %Hh%M'
-        else:
-            date_f = '%x'
-        marks = {}
-        for date in result:
-            marks[unix_time_millis(date)] = str(date.strftime(date_f))
-        return marks
-    return None
-
+from web.utils import card_value_div, dash_date_to_datetime
 
 # pylint: disable=invalid-name
 ERROR_DIV = dbc.Alert("No data to show, there is probably no trips recorded yet", color="danger")
@@ -55,45 +21,59 @@ consumption_fig = ERROR_DIV
 consumption_df = ERROR_DIV
 trips_map = ERROR_DIV
 consumption_fig_by_speed = ERROR_DIV
-consumption_graph_by_temp = ERROR_DIV
+consumption_fig_by_temp = ERROR_DIV
 table_fig = ERROR_DIV
-pandas_options.display.float_format = '${:.2f}'.format
 info = ""
-battery_info = ERROR_DIV
-battery_table = None
+battery_table = ERROR_DIV
 
-SUMMARY_CARDS = {"Average consumption": {"text": None, "src": "static/images/consumption.svg"},
-                 "Average emission": {"text": None, "src": "static/images/pollution.svg"},
-                 "Average charge speed": {"text": None, "src": "static/images/battery-charge-line.svg"},
-                 "Electricity consumption": {"text": None, "src": "static/images/electricity bill.svg"}
+AVG_CHARGE_SPEED = "avg_chg_speed"
+AVG_EMISSION_KM = "avg_emission_km"
+AVG_EMISSION_KW = "avg_emission_kw"
+ELEC_CONSUM_KW = "elec_consum_kw"
+ELEC_CONSUM_PRICE = "elec_consum_price"
+AVG_CONSUM_KW = "avg_consum_kw"
+AVG_CONSUM_PRICE = "avg_consum_price"
+
+SUMMARY_CARDS = {"Average consumption": {"text": [card_value_div(AVG_CONSUM_KW, "kWh/100km"),
+                                                  card_value_div(AVG_CONSUM_PRICE, f"{ElecPrice.currency}/100km")],
+                                         "src": "assets/images/consumption.svg"},
+                 "Average emission": {"text": [card_value_div(AVG_EMISSION_KM, " g/km"),
+                                               card_value_div(AVG_EMISSION_KW, "g/kWh")],
+                                      "src": "assets/images/pollution.svg"},
+                 "Average charge speed": {"text": [card_value_div(AVG_CHARGE_SPEED, " kW")],
+                                          "src": "assets/images/battery-charge-line.svg"},
+                 "Electricity consumption": {"text": [card_value_div(ELEC_CONSUM_KW, "kWh"),
+                                                      card_value_div(ELEC_CONSUM_PRICE, ElecPrice.currency)],
+                                             "src": "assets/images/electricity bill.svg"}
                  }
 
 
 # pylint: disable=too-many-locals
-def get_figures(trips: Trips, charging: List[dict]):
-    global consumption_fig, consumption_df, trips_map, consumption_fig_by_speed, table_fig, info, battery_info, \
-        battery_table, consumption_graph_by_temp
-    lats = []
-    lons = []
-    names = []
-    for trip in trips:
-        for points in trip.positions:
-            lats = np.append(lats, points.latitude)
-            lons = np.append(lons, points.longitude)
-            names = np.append(names, [str(trip.start_at)])
-        lats = np.append(lats, None)
-        lons = np.append(lons, None)
-        names = np.append(names, None)
-    trips_map = px.line_mapbox(lat=lats, lon=lons, hover_name=names,
-                               mapbox_style="stamen-terrain", zoom=12)
+def get_figures(car: Car):
+    global consumption_fig, consumption_df, trips_map, consumption_fig_by_speed, table_fig, info, \
+        battery_table, consumption_fig_by_temp
+    lats = [42, 41]
+    lons = [1, 2]
+    names = ["undefined", "undefined"]
+    trips_map = px.line_mapbox(lat=lats, lon=lons, hover_name=names, zoom=12, mapbox_style="assets/style2.json")
+    trips_map.add_trace(go.Scattermapbox(
+        mode="markers",
+        marker={"symbol": "marker", "size": 20},
+        lon=[lons[0]], lat=[lats[0]],
+        showlegend=False, name="Last Position"))
     # table
     nb_format = Format(precision=2, scheme=Scheme.fixed, symbol=Symbol.yes)  # pylint: disable=no-member
+    style_cell_conditional = []
+    if car.is_electric():
+        style_cell_conditional.append({'if': {'column_id': 'consumption_fuel_km', }, 'display': 'None', })
+    if car.is_thermal():
+        style_cell_conditional.append({'if': {'column_id': 'consumption_km', }, 'display': 'None', })
     table_fig = dash_table.DataTable(
         id='trips-table',
-        sort_action='native',
+        sort_action='custom',
         sort_by=[{'column_id': 'id', 'direction': 'desc'}],
         columns=[{'id': 'id', 'name': '#', 'type': 'numeric'},
-                 {'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
+                 {'id': 'start_at_str', 'name': 'start at', 'type': 'datetime'},
                  {'id': 'duration', 'name': 'duration', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" min").precision(0)},
                  {'id': 'speed_average', 'name': 'average speed', 'type': 'numeric',
@@ -116,51 +96,31 @@ def get_figures(trips: Trips, charging: List[dict]):
                 "text-decoration": "underline"
             }
         ],
-        data=trips.get_info(),
+        style_cell_conditional=style_cell_conditional,
+        data=[],
         page_size=50
     )
     # consumption_fig
-    consumption_df = DataFrame.from_records(trips.get_long_trips())
-    consumption_fig = px.histogram(consumption_df, x="date", y="consumption_km", title='Consumption of the car',
+    consumption_fig = px.histogram(x=[0], y=[1], title='Consumption of the car',
                                    histfunc="avg")
-    consumption_fig.update_layout(yaxis_title="Consumption kWh/100Km")
+    consumption_fig.update_layout(yaxis_title="Consumption kWh/100Km", xaxis_title="date")
 
-    consumption_fig_by_speed = px.histogram(consumption_df, x="speed", y="consumption_km", histfunc="avg",
+    consumption_fig_by_speed = px.histogram(data_frame=[{"start_at": 1, "speed_average": 2}], x="start_at",
+                                            y="speed_average", histfunc="avg",
                                             title="Consumption by speed")
     consumption_fig_by_speed.update_traces(xbins_size=15)
     consumption_fig_by_speed.update_layout(bargap=0.05)
-    consumption_fig_by_speed.add_trace(
-        go.Scatter(mode="markers", x=consumption_df["speed"], y=consumption_df["consumption_km"],
-                   name="Trips"))
+    consumption_fig_by_speed.add_trace(go.Scatter(mode="markers", x=[0],
+                                                  y=[0], name="Trips"))
     consumption_fig_by_speed.update_layout(xaxis_title="average Speed km/h", yaxis_title="Consumption kWh/100Km")
-    kw_per_km = float(consumption_df["consumption_km"].mean())
-    info = "Average consumption: {:.1f} kWh/100km".format(kw_per_km)
 
-    # charging
-    charging_data = DataFrame.from_records(charging)
-    co2_per_kw = __calculate_co2_per_kw(charging_data)
-    co2_per_km = co2_per_kw * kw_per_km / 100
-    try:
-        charge_speed = 3600 * charging_data["kw"].mean() / \
-                       (charging_data["stop_at"] - charging_data["start_at"]).mean().total_seconds()
-        price_kw = (charging_data["price"] / charging_data["kw"]).mean()
-        total_elec = kw_per_km * trips.get_distance() / 100
-    except (TypeError, KeyError, ZeroDivisionError):  # when there is no data yet:
-        charge_speed = 0
-        price_kw = 0
-        total_elec = 0
-
-    SUMMARY_CARDS["Average charge speed"]["text"] = f"{charge_speed:.2f} kW"
-    SUMMARY_CARDS["Average emission"]["text"] = [html.P(f"{co2_per_km:.1f} g/km"), html.P(f"{co2_per_kw:.1f} g/kWh")]
-    SUMMARY_CARDS["Electricity consumption"]["text"] = [f"{total_elec:.0f} kWh", html.Br(), \
-                                                        f"{total_elec * price_kw:.0f} {ElecPrice.currency}"]
-    SUMMARY_CARDS["Average consumption"]["text"] = f"{consumption_df['consumption_km'].mean():.1f} kWh/100km"
+    # battery_table
     battery_table = dash_table.DataTable(
         id='battery-table',
-        sort_action='native',
-        sort_by=[{'column_id': 'start_at', 'direction': 'desc'}],
-        columns=[{'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
-                 {'id': 'stop_at', 'name': 'stop at', 'type': 'datetime'},
+        sort_action='custom',
+        sort_by=[{'column_id': 'start_at_str', 'direction': 'desc'}],
+        columns=[{'id': 'start_at_str', 'name': 'start at', 'type': 'datetime'},
+                 {'id': 'stop_at_str', 'name': 'stop at', 'type': 'datetime'},
                  {'id': 'start_level', 'name': 'start level', 'type': 'numeric'},
                  {'id': 'end_level', 'name': 'end level', 'type': 'numeric'},
                  {'id': 'co2', 'name': 'CO2', 'type': 'numeric',
@@ -170,7 +130,7 @@ def get_figures(trips: Trips, charging: List[dict]):
                  {'id': 'price', 'name': 'price', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" " + ElecPrice.currency).precision(2), 'editable': True}
                  ],
-        data=charging,
+        data=[],
         style_data_conditional=[
             {
                 'if': {'column_id': ['start_level', "end_level"]},
@@ -179,42 +139,25 @@ def get_figures(trips: Trips, charging: List[dict]):
             },
             {
                 'if': {'column_id': 'price'},
-                'backgroundColor': 'rgb(230, 246, 254)'
+                'backgroundColor': '#ABE2FB'
             }
         ],
     )
-    consumption_by_temp_df = consumption_df[consumption_df["consumption_by_temp"].notnull()]
-    if len(consumption_by_temp_df) > 0:
-        consumption_fig_by_temp = px.histogram(consumption_by_temp_df, x="consumption_by_temp", y="consumption_km",
-                                               histfunc="avg", title="Consumption by temperature")
-        consumption_fig_by_temp.update_traces(xbins_size=2)
-        consumption_fig_by_temp.update_layout(bargap=0.05)
-        consumption_fig_by_temp.add_trace(
-            go.Scatter(mode="markers", x=consumption_by_temp_df["consumption_by_temp"],
-                       y=consumption_by_temp_df["consumption_km"], name="Trips"))
-        consumption_fig_by_temp.update_layout(xaxis_title="average temperature in °C",
-                                              yaxis_title="Consumption kWh/100Km")
-        consumption_graph_by_temp = html.Div(Graph(figure=consumption_fig_by_temp), id="consumption_graph_by_temp")
-
-    else:
-        consumption_graph_by_temp = html.Div(Graph(style={'display': 'none'}), id="consumption_graph_by_temp")
+    consumption_fig_by_temp = px.histogram(x=[0], y=[0],
+                                           histfunc="avg", title="Consumption by temperature")
+    consumption_fig_by_temp.update_traces(xbins_size=2)
+    consumption_fig_by_temp.update_layout(bargap=0.05)
+    consumption_fig_by_temp.add_trace(
+        go.Scatter(mode="markers", x=[0],
+                   y=[0], name="Trips"))
+    consumption_fig_by_temp.update_layout(xaxis_title="average temperature in °C",
+                                          yaxis_title="Consumption kWh/100Km")
     return True
 
 
-def __calculate_co2_per_kw(charging_data):
-    try:
-        co2_data = charging_data[charging_data["co2"] > 0]
-        co2_kw_sum = co2_data["kw"].sum()
-        if co2_kw_sum > 0:
-            return co2_data["co2"].sum() / co2_kw_sum
-    except KeyError:
-        return 0
-    return 0
-
-
 def get_battery_curve_fig(row: dict, car: Car):
-    start_date = Database.convert_datetime_from_string(row["start_at"])
-    stop_at = Database.convert_datetime_from_string(row["stop_at"])
+    start_date = dash_date_to_datetime(row["start_at"])
+    stop_at = dash_date_to_datetime(row["stop_at"])
     conn = Database.get_db()
     res = Database.get_battery_curve(conn, start_date, car.vin)
     conn.close()

web/utils.py

@@ -0,0 +1,66 @@
+from datetime import datetime, timedelta
+
+import dash_bootstrap_components as dbc
+import dash_html_components as html
+from dash.development.base_component import Component
+
+
+def unix_time_millis(date):
+    return int(date.timestamp())
+
+
+def get_marks_from_start_end(start, end):
+    nb_marks = 10
+    result = []
+    time_delta = int((end - start).total_seconds() / nb_marks)
+    current = start
+    if time_delta > 0:
+        while current <= end:
+            result.append(current)
+            current += timedelta(seconds=time_delta)
+        result[-1] = end
+        if time_delta < 3600 * 24:
+            if time_delta > 3600:
+                date_f = '%x %Hh'
+            else:
+                date_f = '%x %Hh%M'
+        else:
+            date_f = '%x'
+        marks = {}
+        for date in result:
+            marks[unix_time_millis(date)] = str(date.strftime(date_f))
+        return marks
+    return None
+
+
+def card_value_div(card_id, unit, value="-"):
+    return html.Div([html.Div(value, id=card_id, className="mr-2"), html.Div(unit)],
+                    className="d-flex flex-row justify-content-center")
+
+
+def dash_date_to_datetime(st):
+    return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def create_card(card: dict):
+    res = []
+    for tile, value in card.items():
+        rows = value["text"]
+        # if isinstance(text, str):
+        #     text = html.H3(text)
+        html_text = []
+        for row in rows:
+            html_text.append(html.Div(row, className="d-flex flex-row justify-content-center"))
+        res.append(html.Div(
+            dbc.Card([
+                html.H4(tile, className="card-title text-center"),
+                dbc.Row([
+                    dbc.Col(dbc.CardBody(html_text, style={"whiteSpace": "nowrap", "fontSize": "160%"}),
+                            className="text-center"),
+                    dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"maxHeight": "7rem"}))
+                ],
+                    className="align-items-center flex-nowrap")
+            ], className="h-100 p-2"),
+            className="col-sm-12 col-md-6 col-lg-3 py-2"
+        ))
+    return res

web/views.py

@@ -1,17 +1,17 @@
 import json
-from datetime import datetime, timezone
 from typing import List
 
 import dash_bootstrap_components as dbc
 from dash.dependencies import Output, Input, MATCH, State
-from dash.development.base_component import Component
 from dash.exceptions import PreventUpdate
 import dash_core_components as dcc
 import dash_html_components as html
 import dash_daq as daq
-import pandas as pd
+from deepdiff import DeepDiff
 from flask import jsonify, request, Response as FlaskResponse
 
+import web.utils
+from libs.car import Cars, Car
 from mylogger import logger
 
 from trip import Trips
@@ -23,65 +23,22 @@ from web.app import app, dash_app, myp, chc
 from web.db import Database
 
 # pylint: disable=invalid-name
+from web.figure_filter import Figure_Filter
+from web.utils import dash_date_to_datetime, create_card
+
 RESPONSE = "-response"
 EMPTY_DIV = "empty-div"
 ABRP_SWITCH = 'abrp-switch'
 CALLBACK_CREATED = False
 
-trips: Trips
+trips: Trips = Trips()
 chargings: List[dict]
 min_date = max_date = min_millis = max_millis = step = marks = cached_layout = None
 
 
-def diff_dashtable(data, data_previous, row_id_name="row_id"):
-    df, df_previous = pd.DataFrame(data=data), pd.DataFrame(data_previous)
-    for _df in [df, df_previous]:
-        assert row_id_name in _df.columns
-        _df = _df.set_index(row_id_name)
-    mask = df.ne(df_previous)
-    df_diff = df[mask].dropna(how="all", axis="columns").dropna(how="all", axis="rows")
-    changes = []
-    for idx, row in df_diff.iterrows():
-        row.dropna(inplace=True)
-        for change in row.iteritems():
-            changes.append(
-                {
-                    row_id_name: data[idx][row_id_name],
-                    "column_name": change[0],
-                    "current_value": change[1],
-                    "previous_value": df_previous.at[idx, change[0]],
-                }
-            )
-    return changes
-
-
 def create_callback():  # noqa: MC0001
     global CALLBACK_CREATED
     if not CALLBACK_CREATED:
-        @dash_app.callback(Output('trips_map', 'figure'),
-                           Output('consumption_fig', 'figure'),
-                           Output('consumption_fig_by_speed', 'figure'),
-                           Output('consumption_graph_by_temp', 'children'),
-                           Output('summary-cards', 'children'),
-                           Output('tab_trips_fig', 'children'),
-                           Output('tab_charge', 'children'),
-                           Output('date-slider', 'max'),
-                           Output('date-slider', 'step'),
-                           Output('date-slider', 'marks'),
-                           Input('date-slider', 'value'))
-        def display_value(value):  # pylint: disable=unused-variable
-            mini = datetime.fromtimestamp(value[0], tz=timezone.utc)
-            maxi = datetime.fromtimestamp(value[1], tz=timezone.utc)
-            filtered_trips = Trips()
-            for trip in trips:
-                if mini <= trip.start_at <= maxi:
-                    filtered_trips.append(trip)
-            filtered_chargings = Charging.get_chargings(mini, maxi)
-            figures.get_figures(filtered_trips, filtered_chargings)
-            return figures.trips_map, figures.consumption_fig, figures.consumption_fig_by_speed, \
-                   figures.consumption_graph_by_temp, create_card(figures.SUMMARY_CARDS), \
-                   figures.table_fig, figures.battery_table, max_millis, step, marks
-
         @dash_app.callback(Output(EMPTY_DIV, "children"),
                            [Input("battery-table", "data_timestamp")],
                            [State("battery-table", "data"),
@@ -89,17 +46,22 @@ def create_callback():  # noqa: MC0001
         def capture_diffs_in_battery_table(timestamp, data, data_previous):  # pylint: disable=unused-variable
             if timestamp is None:
                 raise PreventUpdate
-            diff_data = diff_dashtable(data, data_previous, "start_at")
-            for changed_line in diff_data:
-                if changed_line['column_name'] == 'price':
+            diff_data = DeepDiff(data_previous, data, ignore_numeric_type_changes=True, ignore_order=True, view="tree",
+                                 verbose_level=1)
+            for value_changed in diff_data["values_changed"]:
+                index, column_name = value_changed.path(output_format='list')
+                new_value = value_changed.t2
+                if column_name == 'price':
                     conn = Database.get_db()
-                    if not Database.set_chargings_price( conn, changed_line['start_at'],
-                                                        changed_line['current_value']):
+                    date = dash_date_to_datetime(data[index]['start_at'])
+                    if not Database.set_chargings_price(conn, date, new_value):
                         logger.error("Can't find line to update in the database")
+                    else:
+                        logger.debug("update price %s of %s", value_changed, date)
                     conn.close()
-            return ""
+            return ""  # don't need to update dashboard
 
-        @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ],
+        @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open")],
                            [Input("battery-table", "active_cell"),
                             Input("tab_battery_popup-close", "n_clicks")],
                            [State('battery-table', 'data'),
@@ -116,7 +78,7 @@ def create_callback():  # noqa: MC0001
                            [Input("trips-table", "active_cell"),
                             Input("tab_trips_popup-close", "n_clicks")],
                            State("tab_trips_popup", "is_open"))
-        def get_altitude(active_cell, close, is_open):  # pylint: disable=unused-argument, unused-variable
+        def get_altitude_graph(active_cell, close, is_open):  # pylint: disable=unused-argument, unused-variable
             if is_open is None:
                 is_open = False
             if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open:
@@ -160,6 +122,21 @@ def get_vehicle_info(vin):
     return response
 
 
+STYLE_CACHE = None
+
+
+@app.route("/assets/style2.json")
+def get_style():
+    global STYLE_CACHE
+    if not STYLE_CACHE:
+        with open(app.root_path + "/assets/style.json", "r") as f:
+            res = json.loads(f.read())
+            STYLE_CACHE = res
+    url_root = request.url_root
+    STYLE_CACHE["sprite"] = url_root + "assets/sprites/osm-liberty@2x"
+    return jsonify(STYLE_CACHE)
+
+
 @app.route('/charge_now/<string:vin>/<int:charge>')
 def charge_now(vin, charge):
     return jsonify(myp.charge_now(vin, charge != 0))
@@ -248,14 +225,17 @@ def update_trips():
     conn.close()
     min_date = None
     max_date = None
+    car = myp.vehicles_list[0]  # todo handle multiple car
     try:
-        trips_by_vin = Trips.get_trips(myp.vehicles_list)
-        trips = next(iter(trips_by_vin.values()))  # todo handle multiple car
+        trips_by_vin = Trips.get_trips(Cars([car]))
+        trips = trips_by_vin[car.vin]
         assert len(trips) > 0
         min_date = trips[0].start_at
         max_date = trips[-1].start_at
-    except (StopIteration, AssertionError):
+        figures.get_figures(trips[0].car)
+    except (AssertionError, KeyError):
         logger.debug("No trips yet")
+        figures.get_figures(Car("vin","vid","brand"))
     try:
         chargings = Charging.get_chargings()
         assert len(chargings) > 0
@@ -272,11 +252,12 @@ def update_trips():
     # update for slider
     try:
         logger.debug("min_date:%s - max_date:%s", min_date, max_date)
-        min_millis = figures.unix_time_millis(min_date)
-        max_millis = figures.unix_time_millis(max_date)
+        min_millis = web.utils.unix_time_millis(min_date)
+        max_millis = web.utils.unix_time_millis(max_date)
         step = (max_millis - min_millis) / 100
-        marks = figures.get_marks_from_start_end(min_date, max_date)
+        marks = web.utils.get_marks_from_start_end(min_date, max_date)
         cached_layout = None  # force regenerate layout
+        figures.get_figures(car)
     except (ValueError, IndexError):
         logger.error("update_trips (slider): %s", exc_info=True)
     except AttributeError:
@@ -303,40 +284,12 @@ def __get_control_tabs():
     return tabs
 
 
-def create_card(card: dict):
-    res = []
-    for tile, value in card.items():
-        text = value["text"]
-        # if isinstance(text, str):
-        #     text = html.H3(text)
-        res.append(html.Div(
-            dbc.Card([
-                html.H4(tile, className="card-title text-center"),
-                dbc.Row([
-                    dbc.Col(dbc.CardBody(text, style={"white-space": "nowrap", "font-size": "160%"}),
-                            className="text-center"),
-                    dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"max-height": "7rem"}))
-                ],
-                    className="align-items-center flex-nowrap")
-            ], className="h-100 p-2"),
-            className="col-sm-12 col-md-6 col-lg-3 py-2"
-        ))
-    return res
-
-
 def serve_layout():
     global cached_layout
     if cached_layout is None:
         logger.debug("Create new layout")
+        fig_filter = Figure_Filter()
         try:
-            figures.get_figures(trips, chargings)
-            summary_tab = [dbc.Container(dbc.Row(id="summary-cards",
-                                                 children=create_card(figures.SUMMARY_CARDS)), fluid=True),
-                           dcc.Graph(figure=figures.consumption_fig, id="consumption_fig"),
-                           dcc.Graph(figure=figures.consumption_fig_by_speed, id="consumption_fig_by_speed"),
-                           figures.consumption_graph_by_temp]
-            maps = dcc.Graph(figure=figures.trips_map, id="trips_map", style={"height": '90vh'})
-            create_callback()
             range_slider = dcc.RangeSlider(
                 id='date-slider',
                 min=min_millis,
@@ -345,12 +298,31 @@ def serve_layout():
                 marks=marks,
                 value=[min_millis, max_millis],
             )
-        except (IndexError, TypeError, NameError):
+            summary_tab = [
+                dbc.Container(dbc.Row(id="summary-cards",
+                                      children=create_card(figures.SUMMARY_CARDS)), fluid=True),
+                fig_filter.add_graph(dcc.Graph(id="consumption_fig"), "start_at", ["consumption_km"],
+                                     figures.consumption_fig),
+                fig_filter.add_graph(dcc.Graph(id="consumption_fig_by_speed"), "speed_average",
+                                     ["consumption_km"] * 2, figures.consumption_fig_by_speed),
+                fig_filter.add_graph(dcc.Graph(id="consumption_graph_by_temp"), "consumption_by_temp",
+                                     ["consumption_km"] * 2, figures.consumption_fig_by_temp)]
+            maps = fig_filter.add_map(dcc.Graph(id="trips_map", style={"height": '90vh'}), "lat",
+                                      ["long", "start_at"], figures.trips_map)
+            fig_filter.add_table("trips", figures.table_fig)
+            fig_filter.add_table("chargings", figures.battery_table)
+            fig_filter.src = {"trips": trips.get_trips_as_dict(), "chargings": chargings}
+            dash_app.clientside_callback(*fig_filter.get_clientside_callback())
+            create_callback()
+        except (IndexError, TypeError, NameError, AssertionError, NameError):
             summary_tab = figures.ERROR_DIV
             maps = figures.ERROR_DIV
             logger.warning("Failed to generate figure, there is probably not enough data yet", exc_info_debug=True)
             range_slider = html.Div()
+            figures.battery_table = figures.ERROR_DIV
+
         data_div = html.Div([
+            *fig_filter.get_store(),
             range_slider,
             html.Div([
                 dbc.Tabs([
@@ -404,8 +376,8 @@ def serve_layout():
 
 
 try:
-    Database.set_db_callback(update_trips)
     Charging.set_default_price()
+    Database.set_db_callback(update_trips)
     update_trips()
 except (IndexError, TypeError):
     logger.debug("Failed to get trips, there is probably not enough data yet:", exc_info=True)