Merge pull request #89 from flobz/develop

add battery charge plot, add altitude to db & altitude graph

.pre-commit-config.yaml

@@ -1,3 +1,5 @@
+default_language_version:
+      python: python3.6
 repos:
 -   repo: https://github.com/PyCQA/prospector
     rev: 1.3.1 # The version of Prospector to use, at least 1.1.7

.prospector.yaml

@@ -1,4 +1,6 @@
 doc-warnings: false
+use: flask
+max-line-length: 120
 ignore-paths:
   - psa_connectedcar
 pep8:
@@ -13,3 +15,7 @@ pylint:
     - C0114
     - C0115
     - C0116
+    - W0603
+    - I0011
+    - W0511
+

.pylintrc

@@ -1,18 +0,0 @@
-[MASTER]
-disable=
-    C0114,C0115,C0116,W0603,I0011,W0511
-[FORMAT]
-max-line-length=120
-
-[BASIC]
-good-names=i,
-           j,
-           k,
-           ex,
-           Run,
-	   e,
-           f,
-	   t,
-           x,
-           ip,
-	   s

README.md

@@ -1,7 +1,7 @@
 # Remote Control of PSA car
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/4b4b98fe6dc04956a1c9a07b97c46c06)](https://app.codacy.com/gh/flobz/psa_car_controller?utm_source=github.com&utm_medium=referral&utm_content=flobz/psa_car_controller&utm_campaign=Badge_Grade_Settings)
 ### This is a python program to control a psa car with connected_car v4 api. Using android app to retrieve credentials.
-I test it with a Peugeot e-208 but it works with others PSA vehicles (Citroen, Opel, Vauxhall, DS).
+I test it with a Peugeot e-208, but it works with others PSA vehicles (Citroen, Opel, Vauxhall, DS).
 
 With this app  you will be able to :
  - get the status of the car (battery level for electric vehicle, position ... )
@@ -11,8 +11,13 @@ With this app  you will be able to :
  - control air conditioning
  - control lights and horn if your vehicle is compatible (mine isn't) 
  - get consumption statistic
- - visualize your trips on a map
- 
+ - visualize your trips on a map or in a table
+ - get the list of car charging  
+ - visualize battery charging curve
+ - visualize altitude trip curve
+ - get car charging co2 emission
+ - get car charging price
+
 The official api is documented [here](https://developer.groupe-psa.io/webapi/b2c/quickstart/connect/#article) but it is not totally up to date, and contains some errors. 
 
 A video in French was made by vlycop to explain how to use this application : https://youtu.be/XO7-N7G3biU 
@@ -20,7 +25,7 @@ A video in French was made by vlycop to explain how to use this application : ht
 
 ## I. Get credentials
 We need to get credentials from the android app.
-We will retrieve these informations:
+We will retrieve this information:
  - client-id and client-secret  for the api
  - some url to login
 
@@ -49,7 +54,7 @@ We will retrieve these informations:
 
     Your vehicles: {'VINNUBMER': {'id': 'vehicule id'}}
 
-1.4 If it works you will have VIN of your vehicles and there ids in the last line. The script generate a test.json file with all credentials needed.
+1.4 If it works you will have VIN of your vehicles and there ids in the last line. The script generates a test.json file with all credentials needed.
 
  ## II. Use the app
   
@@ -60,7 +65,7 @@ We will retrieve these informations:
 
      ``python3 server.py -f test.json -c charge_config1.json``
      
-     At the first launch you will receive a SMS and you will be asked to give it and also give your pin code (the four-digit code that your use on the android app).
+     At the first launch you will receive an SMS, and you will be asked to give it and also give your pin code (the four-digit code that your use on the android app).
      If it failed you can remove the file otp.bin and retry.
      
      You can see all options available with :
@@ -72,7 +77,7 @@ We will retrieve these informations:
     2.1 Get the car state :
     http://localhost:5000/get_vehicleinfo/YOURVIN
     
-    2.2 Stop charge (only for solution 1)
+    2.2 Stop charge
     http://localhost:5000/charge_now/YOURVIN/0
     
     2.3 Set hour to stop the charge to 6am
@@ -141,7 +146,7 @@ mitmproxy --set client_certs=MWPMYMA1.pem
 ```
 
 ## Donation
-If you want you want to thank me for my work :smile:
+If you want to thank me for my work :smile:
 
 [![donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif)](https://www.paypal.com/donate?hosted_button_id=SM652WPXFNCXS)
 

charge_control.py

@@ -1,6 +1,5 @@
 import json
 import threading
-import traceback
 from copy import copy
 from datetime import datetime, timedelta
 from hashlib import md5
@@ -91,10 +90,10 @@ class ChargeControl:
                 if self._next_stop_hour is not None and self._next_stop_hour < now:
                     self._next_stop_hour += timedelta(days=1)
                 self.retry_count = 0
-        except AttributeError:
-            logger.error("Probably can't retrieve all information from API: %s", traceback.format_exc())
+        except (AttributeError, ValueError):
+            logger.exception("Probably can't retrieve all information from API:")
         except: # pylint: disable=bare-except
-            logger.error(traceback.format_exc())
+            logger.exception("Charge control:")
 
     def get_dict(self):
         chd = copy(self.__dict__)

ecomix.py

@@ -2,7 +2,6 @@ from datetime import datetime, timedelta
 from statistics import mean, StatisticsError
 import xml.etree.cElementTree as ElT
 import numbers
-import traceback
 
 import requests
 import reverse_geocode
@@ -10,10 +9,12 @@ import reverse_geocode
 from mylogger import logger
 
 CO2_SIGNAL_REQ_INTERVAL = 600
+CO2_SIGNAL_URL = "https://api.co2signal.com"
 
 
 class Ecomix:
     _cache = {}
+    co2_signal_key = None
 
     @staticmethod
     def get_data_france(start, end):
@@ -47,18 +48,18 @@ class Ecomix:
             return None
 
     @staticmethod
-    def get_data_from_co2_signal(latitude, longitude, co2_signal_key):
-        if co2_signal_key is not None:
+    def get_data_from_co2_signal(latitude, longitude):
+        if Ecomix.co2_signal_key is not None:
             try:
                 country_code = Ecomix.get_country(latitude, longitude)
                 assert country_code is not None
                 if country_code not in Ecomix._cache:
                     Ecomix._cache[country_code] = []
                 elif len(Ecomix._cache[country_code]) > 0 and \
-                        (datetime.now()-Ecomix._cache[country_code][-1][0]).total_seconds() < CO2_SIGNAL_REQ_INTERVAL:
+                        (datetime.now() - Ecomix._cache[country_code][-1][0]).total_seconds() < CO2_SIGNAL_REQ_INTERVAL:
                     return False
-                res = requests.get("https://api.co2signal.com/v1/latest",
-                                   headers={"auth-token": co2_signal_key},
+                res = requests.get(CO2_SIGNAL_URL + "/v1/latest",
+                                   headers={"auth-token": Ecomix.co2_signal_key},
                                    params={"countryCode": country_code})
                 data = res.json()
                 value = data["data"]["carbonIntensity"]
@@ -66,7 +67,7 @@ class Ecomix:
                 Ecomix._cache[country_code].append([datetime.now(), value])
                 return data["status"] == "ok"
             except (AssertionError, NameError, KeyError):
-                logger.debug(traceback.format_exc())
+                logger.debug("ecomix:", exc_info=True)
                 return False
         else:
             return False
@@ -99,12 +100,12 @@ class Ecomix:
             return None
 
     @staticmethod
-    def get_co2_per_kw(start: datetime, end: datetime, latitude, longitude, from_cache=False):
+    def get_co2_per_kw(start: datetime, end: datetime, latitude, longitude):
         co2_per_kw = None
         country_code = Ecomix.get_country(latitude, longitude)
         if country_code is None:
             return None
-        if from_cache:
+        if Ecomix.co2_signal_key is not None:
             co2_per_kw = Ecomix.get_co2_from_signal_cache(start, end, country_code)
         elif country_code == 'FR':
             co2_per_kw = Ecomix.get_data_france(start, end)

libs/car.py

@@ -42,7 +42,7 @@ class Car:
         if self.status is not None:
             return self.status
         logger.error("status of %s is None", self.vin)
-        raise ValueError("status of %s is None")
+        raise ValueError("status of {} is None".format(self.vin))
 
     @classmethod
     def from_json(cls, data: dict):
@@ -72,11 +72,17 @@ class Car:
             self._status.__class__ = CarStatus
             self._status.correct()
 
+    def get_charge_speed(self, start_level, end_level, duration_in_sec) -> float:
+        duration_in_hour = duration_in_sec / 3600
+        charged_kw = self.battery_power * (end_level - start_level) / 100
+        kw_hour = charged_kw / duration_in_hour
+        return kw_hour
+
 
 class Cars(list):
     def __init__(self, *args):
         list.__init__(self, *args)
-        self.config_filename = "../cars.json"
+        self.config_filename = "cars.json"
 
     def get_car_by_vin(self, vin) -> Car:
         for car in self:

libs/charging.py

@@ -1,15 +1,20 @@
+from datetime import datetime
+from sqlite3 import IntegrityError
+
 from typing import List
 
+from ecomix import Ecomix
 from libs.elec_price import ElecPrice
-from web.db import get_db, set_chargings_price, clean_battery
-
-elec_price = ElecPrice.read_config()
+from mylogger import logger
+from web.db import Database
 
 
 class Charging:
+    elec_price: ElecPrice = ElecPrice(None)
+
     @staticmethod
     def get_chargings(mini=None, maxi=None) -> List[dict]:
-        conn = get_db()
+        conn = Database.get_db()
         if mini is not None:
             if maxi is not None:
                 res = conn.execute("select * from battery WHERE start_at>=? and start_at<=?", (mini, maxi)).fetchall()
@@ -24,19 +29,54 @@ class Charging:
 
     @staticmethod
     def set_default_price():
-        if elec_price.is_enable():
-            conn = get_db()
+        if Charging.elec_price.is_enable():
+            conn = Database.get_db()
             charge_list = list(map(dict, conn.execute("SELECT * FROM battery WHERE price IS NULL").fetchall()))
             for charge in charge_list:
-                charge["price"] = elec_price.get_price(charge["start_at"], charge["stop_at"], charge["kw"])
-                set_chargings_price(conn, charge["start_at"], charge["price"])
+                charge["price"] = Charging.elec_price.get_price(charge["start_at"], charge["stop_at"], charge["kw"])
+                Database.set_chargings_price(conn, charge["start_at"], charge["price"])
             conn.close()
 
     # pylint: disable=too-many-arguments
     @staticmethod
     def update_chargings(conn, start_at, stop_at, level, co2_per_kw, consumption_kw, vin):
-        price = elec_price.get_price(start_at, stop_at, consumption_kw)
+        price = Charging.elec_price.get_price(start_at, stop_at, consumption_kw)
         conn.execute(
             "UPDATE battery set stop_at=?, end_level=?, co2=?, kw=?, price=? WHERE start_at=? and VIN=?",
             (stop_at, level, co2_per_kw, consumption_kw, price, start_at, vin))
-        clean_battery(conn)
+        Database.clean_battery(conn)
+
+    @staticmethod
+    def record_charging(car, charging_status, charge_date: datetime, level, latitude, longitude, charging_mode):
+        conn = Database.get_db()
+        charge_date = charge_date.replace(microsecond=0)
+        if charging_status == "InProgress":
+            res = conn.execute("SELECT stop_at, start_at FROM battery WHERE VIN=? ORDER BY start_at "
+                               "DESC limit 1", (car.vin,)).fetchone()
+            in_progress = res and res[0] is None
+            if in_progress:
+                start_at = res[1]
+                try:
+                    conn.execute("INSERT INTO battery_curve(start_at,VIN,date,level) VALUES(?,?,?,?)",
+                                 (start_at, car.vin, charge_date, level))
+                except IntegrityError:
+                    logger.debug("level already stored")
+            else:
+                conn.execute("INSERT INTO battery(start_at,start_level,charging_mode,VIN) VALUES(?,?,?,?)",
+                             (charge_date, level, charging_mode, car.vin))
+            Ecomix.get_data_from_co2_signal(latitude, longitude)
+        else:
+            try:
+                start_at, stop_at, start_level = conn.execute(
+                    "SELECT start_at, stop_at, start_level from battery WHERE VIN=? ORDER BY start_at "
+                    "DESC limit 1", (car.vin,)).fetchone()
+                in_progress = stop_at is None
+                if in_progress:
+                    co2_per_kw = Ecomix.get_co2_per_kw(start_at, charge_date, latitude, longitude)
+                    consumption_kw = (level - start_level) / 100 * car.battery_power
+
+                    Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, car.vin)
+            except TypeError:
+                logger.debug("battery table is probably empty :", exc_info=True)
+        conn.commit()
+        conn.close()

libs/elec_price.py

@@ -2,7 +2,8 @@ from datetime import datetime, timezone, timedelta
 import configparser
 from statistics import mean
 
-CONFIG_FILENAME = "config.ini"
+from mylogger import logger
+
 
 
 def set_number(value):
@@ -18,13 +19,14 @@ def utc_to_local(utc_dt):
 
 class ElecPrice:
     currency = ""
+    CONFIG_FILENAME = "config.ini"
 
     def __init__(self, day_price, night_price=None, nights_hours=None):
         self.day_price = set_number(day_price)
         self.night_price = set_number(night_price)
         self.nights_hour = None
         self.set_night_hour(nights_hours)
-        self.config_filename = CONFIG_FILENAME
+        self.config_filename = ElecPrice.CONFIG_FILENAME
 
     def set_night_hour(self, value):
         if value is not None and isinstance(value, list):
@@ -52,16 +54,24 @@ class ElecPrice:
     def get_price(self, start, end, consumption):
         prices = []
         date = start
-        while date < end:
-            prices.append(self.get_instant_price(date))
-            date = date + timedelta(minutes=30)
-        return round(consumption * mean(prices), 2)
+        res = None
+        if not (start is None or end is None):
+            while date < end:
+                prices.append(self.get_instant_price(date))
+                date = date + timedelta(minutes=30)
+            try:
+                res = round(consumption * mean(prices), 2)
+            except TypeError:
+                logger.error("Can't get_price of charge, check config")
+        return res
 
     def is_enable(self):
         return self.day_price is not None
 
     @staticmethod
-    def read_config(name=CONFIG_FILENAME):
+    def read_config(name=None):
+        if name is None:
+            name = ElecPrice.CONFIG_FILENAME
         config = configparser.ConfigParser()
         if len(config.read(name)) == 0:
             ElecPrice.write_default_config(name)
@@ -79,7 +89,9 @@ class ElecPrice:
         return ElecPrice(elec_config["day price"], night_price, night_hours)
 
     @staticmethod
-    def write_default_config(name=CONFIG_FILENAME):
+    def write_default_config(name=None):
+        if name is None:
+            name = ElecPrice.CONFIG_FILENAME
         config = configparser.ConfigParser()
         config["General"] = {
             "currency": "€"

my_psacc.py

@@ -1,7 +1,6 @@
 import json
 import re
 import threading
-import traceback
 import uuid
 from datetime import datetime
 from json import JSONEncoder
@@ -10,8 +9,7 @@ from time import sleep
 
 from oauth2_client.credentials_manager import ServiceInformation
 import paho.mqtt.client as mqtt
-from geojson import Feature, Point, FeatureCollection
-from geojson import dumps as geo_dumps
+from requests.exceptions import RequestException
 
 import psa_connectedcar as psac
 from libs.car import Cars, Car
@@ -22,9 +20,9 @@ from otp.otp import load_otp, new_otp_session, save_otp, ConfigException, Otp
 from psa_connectedcar.rest import ApiException
 from mylogger import logger
 
-from utils import get_temp, rate_limit
+from utils import rate_limit
 from web.abrp import Abrp
-from web.db import get_db, clean_position, get_last_temp
+from web.db import Database
 
 PSA_CORRELATION_DATE_FORMAT = "%Y%m%d%H%M%S%f"
 PSA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
@@ -104,15 +102,19 @@ class MyPSACC:
             self.abrp: Abrp = Abrp(**abrp)
         self.set_proxies(proxies)
         self.config_file = DEFAULT_CONFIG_FILENAME
-        self.co2_signal_api = co2_signal_api
+        Ecomix.co2_signal_key = co2_signal_api
 
     def get_app_name(self):
         return realm_info[self.realm]['app_name']
 
     def refresh_token(self):
-        # pylint: disable=protected-access
-        self.manager._refresh_token()
-        self.save_config()
+        try:
+            # pylint: disable=protected-access
+            self.manager._refresh_token()
+            self.save_config()
+        except RequestException as e:
+            logger.error("Can't refresh token %s", e)
+            sleep(60)
 
     def api(self) -> psac.VehiclesApi:
         self.api_config.access_token = self.manager.access_token
@@ -141,8 +143,9 @@ class MyPSACC:
                     if self._record_enabled:
                         self.record_info(car)
                     return res
-            except ApiException:
-                logger.error(traceback.format_exc())
+            except ApiException as ex:
+                logger.error("get_vehicle_info: ApiException: %s", ex)
+                logger.debug(exc_info=True)
         car.status = res
         return res
 
@@ -171,7 +174,7 @@ class MyPSACC:
                 self.vehicles_list.add(Car(vehicle.vin, vehicle.id, vehicle.brand, vehicle.label))
             self.vehicles_list.save_cars()
         except ApiException:
-            logger.error(traceback.format_exc())
+            logger.exception("get_vehicles:")
         return self.vehicles_list
 
     def load_otp(self, force_new=False):
@@ -203,13 +206,18 @@ class MyPSACC:
         return otp_code
 
     def get_remote_access_token(self, password):
-        res = self.manager.post(REMOTE_URL + self.client_id,
-                                json={"grant_type": "password", "password": password},
-                                headers=self.headers)
-        data = res.json()
-        self.remote_access_token = data["access_token"]
-        self.remote_refresh_token = data["refresh_token"]
-        return res
+        try:
+            res = self.manager.post(REMOTE_URL + self.client_id,
+                                    json={"grant_type": "password", "password": password},
+                                    headers=self.headers)
+            data = res.json()
+            self.remote_access_token = data["access_token"]
+            self.remote_refresh_token = data["refresh_token"]
+            return res
+        except RequestException as e:
+            logger.error("Can't refresh remote token %s", e)
+            sleep(60)
+        return None
 
     def refresh_remote_token(self, force=False):
         if not force and self.remote_token_last_update is not None:
@@ -217,26 +225,31 @@ class MyPSACC:
             if (datetime.now() - last_update).total_seconds() < MQTT_TOKEN_TTL:
                 return None
         self.refresh_token()
-        if self.remote_refresh_token is None:
-            logger.error("remote_refresh_token isn't defined")
-            self.load_otp(force_new=True)
-        res = self.manager.post(REMOTE_URL + self.client_id,
-                                json={"grant_type": "refresh_token", "refresh_token": self.remote_refresh_token},
-                                headers=self.headers)
-        data = res.json()
-        logger.debug("refresh_remote_token: %s", data)
-        if "access_token" in data:
-            self.remote_access_token = data["access_token"]
-            self.remote_refresh_token = data["refresh_token"]
-            self.remote_token_last_update = datetime.now()
-        else:
-            logger.error("can't refresh_remote_token: %s\n Create a new one", data)
-            self.remote_token_last_update = datetime.now()
-            otp_code = self.get_otp_code()
-            res = self.get_remote_access_token(otp_code)
-        self.mqtt_client.username_pw_set("IMA_OAUTH_ACCESS_TOKEN", self.remote_access_token)
-        self.save_config()
-        return res
+        try:
+            if self.remote_refresh_token is None:
+                logger.error("remote_refresh_token isn't defined")
+                self.load_otp(force_new=True)
+            res = self.manager.post(REMOTE_URL + self.client_id,
+                                    json={"grant_type": "refresh_token", "refresh_token": self.remote_refresh_token},
+                                    headers=self.headers)
+            data = res.json()
+            logger.debug("refresh_remote_token: %s", data)
+            if "access_token" in data:
+                self.remote_access_token = data["access_token"]
+                self.remote_refresh_token = data["refresh_token"]
+                self.remote_token_last_update = datetime.now()
+            else:
+                logger.error("can't refresh_remote_token: %s\n Create a new one", data)
+                self.remote_token_last_update = datetime.now()
+                otp_code = self.get_otp_code()
+                res = self.get_remote_access_token(otp_code)
+            self.mqtt_client.username_pw_set("IMA_OAUTH_ACCESS_TOKEN", self.remote_access_token)
+            self.save_config()
+            return res
+        except RequestException as e:
+            logger.error("Can't refresh remote token %s", e)
+            sleep(60)
+            return None
 
     # pylint: disable=unused-argument
     def __on_mqtt_connect(self, client, userdata, result_code, _):
@@ -283,7 +296,7 @@ class MyPSACC:
                 sleep(60)
                 self.wakeup(data["vin"])
         except KeyError:
-            logger.error(traceback.format_exc())
+            logger.exception("mqtt message:")
 
     def start_mqtt(self):
         self.load_otp()
@@ -329,8 +342,7 @@ class MyPSACC:
                 minute = hour_minute[1]
             return hour, minute
         except IndexError:
-            logger.error(traceback.format_exc())
-            logger.error("Can't get charge hour: %s", hour_str)
+            logger.exception("Can't get charge hour: %s", hour_str)
             return None
 
     def get_charge_status(self, vin):
@@ -338,8 +350,8 @@ class MyPSACC:
         status = data.get_energy('Electric').charging.status
         return status
 
-    def __veh_charge_request(self, vin, hour, miinute, charge_type):
-        msg = self.mqtt_request(vin, {"program": {"hour": hour, "minute": miinute}, "type": charge_type})
+    def __veh_charge_request(self, vin, hour, minute, charge_type):
+        msg = self.mqtt_request(vin, {"program": {"hour": hour, "minute": minute}, "type": charge_type})
         logger.info(msg)
         self.mqtt_client.publish(MQTT_REQ_TOPIC + self.customer_id + "/VehCharge", msg)
 
@@ -451,96 +463,24 @@ class MyPSACC:
 
         longitude = car.status.last_position.geometry.coordinates[0]
         latitude = car.status.last_position.geometry.coordinates[1]
+        altitude = car.status.last_position.geometry.coordinates[2]
         date = car.status.last_position.properties.updated_at
         if date is None:
             date = charge_date
         logger.debug("vin:%s longitude:%s latitude:%s date:%s mileage:%s level:%s charge_date:%s level_fuel:"
                      "%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel,
                      moving)
-        self.__record_position(car.vin, mileage, latitude, longitude, date, level, level_fuel, moving)
-        self.abrp.call(car, get_last_temp(car.vin))
+        Database.record_position(self.weather_api, car.vin, mileage, latitude, longitude, altitude, date, level,
+                                 level_fuel, moving)
+        self.abrp.call(car, Database.get_last_temp(car.vin))
         try:
             charging_status = car.status.get_energy('Electric').charging.status
-            self.__record_charging(car.vin, charging_status, charge_date, level, latitude, longitude)
+            charging_mode = car.status.get_energy('Electric').charging.charging_mode
+            Charging.record_charging(car, charging_status, charge_date, level, latitude, longitude, charging_mode)
             logger.debug("charging_status:%s ", charging_status)
         except AttributeError:
             logger.error("charging status not available from api")
 
-    def __record_position(self, vin, mileage, latitude, longitude, date, level, level_fuel, moving):
-        conn = get_db()
-        if mileage == 0:  # fix a bug of the api
-            logger.error("The api return a wrong mileage for %s : %f", vin, mileage)
-        else:
-            if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None:
-                temp = get_temp(latitude, longitude, self.weather_api)
-                if level_fuel == 0:  # fix fuel level not provided when car is off
-                    try:
-                        level_fuel = conn.execute(
-                            "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC "
-                            "LIMIT 1",
-                            (vin,)).fetchone()[0]
-                        logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin)
-                    except TypeError:
-                        level_fuel = None
-                        logger.info("level_fuel unfixed for %s", vin)
-
-                conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,mileage,level,level_fuel,moving,"
-                             "temperature) VALUES(?,?,?,?,?,?,?,?,?)",
-                             (date, vin, longitude, latitude, mileage, level, level_fuel, moving, temp))
-
-                conn.commit()
-                logger.info("new position recorded for %s", vin)
-                clean_position(conn)
-                return True
-            logger.debug("position already saved")
-        return False
-
-    def __record_charging(self, vin, charging_status, charge_date, level, latitude, longitude):
-        conn = get_db()
-        if charging_status == "InProgress":
-            try:
-                in_progress = conn.execute("SELECT stop_at FROM battery WHERE VIN=? ORDER BY start_at DESC limit 1",
-                                           (vin,)).fetchone()[0] is None
-            except TypeError:
-                in_progress = False
-            if not in_progress:
-                conn.execute("INSERT INTO battery(start_at,start_level,VIN) VALUES(?,?,?)", (charge_date, level, vin))
-                conn.commit()
-            Ecomix.get_data_from_co2_signal(latitude, longitude, self.co2_signal_api)
-        else:
-            try:
-                start_at, stop_at, start_level = conn.execute(
-                    "SELECT start_at, stop_at, start_level from battery WHERE VIN=? ORDER BY start_at "
-                    "DESC limit 1", (vin,)).fetchone()
-                in_progress = stop_at is None
-                if in_progress:
-                    co2_per_kw = Ecomix.get_co2_per_kw(start_at, charge_date, latitude, longitude,
-                                                       from_cache=self.co2_signal_api is not None)
-                    consumption_kw = (level - start_level) / 100 * self.vehicles_list.get_car_by_vin(vin).battery_power
-
-                    Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, vin)
-                    conn.commit()
-            except TypeError:
-                logger.debug("battery table is empty")
-        conn.close()
-
-    @staticmethod
-    def get_recorded_position():
-        conn = get_db()
-        res = conn.execute('SELECT * FROM position ORDER BY Timestamp')
-        features_list = []
-        for row in res:
-            if row["longitude"] is None or row["latitude"] is None:
-                continue
-            feature = Feature(geometry=Point((row["longitude"], row["latitude"])),
-                              properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"),
-                                          "mileage": row["mileage"],
-                                          "level": row["level"], "level_fuel": row["level_fuel"]})
-            features_list.append(feature)
-        feature_collection = FeatureCollection(features_list)
-        conn.close()
-        return geo_dumps(feature_collection, sort_keys=True)
-
     def __iter__(self):
         for key, value in self.__dict__.items():
             yield key, value
@@ -553,7 +493,7 @@ class MyPeugeotEncoder(JSONEncoder):
         data = dict(mp)
         mpd = {"proxies": data["_proxies"], "refresh_token": mp.manager.refresh_token,
                "client_secret": mp.service_information.client_secret, "abrp": dict(mp.abrp)}
-        for param in ["client_id", "realm", "remote_refresh_token", "customer_id", "weather_api", "country_code",
-                      "co2_signal_api"]:
+        for param in ["client_id", "realm", "remote_refresh_token", "customer_id", "weather_api", "country_code"]:
             mpd[param] = data[param]
+        mpd["co2_signal_api"] = Ecomix.co2_signal_key
         return mpd

mylogger.py

@@ -5,18 +5,26 @@ DEBUG_LEVELV_NUM = 9
 logging.addLevelName(DEBUG_LEVELV_NUM, "DEBUGV")
 
 
-def debugv(self, message, *args, **kws):
-    self.log(DEBUG_LEVELV_NUM, message, *args, **kws)
+class CustomLogger(logging.Logger):
+    # pylint: disable=too-many-arguments
+    def __new_style_log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, **kwargs):
+        if kwargs.pop('style', "%") == "{":  # optional
+            msg = msg.format(*args)
+            args = []
+        super()._log(level, msg, args, exc_info, extra, stack_info)
+
+    def debugv(self, msg, *args, **kwargs):
+        if self.isEnabledFor(DEBUG_LEVELV_NUM):
+            self.__new_style_log(DEBUG_LEVELV_NUM, msg, args, **kwargs)
 
 
-logging.Logger.debugv = debugv
+logging.setLoggerClass(CustomLogger)
 # pylint: disable=invalid-name
 logger = logging.getLogger("log")
 
 
 def my_logger(file='activity.log', handler_level=logging.INFO):
     global logger
-
     logger.setLevel(handler_level)
     formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
     file_handler = RotatingFileHandler(file, 'a', 1000000, 1, encoding='utf8')

otp/otp.py

@@ -1,5 +1,4 @@
 import hashlib
-import traceback
 import pickle
 from secrets import token_hex, token_bytes
 from math import ceil
@@ -278,7 +277,8 @@ class Otp:
 
     def __getstate__(self):
         odict = self.__dict__.copy()
-        del odict['cipher']  # don't pickle this
+        if 'cipher' in odict:
+            del odict['cipher']  # don't pickle this
         return odict
 
     def __setstate__(self, dict_param):
@@ -321,7 +321,7 @@ def load_otp(filename="otp.bin"):
             except ModuleNotFoundError:
                 return RenameUnpickler(input_file).load()
     except FileNotFoundError:
-        logger.debug(traceback.format_exc())
+        logger.debug("", exc_info=True)
     return None
 
 

requirements-dev.txt

@@ -1,2 +1,3 @@
 prospector>=1.3.0
 pre-commit
+deepdiff

server.py

@@ -11,6 +11,8 @@ from oauth2_client.credentials_manager import OAuthError
 
 import web.app
 from charge_control import ChargeControls
+from libs.charging import Charging
+from libs.elec_price import ElecPrice
 from mylogger import my_logger
 from mylogger import logger
 from my_psacc import MyPSACC
@@ -40,7 +42,7 @@ def parse_args():
     return parser.parse_args()
 
 
-# flake8: noqa: C901
+# noqa: MC0001
 if __name__ == "__main__":
     if sys.version_info < (3, 6):
         raise RuntimeError("This application requires Python 3.6+")
@@ -55,6 +57,7 @@ if __name__ == "__main__":
     web.app.myp = MyPSACC.load_config(name=CONFIG_NAME)
     atexit.register(web.app.myp.save_config)
     web.app.myp.set_record(args.record)
+    Charging.elec_price = ElecPrice.read_config()
     if args.offline:
         logger.info("offline mode")
     else:

trip.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import logging
 from statistics import mean
 from typing import List, Dict
 
@@ -9,7 +8,7 @@ from geojson import Feature, FeatureCollection, MultiLineString
 from libs.car import Cars, Car
 from mylogger import logger
 from trip_parser import TripParser
-from web.db import get_db
+from web.db import Database
 
 
 class Points:
@@ -37,6 +36,7 @@ class Trip:
         self.duration = None
         self.mileage = None
         self.car: Car = None
+        self.altitude_diff = None
         self.temperatures = []
 
     def add_points(self, latitude, longitude):
@@ -51,13 +51,14 @@ class Trip:
         return None
 
     def set_consumption(self, diff_level: float) -> float:
-        if self.distance is None:
-            raise ValueError("Distance not set")
         if diff_level < 0:
             logger.debugv("trip has negative consumption")
             diff_level = 0
         self.consumption = diff_level * self.car.battery_power / 100
-        self.consumption_km = 100 * self.consumption / self.distance  # kw/100 km
+        try:
+            self.consumption_km = 100 * self.consumption / self.distance  # kw/100 km
+        except TypeError:
+            raise ValueError("Distance not set")
         return self.consumption_km
 
     def set_fuel_consumption(self, consumption) -> float:
@@ -88,14 +89,22 @@ class Trip:
                                                                "average consumption": self.consumption_km,
                                                                "average consumption fuel": self.consumption_fuel_km})
 
-    def get_info(self):
+    def get_info(self, row_id=None):
         res = {"start_at": self.start_at.astimezone(tz.tzlocal()).replace(tzinfo=None).strftime("%x %X"),
                # convert to naive tz,
                "duration": self.duration * 60, "speed_average": self.speed_average,
                "consumption_km": self.consumption_km, "consumption_fuel_km": self.consumption_fuel_km,
-               "distance": self.distance, "mileage": self.mileage}
+               "distance": self.distance, "mileage": self.mileage, "altitude_diff": self.altitude_diff}
+        if row_id is not None:
+            res["id"] = row_id
         return res
 
+    def set_altitude_diff(self, start, end):
+        try:
+            self.altitude_diff = end - start
+        except (NameError, TypeError):
+            pass
+
 
 class Trips(list):
     def __init__(self, *args):
@@ -124,18 +133,18 @@ class Trips(list):
         logger.debugv("trip discarded")
         return False
 
-    # flake8: noqa: C901
-    @staticmethod
-    def get_trips(vehicles_list: Cars) -> Dict[str, Trips]:
-        # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks
-        conn = get_db()
+    @staticmethod  # noqa: MC0001
+    def get_trips(vehicles_list: Cars) -> Dict[str, "Trips"]:
+        # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks,too-many-branches
+        conn = Database.get_db()
         vehicles = conn.execute(
             "SELECT DISTINCT vin FROM position;").fetchall()
         trips_by_vin = {}
         for vin in vehicles:
             trips = Trips()
             vin = vin[0]
-            res = conn.execute('SELECT * FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall()
+            res = conn.execute('SELECT Timestamp, VIN, longitude, latitude, mileage, level, moving, temperature,'
+                               ' level_fuel, altitude FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall()
             if len(res) > 1:
                 car = vehicles_list.get_car_by_vin(vin)
                 assert car is not None
@@ -145,8 +154,9 @@ class Trips(list):
                 trip = Trip()
                 # for debugging use this line res = list(map(dict,res))
                 for x in range(0, len(res) - 2):
-                    logger.debugv("%s mileage:%.1f level:%s level_fuel:%s",
-                                  res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel'])
+                    if logger.isEnabledFor(logging.DEBUG):  # reduce execution time if debug disabled
+                        logger.debugv("%s mileage:%.1f level:%s level_fuel:%s",
+                                      res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel'])
                     next_el = res[x + 2]
                     distance = end["mileage"] - start["mileage"]
                     duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600
@@ -154,17 +164,11 @@ class Trips(list):
                         speed_average = distance / duration
                     except ZeroDivisionError:
                         speed_average = 0
-                    restart_trip = False
-                    if trip_parser.is_refuel(start, end, distance):
-                        restart_trip = True
-                    elif speed_average < 0.2 and duration > 0.05:
-                        restart_trip = True
-                        logger.debugv("low speed detected")
-                    if restart_trip:
+                    if TripParser.is_low_speed(speed_average, duration) or trip_parser.is_refuel(start, end, distance):
                         start = end
                         trip = Trip()
-                        logger.debugv("restart trip at %s mileage:%.1f level:%s level_fuel:%s",
-                                      start['Timestamp'], start['mileage'], start['level'], start['level_fuel'])
+                        logger.debugv("restart trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}"
+                                      " level_fuel:{0[level_fuel]}", start, style='{')
                     else:
                         distance = next_el["mileage"] - end["mileage"]  # km
                         duration = (next_el["Timestamp"] - end["Timestamp"]).total_seconds() / 3600
@@ -173,13 +177,9 @@ class Trips(list):
                         except ZeroDivisionError:
                             speed_average = 0
                         end_trip = False
-                        if trip_parser.is_refuel(end, next_el, distance):
+                        if trip_parser.is_refuel(end, next_el, distance) or \
+                                TripParser.is_low_speed(speed_average, duration):
                             end_trip = True
-                        elif speed_average < 0.2 and duration > 0.05:
-                            # (distance == 0 and duration > 0.08) or duration > 2 or
-                            # check the speed to handle missing point
-                            end_trip = True
-                            logger.debugv("low speed detected")
                         elif duration > 2:
                             end_trip = True
                             logger.debugv("too much time detected")
@@ -189,8 +189,8 @@ class Trips(list):
                             end_trip = True
                             logger.debugv("last position found")
                         if end_trip:
-                            logger.debugv("stop trip at %s mileage:%.1f level:%s level_fuel:%s",
-                                          end['Timestamp'], end['mileage'], end['level'], end['level_fuel'])
+                            logger.debugv("stop trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}"
+                                          " level_fuel:{0[level_fuel]}", end, style='{')
                             trip.distance = end["mileage"] - start["mileage"]  # km
                             if trip.distance > 0:
                                 trip.start_at = start["Timestamp"]
@@ -201,17 +201,17 @@ class Trips(list):
                                 trip.duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600
                                 trip.speed_average = trip.distance / trip.duration
                                 diff_level, diff_level_fuel = trip_parser.get_level_consumption(start, end)
+                                trip.set_altitude_diff(start["altitude"], end["altitude"])
                                 trip.car = car
                                 if diff_level != 0:
                                     trip.set_consumption(diff_level)  # kw
                                 if diff_level_fuel != 0:
                                     trip.set_fuel_consumption(diff_level_fuel)
                                 trip.mileage = end["mileage"]
-                                logger.debugv("Trip: %s -> %s %.1fkm %.2fh %.0fkm/h %.2fkWh %.2fkWh/100km %.2fL "
-                                              "%.2fL/100km %.1fkm",
-                                              trip.start_at, trip.end_at, trip.distance, trip.duration,
-                                              trip.speed_average, trip.consumption, trip.consumption_km,
-                                              trip.consumption_fuel, trip.consumption_fuel_km, trip.mileage)
+                                logger.debugv("Trip: {0.start_at} -> {0.end_at} {0.distance:.1f}km {0.duration:.2f}h "
+                                              "{0.speed_average:.0f}km/h {0.consumption:.2f}kWh "
+                                              "{0.consumption_km:.2f}kWh/100km {0.consumption_fuel:.2f}L "
+                                              "{0.consumption_fuel_km:.2f}L/100km {0.mileage:.1f}km", trip, style="{")
                                 # filter bad value
                                 trips.check_and_append(trip)
                             start = next_el
@@ -221,3 +221,11 @@ class Trips(list):
                     end = next_el
                 trips_by_vin[vin] = trips
         return trips_by_vin
+
+    def get_info(self):
+        res = []
+        row_id = 1
+        for trip in self:
+            res.append(trip.get_info(row_id))
+            row_id += 1
+        return res

trip_parser.py

@@ -2,9 +2,9 @@ from collections.abc import Callable
 from libs.car import Car
 from mylogger import logger
 
-LEVEL = "level"
+LEVEL = 5
 
-LEVEL_FUEL = "level_fuel"
+LEVEL_FUEL = 8
 
 
 class TripParser:
@@ -67,3 +67,8 @@ class TripParser:
         # If distance is bigger than 0 but charge bigger than five there is probably missing point and we assume that
         # regeneration/temperature can't increase by 5 percent the battery level
         return decharge < -2 and (distance == 0 or decharge < -5)
+
+    @staticmethod
+    def is_low_speed(speed_average, duration):
+        logger.debugv("Low speed detected")
+        return speed_average < 0.2 and duration > 0.05

utils.py

@@ -1,4 +1,3 @@
-import traceback
 from functools import wraps
 from threading import Semaphore, Timer
 import socket
@@ -20,9 +19,9 @@ def get_temp(latitude: str, longitude: str, api_key: str) -> float:
             logger.debug("Temperature :%fc", temp)
             return temp
     except ConnectionError:
-        logger.error("Can't connect to openweathermap :%s", traceback.format_exc())
+        logger.error("Can't connect to openweathermap :", exc_info=True)
     except KeyError:
-        logger.error("Unable to get temperature from openweathermap :%s", traceback.format_exc())
+        logger.error("Unable to get temperature from openweathermap :", exc_info=True)
     return None
 
 

web/abrp.py

@@ -1,5 +1,4 @@
 import json
-import traceback
 from datetime import datetime
 
 import requests
@@ -43,7 +42,7 @@ class Abrp:
                 logger.debug(response.text)
                 return response.json()["status"] == "ok"
         except (AttributeError, IndexError, ValueError):
-            logger.error(traceback.format_exc())
+            logger.exception("abrp:")
         return False
 
     def __iter__(self):

web/app.py

@@ -27,7 +27,7 @@ myp: MyPSACC = None
 chc: ChargeControls = None
 
 
-def start_app(title, base_path, debug: bool, host, port):
+def start_app(title, base_path, debug: bool, host, port, reloader=False):  # pylint: disable=too-many-arguments
     global app, dash_app, dispatcher
     try:
         lang = locale.getlocale()[0].split("_")[0]
@@ -48,7 +48,7 @@ def start_app(title, base_path, debug: bool, host, port):
                          server=app, requests_pathname_prefix=requests_pathname_prefix)
     # keep this line
     import web.views  # pylint: disable=unused-import,import-outside-toplevel
-    return run_simple(host, port, application, use_reloader=False, use_debugger=debug)
+    return run_simple(host, port, application, use_reloader=reloader, use_debugger=debug)
 
 
 def save_config(my_peugeot: MyPSACC, name):

web/db.py

@@ -1,109 +1,237 @@
+import sys
 import sqlite3
 from datetime import datetime
+from time import sleep
 
 from typing import Callable
 import pytz
+import requests
+
+from geojson import Feature, Point, FeatureCollection
+from geojson import dumps as geo_dumps
 
 from mylogger import logger
+from utils import get_temp
 
-callback_fct: Callable[[], None] = lambda: None
-DEFAULT_DB_FILE = 'info.db'
-# pylint: disable=invalid-name
-db_initialized = False
+NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]]
+NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]]
 
 
-def convert_datetime_from_bytes(bytes_string):
-    return datetime.strptime(bytes_string.decode("utf-8"), "%Y-%m-%d %H:%M:%S+00:00").replace(tzinfo=pytz.UTC)
+def convert_sql_res(rows):
+    return list(map(dict, rows))
 
 
-def convert_datetime_from_string(st):
-    return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=pytz.UTC)
+DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00"
 
 
-def update_callback():
-    callback_fct()
+def new_convert_datetime_from_string(string):
+    return datetime.fromisoformat(string)
 
 
-def set_db_callback(callbackfct):
-    global callback_fct
-    callback_fct = callbackfct
+class Database:
+    callback_fct: Callable[[], None] = lambda: None
+    DEFAULT_DB_FILE = 'info.db'
+    db_initialized = False
 
+    @staticmethod
+    def convert_datetime_from_string(string):
+        try:
+            return datetime.strptime(string, DATE_FORMAT).replace(tzinfo=pytz.UTC)
+        except ValueError:
+            return datetime.strptime(string.replace("T", " "), DATE_FORMAT).replace(tzinfo=pytz.UTC)
 
-def backup(conn):
-    back_conn = sqlite3.connect("info_backup.db")
-    conn.backup(back_conn)
-    back_conn.close()
+    @staticmethod
+    def convert_datetime_from_bytes(bytes_string):
+        return Database.convert_datetime_from_string(bytes_string.decode("utf-8"))
 
+    @staticmethod
+    def convert_datetime_to_string(date: datetime):
+        return date.replace(tzinfo=pytz.UTC).isoformat(timespec='seconds', sep=" ")
 
-def init_db(conn):
-    global db_initialized
-    conn.execute("CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, VIN TEXT, longitude REAL, "
-                 "latitude REAL, mileage REAL, level INTEGER, level_fuel INTEGER, moving BOOLEAN,"
-                 " temperature INTEGER);")
-    make_backup = False
-    try:
-        conn.execute("ALTER TABLE position ADD level_fuel INTEGER;")
-        make_backup = True
-    except sqlite3.OperationalError:
-        pass
-    conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, "
-                 "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);")
-    conn.create_function("update_trips", 0, update_callback)
-    conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN "
-                 "SELECT update_trips(); END;")
-    try:
-        conn.execute("ALTER TABLE battery ADD price INTEGER;")
-        make_backup = True
-    except sqlite3.OperationalError:
-        pass
-    if make_backup:
-        backup(conn)
-    clean_battery(conn)
-    conn.commit()
-    db_initialized = True
+    @staticmethod
+    def update_callback():
+        Database.callback_fct()
 
+    @staticmethod
+    def set_db_callback(callbackfct):
+        Database.callback_fct = callbackfct
 
-def get_db(db_file=DEFAULT_DB_FILE):
-    sqlite3.register_converter("DATETIME", convert_datetime_from_bytes)
-    conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
-    conn.row_factory = sqlite3.Row
-    if not db_initialized:
-        init_db(conn)
-    return conn
+    @staticmethod
+    def backup(conn):
+        if sys.version_info < (3, 7):
+            logger.warning("Can't do database backup, please upgrade to python 3.7")
+        else:
+            back_conn = sqlite3.connect("info_backup.db")
+            conn.backup(back_conn)
+            back_conn.close()
 
+    @staticmethod
+    def init_db(conn):
+        conn.execute("""CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY,
+                                                             VIN TEXT, longitude REAL,
+                                                             latitude REAL,
+                                                             mileage REAL,
+                                                             level INTEGER,
+                                                             level_fuel INTEGER,
+                                                             moving BOOLEAN,
+                                                             temperature INTEGER,
+                                                             altitude INTEGER);""")
+        make_backup = False
+        conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, "
+                     "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);")
+        conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN "
+                     "SELECT update_trips(); END;")
+        conn.execute("""CREATE TABLE IF NOT EXISTS battery_curve (start_at DATETIME, VIN TEXT, date DATETIME,
+                        level INTEGER, UNIQUE(start_at, VIN, level));""")
+        for table, columns in [["position", NEW_POSITION_COLUMNS], ["battery", NEW_BATTERY_COLUMNS]]:
+            for column, column_type in columns:
+                try:
+                    conn.execute(f"ALTER TABLE {table} ADD {column} {column_type};")
+                    make_backup = True
+                except sqlite3.OperationalError:
+                    pass
+        if make_backup:
+            Database.backup(conn)
+        Database.clean_battery(conn)
+        Database.add_altitude_to_db(conn)
+        conn.commit()
+        if sys.version_info >= (3, 7):
+            Database.convert_datetime_from_string = new_convert_datetime_from_string
+        sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes)
+        sqlite3.register_adapter(datetime, Database.convert_datetime_to_string)
+        Database.db_initialized = True
 
-def clean_battery(conn):
-    # delete charging longer than 17h
-    conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;")
-    conn.execute("DELETE FROM battery WHERE start_level==end_level;")
-    conn.commit()
+    @staticmethod
+    def get_db(db_file=None, update_callback=True):
+        if db_file is None:
+            db_file = Database.DEFAULT_DB_FILE
+        conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
+        conn.row_factory = sqlite3.Row
+        if update_callback:
+            conn.create_function("update_trips", 0, Database.update_callback)
+        if not Database.db_initialized:
+            Database.init_db(conn)
+        return conn
 
-
-def clean_position(conn):
-    res = conn.execute(
-        "SELECT Timestamp,mileage,level from position ORDER BY Timestamp DESC LIMIT 3;").fetchall()
-    # Clean DB
-    if len(res) == 3 and res[0]["mileage"] == res[1]["mileage"] == res[2]["mileage"] and \
-            res[0]["level"] == res[1]["level"] == res[2]["level"]:
-        logger.debug("Delete duplicate line")
-        conn.execute("DELETE FROM position where Timestamp=?;", (res[1]["Timestamp"],))
+    @staticmethod
+    def clean_battery(conn):
+        # delete charging longer than 17h
+        conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;")
+        conn.execute("DELETE FROM battery WHERE start_level==end_level;")
         conn.commit()
 
+    @staticmethod
+    def clean_position(conn):
+        res = conn.execute(
+            "SELECT Timestamp,mileage,level from position ORDER BY Timestamp DESC LIMIT 3;").fetchall()
+        # Clean DB
+        if len(res) == 3 and res[0]["mileage"] == res[1]["mileage"] == res[2]["mileage"] and \
+                res[0]["level"] == res[1]["level"] == res[2]["level"]:
+            logger.debug("Delete duplicate line")
+            conn.execute("DELETE FROM position where Timestamp=?;", (res[1]["Timestamp"],))
+            conn.commit()
 
-def get_last_temp(vin):
-    conn = get_db()
-    res = conn.execute("SELECT temperature FROM position WHERE VIN=? ORDER BY Timestamp DESC limit 1",
-                       (vin,)).fetchone()
-    if res is None:
-        return None
-    return res[0]
+    @staticmethod
+    def get_last_temp(vin):
+        conn = Database.get_db()
+        res = conn.execute("SELECT temperature FROM position WHERE VIN=? ORDER BY Timestamp DESC limit 1",
+                           (vin,)).fetchone()
+        if res is None:
+            return None
+        return res[0]
 
+    @staticmethod
+    def set_chargings_price(conn, start_at, price):
+        if isinstance(start_at, str):
+            start_at = Database.convert_datetime_from_string(start_at)
+        update = conn.execute("UPDATE battery SET price=? WHERE start_at=?", (price, start_at)).rowcount == 1
+        conn.commit()
+        if not update:
+            logger.error("Can't find line to update in the database")
+        return update
 
-def set_chargings_price(conn, start_at, price):
-    if isinstance(start_at, str):
-        start_at = convert_datetime_from_string(start_at)
-    update = conn.execute("UPDATE battery SET price=? WHERE start_at=?", (price, start_at)).rowcount == 1
-    conn.commit()
-    if not update:
-        logger.error("Can't find line to update in the database")
-    return update
+    @staticmethod
+    def get_battery_curve(conn, start_at, vin):
+        return convert_sql_res(conn.execute("""SELECT date, level FROM battery_curve
+                                                WHERE start_at=? and VIN=?;""", (start_at, vin)).fetchall())
+
+    @staticmethod
+    def add_altitude_to_db(conn):
+        max_pos_by_req = 100
+        nb_null = conn.execute(
+            "SELECT COUNT(1) FROM position WHERE altitude IS NULL;").fetchone()[0]
+        if nb_null > max_pos_by_req:
+            logger.warning("There is %s to fetch from API, it can take some time", nb_null)
+        try:
+            while True:
+                res = conn.execute("SELECT DISTINCT latitude,longitude "
+                                   "FROM position WHERE altitude IS NULL LIMIT ?;", (max_pos_by_req,)).fetchall()
+                nb_res = len(res)
+                if nb_res > 0:
+                    logger.debug("add altitude for %s positions point", len(nb_null))
+                    nb_null -= nb_res
+                    locations_str = ""
+                    for line in res:
+                        locations_str += str(line[0]) + "," + str(line[1]) + "|"
+                    locations_str = locations_str[:-1]
+                    res = requests.get("https://api.opentopodata.org/v1/srtm30m",
+                                       params={"locations": locations_str})
+                    data = res.json()["results"]
+                    for line in data:
+                        conn.execute("UPDATE position SET altitude=? WHERE latitude=? and longitude=?",
+                                     (line["elevation"], line["location"]["lat"], line["location"]["lng"]))
+                    conn.commit()
+                    if nb_res == 100:
+                        sleep(1)  # API is limited to 1 call by sec
+                else:
+                    break
+        except (ValueError, KeyError, requests.exceptions.RequestException):
+            logger.error("Can't get altitude from API")
+
+    @staticmethod
+    def get_recorded_position():
+        conn = Database.get_db()
+        res = conn.execute('SELECT * FROM position ORDER BY Timestamp')
+        features_list = []
+        for row in res:
+            if row["longitude"] is None or row["latitude"] is None:
+                continue
+            feature = Feature(geometry=Point((row["longitude"], row["latitude"])),
+                              properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"),
+                                          "mileage": row["mileage"],
+                                          "level": row["level"], "level_fuel": row["level_fuel"]})
+            features_list.append(feature)
+        feature_collection = FeatureCollection(features_list)
+        conn.close()
+        return geo_dumps(feature_collection, sort_keys=True)
+
+    # pylint: disable=too-many-arguments
+    @staticmethod
+    def record_position(weather_api, vin, mileage, latitude, longitude, altitude, date, level, level_fuel, moving):
+        conn = Database.get_db()
+        if mileage == 0:  # fix a bug of the api
+            logger.error("The api return a wrong mileage for %s : %f", vin, mileage)
+        else:
+            if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None:
+                temp = get_temp(latitude, longitude, weather_api)
+                if level_fuel == 0:  # fix fuel level not provided when car is off
+                    try:
+                        level_fuel = conn.execute(
+                            "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC "
+                            "LIMIT 1",
+                            (vin,)).fetchone()[0]
+                        logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin)
+                    except TypeError:
+                        level_fuel = None
+                        logger.info("level_fuel unfixed for %s", vin)
+
+                conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,altitude,mileage,level,level_fuel,"
+                             "moving,temperature) VALUES(?,?,?,?,?,?,?,?,?,?)",
+                             (date, vin, longitude, latitude, altitude, mileage, level, level_fuel, moving, temp))
+
+                conn.commit()
+                logger.info("new position recorded for %s", vin)
+                Database.clean_position(conn)
+                return True
+            logger.debug("position already saved")
+        return False

web/figures.py

@@ -1,4 +1,5 @@
 from copy import deepcopy
+
 from typing import List
 
 import dash_bootstrap_components as dbc
@@ -13,8 +14,10 @@ from pandas import DataFrame
 from pandas import options as pandas_options
 import dash_html_components as html
 
-from trip import Trips
+from libs.car import Car
 from libs.elec_price import ElecPrice
+from trip import Trips, Trip
+from web.db import Database
 
 
 def unix_time_millis(date):
@@ -82,8 +85,9 @@ def get_figures(trips: Trips, charging: List[dict]):
     table_fig = dash_table.DataTable(
         id='trips-table',
         sort_action='native',
-        # sort_by=[{'column_id': 'start_at', 'direction': 'desc'}],
-        columns=[{'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
+        sort_by=[{'column_id': 'id', 'direction': 'desc'}],
+        columns=[{'id': 'id', 'name': '#', 'type': 'numeric'},
+                 {'id': 'start_at', 'name': 'start at', 'type': 'datetime'},
                  {'id': 'duration', 'name': 'duration', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" min").precision(0)},
                  {'id': 'speed_average', 'name': 'average speed', 'type': 'numeric',
@@ -95,8 +99,18 @@ def get_figures(trips: Trips, charging: List[dict]):
                  {'id': 'distance', 'name': 'distance', 'type': 'numeric',
                   'format': nb_format.symbol_suffix(" km").precision(1)},
                  {'id': 'mileage', 'name': 'mileage', 'type': 'numeric',
-                  'format': nb_format.symbol_suffix(" km").precision(1)}],
-        data=[tr.get_info() for tr in trips[::-1]],
+                  'format': nb_format},
+                 {'id': 'altitude_diff', 'name': 'Altitude diff', 'type': 'numeric',
+                  'format': deepcopy(nb_format).symbol_suffix(" m").precision(0)}
+                 ],
+        style_data_conditional=[
+            {
+                'if': {'column_id': ['altitude_diff']},
+                'color': 'dodgerblue',
+                "text-decoration": "underline"
+            }
+        ],
+        data=trips.get_info(),
         page_size=50
     )
     # consumption_fig
@@ -128,6 +142,7 @@ def get_figures(trips: Trips, charging: List[dict]):
         charge_speed = 0
         price_kw = 0
         total_elec = 0
+
     battery_info = html.Div(children=[
         html.Tr(
             [
@@ -175,7 +190,18 @@ def get_figures(trips: Trips, charging: List[dict]):
                  {'id': 'price', 'name': 'price', 'type': 'numeric',
                   'format': deepcopy(nb_format).symbol_suffix(" " + ElecPrice.currency).precision(2), 'editable': True}
                  ],
-        data=charging
+        data=charging,
+        style_data_conditional=[
+            {
+                'if': {'column_id': ['start_level', "end_level"]},
+                'color': 'dodgerblue',
+                "text-decoration": "underline"
+            },
+            {
+                'if': {'column_id': 'price'},
+                'backgroundColor': 'rgb(230, 246, 254)'
+            }
+        ],
     )
     consumption_by_temp_df = consumption_df[consumption_df["consumption_by_temp"].notnull()]
     if len(consumption_by_temp_df) > 0:
@@ -192,7 +218,7 @@ def get_figures(trips: Trips, charging: List[dict]):
 
     else:
         consumption_graph_by_temp = html.Div(Graph(style={'display': 'none'}), id="consumption_graph_by_temp")
-
+    return True
 
 def __calculate_co2_per_kw(charging_data):
     try:
@@ -203,3 +229,34 @@ def __calculate_co2_per_kw(charging_data):
     except KeyError:
         return 0
     return 0
+
+
+def get_battery_curve_fig(row: dict, car: Car):
+    start_date = Database.convert_datetime_from_string(row["start_at"])
+    stop_at = Database.convert_datetime_from_string(row["stop_at"])
+    res = Database.get_battery_curve(Database.get_db(), start_date, car.vin)
+    res.insert(0, {"level": row["start_level"], "date": start_date})
+    res.append({"level": row["end_level"], "date": stop_at})
+    battery_curves = []
+    speed = 0
+    for x in range(1, len(res)):
+        start_level = res[x - 1]["level"]
+        end_level = res[x]["level"]
+        speed = car.get_charge_speed(start_level, end_level, (res[x]["date"] - res[x - 1]["date"]).total_seconds())
+        battery_curves.append({"level": start_level, "speed": speed})
+    battery_curves.append({"level": row["end_level"], "speed": speed})
+    fig = px.line(battery_curves, x="level", y="speed")
+    fig.update_layout(xaxis_title="Battery %", yaxis_title="Charging speed in kW")
+    return html.Div(Graph(figure=fig))
+
+
+def get_altitude_fig(trip: Trip):
+    conn = Database.get_db()
+    res = list(map(list, conn.execute("SELECT mileage, altitude FROM position WHERE Timestamp>=? and Timestamp<=?;",
+                                      (trip.start_at, trip.end_at)).fetchall()))
+    start_mileage = res[0][0]
+    for line in res:
+        line[0] = line[0] - start_mileage
+    fig = px.line(res, x=0, y=1)
+    fig.update_layout(xaxis_title="Distance km", yaxis_title="Altitude m")
+    return html.Div(Graph(figure=fig))

web/views.py

@@ -1,5 +1,4 @@
 import json
-import traceback
 from datetime import datetime, timezone
 from typing import List
 
@@ -20,10 +19,9 @@ from libs.charging import Charging
 from web import figures
 
 from web.app import app, dash_app, myp, chc
-from web.db import set_chargings_price, get_db, set_db_callback
+from web.db import Database
 
 # pylint: disable=invalid-name
-
 RESPONSE = "-response"
 EMPTY_DIV = "empty-div"
 ABRP_SWITCH = 'abrp-switch'
@@ -56,7 +54,7 @@ def diff_dashtable(data, data_previous, row_id_name="row_id"):
     return changes
 
 
-def create_callback():
+def create_callback():  # noqa: MC0001
     global CALLBACK_CREATED
     if not CALLBACK_CREATED:
         @dash_app.callback(Output('trips_map', 'figure'),
@@ -64,8 +62,8 @@ def create_callback():
                            Output('consumption_fig_by_speed', 'figure'),
                            Output('consumption_graph_by_temp', 'children'),
                            Output('consumption', 'children'),
-                           Output('tab_trips', 'children'),
-                           Output('tab_battery', 'children'),
+                           Output('tab_trips_fig', 'children'),
+                           Output('tab_battery_fig', 'children'),
                            Output('tab_charge', 'children'),
                            Output('date-slider', 'max'),
                            Output('date-slider', 'step'),
@@ -83,23 +81,48 @@ def create_callback():
             consumption = "Average consumption: {:.1f} kWh/100km".format(
                 float(figures.consumption_df["consumption_km"].mean()))
             return figures.trips_map, figures.consumption_fig, figures.consumption_fig_by_speed, \
-                   figures.consumption_graph_by_temp, consumption, figures.table_fig, figures.battery_info, \
-                   figures.battery_table, max_millis, step, marks
+                figures.consumption_graph_by_temp, consumption, figures.table_fig, figures.battery_info, \
+                figures.battery_table, max_millis, step, marks
 
         @dash_app.callback(Output(EMPTY_DIV, "children"),
                            [Input("battery-table", "data_timestamp")],
                            [State("battery-table", "data"),
                             State("battery-table", "data_previous")])
-        def capture_diffs(timestamp, data, data_previous):  # pylint: disable=unused-variable
+        def capture_diffs_in_battery_table(timestamp, data, data_previous):  # pylint: disable=unused-variable
             if timestamp is None:
                 raise PreventUpdate
             diff_data = diff_dashtable(data, data_previous, "start_at")
             for changed_line in diff_data:
                 if changed_line['column_name'] == 'price':
-                    if not set_chargings_price(get_db(), changed_line['start_at'], changed_line['current_value']):
+                    if not Database.set_chargings_price(Database.get_db(), changed_line['start_at'],
+                                                        changed_line['current_value']):
                         logger.error("Can't find line to update in the database")
             return ""
 
+        @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ],
+                           [Input("battery-table", "active_cell"),
+                            Input("tab_battery_popup-close", "n_clicks")],
+                           [State('battery-table', 'data'),
+                            State("tab_battery_popup", "is_open")])
+        def get_battery_curve(active_cell, close, data, is_open):  # pylint: disable=unused-argument, unused-variable
+            if is_open is None:
+                is_open = False
+            if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open:
+                row = data[active_cell["row"]]
+                return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True
+            return "", False
+
+        @dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ],
+                           [Input("trips-table", "active_cell"),
+                            Input("tab_trips_popup-close", "n_clicks")],
+                           State("tab_trips_popup", "is_open"))
+        def get_altitude(active_cell, close, is_open):  # pylint: disable=unused-argument, unused-variable
+            if is_open is None:
+                is_open = False
+            if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open:
+                return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True
+            return "", False
+
         CALLBACK_CREATED = True
 
 
@@ -191,7 +214,7 @@ def get_charge_control():
 
 @app.route('/positions')
 def get_recorded_position():
-    return FlaskResponse(myp.get_recorded_position(), mimetype='application/json')
+    return FlaskResponse(Database.get_recorded_position(), mimetype='application/json')
 
 
 @app.route('/abrp')
@@ -219,6 +242,7 @@ def after_request(response):
 def update_trips():
     global trips, chargings, cached_layout
     logger.info("update_data")
+    Database.add_altitude_to_db(Database.get_db(update_callback=False))
     try:
         trips_by_vin = Trips.get_trips(myp.vehicles_list)
         trips = next(iter(trips_by_vin.values()))  # todo handle multiple car
@@ -238,7 +262,7 @@ def update_trips():
         marks = figures.get_marks_from_start_end(min_date, max_date)
         cached_layout = None  # force regenerate layout
     except (ValueError, IndexError):
-        logger.error("update_trips (slider): %s", traceback.format_exc())
+        logger.error("update_trips (slider): %s", exc_info=True)
     return
 
 
@@ -286,33 +310,61 @@ def serve_layout():
             summary_tab = figures.ERROR_DIV
             maps = figures.ERROR_DIV
             logger.warning("Failed to generate figure, there is probably not enough data yet")
-            logger.debug(traceback.format_exc())
             range_slider = html.Div()
         data_div = html.Div([
             range_slider,
             html.Div([
                 dbc.Tabs([
                     dbc.Tab(label="Summary", tab_id="summary", children=summary_tab),
-                    dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", children=[figures.table_fig]),
-                    dbc.Tab(label="Battery", tab_id="battery", id="tab_battery", children=[figures.battery_info]),
+                    dbc.Tab(label="Trips", tab_id="trips", id="tab_trips",
+                            children=[html.Div(id="tab_trips_fig", children=figures.table_fig),
+                                      dbc.Modal(
+                                          [
+                                              dbc.ModalHeader("Altitude"),
+                                              dbc.ModalBody(html.Div(
+                                                  id="tab_trips_popup_graph")),
+                                              dbc.ModalFooter(
+                                                  dbc.Button("Close",
+                                                             id="tab_trips_popup-close",
+                                                             className="ml-auto")
+                                              ),
+                                          ],
+                                          id="tab_trips_popup",
+                                          size="xl",
+                                      )
+                                      ]),
+                    dbc.Tab(label="Battery", tab_id="battery", id="tab_battery",
+                            children=[html.Div(id="tab_battery_fig", children=[figures.battery_info]),
+                                      dbc.Modal(
+                                          [
+                                              dbc.ModalHeader("Charging speed"),
+                                              dbc.ModalBody(html.Div(id="tab_battery_popup_graph")),
+                                              dbc.ModalFooter(
+                                                  dbc.Button("Close", id="tab_battery_popup-close", className="ml-auto")
+                                              ),
+                                          ],
+                                          id="tab_battery_popup",
+                                          size="xl",
+                                      )]),
                     dbc.Tab(label="Charge", tab_id="charge", id="tab_charge", children=[figures.battery_table]),
                     dbc.Tab(label="Map", tab_id="map", children=[maps]),
                     dbc.Tab(label="Control", tab_id="control", children=dbc.Tabs(id="control-tabs",
                                                                                  children=__get_control_tabs()))],
-                         id="tabs",
-                         active_tab="summary",
-                         persistence=True),
+                    id="tabs",
+                    active_tab="summary",
+                    persistence=True),
                 html.Div(id=EMPTY_DIV),
+                html.Div(id=EMPTY_DIV + "1")
             ])])
         cached_layout = dbc.Container(fluid=True, children=[html.H1('My car info'), data_div])
     return cached_layout
 
 
 try:
-    set_db_callback(update_trips)
+    Database.set_db_callback(update_trips)
     Charging.set_default_price()
     update_trips()
 except (IndexError, TypeError):
-    logger.debug("Failed to get trips, there is probably not enough data yet %s", traceback.format_exc())
+    logger.debug("Failed to get trips, there is probably not enough data yet:", exc_info=True)
 
 dash_app.layout = serve_layout