From cbb593970472b82fbf1806617f1ab987e436c4ea Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Sat, 24 Apr 2021 13:11:35 +0200 Subject: [PATCH 01/15] update README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e34b18a..22fdcf0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Remote Control of PSA car [![Codacy Badge](https://api.codacy.com/project/badge/Grade/4b4b98fe6dc04956a1c9a07b97c46c06)](https://app.codacy.com/gh/flobz/psa_car_controller?utm_source=github.com&utm_medium=referral&utm_content=flobz/psa_car_controller&utm_campaign=Badge_Grade_Settings) ### This is a python program to control a psa car with connected_car v4 api. Using android app to retrieve credentials. -I test it with a Peugeot e-208 but it works with others PSA vehicles (Citroen, Opel, Vauxhall, DS). +I test it with a Peugeot e-208, but it works with others PSA vehicles (Citroen, Opel, Vauxhall, DS). With this app you will be able to : - get the status of the car (battery level for electric vehicle, position ... ) @@ -47,7 +47,7 @@ We will retrieve these informations: Your vehicles: {'VINNUBMER': {'id': 'vehicule id'}} -1.4 If it works you will have VIN of your vehicles and there ids in the last line. The script generate a test.json file with all credentials needed. +1.4 If it works you will have VIN of your vehicles and there ids in the last line. The script generates a test.json file with all credentials needed. ## II. Use the app @@ -58,7 +58,7 @@ We will retrieve these informations: ``python3 server.py -f test.json -c charge_config1.json`` - At the first launch you will receive a SMS and you will be asked to give it and also give your pin code (the four-digit code that your use on the android app). + At the first launch you will receive an SMS, and you will be asked to give it and also give your pin code (the four-digit code that your use on the android app). If it failed you can remove the file otp.bin and retry. You can see all options available with : @@ -70,7 +70,7 @@ We will retrieve these informations: 2.1 Get the car state : http://localhost:5000/get_vehicleinfo/YOURVIN - 2.2 Stop charge (only for solution 1) + 2.2 Stop charge http://localhost:5000/charge_now/YOURVIN/0 2.3 Set hour to stop the charge to 6am @@ -139,6 +139,6 @@ mitmproxy --set client_certs=MWPMYMA1.pem ``` ## Donation -If you want you want to thank me for my work :smile: +If you want to thank me for my work :smile: [![donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif)](https://www.paypal.com/donate?hosted_button_id=SM652WPXFNCXS) From 05ae99f5e33caf440d415cf8c789a4f2c701c950 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Tue, 27 Apr 2021 20:04:08 +0200 Subject: [PATCH 02/15] add battery charge plot --- ecomix.py | 16 ++-- libs/car.py | 6 ++ libs/charging.py | 50 ++++++++++-- my_psacc.py | 52 +++---------- trip.py | 7 +- web/app.py | 5 +- web/db.py | 194 +++++++++++++++++++++++++++-------------------- web/figures.py | 44 ++++++++++- web/views.py | 50 +++++++++--- 9 files changed, 271 insertions(+), 153 deletions(-) diff --git a/ecomix.py b/ecomix.py index 02e2184..ed3634e 100644 --- a/ecomix.py +++ b/ecomix.py @@ -10,10 +10,12 @@ import reverse_geocode from mylogger import logger CO2_SIGNAL_REQ_INTERVAL = 600 +CO2_SIGNAL_URL = "https://api.co2signal.com" class Ecomix: _cache = {} + co2_signal_key = None @staticmethod def get_data_france(start, end): @@ -47,18 +49,18 @@ class Ecomix: return None @staticmethod - def get_data_from_co2_signal(latitude, longitude, co2_signal_key): - if co2_signal_key is not None: + def get_data_from_co2_signal(latitude, longitude): + if Ecomix.co2_signal_key is not None: try: country_code = Ecomix.get_country(latitude, longitude) assert country_code is not None if country_code not in Ecomix._cache: Ecomix._cache[country_code] = [] elif len(Ecomix._cache[country_code]) > 0 and \ - (datetime.now()-Ecomix._cache[country_code][-1][0]).total_seconds() < CO2_SIGNAL_REQ_INTERVAL: + (datetime.now() - Ecomix._cache[country_code][-1][0]).total_seconds() < CO2_SIGNAL_REQ_INTERVAL: return False - res = requests.get("https://api.co2signal.com/v1/latest", - headers={"auth-token": co2_signal_key}, + res = requests.get(CO2_SIGNAL_URL + "/v1/latest", + headers={"auth-token": Ecomix.co2_signal_key}, params={"countryCode": country_code}) data = res.json() value = data["data"]["carbonIntensity"] @@ -99,12 +101,12 @@ class Ecomix: return None @staticmethod - def get_co2_per_kw(start: datetime, end: datetime, latitude, longitude, from_cache=False): + def get_co2_per_kw(start: datetime, end: datetime, latitude, longitude): co2_per_kw = None country_code = Ecomix.get_country(latitude, longitude) if country_code is None: return None - if from_cache: + if Ecomix.co2_signal_key is not None: co2_per_kw = Ecomix.get_co2_from_signal_cache(start, end, country_code) elif country_code == 'FR': co2_per_kw = Ecomix.get_data_france(start, end) diff --git a/libs/car.py b/libs/car.py index d0a129a..92f2ab3 100644 --- a/libs/car.py +++ b/libs/car.py @@ -72,6 +72,12 @@ class Car: self._status.__class__ = CarStatus self._status.correct() + def get_charge_speed(self, start_level, end_level, duration_in_sec) -> float: + duration_in_hour = duration_in_sec / 3600 + charged_kw = self.battery_power * (end_level - start_level) / 100 + kw_hour = charged_kw / duration_in_hour + return kw_hour + class Cars(list): def __init__(self, *args): diff --git a/libs/charging.py b/libs/charging.py index f700c33..a5de899 100644 --- a/libs/charging.py +++ b/libs/charging.py @@ -1,7 +1,12 @@ +from datetime import datetime +from sqlite3 import IntegrityError + from typing import List +from ecomix import Ecomix from libs.elec_price import ElecPrice -from web.db import get_db, set_chargings_price, clean_battery +from mylogger import logger +from web.db import Database elec_price = ElecPrice.read_config() @@ -9,7 +14,7 @@ elec_price = ElecPrice.read_config() class Charging: @staticmethod def get_chargings(mini=None, maxi=None) -> List[dict]: - conn = get_db() + conn = Database.get_db() if mini is not None: if maxi is not None: res = conn.execute("select * from battery WHERE start_at>=? and start_at<=?", (mini, maxi)).fetchall() @@ -25,11 +30,11 @@ class Charging: @staticmethod def set_default_price(): if elec_price.is_enable(): - conn = get_db() + conn = Database.get_db() charge_list = list(map(dict, conn.execute("SELECT * FROM battery WHERE price IS NULL").fetchall())) for charge in charge_list: charge["price"] = elec_price.get_price(charge["start_at"], charge["stop_at"], charge["kw"]) - set_chargings_price(conn, charge["start_at"], charge["price"]) + Database.set_chargings_price(conn, charge["start_at"], charge["price"]) conn.close() # pylint: disable=too-many-arguments @@ -39,4 +44,39 @@ class Charging: conn.execute( "UPDATE battery set stop_at=?, end_level=?, co2=?, kw=?, price=? WHERE start_at=? and VIN=?", (stop_at, level, co2_per_kw, consumption_kw, price, start_at, vin)) - clean_battery(conn) + Database.clean_battery(conn) + + @staticmethod + def record_charging(car, charging_status, charge_date: datetime, level, latitude, longitude, charging_mode): + conn = Database.get_db() + charge_date = charge_date.replace(microsecond=0) + if charging_status == "InProgress": + res = conn.execute("SELECT stop_at, start_at FROM battery WHERE VIN=? ORDER BY start_at " + "DESC limit 1", (car.vin,)).fetchone() + in_progress = res and res[0] is None + if in_progress: + start_at = res[1] + try: + conn.execute("INSERT INTO battery_curve(start_at,VIN,date,level) VALUES(?,?,?,?)", + (start_at, car.vin, charge_date, level)) + except IntegrityError: + logger.debug("level already stored") + else: + conn.execute("INSERT INTO battery(start_at,start_level,charging_mode,VIN) VALUES(?,?,?,?)", + (charge_date, level, charging_mode, car.vin)) + Ecomix.get_data_from_co2_signal(latitude, longitude) + else: + try: + start_at, stop_at, start_level = conn.execute( + "SELECT start_at, stop_at, start_level from battery WHERE VIN=? ORDER BY start_at " + "DESC limit 1", (car.vin,)).fetchone() + in_progress = stop_at is None + if in_progress: + co2_per_kw = Ecomix.get_co2_per_kw(start_at, charge_date, latitude, longitude) + consumption_kw = (level - start_level) / 100 * car.battery_power + + Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, car.vin) + except TypeError: + logger.debug("battery table is empty") + conn.commit() + conn.close() diff --git a/my_psacc.py b/my_psacc.py index 6aa36ea..d4ba842 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -24,7 +24,7 @@ from mylogger import logger from utils import get_temp, rate_limit from web.abrp import Abrp -from web.db import get_db, clean_position, get_last_temp +from web.db import Database PSA_CORRELATION_DATE_FORMAT = "%Y%m%d%H%M%S%f" PSA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" @@ -104,7 +104,7 @@ class MyPSACC: self.abrp: Abrp = Abrp(**abrp) self.set_proxies(proxies) self.config_file = DEFAULT_CONFIG_FILENAME - self.co2_signal_api = co2_signal_api + Ecomix.co2_signal_key = co2_signal_api def get_app_name(self): return realm_info[self.realm]['app_name'] @@ -338,8 +338,8 @@ class MyPSACC: status = data.get_energy('Electric').charging.status return status - def __veh_charge_request(self, vin, hour, miinute, charge_type): - msg = self.mqtt_request(vin, {"program": {"hour": hour, "minute": miinute}, "type": charge_type}) + def __veh_charge_request(self, vin, hour, minute, charge_type): + msg = self.mqtt_request(vin, {"program": {"hour": hour, "minute": minute}, "type": charge_type}) logger.info(msg) self.mqtt_client.publish(MQTT_REQ_TOPIC + self.customer_id + "/VehCharge", msg) @@ -458,16 +458,17 @@ class MyPSACC: "%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel, moving) self.__record_position(car.vin, mileage, latitude, longitude, date, level, level_fuel, moving) - self.abrp.call(car, get_last_temp(car.vin)) + self.abrp.call(car, Database.get_last_temp(car.vin)) try: charging_status = car.status.get_energy('Electric').charging.status - self.__record_charging(car.vin, charging_status, charge_date, level, latitude, longitude) + charging_mode = car.status.get_energy('Electric').charging.charging_mode + Charging.record_charging(car, charging_status, charge_date, level, latitude, longitude, charging_mode) logger.debug("charging_status:%s ", charging_status) except AttributeError: logger.error("charging status not available from api") def __record_position(self, vin, mileage, latitude, longitude, date, level, level_fuel, moving): - conn = get_db() + conn = Database.get_db() if mileage == 0: # fix a bug of the api logger.error("The api return a wrong mileage for %s : %f", vin, mileage) else: @@ -490,43 +491,14 @@ class MyPSACC: conn.commit() logger.info("new position recorded for %s", vin) - clean_position(conn) + Database.clean_position(conn) return True logger.debug("position already saved") return False - def __record_charging(self, vin, charging_status, charge_date, level, latitude, longitude): - conn = get_db() - if charging_status == "InProgress": - try: - in_progress = conn.execute("SELECT stop_at FROM battery WHERE VIN=? ORDER BY start_at DESC limit 1", - (vin,)).fetchone()[0] is None - except TypeError: - in_progress = False - if not in_progress: - conn.execute("INSERT INTO battery(start_at,start_level,VIN) VALUES(?,?,?)", (charge_date, level, vin)) - conn.commit() - Ecomix.get_data_from_co2_signal(latitude, longitude, self.co2_signal_api) - else: - try: - start_at, stop_at, start_level = conn.execute( - "SELECT start_at, stop_at, start_level from battery WHERE VIN=? ORDER BY start_at " - "DESC limit 1", (vin,)).fetchone() - in_progress = stop_at is None - if in_progress: - co2_per_kw = Ecomix.get_co2_per_kw(start_at, charge_date, latitude, longitude, - from_cache=self.co2_signal_api is not None) - consumption_kw = (level - start_level) / 100 * self.vehicles_list.get_car_by_vin(vin).battery_power - - Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, vin) - conn.commit() - except TypeError: - logger.debug("battery table is empty") - conn.close() - @staticmethod def get_recorded_position(): - conn = get_db() + conn = Database.get_db() res = conn.execute('SELECT * FROM position ORDER BY Timestamp') features_list = [] for row in res: @@ -553,7 +525,7 @@ class MyPeugeotEncoder(JSONEncoder): data = dict(mp) mpd = {"proxies": data["_proxies"], "refresh_token": mp.manager.refresh_token, "client_secret": mp.service_information.client_secret, "abrp": dict(mp.abrp)} - for param in ["client_id", "realm", "remote_refresh_token", "customer_id", "weather_api", "country_code", - "co2_signal_api"]: + for param in ["client_id", "realm", "remote_refresh_token", "customer_id", "weather_api", "country_code"]: mpd[param] = data[param] + mpd["co2_signal_api"] = Ecomix.co2_signal_key return mpd diff --git a/trip.py b/trip.py index e58a1a8..7d312cd 100644 --- a/trip.py +++ b/trip.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from statistics import mean from typing import List, Dict @@ -8,8 +6,9 @@ from geojson import Feature, FeatureCollection, MultiLineString from libs.car import Cars, Car from mylogger import logger +from psa_connectedcar import Trips from trip_parser import TripParser -from web.db import get_db +from web.db import Database class Points: @@ -128,7 +127,7 @@ class Trips(list): @staticmethod def get_trips(vehicles_list: Cars) -> Dict[str, Trips]: # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks - conn = get_db() + conn = Database.get_db() vehicles = conn.execute( "SELECT DISTINCT vin FROM position;").fetchall() trips_by_vin = {} diff --git a/web/app.py b/web/app.py index ec428d1..fed1560 100644 --- a/web/app.py +++ b/web/app.py @@ -27,7 +27,7 @@ myp: MyPSACC = None chc: ChargeControls = None -def start_app(title, base_path, debug: bool, host, port): +def start_app(title, base_path, debug: bool, host, port, reloader=False): # pylint: disable=too-many-arguments global app, dash_app, dispatcher try: lang = locale.getlocale()[0].split("_")[0] @@ -47,8 +47,9 @@ def start_app(title, base_path, debug: bool, host, port): dash_app = dash.Dash(external_stylesheets=[dbc.themes.BOOTSTRAP], external_scripts=locale_url, title=title, server=app, requests_pathname_prefix=requests_pathname_prefix) # keep this line + import web.views # pylint: disable=unused-import,import-outside-toplevel - return run_simple(host, port, application, use_reloader=False, use_debugger=debug) + return run_simple(host, port, application, use_reloader=reloader, use_debugger=debug) def save_config(my_peugeot: MyPSACC, name): diff --git a/web/db.py b/web/db.py index 63d6a16..ea831a0 100644 --- a/web/db.py +++ b/web/db.py @@ -1,3 +1,4 @@ +import sys import sqlite3 from datetime import datetime @@ -6,104 +7,131 @@ import pytz from mylogger import logger -callback_fct: Callable[[], None] = lambda: None -DEFAULT_DB_FILE = 'info.db' -# pylint: disable=invalid-name -db_initialized = False +NEW_BATTERY_COLUMNS = [["battery", "INTEGER"], ["charging_mode", "TEXT"]] + +DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" -def convert_datetime_from_bytes(bytes_string): - return datetime.strptime(bytes_string.decode("utf-8"), "%Y-%m-%d %H:%M:%S+00:00").replace(tzinfo=pytz.UTC) +def convert_sql_res(rows): + return list(map(dict, rows)) -def convert_datetime_from_string(st): - return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=pytz.UTC) +class Database: + callback_fct: Callable[[], None] = lambda: None + DEFAULT_DB_FILE = 'info.db' + # pylint: disable=invalid-name + db_initialized = False + + @staticmethod + def convert_datetime_from_bytes(bytes_string): + return datetime.strptime(bytes_string.decode("utf-8"), DATE_FORMAT).replace(tzinfo=pytz.UTC) -def update_callback(): - callback_fct() + @staticmethod + def convert_datetime_from_string(st): + return datetime.strptime(st, DATE_FORMAT).replace(tzinfo=pytz.UTC) + @staticmethod + def convert_datetime_to_string(date: datetime): + return date.replace(tzinfo=pytz.UTC).strftime(DATE_FORMAT) -def set_db_callback(callbackfct): - global callback_fct - callback_fct = callbackfct + @staticmethod + def update_callback(): + Database.callback_fct() + @staticmethod + def set_db_callback(callbackfct): + Database.callback_fct = callbackfct -def backup(conn): - back_conn = sqlite3.connect("info_backup.db") - conn.backup(back_conn) - back_conn.close() + @staticmethod + def backup(conn): + if sys.version_info < (3, 7): + logger.warning("Can't do database backup, please upgrade to python 3.7") + else: + back_conn = sqlite3.connect("info_backup.db") + conn.backup(back_conn) + back_conn.close() + @staticmethod + def init_db(conn): + conn.execute("CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, VIN TEXT, longitude REAL, " + "latitude REAL, mileage REAL, level INTEGER, level_fuel INTEGER, moving BOOLEAN," + " temperature INTEGER);") + make_backup = False + try: + conn.execute("ALTER TABLE position ADD level_fuel INTEGER;") + make_backup = True + except sqlite3.OperationalError: + pass + conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, " + "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);") + conn.create_function("update_trips", 0, Database.update_callback) + conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN " + "SELECT update_trips(); END;") + conn.execute("""CREATE TABLE IF NOT EXISTS battery_curve (start_at DATETIME, VIN TEXT, date DATETIME, + level INTEGER, UNIQUE(start_at, VIN, level));""") + for column, column_type in NEW_BATTERY_COLUMNS: + try: + conn.execute(f"ALTER TABLE battery ADD {column} {column_type};") + make_backup = True + except sqlite3.OperationalError: + pass + if make_backup: + Database.backup(conn) + Database.clean_battery(conn) + conn.commit() + Database.db_initialized = True -def init_db(conn): - global db_initialized - conn.execute("CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, VIN TEXT, longitude REAL, " - "latitude REAL, mileage REAL, level INTEGER, level_fuel INTEGER, moving BOOLEAN," - " temperature INTEGER);") - make_backup = False - try: - conn.execute("ALTER TABLE position ADD level_fuel INTEGER;") - make_backup = True - except sqlite3.OperationalError: - pass - conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, " - "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);") - conn.create_function("update_trips", 0, update_callback) - conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN " - "SELECT update_trips(); END;") - try: - conn.execute("ALTER TABLE battery ADD price INTEGER;") - make_backup = True - except sqlite3.OperationalError: - pass - if make_backup: - backup(conn) - clean_battery(conn) - conn.commit() - db_initialized = True + @staticmethod + def get_db(db_file=None): + if db_file is None: + db_file = Database.DEFAULT_DB_FILE + sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) + sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) + conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) + conn.row_factory = sqlite3.Row + if not Database.db_initialized: + Database.init_db(conn) + return conn - -def get_db(db_file=DEFAULT_DB_FILE): - sqlite3.register_converter("DATETIME", convert_datetime_from_bytes) - conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) - conn.row_factory = sqlite3.Row - if not db_initialized: - init_db(conn) - return conn - - -def clean_battery(conn): - # delete charging longer than 17h - conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;") - conn.execute("DELETE FROM battery WHERE start_level==end_level;") - conn.commit() - - -def clean_position(conn): - res = conn.execute( - "SELECT Timestamp,mileage,level from position ORDER BY Timestamp DESC LIMIT 3;").fetchall() - # Clean DB - if len(res) == 3 and res[0]["mileage"] == res[1]["mileage"] == res[2]["mileage"] and \ - res[0]["level"] == res[1]["level"] == res[2]["level"]: - logger.debug("Delete duplicate line") - conn.execute("DELETE FROM position where Timestamp=?;", (res[1]["Timestamp"],)) + @staticmethod + def clean_battery(conn): + # delete charging longer than 17h + conn.execute("DElETE FROM battery WHERE JULIANDAY(stop_at)-JULIANDAY(start_at)>0.7;") + conn.execute("DELETE FROM battery WHERE start_level==end_level;") conn.commit() + @staticmethod + def clean_position(conn): + res = conn.execute( + "SELECT Timestamp,mileage,level from position ORDER BY Timestamp DESC LIMIT 3;").fetchall() + # Clean DB + if len(res) == 3 and res[0]["mileage"] == res[1]["mileage"] == res[2]["mileage"] and \ + res[0]["level"] == res[1]["level"] == res[2]["level"]: + logger.debug("Delete duplicate line") + conn.execute("DELETE FROM position where Timestamp=?;", (res[1]["Timestamp"],)) + conn.commit() -def get_last_temp(vin): - conn = get_db() - res = conn.execute("SELECT temperature FROM position WHERE VIN=? ORDER BY Timestamp DESC limit 1", - (vin,)).fetchone() - if res is None: - return None - return res[0] + @staticmethod + def get_last_temp(vin): + conn = Database.get_db() + res = conn.execute("SELECT temperature FROM position WHERE VIN=? ORDER BY Timestamp DESC limit 1", + (vin,)).fetchone() + if res is None: + return None + return res[0] + @staticmethod + def set_chargings_price(conn, start_at, price): + if isinstance(start_at, str): + start_at = Database.convert_datetime_from_string(start_at) + update = conn.execute("UPDATE battery SET price=? WHERE start_at=?", (price, start_at)).rowcount == 1 + conn.commit() + if not update: + logger.error("Can't find line to update in the database") + return update -def set_chargings_price(conn, start_at, price): - if isinstance(start_at, str): - start_at = convert_datetime_from_string(start_at) - update = conn.execute("UPDATE battery SET price=? WHERE start_at=?", (price, start_at)).rowcount == 1 - conn.commit() - if not update: - logger.error("Can't find line to update in the database") - return update + @staticmethod + def get_battery_curve(conn, start_at, vin): + return convert_sql_res(conn.execute("""SELECT date, level FROM battery_curve + WHERE start_at=? and VIN=?;""", (start_at, vin)).fetchall()) diff --git a/web/figures.py b/web/figures.py index 0f005a2..3ae912d 100644 --- a/web/figures.py +++ b/web/figures.py @@ -1,4 +1,6 @@ from copy import deepcopy +from datetime import datetime + from typing import List import dash_bootstrap_components as dbc @@ -12,9 +14,12 @@ import plotly.graph_objects as go from pandas import DataFrame from pandas import options as pandas_options import dash_html_components as html +import pytz -from trip import Trips +from libs.car import Car from libs.elec_price import ElecPrice +from trip import Trips +from web.db import Database def unix_time_millis(date): @@ -128,6 +133,7 @@ def get_figures(trips: Trips, charging: List[dict]): charge_speed = 0 price_kw = 0 total_elec = 0 + battery_info = html.Div(children=[ html.Tr( [ @@ -175,7 +181,18 @@ def get_figures(trips: Trips, charging: List[dict]): {'id': 'price', 'name': 'price', 'type': 'numeric', 'format': deepcopy(nb_format).symbol_suffix(" " + ElecPrice.currency).precision(2), 'editable': True} ], - data=charging + data=charging, + style_data_conditional=[ + { + 'if': {'column_id': ['start_level', "end_level"]}, + 'color': 'dodgerblue', + "text-decoration": "underline" + }, + { + 'if': {'column_id': 'price'}, + 'backgroundColor': 'rgb(230, 246, 254)' + } + ], ) consumption_by_temp_df = consumption_df[consumption_df["consumption_by_temp"].notnull()] if len(consumption_by_temp_df) > 0: @@ -203,3 +220,26 @@ def __calculate_co2_per_kw(charging_data): except KeyError: return 0 return 0 + + +def dash_date_to_datetime(dash_date): + return datetime.strptime(dash_date, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=pytz.UTC) + + +def get_battery_curve_fig(row: dict, car: Car): + start_date = dash_date_to_datetime(row["start_at"]) + stop_at = dash_date_to_datetime(row["stop_at"]) + res = Database.get_battery_curve(Database.get_db(), start_date, car.vin) + res.insert(0, {"level": row["start_level"], "date": start_date}) + res.append({"level": row["end_level"], "date": stop_at}) + battery_curves = [] + speed = 0 + for x in range(1, len(res)): + start_level = res[x - 1]["level"] + end_level = res[x]["level"] + speed = car.get_charge_speed(start_level, end_level, (res[x]["date"] - res[x - 1]["date"]).total_seconds()) + battery_curves.append({"level": start_level, "speed": speed}) + battery_curves.append({"level": row["end_level"], "speed": speed}) + fig = px.line(battery_curves, x="level", y="speed") + fig.update_layout(xaxis_title="Battery %", yaxis_title="Charging speed in kW") + return html.Div(Graph(figure=fig)) diff --git a/web/views.py b/web/views.py index 39179f7..9d32728 100644 --- a/web/views.py +++ b/web/views.py @@ -20,10 +20,9 @@ from libs.charging import Charging from web import figures from web.app import app, dash_app, myp, chc -from web.db import set_chargings_price, get_db, set_db_callback +from web.db import Database # pylint: disable=invalid-name - RESPONSE = "-response" EMPTY_DIV = "empty-div" ABRP_SWITCH = 'abrp-switch' @@ -65,7 +64,7 @@ def create_callback(): Output('consumption_graph_by_temp', 'children'), Output('consumption', 'children'), Output('tab_trips', 'children'), - Output('tab_battery', 'children'), + Output('tab_battery_fig', 'children'), Output('tab_charge', 'children'), Output('date-slider', 'max'), Output('date-slider', 'step'), @@ -90,19 +89,37 @@ def create_callback(): [Input("battery-table", "data_timestamp")], [State("battery-table", "data"), State("battery-table", "data_previous")]) - def capture_diffs(timestamp, data, data_previous): # pylint: disable=unused-variable + def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': - if not set_chargings_price(get_db(), changed_line['start_at'], changed_line['current_value']): + if not Database.set_chargings_price(Database.get_db(), + figures.dash_date_to_datetime(changed_line['start_at']), + changed_line['current_value']): logger.error("Can't find line to update in the database") return "" CALLBACK_CREATED = True +@dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ], + [Input("battery-table", "active_cell"), + Input("tab_battery_popup-close", "n_clicks")], + [State('battery-table', 'data'), + State("tab_battery_popup", "is_open")] + ) +def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: + row = data[active_cell["row"]] + print("ok") + return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True + return "", False + + @dash_app.callback(Output({'role': ABRP_SWITCH + RESPONSE, 'vin': MATCH}, 'children'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'id'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'value')) @@ -294,22 +311,35 @@ def serve_layout(): dbc.Tabs([ dbc.Tab(label="Summary", tab_id="summary", children=summary_tab), dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", children=[figures.table_fig]), - dbc.Tab(label="Battery", tab_id="battery", id="tab_battery", children=[figures.battery_info]), + dbc.Tab(label="Battery", tab_id="battery", id="tab_battery", + children=[html.Div(id="tab_battery_fig", children=[figures.battery_info]), + dbc.Modal( + [ + dbc.ModalHeader("Charging speed"), + dbc.ModalBody(html.Div(id="tab_battery_popup_graph")), + dbc.ModalFooter( + dbc.Button("Close", id="tab_battery_popup-close", className="ml-auto") + ), + ], + id="tab_battery_popup", + size="xl", + )]), dbc.Tab(label="Charge", tab_id="charge", id="tab_charge", children=[figures.battery_table]), dbc.Tab(label="Map", tab_id="map", children=[maps]), dbc.Tab(label="Control", tab_id="control", children=dbc.Tabs(id="control-tabs", children=__get_control_tabs()))], - id="tabs", - active_tab="summary", - persistence=True), + id="tabs", + active_tab="summary", + persistence=True), html.Div(id=EMPTY_DIV), + html.Div(id=EMPTY_DIV + "1") ])]) cached_layout = dbc.Container(fluid=True, children=[html.H1('My car info'), data_div]) return cached_layout try: - set_db_callback(update_trips) + Database.set_db_callback(update_trips) Charging.set_default_price() update_trips() except (IndexError, TypeError): From 6d210a7b4ac821ff16ec277f872a321dc8d85067 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 20:09:55 +0200 Subject: [PATCH 03/15] code clean --- web/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/app.py b/web/app.py index fed1560..9c7aebf 100644 --- a/web/app.py +++ b/web/app.py @@ -27,7 +27,7 @@ myp: MyPSACC = None chc: ChargeControls = None -def start_app(title, base_path, debug: bool, host, port, reloader=False): # pylint: disable=too-many-arguments +def start_app(title, base_path, debug: bool, host, port, reloader=False): # pylint: disable=too-many-arguments global app, dash_app, dispatcher try: lang = locale.getlocale()[0].split("_")[0] @@ -47,7 +47,6 @@ def start_app(title, base_path, debug: bool, host, port, reloader=False): # pyli dash_app = dash.Dash(external_stylesheets=[dbc.themes.BOOTSTRAP], external_scripts=locale_url, title=title, server=app, requests_pathname_prefix=requests_pathname_prefix) # keep this line - import web.views # pylint: disable=unused-import,import-outside-toplevel return run_simple(host, port, application, use_reloader=reloader, use_debugger=debug) From 91f72884e66fa602000a412aa0ff965e75f5a5d7 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 20:14:32 +0200 Subject: [PATCH 04/15] add altitude to db & altitude graph --- my_psacc.py | 54 ++------------------- trip.py | 26 ++++++++-- web/db.py | 126 ++++++++++++++++++++++++++++++++++++++++++------- web/figures.py | 33 +++++++++++-- web/views.py | 40 +++++++++++++--- 5 files changed, 197 insertions(+), 82 deletions(-) diff --git a/my_psacc.py b/my_psacc.py index d4ba842..6ca3d6a 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -10,8 +10,6 @@ from time import sleep from oauth2_client.credentials_manager import ServiceInformation import paho.mqtt.client as mqtt -from geojson import Feature, Point, FeatureCollection -from geojson import dumps as geo_dumps import psa_connectedcar as psac from libs.car import Cars, Car @@ -22,7 +20,7 @@ from otp.otp import load_otp, new_otp_session, save_otp, ConfigException, Otp from psa_connectedcar.rest import ApiException from mylogger import logger -from utils import get_temp, rate_limit +from utils import rate_limit from web.abrp import Abrp from web.db import Database @@ -451,13 +449,15 @@ class MyPSACC: longitude = car.status.last_position.geometry.coordinates[0] latitude = car.status.last_position.geometry.coordinates[1] + altitude = car.status.last_position.geometry.coordinates[3] date = car.status.last_position.properties.updated_at if date is None: date = charge_date logger.debug("vin:%s longitude:%s latitude:%s date:%s mileage:%s level:%s charge_date:%s level_fuel:" "%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel, moving) - self.__record_position(car.vin, mileage, latitude, longitude, date, level, level_fuel, moving) + Database.record_position(self.weather_api, car.vin, mileage, latitude, longitude, altitude, date, level, + level_fuel, moving) self.abrp.call(car, Database.get_last_temp(car.vin)) try: charging_status = car.status.get_energy('Electric').charging.status @@ -467,52 +467,6 @@ class MyPSACC: except AttributeError: logger.error("charging status not available from api") - def __record_position(self, vin, mileage, latitude, longitude, date, level, level_fuel, moving): - conn = Database.get_db() - if mileage == 0: # fix a bug of the api - logger.error("The api return a wrong mileage for %s : %f", vin, mileage) - else: - if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None: - temp = get_temp(latitude, longitude, self.weather_api) - if level_fuel == 0: # fix fuel level not provided when car is off - try: - level_fuel = conn.execute( - "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC " - "LIMIT 1", - (vin,)).fetchone()[0] - logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin) - except TypeError: - level_fuel = None - logger.info("level_fuel unfixed for %s", vin) - - conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,mileage,level,level_fuel,moving," - "temperature) VALUES(?,?,?,?,?,?,?,?,?)", - (date, vin, longitude, latitude, mileage, level, level_fuel, moving, temp)) - - conn.commit() - logger.info("new position recorded for %s", vin) - Database.clean_position(conn) - return True - logger.debug("position already saved") - return False - - @staticmethod - def get_recorded_position(): - conn = Database.get_db() - res = conn.execute('SELECT * FROM position ORDER BY Timestamp') - features_list = [] - for row in res: - if row["longitude"] is None or row["latitude"] is None: - continue - feature = Feature(geometry=Point((row["longitude"], row["latitude"])), - properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"), - "mileage": row["mileage"], - "level": row["level"], "level_fuel": row["level_fuel"]}) - features_list.append(feature) - feature_collection = FeatureCollection(features_list) - conn.close() - return geo_dumps(feature_collection, sort_keys=True) - def __iter__(self): for key, value in self.__dict__.items(): yield key, value diff --git a/trip.py b/trip.py index 7d312cd..36dd39c 100644 --- a/trip.py +++ b/trip.py @@ -1,3 +1,4 @@ +import traceback from statistics import mean from typing import List, Dict @@ -6,7 +7,6 @@ from geojson import Feature, FeatureCollection, MultiLineString from libs.car import Cars, Car from mylogger import logger -from psa_connectedcar import Trips from trip_parser import TripParser from web.db import Database @@ -36,6 +36,7 @@ class Trip: self.duration = None self.mileage = None self.car: Car = None + self.altitude_diff = None self.temperatures = [] def add_points(self, latitude, longitude): @@ -87,14 +88,22 @@ class Trip: "average consumption": self.consumption_km, "average consumption fuel": self.consumption_fuel_km}) - def get_info(self): + def get_info(self, row_id=None): res = {"start_at": self.start_at.astimezone(tz.tzlocal()).replace(tzinfo=None).strftime("%x %X"), # convert to naive tz, "duration": self.duration * 60, "speed_average": self.speed_average, "consumption_km": self.consumption_km, "consumption_fuel_km": self.consumption_fuel_km, - "distance": self.distance, "mileage": self.mileage} + "distance": self.distance, "mileage": self.mileage, "altitude_diff": self.altitude_diff} + if row_id is not None: + res["id"] = row_id return res + def set_altitude_diff(self, start, end): + try: + self.altitude_diff = end - start + except (NameError, TypeError): + pass + class Trips(list): def __init__(self, *args): @@ -125,7 +134,7 @@ class Trips(list): # flake8: noqa: C901 @staticmethod - def get_trips(vehicles_list: Cars) -> Dict[str, Trips]: + def get_trips(vehicles_list: Cars) -> Dict[str, "Trips"]: # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks conn = Database.get_db() vehicles = conn.execute( @@ -200,6 +209,7 @@ class Trips(list): trip.duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600 trip.speed_average = trip.distance / trip.duration diff_level, diff_level_fuel = trip_parser.get_level_consumption(start, end) + trip.set_altitude_diff(start["altitude"], end["altitude"]) trip.car = car if diff_level != 0: trip.set_consumption(diff_level) # kw @@ -220,3 +230,11 @@ class Trips(list): end = next_el trips_by_vin[vin] = trips return trips_by_vin + + def get_info(self): + res = [] + id = 1 + for trip in self: + res.append(trip.get_info(id)) + id += 1 + return res diff --git a/web/db.py b/web/db.py index ea831a0..5d8c41e 100644 --- a/web/db.py +++ b/web/db.py @@ -1,13 +1,21 @@ import sys import sqlite3 +import traceback from datetime import datetime +from time import sleep from typing import Callable import pytz +import requests + +from geojson import Feature, Point, FeatureCollection +from geojson import dumps as geo_dumps from mylogger import logger +from utils import get_temp NEW_BATTERY_COLUMNS = [["battery", "INTEGER"], ["charging_mode", "TEXT"]] +NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]] DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" @@ -26,7 +34,6 @@ class Database: def convert_datetime_from_bytes(bytes_string): return datetime.strptime(bytes_string.decode("utf-8"), DATE_FORMAT).replace(tzinfo=pytz.UTC) - @staticmethod def convert_datetime_from_string(st): return datetime.strptime(st, DATE_FORMAT).replace(tzinfo=pytz.UTC) @@ -54,42 +61,46 @@ class Database: @staticmethod def init_db(conn): - conn.execute("CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, VIN TEXT, longitude REAL, " - "latitude REAL, mileage REAL, level INTEGER, level_fuel INTEGER, moving BOOLEAN," - " temperature INTEGER);") + conn.execute("""CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, + VIN TEXT, longitude REAL, + latitude REAL, + mileage REAL, + level INTEGER, + level_fuel INTEGER, + moving BOOLEAN, + temperature INTEGER, + altitude INTEGER);""") make_backup = False - try: - conn.execute("ALTER TABLE position ADD level_fuel INTEGER;") - make_backup = True - except sqlite3.OperationalError: - pass conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, " "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);") - conn.create_function("update_trips", 0, Database.update_callback) conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN " "SELECT update_trips(); END;") conn.execute("""CREATE TABLE IF NOT EXISTS battery_curve (start_at DATETIME, VIN TEXT, date DATETIME, level INTEGER, UNIQUE(start_at, VIN, level));""") - for column, column_type in NEW_BATTERY_COLUMNS: - try: - conn.execute(f"ALTER TABLE battery ADD {column} {column_type};") - make_backup = True - except sqlite3.OperationalError: - pass + for table, columns in [["position", NEW_POSITION_COLUMNS], ["battery", NEW_BATTERY_COLUMNS]]: + for column, column_type in columns: + try: + conn.execute(f"ALTER TABLE {table} ADD {column} {column_type};") + make_backup = True + except sqlite3.OperationalError: + pass if make_backup: Database.backup(conn) Database.clean_battery(conn) + Database.add_altitude_to_db(conn) conn.commit() Database.db_initialized = True @staticmethod - def get_db(db_file=None): + def get_db(db_file=None, update_callback=True): if db_file is None: db_file = Database.DEFAULT_DB_FILE sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) conn.row_factory = sqlite3.Row + if update_callback: + conn.create_function("update_trips", 0, Database.update_callback) if not Database.db_initialized: Database.init_db(conn) return conn @@ -135,3 +146,84 @@ class Database: def get_battery_curve(conn, start_at, vin): return convert_sql_res(conn.execute("""SELECT date, level FROM battery_curve WHERE start_at=? and VIN=?;""", (start_at, vin)).fetchall()) + + @staticmethod + def add_altitude_to_db(conn): + max_pos_by_req = 100 + nb_null = conn.execute( + "SELECT COUNT(1) FROM position WHERE altitude IS NULL;").fetchone()[0] + if nb_null > max_pos_by_req: + logger.warning("There is %s to fetch from API, it can take some time") + try: + while True: + res = conn.execute(f"SELECT DISTINCT latitude,longitude " + f"FROM position WHERE altitude IS NULL LIMIT {max_pos_by_req};").fetchall() + nb_res = len(res) + if nb_res > 0: + logger.info("add altitude for %s", len(res)) + locations_str = "" + for line in res: + locations_str += str(line[0]) + "," + str(line[1]) + "|" + locations_str = locations_str[:-1] + res = requests.get("https://api.opentopodata.org/v1/srtm30m", + params={"locations": locations_str}) + data = res.json()["results"] + for line in data: + conn.execute("UPDATE position SET altitude=? WHERE latitude=? and longitude=?", + (line["elevation"], line["location"]["lat"], line["location"]["lng"])) + conn.commit() + if nb_res == 100: + sleep(1) # API is limited to 1 call by sec + else: + break + except (ValueError, KeyError): + logger.error("Can't get altitude from API") + logger.debug(traceback.format_exc()) + + @staticmethod + def get_recorded_position(): + conn = Database.get_db() + res = conn.execute('SELECT * FROM position ORDER BY Timestamp') + features_list = [] + for row in res: + if row["longitude"] is None or row["latitude"] is None: + continue + feature = Feature(geometry=Point((row["longitude"], row["latitude"])), + properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"), + "mileage": row["mileage"], + "level": row["level"], "level_fuel": row["level_fuel"]}) + features_list.append(feature) + feature_collection = FeatureCollection(features_list) + conn.close() + return geo_dumps(feature_collection, sort_keys=True) + + # pylint: disable=too-many-arguments + @staticmethod + def record_position(weather_api, vin, mileage, latitude, longitude, altitude, date, level, level_fuel, moving): + conn = Database.get_db() + if mileage == 0: # fix a bug of the api + logger.error("The api return a wrong mileage for %s : %f", vin, mileage) + else: + if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None: + temp = get_temp(latitude, longitude, weather_api) + if level_fuel == 0: # fix fuel level not provided when car is off + try: + level_fuel = conn.execute( + "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC " + "LIMIT 1", + (vin,)).fetchone()[0] + logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin) + except TypeError: + level_fuel = None + logger.info("level_fuel unfixed for %s", vin) + + conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,altitude,mileage,level,level_fuel," + "moving,temperature) VALUES(?,?,?,?,?,?,?,?,?,?)", + (date, vin, longitude, latitude, altitude, mileage, level, level_fuel, moving, temp)) + + conn.commit() + logger.info("new position recorded for %s", vin) + Database.clean_position(conn) + return True + logger.debug("position already saved") + return False diff --git a/web/figures.py b/web/figures.py index 3ae912d..ebada39 100644 --- a/web/figures.py +++ b/web/figures.py @@ -18,7 +18,7 @@ import pytz from libs.car import Car from libs.elec_price import ElecPrice -from trip import Trips +from trip import Trips, Trip from web.db import Database @@ -87,8 +87,9 @@ def get_figures(trips: Trips, charging: List[dict]): table_fig = dash_table.DataTable( id='trips-table', sort_action='native', - # sort_by=[{'column_id': 'start_at', 'direction': 'desc'}], - columns=[{'id': 'start_at', 'name': 'start at', 'type': 'datetime'}, + sort_by=[{'column_id': 'id', 'direction': 'desc'}], + columns=[{'id': 'id', 'name': '#', 'type': 'numeric'}, + {'id': 'start_at', 'name': 'start at', 'type': 'datetime'}, {'id': 'duration', 'name': 'duration', 'type': 'numeric', 'format': deepcopy(nb_format).symbol_suffix(" min").precision(0)}, {'id': 'speed_average', 'name': 'average speed', 'type': 'numeric', @@ -100,8 +101,18 @@ def get_figures(trips: Trips, charging: List[dict]): {'id': 'distance', 'name': 'distance', 'type': 'numeric', 'format': nb_format.symbol_suffix(" km").precision(1)}, {'id': 'mileage', 'name': 'mileage', 'type': 'numeric', - 'format': nb_format.symbol_suffix(" km").precision(1)}], - data=[tr.get_info() for tr in trips[::-1]], + 'format': nb_format}, + {'id': 'altitude_diff', 'name': 'Altitude diff', 'type': 'numeric', + 'format': deepcopy(nb_format).symbol_suffix(" m").precision(0)} + ], + style_data_conditional=[ + { + 'if': {'column_id': ['altitude_diff']}, + 'color': 'dodgerblue', + "text-decoration": "underline" + } + ], + data=trips.get_info(), page_size=50 ) # consumption_fig @@ -243,3 +254,15 @@ def get_battery_curve_fig(row: dict, car: Car): fig = px.line(battery_curves, x="level", y="speed") fig.update_layout(xaxis_title="Battery %", yaxis_title="Charging speed in kW") return html.Div(Graph(figure=fig)) + + +def get_altitude_fig(trip:Trip): + conn = Database.get_db() + res = list(map(list, conn.execute("SELECT mileage, altitude FROM position WHERE Timestamp>=? and Timestamp<=?;", + (trip.start_at, trip.end_at)).fetchall())) + start_mileage = res[0][0] + for line in res: + line[0] = line[0] - start_mileage + fig = px.line(res, x=0, y=1) + fig.update_layout(xaxis_title="Distance km", yaxis_title="Altitude m") + return html.Div(Graph(figure=fig)) diff --git a/web/views.py b/web/views.py index 9d32728..47d1722 100644 --- a/web/views.py +++ b/web/views.py @@ -63,7 +63,7 @@ def create_callback(): Output('consumption_fig_by_speed', 'figure'), Output('consumption_graph_by_temp', 'children'), Output('consumption', 'children'), - Output('tab_trips', 'children'), + Output('tab_trips_fig', 'children'), Output('tab_battery_fig', 'children'), Output('tab_charge', 'children'), Output('date-slider', 'max'), @@ -108,18 +108,29 @@ def create_callback(): [Input("battery-table", "active_cell"), Input("tab_battery_popup-close", "n_clicks")], [State('battery-table', 'data'), - State("tab_battery_popup", "is_open")] - ) + State("tab_battery_popup", "is_open")]) def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument if is_open is None: is_open = False if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: row = data[active_cell["row"]] - print("ok") return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True return "", False +@dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], + [Input("trips-table", "active_cell"), + Input("tab_trips_popup-close", "n_clicks")], + State("tab_trips_popup", "is_open")) +def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument + print("altitude") + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: + return figures.get_altitude_fig(trips[active_cell["row_id"]-1]), True + return "", False + + @dash_app.callback(Output({'role': ABRP_SWITCH + RESPONSE, 'vin': MATCH}, 'children'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'id'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'value')) @@ -208,7 +219,7 @@ def get_charge_control(): @app.route('/positions') def get_recorded_position(): - return FlaskResponse(myp.get_recorded_position(), mimetype='application/json') + return FlaskResponse(Database.get_recorded_position(), mimetype='application/json') @app.route('/abrp') @@ -236,6 +247,7 @@ def after_request(response): def update_trips(): global trips, chargings, cached_layout logger.info("update_data") + Database.add_altitude_to_db(Database.get_db(update_callback=False)) try: trips_by_vin = Trips.get_trips(myp.vehicles_list) trips = next(iter(trips_by_vin.values())) # todo handle multiple car @@ -310,7 +322,23 @@ def serve_layout(): html.Div([ dbc.Tabs([ dbc.Tab(label="Summary", tab_id="summary", children=summary_tab), - dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", children=[figures.table_fig]), + dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", + children=[html.Div(id="tab_trips_fig", children=figures.table_fig), + dbc.Modal( + [ + dbc.ModalHeader("Altitude"), + dbc.ModalBody(html.Div( + id="tab_trips_popup_graph")), + dbc.ModalFooter( + dbc.Button("Close", + id="tab_trips_popup-close", + className="ml-auto") + ), + ], + id="tab_trips_popup", + size="xl", + ) + ]), dbc.Tab(label="Battery", tab_id="battery", id="tab_battery", children=[html.Div(id="tab_battery_fig", children=[figures.battery_info]), dbc.Modal( From 248e3b284249323f6667cbdffe56a76ceab2f9c2 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 20:39:56 +0200 Subject: [PATCH 05/15] handle api not responding --- my_psacc.py | 75 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/my_psacc.py b/my_psacc.py index 6ca3d6a..414c30c 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -10,6 +10,7 @@ from time import sleep from oauth2_client.credentials_manager import ServiceInformation import paho.mqtt.client as mqtt +from requests.exceptions import RequestException import psa_connectedcar as psac from libs.car import Cars, Car @@ -108,9 +109,13 @@ class MyPSACC: return realm_info[self.realm]['app_name'] def refresh_token(self): - # pylint: disable=protected-access - self.manager._refresh_token() - self.save_config() + try: + # pylint: disable=protected-access + self.manager._refresh_token() + self.save_config() + except RequestException as e: + logger.error("Can't refresh token %s", e) + sleep(60) def api(self) -> psac.VehiclesApi: self.api_config.access_token = self.manager.access_token @@ -201,13 +206,18 @@ class MyPSACC: return otp_code def get_remote_access_token(self, password): - res = self.manager.post(REMOTE_URL + self.client_id, - json={"grant_type": "password", "password": password}, - headers=self.headers) - data = res.json() - self.remote_access_token = data["access_token"] - self.remote_refresh_token = data["refresh_token"] - return res + try: + res = self.manager.post(REMOTE_URL + self.client_id, + json={"grant_type": "password", "password": password}, + headers=self.headers) + data = res.json() + self.remote_access_token = data["access_token"] + self.remote_refresh_token = data["refresh_token"] + return res + except RequestException as e: + logger.error("Can't refresh remote token %s", e) + sleep(60) + return None def refresh_remote_token(self, force=False): if not force and self.remote_token_last_update is not None: @@ -215,26 +225,31 @@ class MyPSACC: if (datetime.now() - last_update).total_seconds() < MQTT_TOKEN_TTL: return None self.refresh_token() - if self.remote_refresh_token is None: - logger.error("remote_refresh_token isn't defined") - self.load_otp(force_new=True) - res = self.manager.post(REMOTE_URL + self.client_id, - json={"grant_type": "refresh_token", "refresh_token": self.remote_refresh_token}, - headers=self.headers) - data = res.json() - logger.debug("refresh_remote_token: %s", data) - if "access_token" in data: - self.remote_access_token = data["access_token"] - self.remote_refresh_token = data["refresh_token"] - self.remote_token_last_update = datetime.now() - else: - logger.error("can't refresh_remote_token: %s\n Create a new one", data) - self.remote_token_last_update = datetime.now() - otp_code = self.get_otp_code() - res = self.get_remote_access_token(otp_code) - self.mqtt_client.username_pw_set("IMA_OAUTH_ACCESS_TOKEN", self.remote_access_token) - self.save_config() - return res + try: + if self.remote_refresh_token is None: + logger.error("remote_refresh_token isn't defined") + self.load_otp(force_new=True) + res = self.manager.post(REMOTE_URL + self.client_id, + json={"grant_type": "refresh_token", "refresh_token": self.remote_refresh_token}, + headers=self.headers) + data = res.json() + logger.debug("refresh_remote_token: %s", data) + if "access_token" in data: + self.remote_access_token = data["access_token"] + self.remote_refresh_token = data["refresh_token"] + self.remote_token_last_update = datetime.now() + else: + logger.error("can't refresh_remote_token: %s\n Create a new one", data) + self.remote_token_last_update = datetime.now() + otp_code = self.get_otp_code() + res = self.get_remote_access_token(otp_code) + self.mqtt_client.username_pw_set("IMA_OAUTH_ACCESS_TOKEN", self.remote_access_token) + self.save_config() + return res + except RequestException as e: + logger.error("Can't refresh remote token %s", e) + sleep(60) + return None # pylint: disable=unused-argument def __on_mqtt_connect(self, client, userdata, result_code, _): From 4477e5d22a32b858fd76e51aab2efa7e84a24132 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 20:59:53 +0200 Subject: [PATCH 06/15] change log msg --- web/db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web/db.py b/web/db.py index 5d8c41e..8eab1aa 100644 --- a/web/db.py +++ b/web/db.py @@ -153,14 +153,15 @@ class Database: nb_null = conn.execute( "SELECT COUNT(1) FROM position WHERE altitude IS NULL;").fetchone()[0] if nb_null > max_pos_by_req: - logger.warning("There is %s to fetch from API, it can take some time") + logger.warning("There is %s to fetch from API, it can take some time", nb_null) try: while True: res = conn.execute(f"SELECT DISTINCT latitude,longitude " f"FROM position WHERE altitude IS NULL LIMIT {max_pos_by_req};").fetchall() nb_res = len(res) if nb_res > 0: - logger.info("add altitude for %s", len(res)) + logger.debug("add altitude for %s positions point", len(nb_null)) + nb_null -= nb_res locations_str = "" for line in res: locations_str += str(line[0]) + "," + str(line[1]) + "|" From 7664861fbcacd29dcbcb5de34e37297f9fd0fe52 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 21:10:02 +0200 Subject: [PATCH 07/15] fix altitude index --- my_psacc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my_psacc.py b/my_psacc.py index 414c30c..3f81e2e 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -464,7 +464,7 @@ class MyPSACC: longitude = car.status.last_position.geometry.coordinates[0] latitude = car.status.last_position.geometry.coordinates[1] - altitude = car.status.last_position.geometry.coordinates[3] + altitude = car.status.last_position.geometry.coordinates[2] date = car.status.last_position.properties.updated_at if date is None: date = charge_date From 9225dc471c53f35ae603594c4c97ffa49885a410 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 21:20:09 +0200 Subject: [PATCH 08/15] clode clean --- trip.py | 7 +++---- web/db.py | 4 ++-- web/figures.py | 4 ++-- web/views.py | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/trip.py b/trip.py index 36dd39c..158e2ce 100644 --- a/trip.py +++ b/trip.py @@ -1,4 +1,3 @@ -import traceback from statistics import mean from typing import List, Dict @@ -233,8 +232,8 @@ class Trips(list): def get_info(self): res = [] - id = 1 + row_id = 1 for trip in self: - res.append(trip.get_info(id)) - id += 1 + res.append(trip.get_info(row_id)) + row_id += 1 return res diff --git a/web/db.py b/web/db.py index 8eab1aa..df41969 100644 --- a/web/db.py +++ b/web/db.py @@ -156,8 +156,8 @@ class Database: logger.warning("There is %s to fetch from API, it can take some time", nb_null) try: while True: - res = conn.execute(f"SELECT DISTINCT latitude,longitude " - f"FROM position WHERE altitude IS NULL LIMIT {max_pos_by_req};").fetchall() + res = conn.execute("SELECT DISTINCT latitude,longitude " + "FROM position WHERE altitude IS NULL LIMIT ?;", max_pos_by_req).fetchall() nb_res = len(res) if nb_res > 0: logger.debug("add altitude for %s positions point", len(nb_null)) diff --git a/web/figures.py b/web/figures.py index ebada39..a8d8812 100644 --- a/web/figures.py +++ b/web/figures.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import List +import pytz import dash_bootstrap_components as dbc import dash_table import numpy as np @@ -14,7 +15,6 @@ import plotly.graph_objects as go from pandas import DataFrame from pandas import options as pandas_options import dash_html_components as html -import pytz from libs.car import Car from libs.elec_price import ElecPrice @@ -256,7 +256,7 @@ def get_battery_curve_fig(row: dict, car: Car): return html.Div(Graph(figure=fig)) -def get_altitude_fig(trip:Trip): +def get_altitude_fig(trip: Trip): conn = Database.get_db() res = list(map(list, conn.execute("SELECT mileage, altitude FROM position WHERE Timestamp>=? and Timestamp<=?;", (trip.start_at, trip.end_at)).fetchall())) diff --git a/web/views.py b/web/views.py index 47d1722..c312763 100644 --- a/web/views.py +++ b/web/views.py @@ -127,7 +127,7 @@ def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argumen if is_open is None: is_open = False if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: - return figures.get_altitude_fig(trips[active_cell["row_id"]-1]), True + return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True return "", False @@ -356,9 +356,9 @@ def serve_layout(): dbc.Tab(label="Map", tab_id="map", children=[maps]), dbc.Tab(label="Control", tab_id="control", children=dbc.Tabs(id="control-tabs", children=__get_control_tabs()))], - id="tabs", - active_tab="summary", - persistence=True), + id="tabs", + active_tab="summary", + persistence=True), html.Div(id=EMPTY_DIV), html.Div(id=EMPTY_DIV + "1") ])]) From 3b2aba44cf9043c68f20079316d4227c4a9d3b49 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Wed, 28 Apr 2021 21:20:58 +0200 Subject: [PATCH 09/15] handle api not responding --- web/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/db.py b/web/db.py index df41969..f94317e 100644 --- a/web/db.py +++ b/web/db.py @@ -177,7 +177,7 @@ class Database: sleep(1) # API is limited to 1 call by sec else: break - except (ValueError, KeyError): + except (ValueError, KeyError, requests.exceptions.RequestException): logger.error("Can't get altitude from API") logger.debug(traceback.format_exc()) From 40070cfe302c97e36fb3789cf673e82ae1256713 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Thu, 29 Apr 2021 09:59:29 +0200 Subject: [PATCH 10/15] fix brand code for DS --- app_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app_decoder.py b/app_decoder.py index 6a108be..1070523 100755 --- a/app_decoder.py +++ b/app_decoder.py @@ -20,7 +20,7 @@ from my_psacc import MyPSACC BRAND = {"com.psa.mym.myopel": {"realm": "clientsB2COpel", "brand_code": "OP", "app_name": "MyOpel"}, "com.psa.mym.mypeugeot": {"realm": "clientsB2CPeugeot", "brand_code": "AP", "app_name": "MyPeugeot"}, "com.psa.mym.mycitroen": {"realm": "clientsB2CCitroen", "brand_code": "AC", "app_name": "MyCitroen"}, - "com.psa.mym.myds": {"realm": "clientsB2CDS", "brand_code": "AC", "app_name": "MyDS"}, + "com.psa.mym.myds": {"realm": "clientsB2CDS", "brand_code": "DS", "app_name": "MyDS"}, "com.psa.mym.myvauxhall": {"realm": "clientsB2CVauxhall", "brand_code": "0V", "app_name": "MyVauxhall"} } From 86cb20f9e68dc1a64b84b5278ebaf5f41a029a7e Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Thu, 29 Apr 2021 12:10:43 +0200 Subject: [PATCH 11/15] some fix --- libs/charging.py | 13 +++++++------ libs/elec_price.py | 21 ++++++++++++++++----- otp/otp.py | 3 ++- server.py | 3 +++ web/db.py | 4 ++-- web/figures.py | 2 +- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/libs/charging.py b/libs/charging.py index a5de899..f47e45d 100644 --- a/libs/charging.py +++ b/libs/charging.py @@ -1,3 +1,4 @@ +import traceback from datetime import datetime from sqlite3 import IntegrityError @@ -8,10 +9,10 @@ from libs.elec_price import ElecPrice from mylogger import logger from web.db import Database -elec_price = ElecPrice.read_config() - class Charging: + elec_price: ElecPrice = None + @staticmethod def get_chargings(mini=None, maxi=None) -> List[dict]: conn = Database.get_db() @@ -29,18 +30,18 @@ class Charging: @staticmethod def set_default_price(): - if elec_price.is_enable(): + if Charging.elec_price.is_enable(): conn = Database.get_db() charge_list = list(map(dict, conn.execute("SELECT * FROM battery WHERE price IS NULL").fetchall())) for charge in charge_list: - charge["price"] = elec_price.get_price(charge["start_at"], charge["stop_at"], charge["kw"]) + charge["price"] = Charging.elec_price.get_price(charge["start_at"], charge["stop_at"], charge["kw"]) Database.set_chargings_price(conn, charge["start_at"], charge["price"]) conn.close() # pylint: disable=too-many-arguments @staticmethod def update_chargings(conn, start_at, stop_at, level, co2_per_kw, consumption_kw, vin): - price = elec_price.get_price(start_at, stop_at, consumption_kw) + price = Charging.elec_price.get_price(start_at, stop_at, consumption_kw) conn.execute( "UPDATE battery set stop_at=?, end_level=?, co2=?, kw=?, price=? WHERE start_at=? and VIN=?", (stop_at, level, co2_per_kw, consumption_kw, price, start_at, vin)) @@ -77,6 +78,6 @@ class Charging: Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, car.vin) except TypeError: - logger.debug("battery table is empty") + logger.debug("battery table is probably empty : %s", traceback.format_exc()) conn.commit() conn.close() diff --git a/libs/elec_price.py b/libs/elec_price.py index edb17cf..91d660d 100644 --- a/libs/elec_price.py +++ b/libs/elec_price.py @@ -2,7 +2,8 @@ from datetime import datetime, timezone, timedelta import configparser from statistics import mean -CONFIG_FILENAME = "config.ini" +from mylogger import logger + def set_number(value): @@ -18,13 +19,14 @@ def utc_to_local(utc_dt): class ElecPrice: currency = "" + CONFIG_FILENAME = "config.ini" def __init__(self, day_price, night_price=None, nights_hours=None): self.day_price = set_number(day_price) self.night_price = set_number(night_price) self.nights_hour = None self.set_night_hour(nights_hours) - self.config_filename = CONFIG_FILENAME + self.config_filename = ElecPrice.CONFIG_FILENAME def set_night_hour(self, value): if value is not None and isinstance(value, list): @@ -55,13 +57,20 @@ class ElecPrice: while date < end: prices.append(self.get_instant_price(date)) date = date + timedelta(minutes=30) - return round(consumption * mean(prices), 2) + try: + res = round(consumption * mean(prices), 2) + except TypeError: + logger.error("Can't get_price of charge, check config") + res = None + return res def is_enable(self): return self.day_price is not None @staticmethod - def read_config(name=CONFIG_FILENAME): + def read_config(name=None): + if name is None: + name = ElecPrice.CONFIG_FILENAME config = configparser.ConfigParser() if len(config.read(name)) == 0: ElecPrice.write_default_config(name) @@ -79,7 +88,9 @@ class ElecPrice: return ElecPrice(elec_config["day price"], night_price, night_hours) @staticmethod - def write_default_config(name=CONFIG_FILENAME): + def write_default_config(name=None): + if name is None: + name = ElecPrice.CONFIG_FILENAME config = configparser.ConfigParser() config["General"] = { "currency": "€" diff --git a/otp/otp.py b/otp/otp.py index 617d879..63301d9 100644 --- a/otp/otp.py +++ b/otp/otp.py @@ -278,7 +278,8 @@ class Otp: def __getstate__(self): odict = self.__dict__.copy() - del odict['cipher'] # don't pickle this + if 'cipher' in odict: + del odict['cipher'] # don't pickle this return odict def __setstate__(self, dict_param): diff --git a/server.py b/server.py index 0782b03..2f5a164 100755 --- a/server.py +++ b/server.py @@ -11,6 +11,8 @@ from oauth2_client.credentials_manager import OAuthError import web.app from charge_control import ChargeControls +from libs.charging import Charging +from libs.elec_price import ElecPrice from mylogger import my_logger from mylogger import logger from my_psacc import MyPSACC @@ -55,6 +57,7 @@ if __name__ == "__main__": web.app.myp = MyPSACC.load_config(name=CONFIG_NAME) atexit.register(web.app.myp.save_config) web.app.myp.set_record(args.record) + Charging.elec_price = ElecPrice.read_config() if args.offline: logger.info("offline mode") else: diff --git a/web/db.py b/web/db.py index f94317e..9056742 100644 --- a/web/db.py +++ b/web/db.py @@ -14,7 +14,7 @@ from geojson import dumps as geo_dumps from mylogger import logger from utils import get_temp -NEW_BATTERY_COLUMNS = [["battery", "INTEGER"], ["charging_mode", "TEXT"]] +NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]] NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]] DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" @@ -157,7 +157,7 @@ class Database: try: while True: res = conn.execute("SELECT DISTINCT latitude,longitude " - "FROM position WHERE altitude IS NULL LIMIT ?;", max_pos_by_req).fetchall() + "FROM position WHERE altitude IS NULL LIMIT ?;", (max_pos_by_req,)).fetchall() nb_res = len(res) if nb_res > 0: logger.debug("add altitude for %s positions point", len(nb_null)) diff --git a/web/figures.py b/web/figures.py index a8d8812..729bd46 100644 --- a/web/figures.py +++ b/web/figures.py @@ -220,7 +220,7 @@ def get_figures(trips: Trips, charging: List[dict]): else: consumption_graph_by_temp = html.Div(Graph(style={'display': 'none'}), id="consumption_graph_by_temp") - + return True def __calculate_co2_per_kw(charging_data): try: From 5f75d3fb30eaaf23cc041a2ca05fe94932be8d68 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Fri, 30 Apr 2021 10:26:55 +0200 Subject: [PATCH 12/15] improve performance & debug --- charge_control.py | 7 +++-- ecomix.py | 3 +-- libs/car.py | 4 +-- libs/charging.py | 5 ++-- libs/elec_price.py | 17 ++++++------ my_psacc.py | 13 +++++----- mylogger.py | 16 +++++++++--- otp/otp.py | 3 +-- requirements-dev.txt | 1 + trip.py | 49 +++++++++++++++------------------- trip_parser.py | 9 +++++-- utils.py | 5 ++-- web/abrp.py | 3 +-- web/db.py | 31 +++++++++++++--------- web/figures.py | 10 ++----- web/views.py | 62 ++++++++++++++++++++------------------------ 16 files changed, 117 insertions(+), 121 deletions(-) diff --git a/charge_control.py b/charge_control.py index 0a18e74..b91fc69 100644 --- a/charge_control.py +++ b/charge_control.py @@ -1,6 +1,5 @@ import json import threading -import traceback from copy import copy from datetime import datetime, timedelta from hashlib import md5 @@ -91,10 +90,10 @@ class ChargeControl: if self._next_stop_hour is not None and self._next_stop_hour < now: self._next_stop_hour += timedelta(days=1) self.retry_count = 0 - except AttributeError: - logger.error("Probably can't retrieve all information from API: %s", traceback.format_exc()) + except (AttributeError, ValueError): + logger.exception("Probably can't retrieve all information from API:") except: # pylint: disable=bare-except - logger.error(traceback.format_exc()) + logger.exception("Charge control:") def get_dict(self): chd = copy(self.__dict__) diff --git a/ecomix.py b/ecomix.py index ed3634e..ded9671 100644 --- a/ecomix.py +++ b/ecomix.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta from statistics import mean, StatisticsError import xml.etree.cElementTree as ElT import numbers -import traceback import requests import reverse_geocode @@ -68,7 +67,7 @@ class Ecomix: Ecomix._cache[country_code].append([datetime.now(), value]) return data["status"] == "ok" except (AssertionError, NameError, KeyError): - logger.debug(traceback.format_exc()) + logger.debug("ecomix:", exc_info=True) return False else: return False diff --git a/libs/car.py b/libs/car.py index 92f2ab3..27f1d0b 100644 --- a/libs/car.py +++ b/libs/car.py @@ -42,7 +42,7 @@ class Car: if self.status is not None: return self.status logger.error("status of %s is None", self.vin) - raise ValueError("status of %s is None") + raise ValueError("status of {} is None".format(self.vin)) @classmethod def from_json(cls, data: dict): @@ -82,7 +82,7 @@ class Car: class Cars(list): def __init__(self, *args): list.__init__(self, *args) - self.config_filename = "../cars.json" + self.config_filename = "cars.json" def get_car_by_vin(self, vin) -> Car: for car in self: diff --git a/libs/charging.py b/libs/charging.py index f47e45d..367c4e2 100644 --- a/libs/charging.py +++ b/libs/charging.py @@ -1,4 +1,3 @@ -import traceback from datetime import datetime from sqlite3 import IntegrityError @@ -11,7 +10,7 @@ from web.db import Database class Charging: - elec_price: ElecPrice = None + elec_price: ElecPrice = ElecPrice(None) @staticmethod def get_chargings(mini=None, maxi=None) -> List[dict]: @@ -78,6 +77,6 @@ class Charging: Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, car.vin) except TypeError: - logger.debug("battery table is probably empty : %s", traceback.format_exc()) + logger.debug("battery table is probably empty :", exc_info=True) conn.commit() conn.close() diff --git a/libs/elec_price.py b/libs/elec_price.py index 91d660d..1281e80 100644 --- a/libs/elec_price.py +++ b/libs/elec_price.py @@ -54,14 +54,15 @@ class ElecPrice: def get_price(self, start, end, consumption): prices = [] date = start - while date < end: - prices.append(self.get_instant_price(date)) - date = date + timedelta(minutes=30) - try: - res = round(consumption * mean(prices), 2) - except TypeError: - logger.error("Can't get_price of charge, check config") - res = None + res = None + if not (start is None or end is None): + while date < end: + prices.append(self.get_instant_price(date)) + date = date + timedelta(minutes=30) + try: + res = round(consumption * mean(prices), 2) + except TypeError: + logger.error("Can't get_price of charge, check config") return res def is_enable(self): diff --git a/my_psacc.py b/my_psacc.py index 3f81e2e..b03e81c 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -1,7 +1,6 @@ import json import re import threading -import traceback import uuid from datetime import datetime from json import JSONEncoder @@ -144,8 +143,9 @@ class MyPSACC: if self._record_enabled: self.record_info(car) return res - except ApiException: - logger.error(traceback.format_exc()) + except ApiException as ex: + logger.error("get_vehicle_info: ApiException: %s", ex) + logger.debug(exc_info=True) car.status = res return res @@ -174,7 +174,7 @@ class MyPSACC: self.vehicles_list.add(Car(vehicle.vin, vehicle.id, vehicle.brand, vehicle.label)) self.vehicles_list.save_cars() except ApiException: - logger.error(traceback.format_exc()) + logger.exception("get_vehicles:") return self.vehicles_list def load_otp(self, force_new=False): @@ -296,7 +296,7 @@ class MyPSACC: sleep(60) self.wakeup(data["vin"]) except KeyError: - logger.error(traceback.format_exc()) + logger.exception("mqtt message:") def start_mqtt(self): self.load_otp() @@ -342,8 +342,7 @@ class MyPSACC: minute = hour_minute[1] return hour, minute except IndexError: - logger.error(traceback.format_exc()) - logger.error("Can't get charge hour: %s", hour_str) + logger.exception("Can't get charge hour: %s", hour_str) return None def get_charge_status(self, vin): diff --git a/mylogger.py b/mylogger.py index 30c8291..3fba1ff 100644 --- a/mylogger.py +++ b/mylogger.py @@ -5,18 +5,26 @@ DEBUG_LEVELV_NUM = 9 logging.addLevelName(DEBUG_LEVELV_NUM, "DEBUGV") -def debugv(self, message, *args, **kws): - self.log(DEBUG_LEVELV_NUM, message, *args, **kws) +class CustomLogger(logging.Logger): + # pylint: disable=too-many-arguments + def __new_style_log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, **kwargs): + if kwargs.pop('style', "%") == "{": # optional + msg = msg.format(*args) + args = [] + super()._log(level, msg, args, exc_info, extra, stack_info) + + def debugv(self, msg, *args, **kwargs): + if self.isEnabledFor(DEBUG_LEVELV_NUM): + self.__new_style_log(DEBUG_LEVELV_NUM, msg, args, **kwargs) -logging.Logger.debugv = debugv +logging.setLoggerClass(CustomLogger) # pylint: disable=invalid-name logger = logging.getLogger("log") def my_logger(file='activity.log', handler_level=logging.INFO): global logger - logger.setLevel(handler_level) formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s') file_handler = RotatingFileHandler(file, 'a', 1000000, 1, encoding='utf8') diff --git a/otp/otp.py b/otp/otp.py index 63301d9..4cf4816 100644 --- a/otp/otp.py +++ b/otp/otp.py @@ -1,5 +1,4 @@ import hashlib -import traceback import pickle from secrets import token_hex, token_bytes from math import ceil @@ -322,7 +321,7 @@ def load_otp(filename="otp.bin"): except ModuleNotFoundError: return RenameUnpickler(input_file).load() except FileNotFoundError: - logger.debug(traceback.format_exc()) + logger.debug("",exc_info=True) return None diff --git a/requirements-dev.txt b/requirements-dev.txt index b3f631e..2af2088 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ prospector>=1.3.0 pre-commit +deepdiff \ No newline at end of file diff --git a/trip.py b/trip.py index 158e2ce..09a3b92 100644 --- a/trip.py +++ b/trip.py @@ -1,3 +1,4 @@ +import logging from statistics import mean from typing import List, Dict @@ -50,13 +51,14 @@ class Trip: return None def set_consumption(self, diff_level: float) -> float: - if self.distance is None: - raise ValueError("Distance not set") if diff_level < 0: logger.debugv("trip has negative consumption") diff_level = 0 self.consumption = diff_level * self.car.battery_power / 100 - self.consumption_km = 100 * self.consumption / self.distance # kw/100 km + try: + self.consumption_km = 100 * self.consumption / self.distance # kw/100 km + except TypeError: + raise ValueError("Distance not set") return self.consumption_km def set_fuel_consumption(self, consumption) -> float: @@ -142,7 +144,8 @@ class Trips(list): for vin in vehicles: trips = Trips() vin = vin[0] - res = conn.execute('SELECT * FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall() + res = conn.execute('SELECT Timestamp, VIN, longitude, latitude, mileage, level, moving, temperature,' + ' level_fuel, altitude FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall() if len(res) > 1: car = vehicles_list.get_car_by_vin(vin) assert car is not None @@ -152,8 +155,9 @@ class Trips(list): trip = Trip() # for debugging use this line res = list(map(dict,res)) for x in range(0, len(res) - 2): - logger.debugv("%s mileage:%.1f level:%s level_fuel:%s", - res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel']) + if logger.isEnabledFor(logging.DEBUG): # reduce execution time if debug disabled + logger.debugv("%s mileage:%.1f level:%s level_fuel:%s", + res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel']) next_el = res[x + 2] distance = end["mileage"] - start["mileage"] duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600 @@ -161,17 +165,11 @@ class Trips(list): speed_average = distance / duration except ZeroDivisionError: speed_average = 0 - restart_trip = False - if trip_parser.is_refuel(start, end, distance): - restart_trip = True - elif speed_average < 0.2 and duration > 0.05: - restart_trip = True - logger.debugv("low speed detected") - if restart_trip: + if TripParser.is_low_speed(speed_average, duration) or trip_parser.is_refuel(start, end, distance): start = end trip = Trip() - logger.debugv("restart trip at %s mileage:%.1f level:%s level_fuel:%s", - start['Timestamp'], start['mileage'], start['level'], start['level_fuel']) + logger.debugv("restart trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}" + " level_fuel:{0[level_fuel]}", start, style='{') else: distance = next_el["mileage"] - end["mileage"] # km duration = (next_el["Timestamp"] - end["Timestamp"]).total_seconds() / 3600 @@ -180,13 +178,9 @@ class Trips(list): except ZeroDivisionError: speed_average = 0 end_trip = False - if trip_parser.is_refuel(end, next_el, distance): + if trip_parser.is_refuel(end, next_el, distance) or \ + TripParser.is_low_speed(speed_average, duration): end_trip = True - elif speed_average < 0.2 and duration > 0.05: - # (distance == 0 and duration > 0.08) or duration > 2 or - # check the speed to handle missing point - end_trip = True - logger.debugv("low speed detected") elif duration > 2: end_trip = True logger.debugv("too much time detected") @@ -196,8 +190,8 @@ class Trips(list): end_trip = True logger.debugv("last position found") if end_trip: - logger.debugv("stop trip at %s mileage:%.1f level:%s level_fuel:%s", - end['Timestamp'], end['mileage'], end['level'], end['level_fuel']) + logger.debugv("stop trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}" + " level_fuel:{0[level_fuel]}", end, style='{') trip.distance = end["mileage"] - start["mileage"] # km if trip.distance > 0: trip.start_at = start["Timestamp"] @@ -215,11 +209,10 @@ class Trips(list): if diff_level_fuel != 0: trip.set_fuel_consumption(diff_level_fuel) trip.mileage = end["mileage"] - logger.debugv("Trip: %s -> %s %.1fkm %.2fh %.0fkm/h %.2fkWh %.2fkWh/100km %.2fL " - "%.2fL/100km %.1fkm", - trip.start_at, trip.end_at, trip.distance, trip.duration, - trip.speed_average, trip.consumption, trip.consumption_km, - trip.consumption_fuel, trip.consumption_fuel_km, trip.mileage) + logger.debugv("Trip: {0.start_at} -> {0.end_at} {0.distance:.1f}km {0.duration:.2f}h " + "{0.speed_average:.0f}km/h {0.consumption:.2f}kWh " + "{0.consumption_km:.2f}kWh/100km {0.consumption_fuel:.2f}L " + "{0.consumption_fuel_km:.2f}L/100km {0.mileage:.1f}km", trip, style="{") # filter bad value trips.check_and_append(trip) start = next_el diff --git a/trip_parser.py b/trip_parser.py index 5917b13..3426a29 100644 --- a/trip_parser.py +++ b/trip_parser.py @@ -2,9 +2,9 @@ from collections.abc import Callable from libs.car import Car from mylogger import logger -LEVEL = "level" +LEVEL = 5 -LEVEL_FUEL = "level_fuel" +LEVEL_FUEL = 8 class TripParser: @@ -67,3 +67,8 @@ class TripParser: # If distance is bigger than 0 but charge bigger than five there is probably missing point and we assume that # regeneration/temperature can't increase by 5 percent the battery level return decharge < -2 and (distance == 0 or decharge < -5) + + @staticmethod + def is_low_speed(speed_average, duration): + logger.debugv("Low speed detected") + return speed_average < 0.2 and duration > 0.05 diff --git a/utils.py b/utils.py index bd54e91..0058bc3 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,3 @@ -import traceback from functools import wraps from threading import Semaphore, Timer import socket @@ -20,9 +19,9 @@ def get_temp(latitude: str, longitude: str, api_key: str) -> float: logger.debug("Temperature :%fc", temp) return temp except ConnectionError: - logger.error("Can't connect to openweathermap :%s", traceback.format_exc()) + logger.error("Can't connect to openweathermap :", exc_info=True) except KeyError: - logger.error("Unable to get temperature from openweathermap :%s", traceback.format_exc()) + logger.error("Unable to get temperature from openweathermap :", exc_info=True) return None diff --git a/web/abrp.py b/web/abrp.py index 23dba19..db63cc1 100644 --- a/web/abrp.py +++ b/web/abrp.py @@ -1,5 +1,4 @@ import json -import traceback from datetime import datetime import requests @@ -43,7 +42,7 @@ class Abrp: logger.debug(response.text) return response.json()["status"] == "ok" except (AttributeError, IndexError, ValueError): - logger.error(traceback.format_exc()) + logger.exception("abrp:") return False def __iter__(self): diff --git a/web/db.py b/web/db.py index 9056742..9b27d20 100644 --- a/web/db.py +++ b/web/db.py @@ -1,6 +1,5 @@ import sys import sqlite3 -import traceback from datetime import datetime from time import sleep @@ -17,30 +16,37 @@ from utils import get_temp NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]] NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]] -DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" - def convert_sql_res(rows): return list(map(dict, rows)) +DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" + + +def new_convert_datetime_from_string(string): + return datetime.fromisoformat(string) + + class Database: callback_fct: Callable[[], None] = lambda: None DEFAULT_DB_FILE = 'info.db' - # pylint: disable=invalid-name db_initialized = False @staticmethod - def convert_datetime_from_bytes(bytes_string): - return datetime.strptime(bytes_string.decode("utf-8"), DATE_FORMAT).replace(tzinfo=pytz.UTC) + def convert_datetime_from_string(string): + try: + return datetime.strptime(string, DATE_FORMAT).replace(tzinfo=pytz.UTC) + except ValueError: + return datetime.strptime(string.replace("T", " "), DATE_FORMAT).replace(tzinfo=pytz.UTC) @staticmethod - def convert_datetime_from_string(st): - return datetime.strptime(st, DATE_FORMAT).replace(tzinfo=pytz.UTC) + def convert_datetime_from_bytes(bytes_string): + return Database.convert_datetime_from_string(bytes_string.decode("utf-8")) @staticmethod def convert_datetime_to_string(date: datetime): - return date.replace(tzinfo=pytz.UTC).strftime(DATE_FORMAT) + return date.replace(tzinfo=pytz.UTC).isoformat(timespec='seconds', sep=" ") @staticmethod def update_callback(): @@ -89,14 +95,16 @@ class Database: Database.clean_battery(conn) Database.add_altitude_to_db(conn) conn.commit() + if sys.version_info >= (3, 7): + Database.convert_datetime_from_string = new_convert_datetime_from_string + sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) + sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) Database.db_initialized = True @staticmethod def get_db(db_file=None, update_callback=True): if db_file is None: db_file = Database.DEFAULT_DB_FILE - sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) - sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) conn.row_factory = sqlite3.Row if update_callback: @@ -179,7 +187,6 @@ class Database: break except (ValueError, KeyError, requests.exceptions.RequestException): logger.error("Can't get altitude from API") - logger.debug(traceback.format_exc()) @staticmethod def get_recorded_position(): diff --git a/web/figures.py b/web/figures.py index 729bd46..088ce0c 100644 --- a/web/figures.py +++ b/web/figures.py @@ -1,9 +1,7 @@ from copy import deepcopy -from datetime import datetime from typing import List -import pytz import dash_bootstrap_components as dbc import dash_table import numpy as np @@ -233,13 +231,9 @@ def __calculate_co2_per_kw(charging_data): return 0 -def dash_date_to_datetime(dash_date): - return datetime.strptime(dash_date, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=pytz.UTC) - - def get_battery_curve_fig(row: dict, car: Car): - start_date = dash_date_to_datetime(row["start_at"]) - stop_at = dash_date_to_datetime(row["stop_at"]) + start_date = Database.convert_datetime_from_string(row["start_at"]) + stop_at = Database.convert_datetime_from_string(row["stop_at"]) res = Database.get_battery_curve(Database.get_db(), start_date, car.vin) res.insert(0, {"level": row["start_level"], "date": start_date}) res.append({"level": row["end_level"], "date": stop_at}) diff --git a/web/views.py b/web/views.py index c312763..2ffae08 100644 --- a/web/views.py +++ b/web/views.py @@ -1,5 +1,4 @@ import json -import traceback from datetime import datetime, timezone from typing import List @@ -55,7 +54,7 @@ def diff_dashtable(data, data_previous, row_id_name="row_id"): return changes -def create_callback(): +def create_callback(): # flake8: noqa: C901 global CALLBACK_CREATED if not CALLBACK_CREATED: @dash_app.callback(Output('trips_map', 'figure'), @@ -95,42 +94,38 @@ def create_callback(): diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': - if not Database.set_chargings_price(Database.get_db(), - figures.dash_date_to_datetime(changed_line['start_at']), + if not Database.set_chargings_price(Database.get_db(),changed_line['start_at'], changed_line['current_value']): logger.error("Can't find line to update in the database") return "" + @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ], + [Input("battery-table", "active_cell"), + Input("tab_battery_popup-close", "n_clicks")], + [State('battery-table', 'data'), + State("tab_battery_popup", "is_open")]) + def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument, unused-variable + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: + row = data[active_cell["row"]] + return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True + return "", False + + @dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], + [Input("trips-table", "active_cell"), + Input("tab_trips_popup-close", "n_clicks")], + State("tab_trips_popup", "is_open")) + def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument, unused-variable + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: + return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True + return "", False + CALLBACK_CREATED = True -@dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ], - [Input("battery-table", "active_cell"), - Input("tab_battery_popup-close", "n_clicks")], - [State('battery-table', 'data'), - State("tab_battery_popup", "is_open")]) -def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument - if is_open is None: - is_open = False - if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: - row = data[active_cell["row"]] - return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True - return "", False - - -@dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], - [Input("trips-table", "active_cell"), - Input("tab_trips_popup-close", "n_clicks")], - State("tab_trips_popup", "is_open")) -def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument - print("altitude") - if is_open is None: - is_open = False - if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: - return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True - return "", False - - @dash_app.callback(Output({'role': ABRP_SWITCH + RESPONSE, 'vin': MATCH}, 'children'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'id'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'value')) @@ -267,7 +262,7 @@ def update_trips(): marks = figures.get_marks_from_start_end(min_date, max_date) cached_layout = None # force regenerate layout except (ValueError, IndexError): - logger.error("update_trips (slider): %s", traceback.format_exc()) + logger.error("update_trips (slider): %s", exc_info=True) return @@ -315,7 +310,6 @@ def serve_layout(): summary_tab = figures.ERROR_DIV maps = figures.ERROR_DIV logger.warning("Failed to generate figure, there is probably not enough data yet") - logger.debug(traceback.format_exc()) range_slider = html.Div() data_div = html.Div([ range_slider, @@ -371,6 +365,6 @@ try: Charging.set_default_price() update_trips() except (IndexError, TypeError): - logger.debug("Failed to get trips, there is probably not enough data yet %s", traceback.format_exc()) + logger.debug("Failed to get trips, there is probably not enough data yet:", exc_info=True) dash_app.layout = serve_layout From 9228886be72dc7c5c44c71f16c67084ad08f5e14 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Fri, 30 Apr 2021 10:44:32 +0200 Subject: [PATCH 13/15] fix space --- .pre-commit-config.yaml | 2 +- .prospector.yaml | 3 +++ .pylintrc | 2 ++ otp/otp.py | 2 +- web/views.py | 2 +- 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bb311c..bfeb6c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,4 +3,4 @@ repos: rev: 1.3.1 # The version of Prospector to use, at least 1.1.7 hooks: - id: prospector - language: system + language: python diff --git a/.prospector.yaml b/.prospector.yaml index 0af2a38..56712b4 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -13,3 +13,6 @@ pylint: - C0114 - C0115 - C0116 + - W0603 + - I0011 + - W0511 diff --git a/.pylintrc b/.pylintrc index a945bd1..8ea5eb8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,6 +1,8 @@ [MASTER] disable= C0114,C0115,C0116,W0603,I0011,W0511 +enable= + C0326 [FORMAT] max-line-length=120 diff --git a/otp/otp.py b/otp/otp.py index 4cf4816..17d6e21 100644 --- a/otp/otp.py +++ b/otp/otp.py @@ -321,7 +321,7 @@ def load_otp(filename="otp.bin"): except ModuleNotFoundError: return RenameUnpickler(input_file).load() except FileNotFoundError: - logger.debug("",exc_info=True) + logger.debug("", exc_info=True) return None diff --git a/web/views.py b/web/views.py index 2ffae08..3965870 100644 --- a/web/views.py +++ b/web/views.py @@ -91,7 +91,7 @@ def create_callback(): # flake8: noqa: C901 def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate - diff_data = diff_dashtable(data, data_previous, "start_at") + diff_data = diff_dashtable(data, data_previous,"start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': if not Database.set_chargings_price(Database.get_db(),changed_line['start_at'], From a500e309c56af8188218e9f283d391578fca2355 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Fri, 30 Apr 2021 12:06:01 +0200 Subject: [PATCH 14/15] code clean up --- .pre-commit-config.yaml | 4 +++- .prospector.yaml | 3 +++ .pylintrc | 20 -------------------- server.py | 2 +- trip.py | 5 ++--- web/views.py | 16 ++++++++-------- 6 files changed, 17 insertions(+), 33 deletions(-) delete mode 100644 .pylintrc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfeb6c2..2ae7776 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,8 @@ +default_language_version: + python: python3.6 repos: - repo: https://github.com/PyCQA/prospector rev: 1.3.1 # The version of Prospector to use, at least 1.1.7 hooks: - id: prospector - language: python + language: system diff --git a/.prospector.yaml b/.prospector.yaml index 56712b4..5e0b84a 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -1,4 +1,6 @@ doc-warnings: false +use: flask +max-line-length: 120 ignore-paths: - psa_connectedcar pep8: @@ -16,3 +18,4 @@ pylint: - W0603 - I0011 - W0511 + diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 8ea5eb8..0000000 --- a/.pylintrc +++ /dev/null @@ -1,20 +0,0 @@ -[MASTER] -disable= - C0114,C0115,C0116,W0603,I0011,W0511 -enable= - C0326 -[FORMAT] -max-line-length=120 - -[BASIC] -good-names=i, - j, - k, - ex, - Run, - e, - f, - t, - x, - ip, - s diff --git a/server.py b/server.py index 2f5a164..b9e28cb 100755 --- a/server.py +++ b/server.py @@ -42,7 +42,7 @@ def parse_args(): return parser.parse_args() -# flake8: noqa: C901 +# noqa: MC0001 if __name__ == "__main__": if sys.version_info < (3, 6): raise RuntimeError("This application requires Python 3.6+") diff --git a/trip.py b/trip.py index 09a3b92..9960abe 100644 --- a/trip.py +++ b/trip.py @@ -133,10 +133,9 @@ class Trips(list): logger.debugv("trip discarded") return False - # flake8: noqa: C901 - @staticmethod + @staticmethod # noqa: MC0001 def get_trips(vehicles_list: Cars) -> Dict[str, "Trips"]: - # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks + # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks,too-many-branches conn = Database.get_db() vehicles = conn.execute( "SELECT DISTINCT vin FROM position;").fetchall() diff --git a/web/views.py b/web/views.py index 3965870..ac330da 100644 --- a/web/views.py +++ b/web/views.py @@ -54,7 +54,7 @@ def diff_dashtable(data, data_previous, row_id_name="row_id"): return changes -def create_callback(): # flake8: noqa: C901 +def create_callback(): # noqa: MC0001 global CALLBACK_CREATED if not CALLBACK_CREATED: @dash_app.callback(Output('trips_map', 'figure'), @@ -81,8 +81,8 @@ def create_callback(): # flake8: noqa: C901 consumption = "Average consumption: {:.1f} kWh/100km".format( float(figures.consumption_df["consumption_km"].mean())) return figures.trips_map, figures.consumption_fig, figures.consumption_fig_by_speed, \ - figures.consumption_graph_by_temp, consumption, figures.table_fig, figures.battery_info, \ - figures.battery_table, max_millis, step, marks + figures.consumption_graph_by_temp, consumption, figures.table_fig, figures.battery_info, \ + figures.battery_table, max_millis, step, marks @dash_app.callback(Output(EMPTY_DIV, "children"), [Input("battery-table", "data_timestamp")], @@ -91,10 +91,10 @@ def create_callback(): # flake8: noqa: C901 def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate - diff_data = diff_dashtable(data, data_previous,"start_at") + diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': - if not Database.set_chargings_price(Database.get_db(),changed_line['start_at'], + if not Database.set_chargings_price(Database.get_db(), changed_line['start_at'], changed_line['current_value']): logger.error("Can't find line to update in the database") return "" @@ -350,9 +350,9 @@ def serve_layout(): dbc.Tab(label="Map", tab_id="map", children=[maps]), dbc.Tab(label="Control", tab_id="control", children=dbc.Tabs(id="control-tabs", children=__get_control_tabs()))], - id="tabs", - active_tab="summary", - persistence=True), + id="tabs", + active_tab="summary", + persistence=True), html.Div(id=EMPTY_DIV), html.Div(id=EMPTY_DIV + "1") ])]) From d73eea06b64c2c913a02c97add0331c4307d50c3 Mon Sep 17 00:00:00 2001 From: Florian Bezannier Date: Fri, 30 Apr 2021 12:34:19 +0200 Subject: [PATCH 15/15] add features --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 22fdcf0..ed7423f 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,13 @@ With this app you will be able to : - control air conditioning - control lights and horn if your vehicle is compatible (mine isn't) - get consumption statistic - - visualize your trips on a map - + - visualize your trips on a map or in a table + - get the list of car charging + - visualize battery charging curve + - visualize altitude trip curve + - get car charging co2 emission + - get car charging price + The official api is documented [here](https://developer.groupe-psa.io/webapi/b2c/quickstart/connect/#article) but it is not totally up to date, and contains some errors. A video in French was made by vlycop to explain how to use this application : https://youtu.be/XO7-N7G3biU @@ -20,7 +25,7 @@ A video in French was made by vlycop to explain how to use this application : ht ## I. Get credentials We need to get credentials from the android app. -We will retrieve these informations: +We will retrieve this information: - client-id and client-secret for the api - some url to login