diff --git a/my_psacc.py b/my_psacc.py index d4ba842..6ca3d6a 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -10,8 +10,6 @@ from time import sleep from oauth2_client.credentials_manager import ServiceInformation import paho.mqtt.client as mqtt -from geojson import Feature, Point, FeatureCollection -from geojson import dumps as geo_dumps import psa_connectedcar as psac from libs.car import Cars, Car @@ -22,7 +20,7 @@ from otp.otp import load_otp, new_otp_session, save_otp, ConfigException, Otp from psa_connectedcar.rest import ApiException from mylogger import logger -from utils import get_temp, rate_limit +from utils import rate_limit from web.abrp import Abrp from web.db import Database @@ -451,13 +449,15 @@ class MyPSACC: longitude = car.status.last_position.geometry.coordinates[0] latitude = car.status.last_position.geometry.coordinates[1] + altitude = car.status.last_position.geometry.coordinates[3] date = car.status.last_position.properties.updated_at if date is None: date = charge_date logger.debug("vin:%s longitude:%s latitude:%s date:%s mileage:%s level:%s charge_date:%s level_fuel:" "%s moving:%s", car.vin, longitude, latitude, date, mileage, level, charge_date, level_fuel, moving) - self.__record_position(car.vin, mileage, latitude, longitude, date, level, level_fuel, moving) + Database.record_position(self.weather_api, car.vin, mileage, latitude, longitude, altitude, date, level, + level_fuel, moving) self.abrp.call(car, Database.get_last_temp(car.vin)) try: charging_status = car.status.get_energy('Electric').charging.status @@ -467,52 +467,6 @@ class MyPSACC: except AttributeError: logger.error("charging status not available from api") - def __record_position(self, vin, mileage, latitude, longitude, date, level, level_fuel, moving): - conn = Database.get_db() - if mileage == 0: # fix a bug of the api - logger.error("The api return a wrong mileage for %s : %f", vin, mileage) - else: - if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None: - temp = get_temp(latitude, longitude, self.weather_api) - if level_fuel == 0: # fix fuel level not provided when car is off - try: - level_fuel = conn.execute( - "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC " - "LIMIT 1", - (vin,)).fetchone()[0] - logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin) - except TypeError: - level_fuel = None - logger.info("level_fuel unfixed for %s", vin) - - conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,mileage,level,level_fuel,moving," - "temperature) VALUES(?,?,?,?,?,?,?,?,?)", - (date, vin, longitude, latitude, mileage, level, level_fuel, moving, temp)) - - conn.commit() - logger.info("new position recorded for %s", vin) - Database.clean_position(conn) - return True - logger.debug("position already saved") - return False - - @staticmethod - def get_recorded_position(): - conn = Database.get_db() - res = conn.execute('SELECT * FROM position ORDER BY Timestamp') - features_list = [] - for row in res: - if row["longitude"] is None or row["latitude"] is None: - continue - feature = Feature(geometry=Point((row["longitude"], row["latitude"])), - properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"), - "mileage": row["mileage"], - "level": row["level"], "level_fuel": row["level_fuel"]}) - features_list.append(feature) - feature_collection = FeatureCollection(features_list) - conn.close() - return geo_dumps(feature_collection, sort_keys=True) - def __iter__(self): for key, value in self.__dict__.items(): yield key, value diff --git a/trip.py b/trip.py index 7d312cd..36dd39c 100644 --- a/trip.py +++ b/trip.py @@ -1,3 +1,4 @@ +import traceback from statistics import mean from typing import List, Dict @@ -6,7 +7,6 @@ from geojson import Feature, FeatureCollection, MultiLineString from libs.car import Cars, Car from mylogger import logger -from psa_connectedcar import Trips from trip_parser import TripParser from web.db import Database @@ -36,6 +36,7 @@ class Trip: self.duration = None self.mileage = None self.car: Car = None + self.altitude_diff = None self.temperatures = [] def add_points(self, latitude, longitude): @@ -87,14 +88,22 @@ class Trip: "average consumption": self.consumption_km, "average consumption fuel": self.consumption_fuel_km}) - def get_info(self): + def get_info(self, row_id=None): res = {"start_at": self.start_at.astimezone(tz.tzlocal()).replace(tzinfo=None).strftime("%x %X"), # convert to naive tz, "duration": self.duration * 60, "speed_average": self.speed_average, "consumption_km": self.consumption_km, "consumption_fuel_km": self.consumption_fuel_km, - "distance": self.distance, "mileage": self.mileage} + "distance": self.distance, "mileage": self.mileage, "altitude_diff": self.altitude_diff} + if row_id is not None: + res["id"] = row_id return res + def set_altitude_diff(self, start, end): + try: + self.altitude_diff = end - start + except (NameError, TypeError): + pass + class Trips(list): def __init__(self, *args): @@ -125,7 +134,7 @@ class Trips(list): # flake8: noqa: C901 @staticmethod - def get_trips(vehicles_list: Cars) -> Dict[str, Trips]: + def get_trips(vehicles_list: Cars) -> Dict[str, "Trips"]: # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks conn = Database.get_db() vehicles = conn.execute( @@ -200,6 +209,7 @@ class Trips(list): trip.duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600 trip.speed_average = trip.distance / trip.duration diff_level, diff_level_fuel = trip_parser.get_level_consumption(start, end) + trip.set_altitude_diff(start["altitude"], end["altitude"]) trip.car = car if diff_level != 0: trip.set_consumption(diff_level) # kw @@ -220,3 +230,11 @@ class Trips(list): end = next_el trips_by_vin[vin] = trips return trips_by_vin + + def get_info(self): + res = [] + id = 1 + for trip in self: + res.append(trip.get_info(id)) + id += 1 + return res diff --git a/web/db.py b/web/db.py index ea831a0..5d8c41e 100644 --- a/web/db.py +++ b/web/db.py @@ -1,13 +1,21 @@ import sys import sqlite3 +import traceback from datetime import datetime +from time import sleep from typing import Callable import pytz +import requests + +from geojson import Feature, Point, FeatureCollection +from geojson import dumps as geo_dumps from mylogger import logger +from utils import get_temp NEW_BATTERY_COLUMNS = [["battery", "INTEGER"], ["charging_mode", "TEXT"]] +NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]] DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" @@ -26,7 +34,6 @@ class Database: def convert_datetime_from_bytes(bytes_string): return datetime.strptime(bytes_string.decode("utf-8"), DATE_FORMAT).replace(tzinfo=pytz.UTC) - @staticmethod def convert_datetime_from_string(st): return datetime.strptime(st, DATE_FORMAT).replace(tzinfo=pytz.UTC) @@ -54,42 +61,46 @@ class Database: @staticmethod def init_db(conn): - conn.execute("CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, VIN TEXT, longitude REAL, " - "latitude REAL, mileage REAL, level INTEGER, level_fuel INTEGER, moving BOOLEAN," - " temperature INTEGER);") + conn.execute("""CREATE TABLE IF NOT EXISTS position (Timestamp DATETIME PRIMARY KEY, + VIN TEXT, longitude REAL, + latitude REAL, + mileage REAL, + level INTEGER, + level_fuel INTEGER, + moving BOOLEAN, + temperature INTEGER, + altitude INTEGER);""") make_backup = False - try: - conn.execute("ALTER TABLE position ADD level_fuel INTEGER;") - make_backup = True - except sqlite3.OperationalError: - pass conn.execute("CREATE TABLE IF NOT EXISTS battery (start_at DATETIME PRIMARY KEY,stop_at DATETIME,VIN TEXT, " "start_level INTEGER, end_level INTEGER, co2 INTEGER, kw INTEGER);") - conn.create_function("update_trips", 0, Database.update_callback) conn.execute("CREATE TEMP TRIGGER IF NOT EXISTS update_trigger AFTER INSERT ON position BEGIN " "SELECT update_trips(); END;") conn.execute("""CREATE TABLE IF NOT EXISTS battery_curve (start_at DATETIME, VIN TEXT, date DATETIME, level INTEGER, UNIQUE(start_at, VIN, level));""") - for column, column_type in NEW_BATTERY_COLUMNS: - try: - conn.execute(f"ALTER TABLE battery ADD {column} {column_type};") - make_backup = True - except sqlite3.OperationalError: - pass + for table, columns in [["position", NEW_POSITION_COLUMNS], ["battery", NEW_BATTERY_COLUMNS]]: + for column, column_type in columns: + try: + conn.execute(f"ALTER TABLE {table} ADD {column} {column_type};") + make_backup = True + except sqlite3.OperationalError: + pass if make_backup: Database.backup(conn) Database.clean_battery(conn) + Database.add_altitude_to_db(conn) conn.commit() Database.db_initialized = True @staticmethod - def get_db(db_file=None): + def get_db(db_file=None, update_callback=True): if db_file is None: db_file = Database.DEFAULT_DB_FILE sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) conn.row_factory = sqlite3.Row + if update_callback: + conn.create_function("update_trips", 0, Database.update_callback) if not Database.db_initialized: Database.init_db(conn) return conn @@ -135,3 +146,84 @@ class Database: def get_battery_curve(conn, start_at, vin): return convert_sql_res(conn.execute("""SELECT date, level FROM battery_curve WHERE start_at=? and VIN=?;""", (start_at, vin)).fetchall()) + + @staticmethod + def add_altitude_to_db(conn): + max_pos_by_req = 100 + nb_null = conn.execute( + "SELECT COUNT(1) FROM position WHERE altitude IS NULL;").fetchone()[0] + if nb_null > max_pos_by_req: + logger.warning("There is %s to fetch from API, it can take some time") + try: + while True: + res = conn.execute(f"SELECT DISTINCT latitude,longitude " + f"FROM position WHERE altitude IS NULL LIMIT {max_pos_by_req};").fetchall() + nb_res = len(res) + if nb_res > 0: + logger.info("add altitude for %s", len(res)) + locations_str = "" + for line in res: + locations_str += str(line[0]) + "," + str(line[1]) + "|" + locations_str = locations_str[:-1] + res = requests.get("https://api.opentopodata.org/v1/srtm30m", + params={"locations": locations_str}) + data = res.json()["results"] + for line in data: + conn.execute("UPDATE position SET altitude=? WHERE latitude=? and longitude=?", + (line["elevation"], line["location"]["lat"], line["location"]["lng"])) + conn.commit() + if nb_res == 100: + sleep(1) # API is limited to 1 call by sec + else: + break + except (ValueError, KeyError): + logger.error("Can't get altitude from API") + logger.debug(traceback.format_exc()) + + @staticmethod + def get_recorded_position(): + conn = Database.get_db() + res = conn.execute('SELECT * FROM position ORDER BY Timestamp') + features_list = [] + for row in res: + if row["longitude"] is None or row["latitude"] is None: + continue + feature = Feature(geometry=Point((row["longitude"], row["latitude"])), + properties={"vin": row["vin"], "date": row["Timestamp"].strftime("%x %X"), + "mileage": row["mileage"], + "level": row["level"], "level_fuel": row["level_fuel"]}) + features_list.append(feature) + feature_collection = FeatureCollection(features_list) + conn.close() + return geo_dumps(feature_collection, sort_keys=True) + + # pylint: disable=too-many-arguments + @staticmethod + def record_position(weather_api, vin, mileage, latitude, longitude, altitude, date, level, level_fuel, moving): + conn = Database.get_db() + if mileage == 0: # fix a bug of the api + logger.error("The api return a wrong mileage for %s : %f", vin, mileage) + else: + if conn.execute("SELECT Timestamp from position where Timestamp=?", (date,)).fetchone() is None: + temp = get_temp(latitude, longitude, weather_api) + if level_fuel == 0: # fix fuel level not provided when car is off + try: + level_fuel = conn.execute( + "SELECT level_fuel FROM position WHERE level_fuel>0 AND VIN=? ORDER BY Timestamp DESC " + "LIMIT 1", + (vin,)).fetchone()[0] + logger.info("level_fuel fixed with last real value %f for %s", level_fuel, vin) + except TypeError: + level_fuel = None + logger.info("level_fuel unfixed for %s", vin) + + conn.execute("INSERT INTO position(Timestamp,VIN,longitude,latitude,altitude,mileage,level,level_fuel," + "moving,temperature) VALUES(?,?,?,?,?,?,?,?,?,?)", + (date, vin, longitude, latitude, altitude, mileage, level, level_fuel, moving, temp)) + + conn.commit() + logger.info("new position recorded for %s", vin) + Database.clean_position(conn) + return True + logger.debug("position already saved") + return False diff --git a/web/figures.py b/web/figures.py index 3ae912d..ebada39 100644 --- a/web/figures.py +++ b/web/figures.py @@ -18,7 +18,7 @@ import pytz from libs.car import Car from libs.elec_price import ElecPrice -from trip import Trips +from trip import Trips, Trip from web.db import Database @@ -87,8 +87,9 @@ def get_figures(trips: Trips, charging: List[dict]): table_fig = dash_table.DataTable( id='trips-table', sort_action='native', - # sort_by=[{'column_id': 'start_at', 'direction': 'desc'}], - columns=[{'id': 'start_at', 'name': 'start at', 'type': 'datetime'}, + sort_by=[{'column_id': 'id', 'direction': 'desc'}], + columns=[{'id': 'id', 'name': '#', 'type': 'numeric'}, + {'id': 'start_at', 'name': 'start at', 'type': 'datetime'}, {'id': 'duration', 'name': 'duration', 'type': 'numeric', 'format': deepcopy(nb_format).symbol_suffix(" min").precision(0)}, {'id': 'speed_average', 'name': 'average speed', 'type': 'numeric', @@ -100,8 +101,18 @@ def get_figures(trips: Trips, charging: List[dict]): {'id': 'distance', 'name': 'distance', 'type': 'numeric', 'format': nb_format.symbol_suffix(" km").precision(1)}, {'id': 'mileage', 'name': 'mileage', 'type': 'numeric', - 'format': nb_format.symbol_suffix(" km").precision(1)}], - data=[tr.get_info() for tr in trips[::-1]], + 'format': nb_format}, + {'id': 'altitude_diff', 'name': 'Altitude diff', 'type': 'numeric', + 'format': deepcopy(nb_format).symbol_suffix(" m").precision(0)} + ], + style_data_conditional=[ + { + 'if': {'column_id': ['altitude_diff']}, + 'color': 'dodgerblue', + "text-decoration": "underline" + } + ], + data=trips.get_info(), page_size=50 ) # consumption_fig @@ -243,3 +254,15 @@ def get_battery_curve_fig(row: dict, car: Car): fig = px.line(battery_curves, x="level", y="speed") fig.update_layout(xaxis_title="Battery %", yaxis_title="Charging speed in kW") return html.Div(Graph(figure=fig)) + + +def get_altitude_fig(trip:Trip): + conn = Database.get_db() + res = list(map(list, conn.execute("SELECT mileage, altitude FROM position WHERE Timestamp>=? and Timestamp<=?;", + (trip.start_at, trip.end_at)).fetchall())) + start_mileage = res[0][0] + for line in res: + line[0] = line[0] - start_mileage + fig = px.line(res, x=0, y=1) + fig.update_layout(xaxis_title="Distance km", yaxis_title="Altitude m") + return html.Div(Graph(figure=fig)) diff --git a/web/views.py b/web/views.py index 9d32728..47d1722 100644 --- a/web/views.py +++ b/web/views.py @@ -63,7 +63,7 @@ def create_callback(): Output('consumption_fig_by_speed', 'figure'), Output('consumption_graph_by_temp', 'children'), Output('consumption', 'children'), - Output('tab_trips', 'children'), + Output('tab_trips_fig', 'children'), Output('tab_battery_fig', 'children'), Output('tab_charge', 'children'), Output('date-slider', 'max'), @@ -108,18 +108,29 @@ def create_callback(): [Input("battery-table", "active_cell"), Input("tab_battery_popup-close", "n_clicks")], [State('battery-table', 'data'), - State("tab_battery_popup", "is_open")] - ) + State("tab_battery_popup", "is_open")]) def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument if is_open is None: is_open = False if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: row = data[active_cell["row"]] - print("ok") return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True return "", False +@dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], + [Input("trips-table", "active_cell"), + Input("tab_trips_popup-close", "n_clicks")], + State("tab_trips_popup", "is_open")) +def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument + print("altitude") + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: + return figures.get_altitude_fig(trips[active_cell["row_id"]-1]), True + return "", False + + @dash_app.callback(Output({'role': ABRP_SWITCH + RESPONSE, 'vin': MATCH}, 'children'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'id'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'value')) @@ -208,7 +219,7 @@ def get_charge_control(): @app.route('/positions') def get_recorded_position(): - return FlaskResponse(myp.get_recorded_position(), mimetype='application/json') + return FlaskResponse(Database.get_recorded_position(), mimetype='application/json') @app.route('/abrp') @@ -236,6 +247,7 @@ def after_request(response): def update_trips(): global trips, chargings, cached_layout logger.info("update_data") + Database.add_altitude_to_db(Database.get_db(update_callback=False)) try: trips_by_vin = Trips.get_trips(myp.vehicles_list) trips = next(iter(trips_by_vin.values())) # todo handle multiple car @@ -310,7 +322,23 @@ def serve_layout(): html.Div([ dbc.Tabs([ dbc.Tab(label="Summary", tab_id="summary", children=summary_tab), - dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", children=[figures.table_fig]), + dbc.Tab(label="Trips", tab_id="trips", id="tab_trips", + children=[html.Div(id="tab_trips_fig", children=figures.table_fig), + dbc.Modal( + [ + dbc.ModalHeader("Altitude"), + dbc.ModalBody(html.Div( + id="tab_trips_popup_graph")), + dbc.ModalFooter( + dbc.Button("Close", + id="tab_trips_popup-close", + className="ml-auto") + ), + ], + id="tab_trips_popup", + size="xl", + ) + ]), dbc.Tab(label="Battery", tab_id="battery", id="tab_battery", children=[html.Div(id="tab_battery_fig", children=[figures.battery_info]), dbc.Modal(