diff --git a/charge_control.py b/charge_control.py index 0a18e74..b91fc69 100644 --- a/charge_control.py +++ b/charge_control.py @@ -1,6 +1,5 @@ import json import threading -import traceback from copy import copy from datetime import datetime, timedelta from hashlib import md5 @@ -91,10 +90,10 @@ class ChargeControl: if self._next_stop_hour is not None and self._next_stop_hour < now: self._next_stop_hour += timedelta(days=1) self.retry_count = 0 - except AttributeError: - logger.error("Probably can't retrieve all information from API: %s", traceback.format_exc()) + except (AttributeError, ValueError): + logger.exception("Probably can't retrieve all information from API:") except: # pylint: disable=bare-except - logger.error(traceback.format_exc()) + logger.exception("Charge control:") def get_dict(self): chd = copy(self.__dict__) diff --git a/ecomix.py b/ecomix.py index ed3634e..ded9671 100644 --- a/ecomix.py +++ b/ecomix.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta from statistics import mean, StatisticsError import xml.etree.cElementTree as ElT import numbers -import traceback import requests import reverse_geocode @@ -68,7 +67,7 @@ class Ecomix: Ecomix._cache[country_code].append([datetime.now(), value]) return data["status"] == "ok" except (AssertionError, NameError, KeyError): - logger.debug(traceback.format_exc()) + logger.debug("ecomix:", exc_info=True) return False else: return False diff --git a/libs/car.py b/libs/car.py index 92f2ab3..27f1d0b 100644 --- a/libs/car.py +++ b/libs/car.py @@ -42,7 +42,7 @@ class Car: if self.status is not None: return self.status logger.error("status of %s is None", self.vin) - raise ValueError("status of %s is None") + raise ValueError("status of {} is None".format(self.vin)) @classmethod def from_json(cls, data: dict): @@ -82,7 +82,7 @@ class Car: class Cars(list): def __init__(self, *args): list.__init__(self, *args) - self.config_filename = "../cars.json" + self.config_filename = "cars.json" def get_car_by_vin(self, vin) -> Car: for car in self: diff --git a/libs/charging.py b/libs/charging.py index f47e45d..367c4e2 100644 --- a/libs/charging.py +++ b/libs/charging.py @@ -1,4 +1,3 @@ -import traceback from datetime import datetime from sqlite3 import IntegrityError @@ -11,7 +10,7 @@ from web.db import Database class Charging: - elec_price: ElecPrice = None + elec_price: ElecPrice = ElecPrice(None) @staticmethod def get_chargings(mini=None, maxi=None) -> List[dict]: @@ -78,6 +77,6 @@ class Charging: Charging.update_chargings(conn, start_at, charge_date, level, co2_per_kw, consumption_kw, car.vin) except TypeError: - logger.debug("battery table is probably empty : %s", traceback.format_exc()) + logger.debug("battery table is probably empty :", exc_info=True) conn.commit() conn.close() diff --git a/libs/elec_price.py b/libs/elec_price.py index 91d660d..1281e80 100644 --- a/libs/elec_price.py +++ b/libs/elec_price.py @@ -54,14 +54,15 @@ class ElecPrice: def get_price(self, start, end, consumption): prices = [] date = start - while date < end: - prices.append(self.get_instant_price(date)) - date = date + timedelta(minutes=30) - try: - res = round(consumption * mean(prices), 2) - except TypeError: - logger.error("Can't get_price of charge, check config") - res = None + res = None + if not (start is None or end is None): + while date < end: + prices.append(self.get_instant_price(date)) + date = date + timedelta(minutes=30) + try: + res = round(consumption * mean(prices), 2) + except TypeError: + logger.error("Can't get_price of charge, check config") return res def is_enable(self): diff --git a/my_psacc.py b/my_psacc.py index 3f81e2e..b03e81c 100644 --- a/my_psacc.py +++ b/my_psacc.py @@ -1,7 +1,6 @@ import json import re import threading -import traceback import uuid from datetime import datetime from json import JSONEncoder @@ -144,8 +143,9 @@ class MyPSACC: if self._record_enabled: self.record_info(car) return res - except ApiException: - logger.error(traceback.format_exc()) + except ApiException as ex: + logger.error("get_vehicle_info: ApiException: %s", ex) + logger.debug(exc_info=True) car.status = res return res @@ -174,7 +174,7 @@ class MyPSACC: self.vehicles_list.add(Car(vehicle.vin, vehicle.id, vehicle.brand, vehicle.label)) self.vehicles_list.save_cars() except ApiException: - logger.error(traceback.format_exc()) + logger.exception("get_vehicles:") return self.vehicles_list def load_otp(self, force_new=False): @@ -296,7 +296,7 @@ class MyPSACC: sleep(60) self.wakeup(data["vin"]) except KeyError: - logger.error(traceback.format_exc()) + logger.exception("mqtt message:") def start_mqtt(self): self.load_otp() @@ -342,8 +342,7 @@ class MyPSACC: minute = hour_minute[1] return hour, minute except IndexError: - logger.error(traceback.format_exc()) - logger.error("Can't get charge hour: %s", hour_str) + logger.exception("Can't get charge hour: %s", hour_str) return None def get_charge_status(self, vin): diff --git a/mylogger.py b/mylogger.py index 30c8291..3fba1ff 100644 --- a/mylogger.py +++ b/mylogger.py @@ -5,18 +5,26 @@ DEBUG_LEVELV_NUM = 9 logging.addLevelName(DEBUG_LEVELV_NUM, "DEBUGV") -def debugv(self, message, *args, **kws): - self.log(DEBUG_LEVELV_NUM, message, *args, **kws) +class CustomLogger(logging.Logger): + # pylint: disable=too-many-arguments + def __new_style_log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, **kwargs): + if kwargs.pop('style', "%") == "{": # optional + msg = msg.format(*args) + args = [] + super()._log(level, msg, args, exc_info, extra, stack_info) + + def debugv(self, msg, *args, **kwargs): + if self.isEnabledFor(DEBUG_LEVELV_NUM): + self.__new_style_log(DEBUG_LEVELV_NUM, msg, args, **kwargs) -logging.Logger.debugv = debugv +logging.setLoggerClass(CustomLogger) # pylint: disable=invalid-name logger = logging.getLogger("log") def my_logger(file='activity.log', handler_level=logging.INFO): global logger - logger.setLevel(handler_level) formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s') file_handler = RotatingFileHandler(file, 'a', 1000000, 1, encoding='utf8') diff --git a/otp/otp.py b/otp/otp.py index 63301d9..4cf4816 100644 --- a/otp/otp.py +++ b/otp/otp.py @@ -1,5 +1,4 @@ import hashlib -import traceback import pickle from secrets import token_hex, token_bytes from math import ceil @@ -322,7 +321,7 @@ def load_otp(filename="otp.bin"): except ModuleNotFoundError: return RenameUnpickler(input_file).load() except FileNotFoundError: - logger.debug(traceback.format_exc()) + logger.debug("",exc_info=True) return None diff --git a/requirements-dev.txt b/requirements-dev.txt index b3f631e..2af2088 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ prospector>=1.3.0 pre-commit +deepdiff \ No newline at end of file diff --git a/trip.py b/trip.py index 158e2ce..09a3b92 100644 --- a/trip.py +++ b/trip.py @@ -1,3 +1,4 @@ +import logging from statistics import mean from typing import List, Dict @@ -50,13 +51,14 @@ class Trip: return None def set_consumption(self, diff_level: float) -> float: - if self.distance is None: - raise ValueError("Distance not set") if diff_level < 0: logger.debugv("trip has negative consumption") diff_level = 0 self.consumption = diff_level * self.car.battery_power / 100 - self.consumption_km = 100 * self.consumption / self.distance # kw/100 km + try: + self.consumption_km = 100 * self.consumption / self.distance # kw/100 km + except TypeError: + raise ValueError("Distance not set") return self.consumption_km def set_fuel_consumption(self, consumption) -> float: @@ -142,7 +144,8 @@ class Trips(list): for vin in vehicles: trips = Trips() vin = vin[0] - res = conn.execute('SELECT * FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall() + res = conn.execute('SELECT Timestamp, VIN, longitude, latitude, mileage, level, moving, temperature,' + ' level_fuel, altitude FROM position WHERE VIN=? ORDER BY Timestamp', (vin,)).fetchall() if len(res) > 1: car = vehicles_list.get_car_by_vin(vin) assert car is not None @@ -152,8 +155,9 @@ class Trips(list): trip = Trip() # for debugging use this line res = list(map(dict,res)) for x in range(0, len(res) - 2): - logger.debugv("%s mileage:%.1f level:%s level_fuel:%s", - res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel']) + if logger.isEnabledFor(logging.DEBUG): # reduce execution time if debug disabled + logger.debugv("%s mileage:%.1f level:%s level_fuel:%s", + res[x]['Timestamp'], res[x]['mileage'], res[x]['level'], res[x]['level_fuel']) next_el = res[x + 2] distance = end["mileage"] - start["mileage"] duration = (end["Timestamp"] - start["Timestamp"]).total_seconds() / 3600 @@ -161,17 +165,11 @@ class Trips(list): speed_average = distance / duration except ZeroDivisionError: speed_average = 0 - restart_trip = False - if trip_parser.is_refuel(start, end, distance): - restart_trip = True - elif speed_average < 0.2 and duration > 0.05: - restart_trip = True - logger.debugv("low speed detected") - if restart_trip: + if TripParser.is_low_speed(speed_average, duration) or trip_parser.is_refuel(start, end, distance): start = end trip = Trip() - logger.debugv("restart trip at %s mileage:%.1f level:%s level_fuel:%s", - start['Timestamp'], start['mileage'], start['level'], start['level_fuel']) + logger.debugv("restart trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}" + " level_fuel:{0[level_fuel]}", start, style='{') else: distance = next_el["mileage"] - end["mileage"] # km duration = (next_el["Timestamp"] - end["Timestamp"]).total_seconds() / 3600 @@ -180,13 +178,9 @@ class Trips(list): except ZeroDivisionError: speed_average = 0 end_trip = False - if trip_parser.is_refuel(end, next_el, distance): + if trip_parser.is_refuel(end, next_el, distance) or \ + TripParser.is_low_speed(speed_average, duration): end_trip = True - elif speed_average < 0.2 and duration > 0.05: - # (distance == 0 and duration > 0.08) or duration > 2 or - # check the speed to handle missing point - end_trip = True - logger.debugv("low speed detected") elif duration > 2: end_trip = True logger.debugv("too much time detected") @@ -196,8 +190,8 @@ class Trips(list): end_trip = True logger.debugv("last position found") if end_trip: - logger.debugv("stop trip at %s mileage:%.1f level:%s level_fuel:%s", - end['Timestamp'], end['mileage'], end['level'], end['level_fuel']) + logger.debugv("stop trip at {0[Timestamp]} mileage:{0[mileage]:.1f} level:{0[level]}" + " level_fuel:{0[level_fuel]}", end, style='{') trip.distance = end["mileage"] - start["mileage"] # km if trip.distance > 0: trip.start_at = start["Timestamp"] @@ -215,11 +209,10 @@ class Trips(list): if diff_level_fuel != 0: trip.set_fuel_consumption(diff_level_fuel) trip.mileage = end["mileage"] - logger.debugv("Trip: %s -> %s %.1fkm %.2fh %.0fkm/h %.2fkWh %.2fkWh/100km %.2fL " - "%.2fL/100km %.1fkm", - trip.start_at, trip.end_at, trip.distance, trip.duration, - trip.speed_average, trip.consumption, trip.consumption_km, - trip.consumption_fuel, trip.consumption_fuel_km, trip.mileage) + logger.debugv("Trip: {0.start_at} -> {0.end_at} {0.distance:.1f}km {0.duration:.2f}h " + "{0.speed_average:.0f}km/h {0.consumption:.2f}kWh " + "{0.consumption_km:.2f}kWh/100km {0.consumption_fuel:.2f}L " + "{0.consumption_fuel_km:.2f}L/100km {0.mileage:.1f}km", trip, style="{") # filter bad value trips.check_and_append(trip) start = next_el diff --git a/trip_parser.py b/trip_parser.py index 5917b13..3426a29 100644 --- a/trip_parser.py +++ b/trip_parser.py @@ -2,9 +2,9 @@ from collections.abc import Callable from libs.car import Car from mylogger import logger -LEVEL = "level" +LEVEL = 5 -LEVEL_FUEL = "level_fuel" +LEVEL_FUEL = 8 class TripParser: @@ -67,3 +67,8 @@ class TripParser: # If distance is bigger than 0 but charge bigger than five there is probably missing point and we assume that # regeneration/temperature can't increase by 5 percent the battery level return decharge < -2 and (distance == 0 or decharge < -5) + + @staticmethod + def is_low_speed(speed_average, duration): + logger.debugv("Low speed detected") + return speed_average < 0.2 and duration > 0.05 diff --git a/utils.py b/utils.py index bd54e91..0058bc3 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,3 @@ -import traceback from functools import wraps from threading import Semaphore, Timer import socket @@ -20,9 +19,9 @@ def get_temp(latitude: str, longitude: str, api_key: str) -> float: logger.debug("Temperature :%fc", temp) return temp except ConnectionError: - logger.error("Can't connect to openweathermap :%s", traceback.format_exc()) + logger.error("Can't connect to openweathermap :", exc_info=True) except KeyError: - logger.error("Unable to get temperature from openweathermap :%s", traceback.format_exc()) + logger.error("Unable to get temperature from openweathermap :", exc_info=True) return None diff --git a/web/abrp.py b/web/abrp.py index 23dba19..db63cc1 100644 --- a/web/abrp.py +++ b/web/abrp.py @@ -1,5 +1,4 @@ import json -import traceback from datetime import datetime import requests @@ -43,7 +42,7 @@ class Abrp: logger.debug(response.text) return response.json()["status"] == "ok" except (AttributeError, IndexError, ValueError): - logger.error(traceback.format_exc()) + logger.exception("abrp:") return False def __iter__(self): diff --git a/web/db.py b/web/db.py index 9056742..9b27d20 100644 --- a/web/db.py +++ b/web/db.py @@ -1,6 +1,5 @@ import sys import sqlite3 -import traceback from datetime import datetime from time import sleep @@ -17,30 +16,37 @@ from utils import get_temp NEW_BATTERY_COLUMNS = [["price", "INTEGER"], ["charging_mode", "TEXT"]] NEW_POSITION_COLUMNS = [["level_fuel", "INTEGER"], ["altitude", "INTEGER"]] -DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" - def convert_sql_res(rows): return list(map(dict, rows)) +DATE_FORMAT = "%Y-%m-%d %H:%M:%S+00:00" + + +def new_convert_datetime_from_string(string): + return datetime.fromisoformat(string) + + class Database: callback_fct: Callable[[], None] = lambda: None DEFAULT_DB_FILE = 'info.db' - # pylint: disable=invalid-name db_initialized = False @staticmethod - def convert_datetime_from_bytes(bytes_string): - return datetime.strptime(bytes_string.decode("utf-8"), DATE_FORMAT).replace(tzinfo=pytz.UTC) + def convert_datetime_from_string(string): + try: + return datetime.strptime(string, DATE_FORMAT).replace(tzinfo=pytz.UTC) + except ValueError: + return datetime.strptime(string.replace("T", " "), DATE_FORMAT).replace(tzinfo=pytz.UTC) @staticmethod - def convert_datetime_from_string(st): - return datetime.strptime(st, DATE_FORMAT).replace(tzinfo=pytz.UTC) + def convert_datetime_from_bytes(bytes_string): + return Database.convert_datetime_from_string(bytes_string.decode("utf-8")) @staticmethod def convert_datetime_to_string(date: datetime): - return date.replace(tzinfo=pytz.UTC).strftime(DATE_FORMAT) + return date.replace(tzinfo=pytz.UTC).isoformat(timespec='seconds', sep=" ") @staticmethod def update_callback(): @@ -89,14 +95,16 @@ class Database: Database.clean_battery(conn) Database.add_altitude_to_db(conn) conn.commit() + if sys.version_info >= (3, 7): + Database.convert_datetime_from_string = new_convert_datetime_from_string + sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) + sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) Database.db_initialized = True @staticmethod def get_db(db_file=None, update_callback=True): if db_file is None: db_file = Database.DEFAULT_DB_FILE - sqlite3.register_converter("DATETIME", Database.convert_datetime_from_bytes) - sqlite3.register_adapter(datetime, Database.convert_datetime_to_string) conn = sqlite3.connect(db_file, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) conn.row_factory = sqlite3.Row if update_callback: @@ -179,7 +187,6 @@ class Database: break except (ValueError, KeyError, requests.exceptions.RequestException): logger.error("Can't get altitude from API") - logger.debug(traceback.format_exc()) @staticmethod def get_recorded_position(): diff --git a/web/figures.py b/web/figures.py index 729bd46..088ce0c 100644 --- a/web/figures.py +++ b/web/figures.py @@ -1,9 +1,7 @@ from copy import deepcopy -from datetime import datetime from typing import List -import pytz import dash_bootstrap_components as dbc import dash_table import numpy as np @@ -233,13 +231,9 @@ def __calculate_co2_per_kw(charging_data): return 0 -def dash_date_to_datetime(dash_date): - return datetime.strptime(dash_date, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=pytz.UTC) - - def get_battery_curve_fig(row: dict, car: Car): - start_date = dash_date_to_datetime(row["start_at"]) - stop_at = dash_date_to_datetime(row["stop_at"]) + start_date = Database.convert_datetime_from_string(row["start_at"]) + stop_at = Database.convert_datetime_from_string(row["stop_at"]) res = Database.get_battery_curve(Database.get_db(), start_date, car.vin) res.insert(0, {"level": row["start_level"], "date": start_date}) res.append({"level": row["end_level"], "date": stop_at}) diff --git a/web/views.py b/web/views.py index c312763..2ffae08 100644 --- a/web/views.py +++ b/web/views.py @@ -1,5 +1,4 @@ import json -import traceback from datetime import datetime, timezone from typing import List @@ -55,7 +54,7 @@ def diff_dashtable(data, data_previous, row_id_name="row_id"): return changes -def create_callback(): +def create_callback(): # flake8: noqa: C901 global CALLBACK_CREATED if not CALLBACK_CREATED: @dash_app.callback(Output('trips_map', 'figure'), @@ -95,42 +94,38 @@ def create_callback(): diff_data = diff_dashtable(data, data_previous, "start_at") for changed_line in diff_data: if changed_line['column_name'] == 'price': - if not Database.set_chargings_price(Database.get_db(), - figures.dash_date_to_datetime(changed_line['start_at']), + if not Database.set_chargings_price(Database.get_db(),changed_line['start_at'], changed_line['current_value']): logger.error("Can't find line to update in the database") return "" + @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ], + [Input("battery-table", "active_cell"), + Input("tab_battery_popup-close", "n_clicks")], + [State('battery-table', 'data'), + State("tab_battery_popup", "is_open")]) + def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument, unused-variable + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: + row = data[active_cell["row"]] + return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True + return "", False + + @dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], + [Input("trips-table", "active_cell"), + Input("tab_trips_popup-close", "n_clicks")], + State("tab_trips_popup", "is_open")) + def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument, unused-variable + if is_open is None: + is_open = False + if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: + return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True + return "", False + CALLBACK_CREATED = True -@dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open"), ], - [Input("battery-table", "active_cell"), - Input("tab_battery_popup-close", "n_clicks")], - [State('battery-table', 'data'), - State("tab_battery_popup", "is_open")]) -def get_battery_curve(active_cell, close, data, is_open): # pylint: disable=unused-argument - if is_open is None: - is_open = False - if active_cell is not None and active_cell["column_id"] in ["start_level", "end_level"] and not is_open: - row = data[active_cell["row"]] - return figures.get_battery_curve_fig(row, myp.vehicles_list[0]), True - return "", False - - -@dash_app.callback([Output("tab_trips_popup_graph", "children"), Output("tab_trips_popup", "is_open"), ], - [Input("trips-table", "active_cell"), - Input("tab_trips_popup-close", "n_clicks")], - State("tab_trips_popup", "is_open")) -def get_altitude(active_cell, close, is_open): # pylint: disable=unused-argument - print("altitude") - if is_open is None: - is_open = False - if active_cell is not None and active_cell["column_id"] in ["altitude_diff"] and not is_open: - return figures.get_altitude_fig(trips[active_cell["row_id"] - 1]), True - return "", False - - @dash_app.callback(Output({'role': ABRP_SWITCH + RESPONSE, 'vin': MATCH}, 'children'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'id'), Input({'role': ABRP_SWITCH, 'vin': MATCH}, 'value')) @@ -267,7 +262,7 @@ def update_trips(): marks = figures.get_marks_from_start_end(min_date, max_date) cached_layout = None # force regenerate layout except (ValueError, IndexError): - logger.error("update_trips (slider): %s", traceback.format_exc()) + logger.error("update_trips (slider): %s", exc_info=True) return @@ -315,7 +310,6 @@ def serve_layout(): summary_tab = figures.ERROR_DIV maps = figures.ERROR_DIV logger.warning("Failed to generate figure, there is probably not enough data yet") - logger.debug(traceback.format_exc()) range_slider = html.Div() data_div = html.Div([ range_slider, @@ -371,6 +365,6 @@ try: Charging.set_default_price() update_trips() except (IndexError, TypeError): - logger.debug("Failed to get trips, there is probably not enough data yet %s", traceback.format_exc()) + logger.debug("Failed to get trips, there is probably not enough data yet:", exc_info=True) dash_app.layout = serve_layout