diff --git a/docs/Install.md b/docs/Install.md index 7b9cb40..6467e27 100644 --- a/docs/Install.md +++ b/docs/Install.md @@ -13,7 +13,7 @@ We will retrieve this information: - On debian based distribution you can install some requirement from repos, it's faster than installtion with pip: ``` - sudo apt-get install python3-typing-extensions python3-pandas python3-plotly python3-paho-mqtt python3-six python3-dateutil python3-brotli libblas-dev liblapack-dev gfortran python3-pycryptodome python3-numpy libatlas3-base python3-cryptography + sudo apt-get install python3-typing-extensions python3-plotly python3-paho-mqtt python3-six python3-dateutil python3-brotli libblas-dev liblapack-dev gfortran python3-pycryptodome python3-cryptography ``` - For everyone : diff --git a/requirements-dev.txt b/requirements-dev.txt index 2af2088..b3f631e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,2 @@ prospector>=1.3.0 pre-commit -deepdiff \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e34af87..bb3e91c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ dash_daq plotly>=4 cryptography>=2.6 Werkzeug>=1.0.0 -pandas oauth2_client requests pytz @@ -16,6 +15,7 @@ geojson reverse_geocode androguard pycryptodomex +deepdiff #swagger req certifi >= 14.05.14 diff --git a/web/figures.py b/web/figures.py index 50df6e3..67ae795 100644 --- a/web/figures.py +++ b/web/figures.py @@ -4,7 +4,6 @@ import dash_bootstrap_components as dbc import dash_table from dash_core_components import Graph from dash_table.Format import Format, Scheme, Symbol -from dateutil.relativedelta import relativedelta import plotly.express as px import plotly.graph_objects as go import dash_html_components as html @@ -13,35 +12,7 @@ from libs.car import Car from libs.elec_price import ElecPrice from trip import Trip from web.db import Database - - -def unix_time_millis(date): - return int(date.timestamp()) - - -def get_marks_from_start_end(start, end): - nb_marks = 10 - result = [] - time_delta = int((end - start).total_seconds() / nb_marks) - current = start - if time_delta > 0: - while current <= end: - result.append(current) - current += relativedelta(seconds=time_delta) - result[-1] = end - if time_delta < 3600 * 24: - if time_delta > 3600: - date_f = '%x %Hh' - else: - date_f = '%x %Hh%M' - else: - date_f = '%x' - marks = {} - for date in result: - marks[unix_time_millis(date)] = str(date.strftime(date_f)) - return marks - return None - +from web.utils import card_value_div, dash_date_to_datetime # pylint: disable=invalid-name ERROR_DIV = dbc.Alert("No data to show, there is probably no trips recorded yet", color="danger") @@ -57,12 +28,6 @@ battery_info = ERROR_DIV battery_table = None consumption_df_dict = None - -def card_value_div(card_id, unit, value="-"): - return html.Div([html.Div(value, id=card_id, className="mr-2"), html.Div(unit)], - className="d-flex flex-row justify-content-center") - - AVG_CHARGE_SPEED = "avg_chg_speed" AVG_EMISSION_KM = "avg_emission_km" AVG_EMISSION_KW = "avg_emission_kw" @@ -192,20 +157,9 @@ def get_figures(car: Car): return True -def __calculate_co2_per_kw(charging_data): - try: - co2_data = charging_data[charging_data["co2"] > 0] - co2_kw_sum = co2_data["kw"].sum() - if co2_kw_sum > 0: - return co2_data["co2"].sum() / co2_kw_sum - except KeyError: - return 0 - return 0 - - def get_battery_curve_fig(row: dict, car: Car): - start_date = Database.convert_datetime_from_string(row["start_at"]) - stop_at = Database.convert_datetime_from_string(row["stop_at"]) + start_date = dash_date_to_datetime(row["start_at"]) + stop_at = dash_date_to_datetime(row["stop_at"]) conn = Database.get_db() res = Database.get_battery_curve(conn, start_date, car.vin) conn.close() diff --git a/web/utils.py b/web/utils.py new file mode 100644 index 0000000..aba5fc6 --- /dev/null +++ b/web/utils.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta + +import dash_bootstrap_components as dbc +import dash_html_components as html +from dash.development.base_component import Component + + +def unix_time_millis(date): + return int(date.timestamp()) + + +def get_marks_from_start_end(start, end): + nb_marks = 10 + result = [] + time_delta = int((end - start).total_seconds() / nb_marks) + current = start + if time_delta > 0: + while current <= end: + result.append(current) + current += timedelta(seconds=time_delta) + result[-1] = end + if time_delta < 3600 * 24: + if time_delta > 3600: + date_f = '%x %Hh' + else: + date_f = '%x %Hh%M' + else: + date_f = '%x' + marks = {} + for date in result: + marks[unix_time_millis(date)] = str(date.strftime(date_f)) + return marks + return None + + +def card_value_div(card_id, unit, value="-"): + return html.Div([html.Div(value, id=card_id, className="mr-2"), html.Div(unit)], + className="d-flex flex-row justify-content-center") + + +def dash_date_to_datetime(st): + return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z") + + +def create_card(card: dict): + res = [] + for tile, value in card.items(): + rows = value["text"] + # if isinstance(text, str): + # text = html.H3(text) + html_text = [] + for row in rows: + html_text.append(html.Div(row, className="d-flex flex-row justify-content-center")) + res.append(html.Div( + dbc.Card([ + html.H4(tile, className="card-title text-center"), + dbc.Row([ + dbc.Col(dbc.CardBody(html_text, style={"whiteSpace": "nowrap", "fontSize": "160%"}), + className="text-center"), + dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"maxHeight": "7rem"})) + ], + className="align-items-center flex-nowrap") + ], className="h-100 p-2"), + className="col-sm-12 col-md-6 col-lg-3 py-2" + )) + return res diff --git a/web/views.py b/web/views.py index dc229bf..383aae7 100644 --- a/web/views.py +++ b/web/views.py @@ -3,14 +3,14 @@ from typing import List import dash_bootstrap_components as dbc from dash.dependencies import Output, Input, MATCH, State -from dash.development.base_component import Component from dash.exceptions import PreventUpdate import dash_core_components as dcc import dash_html_components as html import dash_daq as daq -import pandas as pd +from deepdiff import DeepDiff from flask import jsonify, request, Response as FlaskResponse +import web.utils from libs.car import Cars from mylogger import logger @@ -24,6 +24,7 @@ from web.db import Database # pylint: disable=invalid-name from web.figure_filter import Figure_Filter +from web.utils import dash_date_to_datetime, create_card RESPONSE = "-response" EMPTY_DIV = "empty-div" @@ -35,28 +36,6 @@ chargings: List[dict] min_date = max_date = min_millis = max_millis = step = marks = cached_layout = None -def diff_dashtable(data, data_previous, row_id_name="row_id"): - df, df_previous = pd.DataFrame(data=data), pd.DataFrame(data_previous) - for _df in [df, df_previous]: - assert row_id_name in _df.columns - _df = _df.set_index(row_id_name) - mask = df.ne(df_previous) - df_diff = df[mask].dropna(how="all", axis="columns").dropna(how="all", axis="rows") - changes = [] - for idx, row in df_diff.iterrows(): - row.dropna(inplace=True) - for change in row.iteritems(): - changes.append( - { - row_id_name: data[idx][row_id_name], - "column_name": change[0], - "current_value": change[1], - "previous_value": df_previous.at[idx, change[0]], - } - ) - return changes - - def create_callback(): # noqa: MC0001 global CALLBACK_CREATED if not CALLBACK_CREATED: @@ -67,15 +46,20 @@ def create_callback(): # noqa: MC0001 def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate - diff_data = diff_dashtable(data, data_previous, "start_at") - for changed_line in diff_data: - if changed_line['column_name'] == 'price': + diff_data = DeepDiff(data_previous, data, ignore_numeric_type_changes=True, ignore_order=True, view="tree", + verbose_level=1) + for value_changed in diff_data["values_changed"]: + index, column_name = value_changed.path(output_format='list') + new_value = value_changed.t2 + if column_name == 'price': conn = Database.get_db() - if not Database.set_chargings_price(conn, changed_line['start_at'], - changed_line['current_value']): + date = dash_date_to_datetime(data[index]['start_at']) + if not Database.set_chargings_price(conn, date, new_value): logger.error("Can't find line to update in the database") + else: + logger.debug("update price %s of %s", value_changed, date) conn.close() - return "" + return "" # don't need to update dashboard @dash_app.callback([Output("tab_battery_popup_graph", "children"), Output("tab_battery_popup", "is_open")], [Input("battery-table", "active_cell"), @@ -266,10 +250,10 @@ def update_trips(): # update for slider try: logger.debug("min_date:%s - max_date:%s", min_date, max_date) - min_millis = figures.unix_time_millis(min_date) - max_millis = figures.unix_time_millis(max_date) + min_millis = web.utils.unix_time_millis(min_date) + max_millis = web.utils.unix_time_millis(max_date) step = (max_millis - min_millis) / 100 - marks = figures.get_marks_from_start_end(min_date, max_date) + marks = web.utils.get_marks_from_start_end(min_date, max_date) cached_layout = None # force regenerate layout figures.get_figures(car) except (ValueError, IndexError): @@ -298,30 +282,6 @@ def __get_control_tabs(): return tabs -def create_card(card: dict): - res = [] - for tile, value in card.items(): - rows = value["text"] - # if isinstance(text, str): - # text = html.H3(text) - html_text = [] - for row in rows: - html_text.append(html.Div(row, className="d-flex flex-row justify-content-center")) - res.append(html.Div( - dbc.Card([ - html.H4(tile, className="card-title text-center"), - dbc.Row([ - dbc.Col(dbc.CardBody(html_text, style={"whiteSpace": "nowrap", "fontSize": "160%"}), - className="text-center"), - dbc.Col(dbc.CardImg(src=value.get("src", Component.UNDEFINED), style={"maxHeight": "7rem"})) - ], - className="align-items-center flex-nowrap") - ], className="h-100 p-2"), - className="col-sm-12 col-md-6 col-lg-3 py-2" - )) - return res - - def serve_layout(): global cached_layout if cached_layout is None: