From 822871e4d4c06ea555a4b2982c02fef3486db32c Mon Sep 17 00:00:00 2001 From: Florian BEZANNIER <48728684+flobz@users.noreply.github.com> Date: Tue, 11 May 2021 18:54:59 +0200 Subject: [PATCH] add test action (#118) * Create main.yml * Update main.yml * add files for test * Update main.yml * add init * Update main.yml * re-add pandas needed because it's needed by plotly.express & some fix * add missing file * Create main.yml --- .github/workflows/main.yml | 36 +++++++++++++++++++++++++++++++++++ docs/Install.md | 2 +- requirements-dev.txt | 2 ++ requirements.txt | 2 +- test/__init__.py | 0 test/data/cars.json | 13 +++++++++++++ test/data/config.ini | 9 +++++++++ test/data/config.json | 15 +++++++++++++++ test/data/otp_test.bin | Bin 0 -> 2002 bytes test/test_unit.py | 4 ++-- web/figure_filter.py | 38 ++++++++++++++++++++----------------- web/utils.py | 26 ++++++++++++++++++++++++- web/views.py | 21 +++++++++----------- 13 files changed, 134 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 test/__init__.py create mode 100644 test/data/cars.json create mode 100644 test/data/config.ini create mode 100644 test/data/config.json create mode 100644 test/data/otp_test.bin diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..58e17d8 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,36 @@ +# This is a basic workflow to help you get started with Actions + +name: Unit test + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + test: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + # Runs a set of commands using the runners shell + - name: Install requirements + run: pip install -r requirements.txt -r requirements-dev.txt + - name: test app + env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + run: | + echo Test + coverage run -m unittest || exit 1 + coverage xml -o cobertura.xml && bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r cobertura.xml diff --git a/docs/Install.md b/docs/Install.md index 6467e27..bee07c7 100644 --- a/docs/Install.md +++ b/docs/Install.md @@ -13,7 +13,7 @@ We will retrieve this information: - On debian based distribution you can install some requirement from repos, it's faster than installtion with pip: ``` - sudo apt-get install python3-typing-extensions python3-plotly python3-paho-mqtt python3-six python3-dateutil python3-brotli libblas-dev liblapack-dev gfortran python3-pycryptodome python3-cryptography + sudo apt-get install python3-typing-extensions python3-pandas python3-plotly python3-paho-mqtt python3-six python3-dateutil python3-brotli libblas-dev liblapack-dev gfortran python3-pycryptodome libatlas3-base python3-cryptography ``` - For everyone : diff --git a/requirements-dev.txt b/requirements-dev.txt index b3f631e..6bcd162 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,4 @@ prospector>=1.3.0 pre-commit +coverage +deepdiff diff --git a/requirements.txt b/requirements.txt index bb3e91c..e34af87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ dash_daq plotly>=4 cryptography>=2.6 Werkzeug>=1.0.0 +pandas oauth2_client requests pytz @@ -15,7 +16,6 @@ geojson reverse_geocode androguard pycryptodomex -deepdiff #swagger req certifi >= 14.05.14 diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/data/cars.json b/test/data/cars.json new file mode 100644 index 0000000..41cfe5c --- /dev/null +++ b/test/data/cars.json @@ -0,0 +1,13 @@ +[ + { + "abrp_name": "peugeot:e208:20:50", + "battery_power": 46, + "brand": "Peugeot", + "fuel_capacity": 0, + "label": "e-208", + "max_elec_consumption": 70, + "max_fuel_consumption": 0, + "vehicle_id": "vid", + "vin": "VR3UHZKXZL" + } +] \ No newline at end of file diff --git a/test/data/config.ini b/test/data/config.ini new file mode 100644 index 0000000..537e577 --- /dev/null +++ b/test/data/config.ini @@ -0,0 +1,9 @@ +[General] +currency = € + +[Electricity config] +day price = 0.1853 +night price = 0.1353 +night hour start = 22h30 +night hour end = 6h00 + diff --git a/test/data/config.json b/test/data/config.json new file mode 100644 index 0000000..127e047 --- /dev/null +++ b/test/data/config.json @@ -0,0 +1,15 @@ +{ + "abrp": { + "token": null + }, + "client_id": "cid", + "client_secret": "csec", + "co2_signal_api": null, + "country_code": "FR", + "customer_id": "AP-ACNT1234", + "proxies": null, + "realm": "clientsB2CPeugeot", + "refresh_token": "aasds-aaa", + "remote_refresh_token": "aa", + "weather_api": null +} \ No newline at end of file diff --git a/test/data/otp_test.bin b/test/data/otp_test.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5bccb45708ba28c8e3e0a78274c279ed6e42b3f GIT binary patch literal 2002 zcmd^A`FGny5O$iLZfOdX`zlvRIUGqVt)y}kXo04oDMz52<(+SLMl;_W`{%l<+QR|=`gi$9y{muG+5wC!= zULNME0++aP+73P}Gbf=-np`o*T$!Xrr z=TIQfx#S890;LzaE;?bD%Fx5e2_(c4go?r-kS_JDMUBSbeYSW$xx{L&%%pZ>a6w7e zSAs5BCl!0hj?!bDA8G_ef#m_x#e|i5v|$}!&-1FJOQVhCB%_Tn`*;`S4C}`1Jz)-QT%)laW3>&GbF7{8AMf<(1=fD?0ll=9tQo)PxO=W3YBm?2g2BQ{n!yAlb}IHvDB&5p-3a`^FzoWP0sSTDYFiXyAd$BAq%eNa--&eq$^~}O%h+lh zSROh~%yY@h!;5bFVT9M-z!~u0sBI6gZEsHoc{#_l8mMZry}3!REUK-@Cwa@sx?=Ol z8?Po8I3;RY*I2eYrN$$A?ap{@n%Z*6=UhUsk2Xfo~8vguA$T<*_EycEJh*AT`;#OTDsZPvyx&~UmWGR zs_EL0-d|Li*@p?z2SfUB1r^mnMIR06`Y~ViY+4ue@sK`AE_Mvov#Koe5_#~`r$f52 z!dHaf9MWgUe%G@Gx%!blAJP{qv*Sk5ts(6!SmiS%-Cntiirw8E(wD~?sAf>mogp1x z1(W8Wq`R!qbq$Mc&hdP;w8045_?j8%y2zer3#4xtGCi5k8DghfZcO@?!LCLaWU6VO zfiC0;{8pK=vGe^qw%;k2I6nrzw_Rul1l>zEtnJmdVK7N9Gx}e%gMP3~O{PsdJv^v^ zZ^|G4u|b)uqOGQCL_h8JcwU%JpMW9eXPfY&qazyH{5F&3hPh&Y*|S5%UbpvcIOfKh aL5uWjcl%(L6D&yM4Bl7Ol!H%={lUL+uqF5a literal 0 HcmV?d00001 diff --git a/test/test_unit.py b/test/test_unit.py index 86ca49c..60ab5ad 100644 --- a/test/test_unit.py +++ b/test/test_unit.py @@ -214,8 +214,8 @@ class TestUnit(unittest.TestCase): 'price': 3.84, 'charging_mode': 'slow'}]) assert get_figures(car) - row = {"start_at": date0.strftime("%Y-%m-%dT%H:%M:%S+00:00"), - "stop_at": date3.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "start_level": start_level, "end_level": end_level} + row = {"start_at": date0.strftime('%Y-%m-%dT%H:%M:%S.000Z'), + "stop_at": date3.strftime('%Y-%m-%dT%H:%M:%S.000Z'), "start_level": start_level, "end_level": end_level} assert get_battery_curve_fig(row, car) is not None assert get_altitude_fig(trip) is not None diff --git a/web/figure_filter.py b/web/figure_filter.py index 83f1d8e..96c0134 100644 --- a/web/figure_filter.py +++ b/web/figure_filter.py @@ -1,6 +1,7 @@ import json from logging import DEBUG +from dash._utils import create_callback_id from dash.dependencies import Output, Input from dash_core_components import Store from mylogger import logger @@ -98,23 +99,26 @@ class Figure_Filter: }, indent=4) return params - def get_clientside_callback(self): - if logger.isEnabledFor(DEBUG): - log_level = 10 - else: - log_level = 20 - fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{ - const params={self.get_params()}; - const logLevel={log_level}; - return filterAndSort(data, range, figures, params, logLevel); - }}""" - res = [fct_def, - *self.__get_output(), - Input('clientside-data-store', 'data'), - Input('date-slider', 'value'), - Input('clientside-figure-store', 'data'), - *self.__get_table_input_sort_by()] - return res + def set_clientside_callback(self, dash_app): + callback_id = create_callback_id(self.__get_output()) + if callback_id not in dash_app.callback_map: + if logger.isEnabledFor(DEBUG): + log_level = 10 + else: + log_level = 20 + fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{ + const params={self.get_params()}; + const logLevel={log_level}; + return filterAndSort(data, range, figures, params, logLevel); + }}""" + dash_app.clientside_callback(fct_def, + *self.__get_output(), + Input('clientside-data-store', 'data'), + Input('date-slider', 'value'), + Input('clientside-figure-store', 'data'), + *self.__get_table_input_sort_by()) + return True + return False def get_store(self): return [Store(id='clientside-figure-store', data=self.__get_figures()), diff --git a/web/utils.py b/web/utils.py index aba5fc6..c584bd1 100644 --- a/web/utils.py +++ b/web/utils.py @@ -3,6 +3,8 @@ from datetime import datetime, timedelta import dash_bootstrap_components as dbc import dash_html_components as html from dash.development.base_component import Component +from pandas import DataFrame +from pytz import UTC def unix_time_millis(date): @@ -39,7 +41,7 @@ def card_value_div(card_id, unit, value="-"): def dash_date_to_datetime(st): - return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z") + return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z").replace(tzinfo=UTC) def create_card(card: dict): @@ -64,3 +66,25 @@ def create_card(card: dict): className="col-sm-12 col-md-6 col-lg-3 py-2" )) return res + + +def diff_dashtable(data, data_previous, row_id_name="row_id"): + df, df_previous = DataFrame(data=data), DataFrame(data_previous) + for _df in [df, df_previous]: + assert row_id_name in _df.columns + _df = _df.set_index(row_id_name) + mask = df.ne(df_previous) + df_diff = df[mask].dropna(how="all", axis="columns").dropna(how="all", axis="rows") + changes = [] + for idx, row in df_diff.iterrows(): + row.dropna(inplace=True) + for change in row.iteritems(): + changes.append( + { + row_id_name: data[idx][row_id_name], + "column_name": change[0], + "current_value": change[1], + "previous_value": df_previous.at[idx, change[0]], + } + ) + return changes diff --git a/web/views.py b/web/views.py index a11579a..5a301da 100644 --- a/web/views.py +++ b/web/views.py @@ -7,7 +7,6 @@ from dash.exceptions import PreventUpdate import dash_core_components as dcc import dash_html_components as html import dash_daq as daq -from deepdiff import DeepDiff from flask import jsonify, request, Response as FlaskResponse import web.utils @@ -21,10 +20,11 @@ from web import figures from web.app import app, dash_app, myp, chc from web.db import Database +from web.utils import diff_dashtable, dash_date_to_datetime # pylint: disable=invalid-name from web.figure_filter import Figure_Filter -from web.utils import dash_date_to_datetime, create_card +from web.utils import create_card RESPONSE = "-response" EMPTY_DIV = "empty-div" @@ -46,18 +46,15 @@ def create_callback(): # noqa: MC0001 def capture_diffs_in_battery_table(timestamp, data, data_previous): # pylint: disable=unused-variable if timestamp is None: raise PreventUpdate - diff_data = DeepDiff(data_previous, data, ignore_numeric_type_changes=True, ignore_order=True, view="tree", - verbose_level=1) - for value_changed in diff_data["values_changed"]: - index, column_name = value_changed.path(output_format='list') - new_value = value_changed.t2 - if column_name == 'price': + diff_data = diff_dashtable(data, data_previous, "start_at") + for changed_line in diff_data: + if changed_line['column_name'] == 'price': conn = Database.get_db() - date = dash_date_to_datetime(data[index]['start_at']) - if not Database.set_chargings_price(conn, date, new_value): + if not Database.set_chargings_price(conn, dash_date_to_datetime(changed_line['start_at']), + changed_line['current_value']): logger.error("Can't find line to update in the database") else: - logger.debug("update price %s of %s", value_changed, date) + logger.debug("update price %s of %s", changed_line['current_value'], changed_line['start_at']) conn.close() return "" # don't need to update dashboard @@ -312,7 +309,7 @@ def serve_layout(): fig_filter.add_table("trips", figures.table_fig) fig_filter.add_table("chargings", figures.battery_table) fig_filter.src = {"trips": trips.get_trips_as_dict(), "chargings": chargings} - dash_app.clientside_callback(*fig_filter.get_clientside_callback()) + fig_filter.set_clientside_callback(dash_app) create_callback() except (IndexError, TypeError, NameError, AssertionError, NameError): summary_tab = figures.ERROR_DIV