add test action (#118)

* Create main.yml

* Update main.yml

* add files for test

* Update main.yml

* add init

* Update main.yml

* re-add pandas needed because it's needed by plotly.express & some fix

* add missing file

* Create main.yml

.github/workflows/main.yml

@@ -0,0 +1,36 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Unit test
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  test:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      # Runs a set of commands using the runners shell
+      - name: Install requirements
+        run: pip install -r requirements.txt -r requirements-dev.txt
+      - name: test app
+        env:
+          CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+        run: |
+          echo Test
+          coverage run -m unittest || exit 1
+          coverage xml -o cobertura.xml && bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r cobertura.xml

docs/Install.md

@@ -13,7 +13,7 @@ We will retrieve this information:
 - On debian based distribution you can install some requirement from repos, it's faster than installtion with pip: 
  
  ```
- sudo apt-get install python3-typing-extensions python3-plotly python3-paho-mqtt  python3-six python3-dateutil python3-brotli  libblas-dev  liblapack-dev gfortran python3-pycryptodome python3-cryptography
+ sudo apt-get install python3-typing-extensions python3-pandas python3-plotly python3-paho-mqtt  python3-six python3-dateutil python3-brotli  libblas-dev  liblapack-dev gfortran python3-pycryptodome libatlas3-base python3-cryptography
  ```
     
 - For everyone :

requirements-dev.txt

@@ -1,2 +1,4 @@
 prospector>=1.3.0
 pre-commit
+coverage
+deepdiff

requirements.txt

@@ -4,6 +4,7 @@ dash_daq
 plotly>=4
 cryptography>=2.6
 Werkzeug>=1.0.0
+pandas
 oauth2_client
 requests
 pytz
@@ -15,7 +16,6 @@ geojson
 reverse_geocode
 androguard
 pycryptodomex
-deepdiff
 
 #swagger req
 certifi >= 14.05.14

test/data/cars.json

@@ -0,0 +1,13 @@
+[
+    {
+        "abrp_name": "peugeot:e208:20:50",
+        "battery_power": 46,
+        "brand": "Peugeot",
+        "fuel_capacity": 0,
+        "label": "e-208",
+        "max_elec_consumption": 70,
+        "max_fuel_consumption": 0,
+        "vehicle_id": "vid",
+        "vin": "VR3UHZKXZL"
+    }
+]

test/data/config.ini

@@ -0,0 +1,9 @@
+[General]
+currency = €
+
+[Electricity config]
+day price = 0.1853
+night price = 0.1353
+night hour start = 22h30
+night hour end = 6h00
+

test/data/config.json

@@ -0,0 +1,15 @@
+{
+    "abrp": {
+        "token": null
+    },
+    "client_id": "cid",
+    "client_secret": "csec",
+    "co2_signal_api": null,
+    "country_code": "FR",
+    "customer_id": "AP-ACNT1234",
+    "proxies": null,
+    "realm": "clientsB2CPeugeot",
+    "refresh_token": "aasds-aaa",
+    "remote_refresh_token": "aa",
+    "weather_api": null
+}

test/test_unit.py

@@ -214,8 +214,8 @@ class TestUnit(unittest.TestCase):
                                          'price': 3.84,
                                          'charging_mode': 'slow'}])
         assert get_figures(car)
-        row = {"start_at": date0.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-               "stop_at": date3.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "start_level": start_level, "end_level": end_level}
+        row = {"start_at": date0.strftime('%Y-%m-%dT%H:%M:%S.000Z'),
+               "stop_at": date3.strftime('%Y-%m-%dT%H:%M:%S.000Z'), "start_level": start_level, "end_level": end_level}
         assert get_battery_curve_fig(row, car) is not None
         assert get_altitude_fig(trip) is not None
 

web/figure_filter.py

@@ -1,6 +1,7 @@
 import json
 from logging import DEBUG
 
+from dash._utils import create_callback_id
 from dash.dependencies import Output, Input
 from dash_core_components import Store
 from mylogger import logger
@@ -98,23 +99,26 @@ class Figure_Filter:
         }, indent=4)
         return params
 
-    def get_clientside_callback(self):
-        if logger.isEnabledFor(DEBUG):
-            log_level = 10
-        else:
-            log_level = 20
-        fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{
-                        const params={self.get_params()};
-                        const logLevel={log_level};
-                        return filterAndSort(data, range, figures, params, logLevel);
-                      }}"""
-        res = [fct_def,
-               *self.__get_output(),
-               Input('clientside-data-store', 'data'),
-               Input('date-slider', 'value'),
-               Input('clientside-figure-store', 'data'),
-               *self.__get_table_input_sort_by()]
-        return res
+    def set_clientside_callback(self, dash_app):
+        callback_id = create_callback_id(self.__get_output())
+        if callback_id not in dash_app.callback_map:
+            if logger.isEnabledFor(DEBUG):
+                log_level = 10
+            else:
+                log_level = 20
+            fct_def = f"""function(data,range, figures, {self.gen_unused_variable()}) {{
+                            const params={self.get_params()};
+                            const logLevel={log_level};
+                            return filterAndSort(data, range, figures, params, logLevel);
+                          }}"""
+            dash_app.clientside_callback(fct_def,
+                   *self.__get_output(),
+                   Input('clientside-data-store', 'data'),
+                   Input('date-slider', 'value'),
+                   Input('clientside-figure-store', 'data'),
+                   *self.__get_table_input_sort_by())
+            return True
+        return False
 
     def get_store(self):
         return [Store(id='clientside-figure-store', data=self.__get_figures()),

web/utils.py

@@ -3,6 +3,8 @@ from datetime import datetime, timedelta
 import dash_bootstrap_components as dbc
 import dash_html_components as html
 from dash.development.base_component import Component
+from pandas import DataFrame
+from pytz import UTC
 
 
 def unix_time_millis(date):
@@ -39,7 +41,7 @@ def card_value_div(card_id, unit, value="-"):
 
 
 def dash_date_to_datetime(st):
-    return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z")
+    return datetime.strptime(st, "%Y-%m-%dT%H:%M:%S.000Z").replace(tzinfo=UTC)
 
 
 def create_card(card: dict):
@@ -64,3 +66,25 @@ def create_card(card: dict):
             className="col-sm-12 col-md-6 col-lg-3 py-2"
         ))
     return res
+
+
+def diff_dashtable(data, data_previous, row_id_name="row_id"):
+    df, df_previous = DataFrame(data=data), DataFrame(data_previous)
+    for _df in [df, df_previous]:
+        assert row_id_name in _df.columns
+        _df = _df.set_index(row_id_name)
+    mask = df.ne(df_previous)
+    df_diff = df[mask].dropna(how="all", axis="columns").dropna(how="all", axis="rows")
+    changes = []
+    for idx, row in df_diff.iterrows():
+        row.dropna(inplace=True)
+        for change in row.iteritems():
+            changes.append(
+                {
+                    row_id_name: data[idx][row_id_name],
+                    "column_name": change[0],
+                    "current_value": change[1],
+                    "previous_value": df_previous.at[idx, change[0]],
+                }
+            )
+    return changes

web/views.py

@@ -7,7 +7,6 @@ from dash.exceptions import PreventUpdate
 import dash_core_components as dcc
 import dash_html_components as html
 import dash_daq as daq
-from deepdiff import DeepDiff
 from flask import jsonify, request, Response as FlaskResponse
 
 import web.utils
@@ -21,10 +20,11 @@ from web import figures
 
 from web.app import app, dash_app, myp, chc
 from web.db import Database
+from web.utils import diff_dashtable, dash_date_to_datetime
 
 # pylint: disable=invalid-name
 from web.figure_filter import Figure_Filter
-from web.utils import dash_date_to_datetime, create_card
+from web.utils import create_card
 
 RESPONSE = "-response"
 EMPTY_DIV = "empty-div"
@@ -46,18 +46,15 @@ def create_callback():  # noqa: MC0001
         def capture_diffs_in_battery_table(timestamp, data, data_previous):  # pylint: disable=unused-variable
             if timestamp is None:
                 raise PreventUpdate
-            diff_data = DeepDiff(data_previous, data, ignore_numeric_type_changes=True, ignore_order=True, view="tree",
-                                 verbose_level=1)
-            for value_changed in diff_data["values_changed"]:
-                index, column_name = value_changed.path(output_format='list')
-                new_value = value_changed.t2
-                if column_name == 'price':
+            diff_data = diff_dashtable(data, data_previous, "start_at")
+            for changed_line in diff_data:
+                if changed_line['column_name'] == 'price':
                     conn = Database.get_db()
-                    date = dash_date_to_datetime(data[index]['start_at'])
-                    if not Database.set_chargings_price(conn, date, new_value):
+                    if not Database.set_chargings_price(conn, dash_date_to_datetime(changed_line['start_at']),
+                                                        changed_line['current_value']):
                         logger.error("Can't find line to update in the database")
                     else:
-                        logger.debug("update price %s of %s", value_changed, date)
+                        logger.debug("update price %s of %s", changed_line['current_value'], changed_line['start_at'])
                     conn.close()
             return ""  # don't need to update dashboard
 
@@ -312,7 +309,7 @@ def serve_layout():
             fig_filter.add_table("trips", figures.table_fig)
             fig_filter.add_table("chargings", figures.battery_table)
             fig_filter.src = {"trips": trips.get_trips_as_dict(), "chargings": chargings}
-            dash_app.clientside_callback(*fig_filter.get_clientside_callback())
+            fig_filter.set_clientside_callback(dash_app)
             create_callback()
         except (IndexError, TypeError, NameError, AssertionError, NameError):
             summary_tab = figures.ERROR_DIV