Compare commits

...

2 Commits

Author SHA1 Message Date
AmitUp9
4d64dd4b04 TRA-4602_timeline bar charts to traffic statistics (#1159)
* pie chart for protocols and methods by requests and volume

* protocols legend

* timeline bar chart component created

* timeline can view requests and volume

* sorting the bra charts by timestemp

* disable view of <1% pieces in pie

* space added to the end of the file

* package.json update

* cr fixes

* remove spave

* remove unnecessary react fragment

Co-authored-by: Liraz Yehezkel <lirazy@up9.com>
2022-06-21 14:54:56 +03:00
gadotroee
9a40895e9c Add timing endpoint (timeline data) for stats (#1157) 2022-06-21 14:33:04 +03:00
10 changed files with 297 additions and 59 deletions

View File

@@ -83,6 +83,11 @@ func GetAccumulativeStats(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetAccumulativeStats())
}
func GetAccumulativeStatsTiming(c *gin.Context) {
// for now hardcoded 10 bars of 5 minutes interval
c.JSON(http.StatusOK, providers.GetAccumulativeStatsTiming(300, 10))
}
func GetCurrentResolvingInformation(c *gin.Context) {
c.JSON(http.StatusOK, holder.GetResolver().GetMap())
}

View File

@@ -2,6 +2,7 @@ package providers
import (
"reflect"
"sync"
"time"
"github.com/jinzhu/copier"
@@ -19,18 +20,18 @@ type GeneralStats struct {
type BucketStats []*TimeFrameStatsValue
type TimeFrameStatsValue struct {
BucketTime time.Time
ProtocolStats map[string]ProtocolStats
BucketTime time.Time `json:"timestamp"`
ProtocolStats map[string]ProtocolStats `json:"protocols"`
}
type ProtocolStats struct {
MethodsStats map[string]*SizeAndEntriesCount
Color string
MethodsStats map[string]*SizeAndEntriesCount `json:"methods"`
Color string `json:"color"`
}
type SizeAndEntriesCount struct {
EntriesCount int
VolumeInBytes int
EntriesCount int `json:"entriesCount"`
VolumeInBytes int `json:"volumeInBytes"`
}
type AccumulativeStatsCounter struct {
@@ -45,9 +46,19 @@ type AccumulativeStatsProtocol struct {
Methods []*AccumulativeStatsCounter `json:"methods"`
}
type AccumulativeStatsProtocolTime struct {
ProtocolsData []*AccumulativeStatsProtocol `json:"protocols"`
Time int64 `json:"timestamp"`
}
var (
generalStats = GeneralStats{}
bucketsStats = BucketStats{}
generalStats = GeneralStats{}
bucketsStats = BucketStats{}
bucketStatsLocker = sync.Mutex{}
)
const (
InternalBucketThreshold = time.Minute * 1
)
func ResetGeneralStats() {
@@ -58,33 +69,34 @@ func GetGeneralStats() GeneralStats {
return generalStats
}
func GetAccumulativeStats() []*AccumulativeStatsProtocol {
func getBucketStatsCopy() BucketStats {
bucketStatsCopy := BucketStats{}
bucketStatsLocker.Lock()
if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
logger.Log.Errorf("Error while copying src stats into temporary copied object")
return nil
}
bucketStatsLocker.Unlock()
return bucketStatsCopy
}
func GetAccumulativeStats() []*AccumulativeStatsProtocol {
bucketStatsCopy := getBucketStatsCopy()
if bucketStatsCopy == nil {
return make([]*AccumulativeStatsProtocol, 0)
}
result := make(map[string]*AccumulativeStatsProtocol, 0)
protocolToColor := make(map[string]string, 0)
methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
for _, countersOfTimeFrame := range bucketStatsCopy {
for protocolName, value := range countersOfTimeFrame.ProtocolStats {
if _, found := result[protocolName]; !found {
result[protocolName] = &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
EntriesCount: 0,
VolumeSizeBytes: 0,
},
Color: value.Color,
}
}
if _, found := methodsPerProtocolAggregated[protocolName]; !found {
methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
if _, ok := protocolToColor[protocolName]; !ok {
protocolToColor[protocolName] = value.Color
}
for method, countersValue := range value.MethodsStats {
if _, found := methodsPerProtocolAggregated[protocolName]; !found {
methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
}
if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
Name: method,
@@ -92,23 +104,116 @@ func GetAccumulativeStats() []*AccumulativeStatsProtocol {
VolumeSizeBytes: 0,
}
}
result[protocolName].AccumulativeStatsCounter.EntriesCount += countersValue.EntriesCount
methodsPerProtocolAggregated[protocolName][method].EntriesCount += countersValue.EntriesCount
result[protocolName].AccumulativeStatsCounter.VolumeSizeBytes += countersValue.VolumeInBytes
methodsPerProtocolAggregated[protocolName][method].VolumeSizeBytes += countersValue.VolumeInBytes
}
}
}
finalResult := make([]*AccumulativeStatsProtocol, 0)
for _, value := range result {
methodsForProtocol := make([]*AccumulativeStatsCounter, 0)
for _, methodValue := range methodsPerProtocolAggregated[value.Name] {
methodsForProtocol = append(methodsForProtocol, methodValue)
return ConvertToPieData(methodsPerProtocolAggregated, protocolToColor)
}
func ConvertToPieData(methodsPerProtocolAggregated map[string]map[string]*AccumulativeStatsCounter, protocolToColor map[string]string) []*AccumulativeStatsProtocol {
protocolsData := make([]*AccumulativeStatsProtocol, 0)
for protocolName, value := range methodsPerProtocolAggregated {
entriesCount := 0
volumeSizeBytes := 0
methods := make([]*AccumulativeStatsCounter, 0)
for _, methodAccData := range value {
entriesCount += methodAccData.EntriesCount
volumeSizeBytes += methodAccData.VolumeSizeBytes
methods = append(methods, methodAccData)
}
value.Methods = methodsForProtocol
finalResult = append(finalResult, value)
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Color: protocolToColor[protocolName],
Methods: methods,
})
}
return protocolsData
}
func GetAccumulativeStatsTiming(intervalSeconds int, numberOfBars int) []*AccumulativeStatsProtocolTime {
bucketStatsCopy := getBucketStatsCopy()
if len(bucketStatsCopy) == 0 {
return make([]*AccumulativeStatsProtocolTime, 0)
}
protocolToColor := make(map[string]string, 0)
methodsPerProtocolPerTimeAggregated := make(map[time.Time]map[string]map[string]*AccumulativeStatsCounter, 0)
// TODO: Extract to function and add tests for those values
lastBucketTime := time.Now().UTC().Add(-1 * InternalBucketThreshold / 2).Round(InternalBucketThreshold)
firstBucketTime := lastBucketTime.Add(-1 * time.Second * time.Duration(intervalSeconds*(numberOfBars-1)))
bucketStatsIndex := len(bucketStatsCopy) - 1
for bucketStatsIndex >= 0 {
currentBucketTime := bucketStatsCopy[bucketStatsIndex].BucketTime
if currentBucketTime.After(firstBucketTime) || currentBucketTime.Equal(firstBucketTime) {
resultBucketRoundedKey := currentBucketTime.Add(-1 * time.Second * time.Duration(intervalSeconds) / 2).Round(time.Second * time.Duration(intervalSeconds))
for protocolName, data := range bucketStatsCopy[bucketStatsIndex].ProtocolStats {
if _, ok := protocolToColor[protocolName]; !ok {
protocolToColor[protocolName] = data.Color
}
for methodName, dataOfMethod := range data.MethodsStats {
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey] = map[string]map[string]*AccumulativeStatsCounter{}
}
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName] = map[string]*AccumulativeStatsCounter{}
}
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
Name: methodName,
EntriesCount: 0,
VolumeSizeBytes: 0,
}
}
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].EntriesCount += dataOfMethod.EntriesCount
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].VolumeSizeBytes += dataOfMethod.VolumeInBytes
}
}
}
bucketStatsIndex--
}
return ConvertToTimelineData(methodsPerProtocolPerTimeAggregated, protocolToColor)
}
func ConvertToTimelineData(methodsPerProtocolPerTimeAggregated map[time.Time]map[string]map[string]*AccumulativeStatsCounter, protocolToColor map[string]string) []*AccumulativeStatsProtocolTime {
finalResult := make([]*AccumulativeStatsProtocolTime, 0)
for timeKey, item := range methodsPerProtocolPerTimeAggregated {
protocolsData := make([]*AccumulativeStatsProtocol, 0)
for protocolName := range item {
entriesCount := 0
volumeSizeBytes := 0
methods := make([]*AccumulativeStatsCounter, 0)
for _, methodAccData := range methodsPerProtocolPerTimeAggregated[timeKey][protocolName] {
entriesCount += methodAccData.EntriesCount
volumeSizeBytes += methodAccData.VolumeSizeBytes
methods = append(methods, methodAccData)
}
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Color: protocolToColor[protocolName],
Methods: methods,
})
}
finalResult = append(finalResult, &AccumulativeStatsProtocolTime{
Time: timeKey.UnixMilli(),
ProtocolsData: protocolsData,
})
}
return finalResult
}
@@ -128,8 +233,15 @@ func EntryAdded(size int, summery *api.BaseEntry) {
generalStats.LastEntryTimestamp = currentTimestamp
}
//GetBucketOfTimeStamp Round the entry to the nearest threshold (one minute) floored (e.g: 15:31:45 -> 15:31:00)
func GetBucketOfTimeStamp(timestamp int64) time.Time {
entryTimeStampAsTime := time.UnixMilli(timestamp)
return entryTimeStampAsTime.Add(-1 * InternalBucketThreshold / 2).Round(InternalBucketThreshold)
}
func addToBucketStats(size int, summery *api.BaseEntry) {
entryTimeBucketRounded := time.Unix(summery.Timestamp, 0).Round(time.Minute * 1)
entryTimeBucketRounded := GetBucketOfTimeStamp(summery.Timestamp)
if len(bucketsStats) == 0 {
bucketsStats = append(bucketsStats, &TimeFrameStatsValue{
BucketTime: entryTimeBucketRounded,

View File

@@ -83,3 +83,22 @@ func TestEntryAddedVolume(t *testing.T) {
}
}
func TestGetBucketOfTimeStamp(t *testing.T) {
tests := map[int64]time.Time{
time.Date(2022, time.Month(1), 1, 10, 34, 45, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
time.Date(2022, time.Month(1), 1, 10, 59, 01, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 59, 00, 0, time.Local),
}
for key, value := range tests {
t.Run(fmt.Sprintf("%v", key), func(t *testing.T) {
actual := providers.GetBucketOfTimeStamp(key)
if actual != value {
t.Errorf("unexpected result - expected: %v, actual: %v", value, actual)
}
})
}
}

View File

@@ -16,7 +16,8 @@ func StatusRoutes(ginApp *gin.Engine) {
routeGroup.GET("/tap", controllers.GetTappingStatus)
routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.GET("/accumulative", controllers.GetAccumulativeStats) // get general stats about entries in DB
routeGroup.GET("/accumulative", controllers.GetAccumulativeStats)
routeGroup.GET("/accumulativeTiming", controllers.GetAccumulativeStatsTiming)
routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
}

View File

@@ -0,0 +1,4 @@
.barChartContainer
width: 100%
display: flex
justify-content: center

View File

@@ -0,0 +1,83 @@
import styles from "./TimelineBarChart.module.sass";
import { StatsMode } from "../TrafficStatsModal"
import React, { useCallback, useEffect, useMemo, useState } from "react";
import {
BarChart,
Bar,
XAxis,
YAxis,
Tooltip,
Legend
} from "recharts";
import { Utils } from "../../../../helpers/Utils";
interface TimelineBarChartProps {
timeLineBarChartMode: string;
data: any;
}
export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarChartMode, data }) => {
const [protocolStats, setProtocolStats] = useState([]);
const [protocolsNamesAndColors, setProtocolsNamesAndColors] = useState([]);
const padTo2Digits = useCallback((num) => {
return String(num).padStart(2, '0');
}, [])
const getHoursAndMinutes = useCallback((protocolTimeKey) => {
const time = new Date(protocolTimeKey)
const hoursAndMinutes = padTo2Digits(time.getHours()) + ':' + padTo2Digits(time.getMinutes());
return hoursAndMinutes;
}, [padTo2Digits])
const creatUniqueObjArray = useCallback((objArray) => {
return [
...new Map(objArray.map((item) => [item["name"], item])).values(),
];
}, [])
useEffect(() => {
if (!data) return;
const protocolsBarsData = [];
const prtcNames = [];
data.map(protocolObj => {
let obj: { [k: string]: any } = {};
obj.timestamp = getHoursAndMinutes(protocolObj.timestamp);
protocolObj.protocols.forEach(protocol => {
obj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
prtcNames.push({ name: protocol.name, color: protocol.color });
})
protocolsBarsData.push(obj);
})
const uniqueObjArray = creatUniqueObjArray(prtcNames);
protocolsBarsData.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1);
setProtocolStats(protocolsBarsData);
setProtocolsNamesAndColors(uniqueObjArray);
}, [data, timeLineBarChartMode, setProtocolStats, setProtocolsNamesAndColors, creatUniqueObjArray, getHoursAndMinutes])
const bars = useMemo(() => protocolsNamesAndColors.map((protocolToDIsplay) => {
return <Bar key={protocolToDIsplay.name} dataKey={protocolToDIsplay.name} stackId="a" fill={protocolToDIsplay.color} />
}), [protocolsNamesAndColors])
return (
<div className={styles.barChartContainer}>
<BarChart
width={730}
height={250}
data={protocolStats}
margin={{
top: 20,
right: 30,
left: 20,
bottom: 5
}}
>
<XAxis dataKey="timestamp" />
<YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} />
<Tooltip formatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value + " Requests"} />
<Legend />
{bars}
</BarChart>
</div>
);
}

View File

@@ -2,11 +2,7 @@ import React, {useEffect, useMemo, useState} from "react";
import styles from "./TrafficPieChart.module.sass";
import {Cell, Legend, Pie, PieChart, Tooltip} from "recharts";
import {Utils} from "../../../../helpers/Utils";
enum PieChartMode {
REQUESTS = "entriesCount",
VOLUME = "volumeSizeBytes"
}
import {StatsMode as PieChartMode} from "../TrafficStatsModal"
const COLORS = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080', '#ffffff', '#000000'];
@@ -24,6 +20,8 @@ const renderCustomizedLabel = ({
const x = cx + radius * Math.cos(-midAngle * RADIAN);
const y = cy + radius * Math.sin(-midAngle * RADIAN);
if (Number((percent * 100).toFixed(0)) <= 1) return;
return (
<text
x={x}

View File

@@ -1,8 +1,9 @@
import React, {useEffect, useState} from "react";
import {Backdrop, Box, Fade, Modal} from "@mui/material";
import React, { useEffect, useState } from "react";
import { Backdrop, Box, Fade, Modal } from "@mui/material";
import styles from "./TrafficStatsModal.module.sass";
import closeIcon from "assets/close.svg";
import {TrafficPieChart} from "./TrafficPieChart/TrafficPieChart";
import { TrafficPieChart } from "./TrafficPieChart/TrafficPieChart";
import { TimelineBarChart } from "./TimelineBarChart/TimelineBarChart";
import spinnerImg from "assets/spinner.svg";
const modalStyle = {
@@ -19,7 +20,7 @@ const modalStyle = {
color: '#000',
};
enum StatsMode {
export enum StatsMode {
REQUESTS = "entriesCount",
VOLUME = "volumeSizeBytes"
}
@@ -27,23 +28,27 @@ enum StatsMode {
interface TrafficStatsModalProps {
isOpen: boolean;
onClose: () => void;
getTrafficStatsDataApi: () => Promise<any>
getPieStatsDataApi: () => Promise<any>
getTimelineStatsDataApi: () => Promise<any>
}
export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getPieStatsDataApi, getTimelineStatsDataApi }) => {
const modes = Object.keys(StatsMode).filter(x => !(parseInt(x) >= 0));
const [statsMode, setStatsMode] = useState(modes[0]);
const [statsData, setStatsData] = useState(null);
const [pieStatsData, setPieStatsData] = useState(null);
const [timelineStatsData, setTimelineStatsData] = useState(null);
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
if(isOpen && getTrafficStatsDataApi) {
if (isOpen && getPieStatsDataApi) {
(async () => {
try {
setIsLoading(true);
const data = await getTrafficStatsDataApi();
setStatsData(data);
const pieData = await getPieStatsDataApi();
setPieStatsData(pieData);
const timelineData = await getTimelineStatsDataApi();
setTimelineStatsData(timelineData);
} catch (e) {
console.error(e)
} finally {
@@ -51,7 +56,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
}
})()
}
}, [isOpen, getTrafficStatsDataApi])
}, [isOpen, getPieStatsDataApi, getTimelineStatsDataApi, setPieStatsData, setTimelineStatsData])
return (
<Modal
@@ -65,19 +70,25 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
<Fade in={isOpen}>
<Box sx={modalStyle}>
<div className={styles.closeIcon}>
<img src={closeIcon} alt="close" onClick={() => onClose()} style={{ cursor: "pointer", userSelect: "none" }}/>
<img src={closeIcon} alt="close" onClick={() => onClose()} style={{ cursor: "pointer", userSelect: "none" }} />
</div>
<div className={styles.title}>Traffic Statistics</div>
<div className={styles.mainContainer}>
<div>
<span style={{marginRight: 15}}>Breakdown By</span>
<span style={{ marginRight: 15 }}>Breakdown By</span>
<select className={styles.select} value={statsMode} onChange={(e) => setStatsMode(e.target.value)}>
{modes.map(mode => <option value={mode}>{mode}</option>)}
{modes.map(mode => <option key={mode} value={mode}>{mode}</option>)}
</select>
</div>
{isLoading ? <div style={{textAlign: "center", marginTop: 20}}>
<div>
{isLoading ? <div style={{ textAlign: "center", marginTop: 20 }}>
<img alt="spinner" src={spinnerImg} style={{ height: 50 }} />
</div> : <TrafficPieChart pieChartMode={statsMode} data={statsData}/>}
</div> :
<div>
<TrafficPieChart pieChartMode={statsMode} data={pieStatsData} />
<TimelineBarChart timeLineBarChartMode={statsMode} data={timelineStatsData} />
</div>}
</div>
</div>
</Box>
</Fade>

View File

@@ -36,7 +36,7 @@ const App = () => {
openModal={oasModalOpen}
handleCloseModal={() => setOasModalOpen(false)}
/>}
<TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getTrafficStatsDataApi={api.getStats}/>
<TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getPieStatsDataApi={api.getPieStats} getTimelineStatsDataApi={api.getTimelineStats}/>
</div>
</ThemeProvider>
</StyledEngineProvider>

View File

@@ -111,8 +111,13 @@ export default class Api {
});
}
getStats = async () => {
getPieStats = async () => {
const response = await client.get("/status/accumulative");
return response.data;
}
getTimelineStats = async () => {
const response = await client.get("/status/accumulativeTiming");
return response.data;
}
}