Gsoc2021 - Phase 3

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

examples/logging_example.py

@@ -1,12 +1,13 @@
 import sem
 import pprint
+import os
 
 #######################
 # Create the campaign #
 #######################
 
 script = 'wifi-power-adaptation-distance'
-ns_path = 'ns-3'
+ns_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ns-3')
 campaign_dir = '/tmp/sem-test/wifi-plotting-example'
 
 campaign = sem.CampaignManager.new(ns_path, script, campaign_dir,

examples/logging_example2.py

@@ -1,11 +1,12 @@
-from sem import logging
+from sem import utils, logging
 import sem
+import os
 
 #######################
 # Create the campaign #
 #######################
 script = 'wifi-power-adaptation-distance'
-ns_path = './examples/ns-3'
+ns_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ns-3')
 campaign_dir = "/tmp/sem-test/wifi-plotting-example"
 
 campaign = sem.CampaignManager.new(ns_path, script, campaign_dir,
@@ -31,18 +32,21 @@
     'STA1_x': 5.0,
     'stepsTime': 1,
     'minPower': 0,
-    'steps': 2,
+    'steps': [2],
     'rtsThreshold': 2346,
     'stepsSize': 1
 }
-
 # Log Component in both formats
+
 log_components = {
     'PowerAdaptationDistance': 'all',
-    'FrameExchangeManager': 'info',
+    'ParfWifiManager': 'all',
+    'WifiPhy': 'level_all',
+    'FrameExchangeManager': 'level_all',
+    'OnOffApplication': 'level_all',
+    'MinstrelWifiManager': 'all'
 }
-# log_components = 'NS_LOG="PowerAdaptationDistance=all:FrameExchangeManager=info"'
-
+# log_components = 'NS_LOG="PowerAdaptationDistance=debug:ParfWifiManager=info"'
 runs = 1  # Number of runs to perform for each combination
 
 # Actually run the simulations
@@ -52,9 +56,14 @@
     runs=runs, log_components=log_components)
 
 if log_path:
-    print(log_path)
-    db, db_path = logging.process_logs(log_path[0])
-    print(logging.filter_logs(db, severity_class='debug',
-                              components={'PowerAdaptationDistance': 'info'}))
-
-    logging.wipe_results(db, db_path)
+    db, data_dir = logging.process_logs(log_path[0])
+    print('Filtered Logs:')
+    print(logging.filter_logs(db,
+                              components={'MinstrelWifiManager': 'debug'},
+                              time_begin=0.8,
+                              time_end=0.81))
+    try:
+        # Creates a dashboard the visualize the log file passed
+        utils.visualize_logs(log_path[0])
+    finally:
+        logging.wipe_results(db, data_dir)

pyproject.toml

@@ -20,6 +20,8 @@ numpy = "*"
 pandas = "*"
 click = "*"
 salib = "^1.3.8"
+Flask = "^2.0.1"
+tinydb-smartcache = "^2.0.0"
 
 [tool.poetry.dev-dependencies]
 sphinx = "*"

sem/dashboard/README.md

@@ -0,0 +1,36 @@
+# Overview of Dashboard features
+The objective of this interactive dashboard is to allow users to visualize the logs generated by ns-3 efficiently. 
+
+### Snapshot of the dashboard:
+![alt text](https://github.com/akshitpatel01/sem/blob/gsoc-phase2/res/dashboard.png)
+
+### The layout and features of the dashboard:
+Starting from the top, the dashboard is divided into the following sections:
+###### Dropdown filters along with a time filter:
+All the filter values in the dropdown are populated (according to the log file passed) as the dashboard first initializes. The dropdown filters have an inbuilt search box as well as 'Select All' and 'Deselect All' buttons. A few points to note about the filters:
+- These filters act as global filters for both the graph and the table.
+- Any unspecified filters are assumed to be 'free' and can take any value. For example, if 'context=[0,1]', 'function=['f1']' and 'Upper Time Limit:1', then the logs with (context=0 or context=1) and (function=f1) and (timestamp<1) will be displayed.
+- The log severity class filter acts as a filter for all the components selected by the component filter. For example, if 'componentA' and 'componentB' are selected along with severity class 'INFO', then the logs of 'componentA' with 'INFO' and logs of 'componentB' with 'INFO' will be displayed.
+- If no severity class is selected from the severity class filter, then by default all severity classes will be enabled for the selected components. 
+- If no component is selected from the component filter, then the severity class filter acts as an independent filter.
+
+###### The Graph:
+This is a line graph that allows users to easily analyze logs with respect to a time axis. A few points to note about the graph:
+- The graph is plotted on a timestamp(X-axis) vs context axis(Y-axis). 
+- As there can be many logs with the same timestamp and context value, for plotting the graph the logs are 'jittered along the context(or Y-axis) axis'. In other words, for logs with the same timestamp and context value, the context value is scattered between +0.2 and -0.2 of the actual context value. 
+- Each point on the graph represents a log line. Hovering the cursor on any point shows the metadata of that particular log.
+- Graph movement and zoom controls
+  - The graph can be zoomed in using the mouse wheel or dragging the mouse and selecting a box to enlarge that area to the entire graph space.
+  - Hold Ctrl and drag the mouse to pan the graph.
+
+###### The Table:
+The table allows users to effectively visualize the logs in a tabular way. A few points to note about the table:
+- The 'Show entries' dropdown menu allows the users to select the number of logs to be displayed on each page.
+- The search box above the table allows the users to search in specific columns. the searchable columns can be selected/deselected using the 'Search columns' dropdown as needed.
+- The search box is specific to the table and has no effect on the graph.
+- Clicking on a row in the table jumps to the page where that particular row is displayed in the table. For example, if the user searches for 'xyz' and logs are displayed based on this search query, then on clicking any row, the table will jump to the page (in the original table without the search query) which contains that particular row. 
+- All the columns of the table (except for extended_context) can be sorted either in ascending or descending order. 
+- The user can switch pages using the pagination numbers at the bottom-right of the table. 
+
+### Steps to run dashboard:
+Refer [examples/logging_example2.py](./../../examples/logging_example2.py)

sem/dashboard/__init__.py

@@ -0,0 +1,67 @@
+from flask import Flask, render_template, jsonify, request
+
+from sem.dashboard.dashboard import Dashboard
+
+
+flask_app = Flask(__name__)
+dashboard = Dashboard()
+
+
+@flask_app.route("/")
+def index():
+    return render_template("dashboard.html")
+
+
+@flask_app.route("/serverside_table", methods=['GET'])
+def serverside_table_content():
+    dashboard.set_request(request)
+    data = dashboard.build_datatable()
+    return jsonify(data)
+
+
+@flask_app.route("/filters", methods=['GET'])
+def filter():
+    dashboard.set_filter_request(request)
+
+    data = dashboard.buildchart()
+    plot_data = []
+    plot_data = [{'x': row['time'],
+                 'y': float(row['jitter_context'])}
+                 for row in data]
+    ret_dict = {
+            'plot': plot_data,
+            'data': data
+            }
+    return jsonify(ret_dict)
+
+
+@flask_app.route("/unique_values", methods=['GET'])
+def get_unique():
+    return jsonify(dashboard.get_metadata())
+
+
+@flask_app.route("/chart", methods=['GET'])
+def make_chart():
+    data = dashboard.buildchart()
+    plot_data = []
+    plot_data = [{'x': row['time'],
+                 'y': float(row['jitter_context'])}
+                 for row in data]
+    ret_dict = {
+            'plot': plot_data,
+            'data': data,
+            }
+    return jsonify(ret_dict)
+
+
+@flask_app.route("/update_search_column", methods=['GET'])
+def set_search_columns():
+    dashboard.set_search_columns(request)
+    return jsonify('SUCCESS')
+
+
+@flask_app.route("/get_index", methods=['GET'])
+def get_index():
+    print(request.values)
+    actual_index = dashboard.get_index_in_filtered_data(int(request.values['data_index']))
+    return jsonify(int(actual_index))

sem/dashboard/dashboard.py

@@ -0,0 +1,186 @@
+import re
+import numpy as np
+
+from copy import deepcopy
+from sem.logging import process_logs, filter_logs
+from bisect import bisect_left, bisect_right
+
+
+class dict_list_index_get_member(object):
+    def __init__(self, dict_list, member):
+        self.dict_list = dict_list
+        self.member = member
+
+    def __getitem__(self, index):
+        return self.dict_list[index][self.member]
+
+    def __len__(self):
+        return self.dict_list.__len__()
+
+
+class Dashboard(object):
+    def __init__(self):
+        self.columns = ['time', 'context', 'extended_context', 'component', 'function', 'arguments', 'severity_class', 'message']
+        self.col_search = ['extended_context', 'arguments', 'message']
+        self.log_class = ['ERROR', 'WARN', 'DEBUG', 'INFO', 'FUNCTION', 'LOGIC']
+        self.filter_request_values = None
+
+    def set_log_path(self, log_path):
+        db, data_dir = process_logs(log_path)
+        data = db.table('logs').all()
+        self.cardinality = len(data)
+        self.cardinality_filtered = len(data)
+        self.db = db
+        self.data = data
+        self.old_filter_data = None
+        self.unique_context = set(ctx['context'] for ctx in data)
+        self.unique_func = set(fun['function'] for fun in data)
+        self.unique_component = set(comp["component"] for comp in data)
+
+    def set_request(self, request):
+        self.request_values = request.values
+
+    def getTotalTime(self):
+        return self.data[-1]['time']
+
+    def get_index_in_filtered_data(self, idx):
+        if self.old_filter_data is not None:
+            actual_index = bisect_left(dict_list_index_get_member(self.old_filter_data, 'index'), idx)
+        else:
+            actual_index = bisect_left(dict_list_index_get_member(self.data, 'index'), idx)
+
+        return actual_index
+
+    def jitter_logs(self, orig_data):
+        unique_time = list(set([data['time'] for data in orig_data]))
+        time_column = dict_list_index_get_member(orig_data, 'time')
+        for timestamp in unique_time:
+            un_t = orig_data[bisect_left(time_column, timestamp):bisect_right(time_column, timestamp)]
+            for ctx in {i['context'] for i in un_t}:
+                data = [item for item in un_t if item['context'] == ctx]
+                if len(data) > 1:
+                    offsets = np.linspace(-0.2, 0.2, len(data))
+                    for idx, unique_context_item in enumerate(data):
+                        unique_context_item['jitter_context'] = str(float(unique_context_item['context']) + offsets[idx])
+                else:
+                    data[0]['jitter_context'] = data[0]['context']
+
+    def buildchart(self):
+        orig_data = self._filter_logs()
+        self.jitter_logs(orig_data)
+        self.old_filter_data = orig_data
+        return orig_data
+
+    def build_datatable(self):
+        data = self._filter_logs()
+        data = self._custom_filter(data)
+        self.cardinality_filtered = len(data)
+        data = self._custom_sort(data)
+        data = self._custom_paging(data)
+        output = {}
+        output['sEcho'] = str(int(self.request_values['sEcho']))
+        output['iTotalRecords'] = str(self.cardinality)
+        output['iTotalDisplayRecords'] = str(self.cardinality_filtered)
+        output['data'] = data
+        return output
+
+    def _custom_paging(self, data):
+        if self.request_values['iDisplayStart'] != "" and int(self.request_values['iDisplayLength']) != -1:
+            start = int(self.request_values['iDisplayStart'])
+            length = int(self.request_values['iDisplayLength'])
+            if len(data) <= length:
+                return data[start:]
+            else:
+                limit = start + length - len(data)
+                if limit < 0:
+                    return data[start:limit]
+                else:
+                    return data[start:]
+
+    def _custom_filter(self, data):
+        def check_row(row):
+            for i in range(len(self.col_search)):
+                value = row[self.col_search[i]]
+                regex = self.request_values['sSearch']
+                if re.compile(regex).search(str(value)):
+                    return True
+            return False
+
+        if self.request_values.get('sSearch', ""):
+            return [row for row in data if check_row(row)]
+        else:
+            return data
+
+    def _custom_sort(self, data):
+
+        if (self.request_values['iSortCol_0'] != "") and (int(self.request_values['iSortingCols']) > 0):
+            for i in range(0, int(self.request_values['iSortingCols'])):
+                column_number = int(self.request_values['iSortCol_' + str(i)])
+                column_name = self.columns[column_number]
+                sort_direction = self.request_values['sSortDir_' + str(i)]
+                # print(len(data))
+                data = sorted(data,
+                        key=lambda x: x[column_name],
+                        reverse=True if sort_direction == 'desc' else False)
+            return data
+        else:
+            return data
+
+    def set_filter_request(self, request):
+        self.filter_request_values = deepcopy(request.values.to_dict(flat=False))
+        if 'severity_class' not in self.filter_request_values:
+            self.filter_request_values['severity_class'] = None
+
+        if 'context' not in self.filter_request_values:
+            self.filter_request_values['context'] = None
+
+        if 'func' not in self.filter_request_values:
+            self.filter_request_values['func'] = None
+
+        if 'component' not in self.filter_request_values:
+            self.filter_request_values['component'] = None
+
+        if self.filter_request_values['time_begin'][0] == '':
+            self.filter_request_values['time_begin'] = None
+        else:
+            self.filter_request_values['time_begin'] = self.filter_request_values['time_begin'][0]
+
+        if self.filter_request_values['time_end'][0] == '':
+            self.filter_request_values['time_end'] = None
+        else:
+            self.filter_request_values['time_end'] = self.filter_request_values['time_end'][0]
+
+
+    def get_metadata(self):
+        return{
+                'context': list(self.unique_context),
+                'function': list(self.unique_func),
+                'component': list(self.unique_component),
+                'search_column': list(self.col_search),
+                'all_columns': list(self.columns),
+                'log_class': list(self.log_class)
+                }
+
+    def set_search_columns(self, request):
+        if request.values.__contains__('search_column'):
+            self.col_search = request.values.to_dict(flat=False)['search_column']
+        else:
+            self.col_search = []
+
+    def _filter_logs(self):
+        if self.filter_request_values is not None:
+            component = None
+            if self.filter_request_values['component'] is not None and self.filter_request_values['severity_class'] is not None:
+                component = {}
+                for comp in self.filter_request_values['component']:
+                    component[comp] = self.filter_request_values['severity_class']
+                return filter_logs(self.db, context=self.filter_request_values['context'], function=self.filter_request_values['func'], time_begin=self.filter_request_values['time_begin'], time_end=self.filter_request_values['time_end'], components=component)
+            elif self.filter_request_values['component'] is not None and self.filter_request_values['severity_class'] is None:
+                component = {}
+                for comp in self.filter_request_values['component']:
+                    component[comp] = self.log_class
+                return filter_logs(self.db, context=self.filter_request_values['context'], function=self.filter_request_values['func'], time_begin=self.filter_request_values['time_begin'], time_end=self.filter_request_values['time_end'], components=component)
+            else:
+                return filter_logs(self.db, severity_class=self.filter_request_values['severity_class'], context=self.filter_request_values['context'], function=self.filter_request_values['func'], time_begin=self.filter_request_values['time_begin'], time_end=self.filter_request_values['time_end'])
+        else:
+            return self.data