gpu-plot

#!/usr/bin/env python3
""" gpu-plot  -  Plot GPU parameter values over time

    Part of the rickslab-gpu-utils package which includes gpu-ls, gpu-mon,
    gpu-pac, and gpu-plot.

    A utility to continuously plot the trend of critical GPU parameters for all
    compatible GPUs. The *--sleep N* can be used to specify the update interval.
    The *gpu-plot* utility has 2 modes of operation.  The default mode is to
    read the GPU driver details directly, which is useful as a standalone
    utility.  The *--stdin* option causes *gpu-plot* to read GPU data from
    stdin.  This is how *gpu-mon* produces the plot and can also be used to
    pipe your own data into the process.  The *--simlog* option can be used
    with the *--stdin* when a monitor log file is piped as stdin. This is
    useful for troubleshooting and can be used to display saved log results.
    The *--ltz* option results in the use of local time instead of UTC.  If you
    plan to run both *gpu-plot* and *gpu-mon*, then the *--plot* option of the
    *gpu-mon* utility should be used instead of both utilities in order reduce
    data reads by a factor of 2.  The *--verbose* option will display progress
    and informational messages generated by the utilities.

    Copyright (C) 2019  RicksLab

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by the Free
    Software Foundation, either version 3 of the License, or (at your option)
    any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program.  If not, see <https://www.gnu.org/licenses/>.
"""
__author__ = 'RicksLab'
__copyright__ = 'Copyright (C) 2019 RicksLab'
__license__ = 'GNU General Public License'
__program_name__ = 'gpu-plot'
__maintainer__ = 'RicksLab'
__docformat__ = 'reStructuredText'
# pylint: disable=multiple-statements
# pylint: disable=line-too-long
# pylint: disable=consider-using-f-string

import sys
from gc import collect as garb_collect
import argparse
import re
import threading
import os
import logging
from time import sleep
from typing import Dict, Set, Tuple, List, Union
import warnings
import numpy as np

try:
    import gi
    gi.require_version('Gtk', '3.0')
    from gi.repository import GLib, Gtk
except ModuleNotFoundError as error:
    print('gi import error: {}'.format(error))
    print('gi is required for {}'.format(__program_name__))
    print('   In a venv, first install vext:  pip install --no-cache-dir vext')
    print('   Then install vext.gi:  pip install --no-cache-dir vext.gi')
    sys.exit(0)
except ImportError as error:
    print('gi import error: {}'.format(error))
    print('If not using system python version, you may get a circular import error.')
    sys.exit(0)

try:
    from matplotlib.backends.backend_gtk3cairo import FigureCanvasGTK3Cairo as FigureCanvas
    import matplotlib.pyplot as plt
except (ModuleNotFoundError, ImportError) as error:
    print('matplotlib import error: {}'.format(error))
    print('matplotlib is required for {}'.format(__program_name__))
    print('Use \'sudo apt install python3-matplotlib\' to install')
    sys.exit(0)

try:
    import pandas as pd
except (ModuleNotFoundError, ImportError) as error:
    print('Pandas import error: {}'.format(error))
    print('Pandas is required for {}'.format(__program_name__))
    print('Install pip3 if needed: \'sudo apt install python3-pip\'')
    print('Then pip install pandas: \'pip3 install pandas\'')
    sys.exit(0)
from pandas.plotting import register_matplotlib_converters

from GPUmodules import __version__, __status__, __credits__
from GPUmodules import GPUgui
from GPUmodules import GPUmodule as Gpu
from GPUmodules.env import GUT_CONST
from GPUmodules.GPUKeys import SensorSet
from GPUmodules.RegexPatterns import PatternKeys as PK

warnings.simplefilter(action='ignore', category=FutureWarning)

register_matplotlib_converters()
set_gtk_prop = GPUgui.GuiProps.set_gtk_prop
LOGGER = logging.getLogger('gpu-utils')
PATTERNS = GUT_CONST.PATTERNS

# SEMAPHORE ############
PD_SEM = threading.Semaphore()
########################


def get_stack_size() -> int:
    """
    Get stack size for caller's frame. Code copied from Stack Overflow.

    :return: Stack size
    """
    size = 2  # current frame and caller's frame always exist
    while True:
        try:
            sys._getframe(size)
            size += 1
        except ValueError:
            return size - 1  # subtract current frame


class PlotData:
    """
    Plot data object.
    """
    plot_list: Dict[str, Set[str]] = {'ax1': {'loading', 'power_cap', 'power', 'temp_val'},
                                      'ax2': {'vddgfx_val', 'sclk_f_val', 'mclk_f_val'}}

    def __init__(self):
        self.df: pd.DataFrame = pd.DataFrame()
        self.pcie_dict: dict = {}
        self.gui_comp = None
        self.gui_ready: bool = False
        self.length: int = 200
        self.quit: bool = False
        self.writer: bool = False
        self.reader: bool = False
        self.consec_writer: int = 0
        self.consec_reader: int = 0
        self.gpu_name_list: List[str] = []
        self.num_gpus: int = 1
        self.com_gpu_list: Gpu.GpuList = Gpu.GpuList()

    def set_gpus(self) -> None:
        """
        Populate num_gpus and gpu_name_list from dataframe member.
        """
        self.num_gpus = self.df['Card#'].nunique()
        self.gpu_name_list = self.df['Card#'].unique()

    def set_com_gpu_list(self, gpu_list: Gpu.GpuList) -> None:
        """
        Set plot data gpu_list object and initialize pcie decode dict.

        :param gpu_list:
        """
        self.com_gpu_list = gpu_list
        self.pcie_dict = gpu_list.get_pcie_map()

    def get_gpu_pcieid(self, card_num: int) -> str:
        """
        Return the pcie id for a given card number.

        :param card_num:
        :return: the pcie id as a string
        """
        if card_num in self.pcie_dict:
            return self.pcie_dict[card_num]
        return 'Error'

    def get_plot_data(self) -> pd.DataFrame:
        """
        Get deep copy of plot data df.

        :return: deep copy of the plot data dataframe
        """
        # SEMAPHORE ############
        PD_SEM.acquire()
        ########################
        ndf = self.df.copy()
        # SEMAPHORE ############
        PD_SEM.release()
        ########################
        return ndf

    def kill_thread(self) -> None:
        """
        Sets flags that result in reader thread death.
        """
        self.reader = False
        self.quit = True
        print('Stopping reader thread')
        sleep(0.2)


class GuiComponents:
    """
    Define the gui components of the plot window.
    """
    _colors: Dict[str, str] = {'plotface':        GPUgui.GuiProps.color_name_to_hex('slate_vdk'),
                               'figface':         GPUgui.GuiProps.color_name_to_hex('slate_md'),
                               'disable_but':     GPUgui.GuiProps.color_name_to_hex('gray70'),
                               'sclk_f_val':      GPUgui.GuiProps.color_name_to_hex('br_green'),
                               'mclk_f_val':      GPUgui.GuiProps.color_name_to_hex('br_yellow'),
                               'loading':         GPUgui.GuiProps.color_name_to_hex('br_pink'),
                               'power':           GPUgui.GuiProps.color_name_to_hex('br_orange'),
                               'power_cap':       GPUgui.GuiProps.color_name_to_hex('br_red'),
                               'vddgfx_val':      GPUgui.GuiProps.color_name_to_hex('br_blue'),
                               'temp_val':        GPUgui.GuiProps.color_name_to_hex('slate_md')}

    _font_colors: Dict[str, str] = {'plotface':   GPUgui.GuiProps.color_name_to_hex('black'),
                                    'figface':    GPUgui.GuiProps.color_name_to_hex('black'),
                                    'sclk_f_val': GPUgui.GuiProps.color_name_to_hex('gray95'),
                                    'mclk_f_val': GPUgui.GuiProps.color_name_to_hex('gray95'),
                                    'loading':    GPUgui.GuiProps.color_name_to_hex('white_off'),
                                    'power':      GPUgui.GuiProps.color_name_to_hex('white_off'),
                                    'power_cap':  GPUgui.GuiProps.color_name_to_hex('white_off'),
                                    'vddgfx_val': GPUgui.GuiProps.color_name_to_hex('gray95'),
                                    'temp_val':   GPUgui.GuiProps.color_name_to_hex('white_off')}

    _gpu_color_list: Tuple[str] = (GPUgui.GuiProps.color_name_to_hex('red'),
                                   GPUgui.GuiProps.color_name_to_hex('green_dk'),
                                   GPUgui.GuiProps.color_name_to_hex('yellow'),
                                   GPUgui.GuiProps.color_name_to_hex('orange'),
                                   GPUgui.GuiProps.color_name_to_hex('purple'),
                                   GPUgui.GuiProps.color_name_to_hex('blue'),
                                   GPUgui.GuiProps.color_name_to_hex('teal'),
                                   GPUgui.GuiProps.color_name_to_hex('olive'))

    def __init__(self, plot_data: PlotData):
        """
        Initialize GUI Components to support plot window.

        :param plot_data:
        """
        plot_data.gui_comp = self
        self.ready = False
        self.gpu_name_list = plot_data.gpu_name_list
        self.num_gpus = plot_data.num_gpus
        self.gui_components = {}
        self.gpu_color = {}
        gpu_color_list = self._gpu_color_list
        plot_item_list = ('loading', 'power', 'power_cap', 'temp_val', 'vddgfx_val', 'sclk_f_val', 'mclk_f_val')

        self.plot_items = {'loading': True, 'power': True, 'power_cap': True,
                           'temp_val': True, 'vddgfx_val': True, 'sclk_f_val': True, 'mclk_f_val': True}

        self.gui_components['info_bar'] = {}
        self.gui_components['legend'] = {}
        self.gui_components['legend']['buttons'] = {}
        self.gui_components['legend']['plot_items'] = {}
        for plotitem in plot_item_list:
            self.gui_components['legend']['plot_items'][plotitem] = True
        self.gui_components['sclk_pstate_status'] = {}
        self.gui_components['sclk_pstate_status']['df_name'] = 'sclk_ps_val'
        self.gui_components['mclk_pstate_status'] = {}
        self.gui_components['mclk_pstate_status']['df_name'] = 'mclk_ps_val'
        self.gui_components['temp_status'] = {}
        self.gui_components['temp_status']['df_name'] = 'temp_val'
        self.gui_components['card_plots'] = {}
        for i, gpu_i in enumerate(self.gpu_name_list):
            self.gui_components['card_plots'][gpu_i] = {}
            self.gui_components['card_plots'][gpu_i]['color'] = gpu_color_list[i]
            self.gpu_color[gpu_i] = gpu_color_list[i]

    def get_color(self, color_name: str) -> str:
        """
        Get color RGB hex code for the given color name.

        :param color_name: Color Name
        :return: Color RGB hex code
        """
        if color_name not in self._colors:
            raise KeyError('color name {} not in color dict {}'.format(color_name, self._colors))
        return self._colors[color_name]

    def get_font_color(self, color_name: str) -> str:
        """
        Get font color RGB hex code for the given color name.

        :param color_name: Color Name
        :return: Color RGB hex code
        """
        if color_name not in self._font_colors:
            raise KeyError('color name {} not in color dict {}'.format(color_name, self._font_colors))
        return self._font_colors[color_name]

    def set_ready(self, mode: bool) -> None:
        """
        Set flag to indicate gui is ready.

        :param mode: True if gui is ready
        """
        self.ready = mode

    def is_ready(self) -> bool:
        """
        Return the ready status of the plot gui.

        :return: True if ready
        """
        return self.ready


class GPUPlotWindow(Gtk.Window):
    """
    Plot window object.
    """
    def __init__(self, gc: GuiComponents, plot_data: PlotData):
        """
        Initialize and open the plot Window.

        :param gc:
        :param plot_data:
        """
        init_chk_value = Gtk.init_check(sys.argv)
        LOGGER.debug('init_check: %s', init_chk_value)
        if not init_chk_value[0]:
            print('Gtk Error, Exiting')
            sys.exit(-1)
        box_spacing_val = 5
        num_bar_plots = 3
        if gc.num_gpus > 4:
            def_gp_y_size = 150
            def_bp_y_size = 200
        elif gc.num_gpus == 4:
            def_gp_y_size = 200
            def_bp_y_size = 200
        else:
            def_gp_y_size = 250
            def_bp_y_size = 250
        def_gp_x_size = 650
        def_bp_x_size = 250
        def_lab_y_size = 28
        if gc.num_gpus > num_bar_plots:
            tot_y_size = gc.num_gpus * (def_gp_y_size + def_lab_y_size)
            gp_y_size = def_gp_y_size
            bp_y_size = (tot_y_size - (num_bar_plots * def_lab_y_size))/num_bar_plots
        elif gc.num_gpus < num_bar_plots:
            tot_y_size = num_bar_plots * (def_bp_y_size + def_lab_y_size)
            bp_y_size = def_bp_y_size
            gp_y_size = (tot_y_size - (gc.num_gpus * def_lab_y_size))/gc.num_gpus
        else:
            gp_y_size = def_gp_y_size
            bp_y_size = def_bp_y_size

        Gtk.Window.__init__(self, title=GUT_CONST.gui_window_title)
        self.set_border_width(0)
        self.set_resizable(False)
        GPUgui.GuiProps.set_style()

        if GUT_CONST.icon_file:
            LOGGER.debug('Icon file: [%s]', GUT_CONST.icon_file)
            if os.path.isfile(GUT_CONST.icon_file):
                self.set_icon_from_file(GUT_CONST.icon_file)

        grid = Gtk.Grid()
        self.add(grid)

        # Get deep copy of current df
        ldf = plot_data.get_plot_data()

        row = 0
        # Top Bar - info
        gc.gui_components['info_bar']['gtk_obj'] = Gtk.Label(name='white_label')
        gc.gui_components['info_bar']['gtk_obj'].set_markup('<big><b>{} Plot</b></big>'.format(__program_name__))
        set_gtk_prop(gc.gui_components['info_bar']['gtk_obj'], align=(0.5, 0.5), top=1, bottom=1, right=4, left=4)
        lbox = Gtk.Box(spacing=box_spacing_val, name='head_box')
        set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
        lbox.pack_start(gc.gui_components['info_bar']['gtk_obj'], True, True, 0)
        grid.attach(lbox, 1, row, 4, 1)
        row += 1

        # Legend
        gc.gui_components['legend']['gtk_obj'] = Gtk.Label(name='white_label')
        gc.gui_components['legend']['gtk_obj'].set_markup('<big><b>Plot Items</b></big>')
        set_gtk_prop(gc.gui_components['legend']['gtk_obj'], align=(0.5, 0.5), top=1, bottom=1, right=4, left=4)
        lbox = Gtk.Box(spacing=box_spacing_val, name='dark_box')
        set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
        lbox.pack_start(gc.gui_components['legend']['gtk_obj'], True, True, 0)
        for comp_name in gc.gui_components['legend']['plot_items']:
            but_label = Gpu.GpuItem.get_button_label(comp_name)
            but_name = 'but_{}'.format(comp_name)
            but_color = gc.get_color(comp_name)
            but_font_color = gc.get_font_color(comp_name)
            gc.gui_components['legend']['buttons'][comp_name] = Gtk.Button(name=but_name, label='')
            GPUgui.GuiProps.set_style(css_str="#%s { background-image: image(%s); color: %s; }" % (
                but_name, but_color, but_font_color))
            for child in gc.gui_components['legend']['buttons'][comp_name].get_children():
                child.set_label('<big><b>{}</b></big>'.format(but_label))
                child.set_use_markup(True)
            gc.gui_components['legend']['buttons'][comp_name].connect('clicked', self.toggle_plot_item, gc, comp_name)
            lbox.pack_start(gc.gui_components['legend']['buttons'][comp_name], True, True, 0)
        grid.attach(lbox, 1, row, 4, 1)
        row += 1
        main_last_row = row

        # Set up bar plots
        grid_bar = Gtk.Grid(name='dark_grid')
        grid.attach(grid_bar, 1, main_last_row, 1, 1)
        brow = 0
        fig_num = 0
        # plot_top_row = row
        for comp_item in (gc.gui_components['sclk_pstate_status'],
                          gc.gui_components['mclk_pstate_status'],
                          gc.gui_components['temp_status']):
            # Add Bar Plots Titles
            bar_plot_name = Gpu.GpuItem.get_button_label(comp_item['df_name'])
            comp_item['title_obj'] = Gtk.Label(name='white_label')
            comp_item['title_obj'].set_markup('<big><b>Card {}</b></big>'.format(bar_plot_name))
            set_gtk_prop(comp_item['title_obj'], align=(0.5, 0.5), top=1, bottom=1, right=4, left=4)
            lbox = Gtk.Box(spacing=box_spacing_val, name='head_box')
            set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
            lbox.pack_start(comp_item['title_obj'], True, True, 0)

            grid_bar.attach(lbox, 1, brow, 1, 1)
            brow += 1

            # Add Bar Plots
            # Set up plot figure and canvas
            comp_item['figure_num'] = 100 + fig_num
            fig_num += 1
            comp_item['figure'], comp_item['ax1'] = plt.subplots(num=comp_item['figure_num'])
            comp_item['figure'].set_facecolor(gc.get_color('figface'))

            plt.figure(comp_item['figure_num'])
            plt.subplots_adjust(left=0.13, right=0.97, top=0.97, bottom=0.1)
            comp_item['ax1'].set_facecolor(gc.get_color('plotface'))
            if comp_item['df_name'] == 'temp_val':
                plt.yticks(np.arange(15, 99, 10))
            else:
                plt.yticks(np.arange(0, 9, 1))

            comp_item['canvas'] = FigureCanvas(comp_item['figure'])
            comp_item['canvas'].set_size_request(def_bp_x_size, bp_y_size)

            lbox = Gtk.Box(spacing=box_spacing_val, name='med_box')
            set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
            lbox.pack_start(comp_item['canvas'], True, True, 0)

            grid_bar.attach(lbox, 1, brow, 1, 1)
            brow += 1

        # Set up gpu plots
        grid_plot = Gtk.Grid(name='dark_grid')
        grid.attach(grid_plot, 2, main_last_row, 3, 1)
        prow = 0
        # row = plot_top_row
        for comp_num, comp_item in gc.gui_components['card_plots'].items():
            data_val = ldf[ldf['Card#'].isin([comp_num])]['energy'].iloc[-1]
            data_val = Gpu.format_table_value(data_val, 'energy')
            model_val = ldf[ldf['Card#'].isin([comp_num])]['model_display'].iloc[-1]
            # Add GPU Plots Titles
            comp_item['title_obj'] = Gtk.Label(name='white_label')
            if not model_val or isinstance(model_val, type(np.nan)): model_val = 'UNKNOWN'
            if len(model_val) > 30: model_val = model_val[:30]
            comp_item['title_obj'].set_markup('<big><b>Card{}  [{}]    {}    Energy:  {} kWh</b></big>'.format(
                                      comp_num, plot_data.get_gpu_pcieid(comp_num), model_val, data_val))
            set_gtk_prop(comp_item['title_obj'], align=(0.5, 0.5), top=1, bottom=1, right=4, left=4)
            box_name = comp_item['color'][1:]
            lbox = Gtk.Box(spacing=box_spacing_val, name=box_name)
            GPUgui.GuiProps.set_style(css_str="#%s { background-image: image(%s); }" % (box_name, comp_item['color']))
            set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
            lbox.pack_start(comp_item['title_obj'], True, True, 0)

            grid_plot.attach(lbox, 1, prow, 1, 1)
            prow += 1

            # Add GPU Plots
            # Set up plot figure and canvas
            comp_item['figure_num'] = 500 + comp_num
            comp_item['figure'], comp_item['ax1'] = plt.subplots(num=comp_item['figure_num'])
            comp_item['figure'].set_facecolor(gc.get_color('figface'))
            plt.figure(comp_item['figure_num'])
            plt.subplots_adjust(left=0.1, right=0.9, top=0.97, bottom=0.03)

            comp_item['ax1'].set_facecolor(gc.get_color('plotface'))
            comp_item['ax1'].set_xticks([])
            comp_item['ax1'].set_xticklabels([])
            comp_item['ax1'].set_yticks(np.arange(0, 250, 20))
            comp_item['ax1'].tick_params(axis='y', which='major', labelsize=8)

            comp_item['ax2'] = comp_item['ax1'].twinx()
            comp_item['ax2'].set_xticks([])
            comp_item['ax2'].set_xticklabels([])
            comp_item['ax2'].set_yticks(np.arange(500, 1500, 100))
            comp_item['ax2'].tick_params(axis='y', which='major', labelsize=8)

            comp_item['canvas'] = FigureCanvas(comp_item['figure'])  # a Gtk.DrawingArea
            comp_item['canvas'].set_size_request(def_gp_x_size, gp_y_size)

            lbox = Gtk.Box(spacing=box_spacing_val, name='light_box')
            set_gtk_prop(lbox, top=1, bottom=1, right=1, left=1)
            lbox.pack_start(comp_item['canvas'], True, True, 0)

            grid_plot.attach(lbox, 1, prow, 1, 1)
            prow += 1

    @staticmethod
    def toggle_plot_item(_, gc: GuiComponents, k: str) -> None:
        """
        Toggle specified plot item.

        :param _: parent
        :param gc: gui components object
        :param k:  Name of plot item to toggle
        """
        but_name = 'but_{}'.format(k)
        but_color = gc.get_color('disable_but') if gc.plot_items[k] else gc.get_color(k)
        GPUgui.GuiProps.set_style(css_str="#%s { background-image: image(%s); }" % (but_name, but_color))
        gc.plot_items[k] = not gc.plot_items[k]


def update_data(gc: GuiComponents, plot_data: PlotData) -> None:
    """
    Update plot data.

    :param gc:
    :param plot_data:
    """
    # SEMAPHORE ###########
    PD_SEM.acquire()
    #######################
    ldf = plot_data.df
    tick_inc = None
    plot_limits: Dict[str, Dict[str, Union[float, int, None]]] = {}
    try:
        time_val = ldf[ldf['Card#'].isin([plot_data.gpu_name_list[0]])]['Time'].iloc[-1]
        gc.gui_components['info_bar']['gtk_obj'].set_markup('<big><b>Time   {}</b></big>'.format(time_val))
        # Update Bar Plots
        bar_plot_types = {'sclk_pstate_status': 'sclk_ps_val',
                          'mclk_pstate_status': 'mclk_ps_val',
                          'temp_status': 'temp_val'}
        for item_name, sensor_name in bar_plot_types.items():
            comp_item = gc.gui_components[item_name]
            data_val = []
            label_val = []
            bar_col = []
            # Set Plot Parameters
            for card_num in plot_data.gpu_name_list:
                active_gpu = plot_data.com_gpu_list.select_gpu(int(card_num))
                if not active_gpu.table_parameters_status[sensor_name]: continue
                l, d = ldf[ldf['Card#'].isin([card_num])][['Card#', comp_item['df_name']]].iloc[-1]
                try:
                    label_val.append(int(l))
                    data_val.append(float(d))
                except:
                    print('Fatal Data Error - Exiting')
                    plot_data.kill_thread()
                bar_col.append(gc.gpu_color[l])

            x_index = np.arange(gc.num_gpus)  # the x locations for the groups
            width = 0.65       # the width of the bars

            # Do bar plot
            plt.figure(comp_item['figure_num'])
            comp_item['ax1'].clear()
            _rects1 = comp_item['ax1'].bar(x_index, data_val, width, color=bar_col, tick_label=label_val)
            if comp_item['df_name'] == 'temp_val':
                for a, b in zip(x_index, data_val):
                    comp_item['ax1'].text(x=a, y=b-5, s=str(b), fontsize=8, ha='center')
                plt.ylim((15, 99))
            else:
                try:
                    data_val = list(map(int, data_val))
                except:
                    pass
                for a, b in zip(x_index, data_val):
                    y_val = b + width if b == 0 else b - width
                    comp_item['ax1'].text(x=a, y=y_val, s=str(b), fontsize=10, ha='center')
                plt.ylim((0, 9))
            comp_item['canvas'].draw()
            comp_item['canvas'].flush_events()

        # Update GPU Plots
        # Setting limits may be based on all data
        for comp_num in gc.gui_components['card_plots']:
            active_gpu = plot_data.com_gpu_list.select_gpu(comp_num)
            for axis_name, plot_items in plot_data.plot_list.items():
                active_plot_items = []
                plot_limits.update({axis_name: {'min': None, 'max': None}})
                for test_item in plot_items:
                    if active_gpu.table_parameters_status[test_item]: active_plot_items.append(test_item)
                if axis_name == 'ax1':
                    max_val = 10*(np.nanmax(ldf.loc[:, active_plot_items]).max(initial=100.0) // 10) + 10
                    min_val = 10*(np.nanmin(ldf.loc[:, active_plot_items]).min(initial=0.0) // 10) - 5
                else:
                    max_val = 100*(np.nanmax(ldf.loc[:, active_plot_items]).max(initial=100.0) // 100) + 300
                    min_val = 100*(np.nanmin(ldf.loc[:, active_plot_items]).min(initial=0.0) // 100) - 100
                if not plot_limits[axis_name]['min'] or min_val < plot_limits[axis_name]['min']:
                    plot_limits[axis_name]['min'] = min_val
                if not plot_limits[axis_name]['max'] or max_val > plot_limits[axis_name]['max']:
                    plot_limits[axis_name]['max'] = max_val

        for comp_num, comp_item in gc.gui_components['card_plots'].items():
            data_val = ldf[ldf['Card#'].isin([comp_num])]['energy'].iloc[-1]
            data_val = Gpu.format_table_value(data_val, 'energy')
            model_val = ldf[ldf['Card#'].isin([comp_num])]['model_display'].iloc[-1]
            comp_item['title_obj'].set_markup('<big><b>Card{}  [{}]    {}    Energy:  {} kWh</b></big>'.format(
                                              comp_num, plot_data.get_gpu_pcieid(comp_num), model_val[:30], data_val))

            # Select plot items with data
            active_gpu = plot_data.com_gpu_list.select_gpu(comp_num)
            for axis_name, plot_items in plot_data.plot_list.items():
                active_plot_items = []
                for test_item in plot_items:
                    if active_gpu.table_parameters_status[test_item]: active_plot_items.append(test_item)
                if not active_plot_items:
                    continue

                # Plot GPUs
                plt.figure(comp_item['figure_num'])
                axis_label = 'Loading/Power/Temp' if axis_name == 'ax1' else 'MHz/mV'
                axis_label_col = 'white_off' if axis_name == 'ax1' else 'gray95'
                comp_item[axis_name].clear()
                comp_item[axis_name].set_xticklabels([])
                comp_item[axis_name].set_ylabel(axis_label,
                                                color=GPUgui.GuiProps.color_name_to_hex(axis_label_col), fontsize=10)
                for plot_item in active_plot_items:
                    if gc.plot_items[plot_item]:
                        comp_item[axis_name].plot(ldf[ldf['Card#'].isin([comp_num])]['datetime'].values,
                                                  ldf[ldf['Card#'].isin([comp_num])][plot_item].values,
                                                  color=gc.get_color(plot_item), linewidth=0.5)
                        comp_item[axis_name].text(x=ldf[ldf['Card#'].isin([comp_num])]['datetime'].iloc[-1],
                                                  y=ldf[ldf['Card#'].isin([comp_num])][plot_item].iloc[-1],
                                                  s=str(int(ldf[ldf['Card#'].isin([comp_num])][plot_item].iloc[-1])),
                                                  bbox={'boxstyle': 'round,pad=0.2',
                                                        'facecolor': gc.get_color(plot_item)},
                                                  fontsize=6)

                # Set axis range and tick increments
                tick_inc = int(10 * round(((plot_limits[axis_name]['max'] - plot_limits[axis_name]['min']) // 12)/10.0, 0))
                tick_inc = max(tick_inc, 5) if axis_name == 'ax1' else max(tick_inc, 50)
                comp_item[axis_name].set_yticks(np.arange(plot_limits[axis_name]['min'],
                                                plot_limits[axis_name]['max'], tick_inc))

            comp_item['canvas'].draw()
            comp_item['canvas'].flush_events()
    except (OSError, ArithmeticError, NameError, TypeError, ValueError) as err:
        LOGGER.exception('plot exception: %s', err)
        LOGGER.debug('Plot limits min_max: %s', plot_limits)
        LOGGER.debug('ticker increment: %s', tick_inc)
        print('matplotlib error: {}'.format(err))
        print('matplotlib error, stack size is {}'.format(get_stack_size()))
        plot_data.kill_thread()

    # SEMAPHORE ###########
    PD_SEM.release()
    #######################


def read_from_stdin(refresh_time: int, plot_data: PlotData) -> None:
    """
    Read plot data from stdin.

    :param refresh_time:
    :param plot_data:
    .. note:: this should continuously read from stdin and populate df and call plot/gui update
    """
    header_item = ''
    first_update = True
    header = True
    sync_add = 0
    while not plot_data.quit:
        if GUT_CONST.simlog: sleep(refresh_time/4.0)
        ndf = pd.DataFrame()

        # Process a set of GPUs at a time
        skip_update = False
        read_time = 0.0
        for _gpu_index in range(0, plot_data.num_gpus + sync_add):
            start_time = GUT_CONST.now(GUT_CONST.useltz)
            line = sys.stdin.readline()
            tmp_read_time = (GUT_CONST.now(GUT_CONST.useltz) - start_time).total_seconds()
            if tmp_read_time > read_time:
                read_time = tmp_read_time

            if line == '':
                LOGGER.debug('Error: Null input line')
                plot_data.kill_thread()
                break
            if header:
                header_item = list(line.strip().split('|'))
                header = False
                continue
            line_items = list(line.strip().split('|'))
            new_line_items = []
            for item in line_items:
                item = item.strip()
                if item == 'nan':
                    new_line_items.append(np.nan)
                elif item.isnumeric():
                    new_line_items.append(int(item))
                elif re.fullmatch(PATTERNS[PK.IS_FLOAT], item):
                    new_line_items.append(float(item))
                elif item in ('', '-1', 'NA', 'None') or item is None:
                    new_line_items.append(np.nan)
                else:
                    new_line_items.append(item)
            line_items = tuple(new_line_items)
            rdf = pd.DataFrame.from_records([line_items], columns=header_item)
            rdf['datetime'] = pd.to_datetime(rdf['Time'], format=GUT_CONST.TIME_FORMAT, exact=False)
            ndf = pd.concat([ndf, rdf], ignore_index=True)
            del rdf
            sync_add = 1 if ndf['Time'].tail(plot_data.num_gpus).nunique() > 1 else 0

        LOGGER.debug('dataFrame %s:\n%s',
                     GUT_CONST.now(GUT_CONST.useltz).strftime(GUT_CONST.TIME_FORMAT), ndf.to_string())

        if not GUT_CONST.simlog:
            if read_time < 0.003:
                skip_update = True
                LOGGER.debug('skipping update')

        # SEMAPHORE ############
        PD_SEM.acquire()
        ########################
        # Concatenate new data on plot_data dataframe and truncate
        plot_data.df = pd.concat([plot_data.df, ndf], ignore_index=True)
        plot_data.df.reset_index(drop=True, inplace=True)

        # Truncate df in place
        plot_length = int(len(plot_data.df.index) / plot_data.num_gpus)
        if plot_length > plot_data.length:
            trun_index = plot_length - plot_data.length
            plot_data.df.drop(np.arange(0, trun_index), inplace=True)
            plot_data.df.reset_index(drop=True, inplace=True)
        # SEMAPHORE ############
        PD_SEM.release()
        ########################
        del ndf

        #########################
        # Update plots
        #########################
        if plot_data.quit:
            GUT_CONST.process_message('Exit stack size: {}'.format(get_stack_size()))
            PD_SEM.acquire()
            PD_SEM.release()
            sys.exit(0)
        if skip_update:
            continue
        if plot_data.gui_comp is None:
            continue
        if plot_data.gui_comp.is_ready():
            if first_update:
                sleep(refresh_time)
                first_update = False
            GLib.idle_add(update_data, plot_data.gui_comp, plot_data)
            while Gtk.events_pending():
                Gtk.main_iteration_do(True)
            # SEMAPHORE ############
            sleep(0.01)
            PD_SEM.acquire()
            PD_SEM.release()
            ########################
            garb_collect()
        LOGGER.debug('update stack size: %s', get_stack_size())

    # Quit
    print('Exit stack size: {}'.format(get_stack_size()))
    sys.exit(0)


def read_from_gpus(refresh_time: int, plot_data: PlotData) -> None:
    """
    Read plot data from stdin.

    :param refresh_time:
    :param plot_data:
    .. note:: this should continuously read from GPUs and populate df and call plot/gui update
    """
    first_update = True
    while not plot_data.quit:
        ndf = pd.DataFrame()

        plot_data.com_gpu_list.read_gpu_sensor_set(data_type=SensorSet.Monitor)

        # Process a set of GPUs at a time
        skip_update = False
        for gpu in plot_data.com_gpu_list.gpus():
            gpu_plot_data = gpu.get_plot_data()
            LOGGER.debug('gpu_plot_data: %s', gpu_plot_data)

            rdf = pd.DataFrame.from_records([tuple(gpu_plot_data.values())], columns=tuple(gpu_plot_data))
            rdf['datetime'] = pd.to_datetime(rdf['Time'], format=GUT_CONST.TIME_FORMAT, exact=False)
            ndf = pd.concat([ndf, rdf], ignore_index=True)
            del rdf

        # SEMAPHORE ############
        PD_SEM.acquire()
        ########################
        # Concatenate new data on plot_data dataframe and truncate
        plot_data.df = pd.concat([plot_data.df, ndf], ignore_index=True)
        plot_data.df.reset_index(drop=True, inplace=True)

        # Truncate df in place
        try:
            plot_length = int(len(plot_data.df.index) / plot_data.num_gpus)
        except TypeError:
            plot_length = 1
        if plot_length > plot_data.length:
            trun_index = plot_length - plot_data.length
            plot_data.df.drop(np.arange(0, trun_index), inplace=True)
            plot_data.df.reset_index(drop=True, inplace=True)
        # SEMAPHORE ############
        PD_SEM.release()
        ########################
        del ndf

        #########################
        # Update plots
        #########################
        if skip_update:
            continue
        if plot_data.gui_comp is None:
            sleep(refresh_time)
            continue
        if plot_data.gui_comp.is_ready():
            if first_update:
                sleep(refresh_time)
                first_update = False
            GLib.idle_add(update_data, plot_data.gui_comp, plot_data)
            while Gtk.events_pending():
                Gtk.main_iteration_do(True)
            # SEMAPHORE ############
            sleep(0.01)
            PD_SEM.acquire()
            PD_SEM.release()
            ########################
            garb_collect()
        LOGGER.debug('update stack size: %s', get_stack_size())
        sleep(refresh_time)

    # Quit
    print('Exit stack size: {}'.format(get_stack_size()))
    sys.exit(0)


def main() -> None:
    """ Main flow for plot."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--about', help='README', action='store_true', default=False)

    # Mutually exclusive input methods
    in_group = parser.add_mutually_exclusive_group(required=False)
    in_group.add_argument('--stdin', help='Read from stdin', action='store_true', default=False)
    in_group.add_argument('--simlog', help='Simulate with piped log file', action='store_true', default=False)

    parser.add_argument('--ltz', help='Use local time zone instead of UTC', action='store_true', default=False)
    parser.add_argument('--sleep', help='Number of seconds to sleep between updates', type=int, default=3)
    parser.add_argument('--verbose', help='Display informational message of GPU util progress',
                        action='store_true', default=False)
    parser.add_argument('-d', '--debug', help='Debug output', action='store_true', default=False)
    args = parser.parse_args()

    # About me
    if args.about:
        print(__doc__)
        print('Author: ', __author__)
        print('Copyright: ', __copyright__)
        print('Credits: ', *['\n      {}'.format(item) for item in __credits__])
        print('License: ', __license__)
        print('Version: ', __version__)
        print('Install Type: ', GUT_CONST.install_type)
        print('Maintainer: ', __maintainer__)
        print('Status: ', __status__)
        import matplotlib
        print('matplotlib version: ', matplotlib.__version__)
        print('pandas version: ', pd.__version__)
        print('numpy version: ', np.__version__)
        sys.exit(0)

    GUT_CONST.set_args(args, __program_name__)
    LOGGER.debug('########## %s %s', __program_name__, __version__)
    LOGGER.debug('pandas version: %s', pd.__version__)
    LOGGER.debug('numpy version: %s', np.__version__)

    if GUT_CONST.check_env() < 0:
        print('Error in environment. Exiting...')
        sys.exit(-1)

    # Get list of GPUs and exit if no GPUs detected
    gpu_list = Gpu.GpuList()
    gpu_list.set_gpu_list()
    num_gpus = gpu_list.num_gpus()
    if num_gpus['total'] == 0:
        print('No GPUs detected, exiting...')
        sys.exit(-1)

    # Read data static/dynamic/info/state driver information for GPUs
    gpu_list.read_gpu_sensor_set(data_type=SensorSet.All)

    # Select GPU's appropriate for monitor
    com_gpu_list = Gpu.set_mon_plot_compatible_gpu_list(gpu_list)
    num_gpus = com_gpu_list.num_gpus()
    if num_gpus['total'] == 0:
        print('No readable and compatible GPUs detected, exiting...')
        sys.exit(-1)

    # Define graph gui and data components
    plot_data = PlotData()
    plot_data.set_com_gpu_list(com_gpu_list)
    if not args.stdin:
        # Check list of GPUs and display vendor and driver details
        Gpu.print_driver_vendor_summary(gpu_list)

        # Set gpu quantity in plot_data
        plot_data.num_gpus = num_gpus['total']
        plot_data.com_gpu_list = com_gpu_list
    # end of if args.stdin == False

    if args.stdin or args.simlog:
        threading.Thread(target=read_from_stdin, daemon=True, args=[args.sleep, plot_data]).start()
    else:
        print('Compatible GPUs:\n    {}'.format(com_gpu_list))
        threading.Thread(target=read_from_gpus, daemon=True, args=[args.sleep, plot_data]).start()

    print('{} waiting for initial data'.format(__program_name__), end='', flush=True)
    while len(plot_data.df.index) < 4:
        valid_items = []
        for gpu in plot_data.com_gpu_list:
            for item_list in plot_data.plot_list.values():
                for test_item in item_list:
                    if gpu.table_parameters_status[test_item] and test_item not in valid_items:
                        valid_items.append(test_item)
        if not valid_items:
            plot_data.kill_thread()
            PD_SEM.acquire()
            PD_SEM.release()
            print('Insufficient valid data: Exiting')
            break
        print('.', end='', flush=True)
        sleep(args.sleep/2.0)
    print('')

    # After reading initial data, set gpus
    plot_data.set_gpus()

    if not plot_data.quit:
        gc = GuiComponents(plot_data)
        gplot = GPUPlotWindow(gc, plot_data)
        gplot.connect('delete-event', Gtk.main_quit)
        gplot.show_all()
        gc.set_ready(True)
        Gtk.main()
    plot_data.kill_thread()


if __name__ == '__main__':
    main()