From a266b8728823478a5a9ecbc5cb91684446551efe Mon Sep 17 00:00:00 2001 From: detlefarend Date: Wed, 22 May 2024 20:57:39 +0200 Subject: [PATCH 01/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- src/mlpro/bf/streams/basics.py | 3 --- src/mlpro/bf/streams/tasks/windows/ringbuffer.py | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mlpro/bf/streams/basics.py b/src/mlpro/bf/streams/basics.py index c68f97b4a..75b5a2c66 100644 --- a/src/mlpro/bf/streams/basics.py +++ b/src/mlpro/bf/streams/basics.py @@ -1403,9 +1403,6 @@ def _update_plot_nd( self, except: pass - if len(self._plot_nd_xdata)==0: - return - # 4 If buffer size is limited, remove obsolete data if p_settings.data_horizon > 0: diff --git a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py index afdbd6bfb..5da4ad145 100644 --- a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py +++ b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py @@ -154,6 +154,11 @@ def _run(self, p_inst : InstDict ): p_inst[inst_del.id] = ( InstTypeDel, inst_del ) self._raise_event_data_removed = True + p_inst[inst.id] = ( InstTypeNew, inst ) + + elif not self._delay: + p_inst[inst.id] = ( InstTypeNew, inst ) + # 1.4 New instance is buffered self._buffer[self._buffer_pos] = inst From 8ae269ca16dc6b97778f69e443ed010570fd4866 Mon Sep 17 00:00:00 2001 From: detlefarend Date: Thu, 23 May 2024 17:59:29 +0200 Subject: [PATCH 02/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- src/mlpro/bf/events.py | 7 +- src/mlpro/bf/math/normalizers/basics.py | 11 +- src/mlpro/bf/math/normalizers/minmax.py | 1 - src/mlpro/bf/math/normalizers/ztrans.py | 135 ++++++++++-------- src/mlpro/bf/streams/basics.py | 17 +-- .../bf/streams/tasks/windows/ringbuffer.py | 86 +++++------ .../oa/streams/tasks/normalizers/ztrans.py | 18 +-- ...howto_bf_streams_110_stream_task_window.py | 2 +- ..._003_rearranger_window_bd_normminmax_2d.py | 4 +- ..._004_rearranger_window_bd_normminmax_3d.py | 4 +- ..._005_rearranger_window_bd_normminmax_nd.py | 14 +- .../howto_oa_pp_009_complex_preprocessing.py | 20 +-- 12 files changed, 168 insertions(+), 151 deletions(-) diff --git a/src/mlpro/bf/events.py b/src/mlpro/bf/events.py index 5f31733fe..d9a70d131 100644 --- a/src/mlpro/bf/events.py +++ b/src/mlpro/bf/events.py @@ -11,10 +11,11 @@ ## -- 2023-03-25 1.1.1 DA Class EventManager: correction in constructor ## -- 2023-11-17 1.2.0 DA Class Event: new time stamp functionality ## -- 2023-11-18 1.2.1 DA Class Event: time stamp is set to now() if not provided +## -- 2024-05-23 1.3.0 DA Method EventManger._raise_event(): reduction to TypeError ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.2.1 (2023-11-18) +Ver. 1.3.0 (2024-05-23) This module provides classes for event handling. To this regard, the property class Eventmanager is provided to add event functionality to child classes by inheritence. @@ -160,6 +161,6 @@ def _raise_event(self, p_event_id:str, p_event_object:Event): try: self.log(Log.C_LOG_TYPE_I, 'Calling handler', i) handler( p_event_id=p_event_id, p_event_object=p_event_object ) - except: + except TypeError: self.log(Log.C_LOG_TYPE_E, 'Handler not compatible! Check your code!') - raise Error \ No newline at end of file + raise TypeError \ No newline at end of file diff --git a/src/mlpro/bf/math/normalizers/basics.py b/src/mlpro/bf/math/normalizers/basics.py index 4f9d518bb..8c893b468 100644 --- a/src/mlpro/bf/math/normalizers/basics.py +++ b/src/mlpro/bf/math/normalizers/basics.py @@ -22,10 +22,11 @@ ## -- 2023-01-12 1.0.13 LSB Bug Fix ## -- 2023-02-13 1.0.14 LSB BugFix: Changed the direct reference to p_param to a copy object ## -- 2024-04-30 1.1.0 DA Refactoring and new class Renormalizable +## -- 2024-05-23 1.2.0 DA Method Normalizer._set_parameters(): little optimization ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2024-04-30) +Ver. 1.2.0 (2024-05-23) This module provides base class for Normalizers and normalizer objects including MinMax normalization and normalization by Z transformation. @@ -44,13 +45,13 @@ ## ------------------------------------------------------------------------------------------------- class Normalizer: """ - Base template class for normalizer objects. + Base class for normalizers. """ ## ------------------------------------------------------------------------------------------------- def __init__(self): - self._param = None + self._param = None self._param_old = None self._param_new = None @@ -58,7 +59,7 @@ def __init__(self): ## ------------------------------------------------------------------------------------------------- def _set_parameters(self, p_param): """ - custom method to set the normalization parameters + Custom method to set the normalization parameters Parameters ---------- @@ -70,7 +71,7 @@ def _set_parameters(self, p_param): boolean:True Returns true after setting the parameters """ - self._param = p_param.copy() + self._param = p_param ## ------------------------------------------------------------------------------------------------- diff --git a/src/mlpro/bf/math/normalizers/minmax.py b/src/mlpro/bf/math/normalizers/minmax.py index ae39c9279..09dbbbead 100644 --- a/src/mlpro/bf/math/normalizers/minmax.py +++ b/src/mlpro/bf/math/normalizers/minmax.py @@ -87,4 +87,3 @@ def update_parameters(self, p_set: Set = None, p_boundaries: Union[list, np.ndar else: self._param_old = self._param_new.copy() self._param = self._param_new.copy() - pass diff --git a/src/mlpro/bf/math/normalizers/ztrans.py b/src/mlpro/bf/math/normalizers/ztrans.py index 0dc009e7d..deceba856 100644 --- a/src/mlpro/bf/math/normalizers/ztrans.py +++ b/src/mlpro/bf/math/normalizers/ztrans.py @@ -22,10 +22,11 @@ ## -- 2023-01-12 1.0.13 LSB Bug Fix ## -- 2023-02-13 1.0.14 LSB BugFix: Changed the direct reference to p_param to a copy object ## -- 2024-04-30 1.1.0 DA Refactoring/separation +## -- 2024-05-23 1.2.0 DA Refactoring (not yet finished) ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2024-04-30) +Ver. 1.2.0 (2024-05-23) This module provides a class for Z transformation. """ @@ -39,12 +40,18 @@ ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- -class NormalizerZTrans(Normalizer): +class NormalizerZTrans (Normalizer): """ Class for Normalization based on Z transformation. """ - ## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- + def __init__(self): + super().__init__() + self._n = 0 + + +## ------------------------------------------------------------------------------------------------- def update_parameters(self, p_dataset: np.ndarray = None, p_data_new: Union[Element, np.ndarray] = None, @@ -64,62 +71,78 @@ def update_parameters(self, Old element that is replaced with the new element. """ - try: - data_new = p_data_new.get_values() - except: - data_new = p_data_new - - try: - data_del = p_data_del.get_values() - except: - data_del = p_data_del - - if self._param_new is not None: self._param_old = self._param_new.copy() - - if data_new is None and data_del is None and isinstance(p_dataset, np.ndarray): - self._std = np.std(p_dataset, axis=0, dtype=np.float64) - self._mean = np.mean(p_dataset, axis=0, dtype=np.float64) - self._n = len(p_dataset) - if self._param_new is None: self._param_new = np.zeros([2, self._std.shape[-1]]) - - elif isinstance(data_new, np.ndarray) and data_del is None and p_dataset is None: - # this try/except block checks if the parameters are previously set with a dataset, otherwise sets the - # parameters based on a single element - try: - old_mean = self._mean.copy() - self._n += 1 - self._mean = (old_mean * (self._n - 1) + data_new) / (self._n) - self._std = np.sqrt((np.square(self._std) * (self._n - 1) - + (data_new - self._mean) * (data_new - old_mean)) / (self._n)) - - except: - self._n = 1 - self._mean = data_new.copy() - self._std = np.zeros(shape=data_new.shape) - - if self._param_new is None: self._param_new = np.zeros([2, data_new.shape[-1]]) - - elif isinstance(data_new, np.ndarray) and isinstance(data_del, np.ndarray) and not p_dataset: - try: - old_mean = self._mean.copy() - self._mean = old_mean + ((data_new - data_del) / (self._n)) - self._std = np.sqrt(np.square(self._std) + ( - ((np.square(data_new) - np.square(data_del)) - self._n * (np.square( - self._mean) - np.square(old_mean)))) / self._n) - except: - raise ParamError("Normalization parameters are not initialised prior to updating with replacing a " - "data element") + # 2024-0523/DA - Needs to be reviewed...!!! + raise Error('To be reviewed!!') + + + # 0 Backup current parameters + if self._param_new is not None: + self._param_old = self._param_new.copy() + + # 1 Update on dataset + if p_dataset is not None: + self._std = np.std(p_dataset, axis=0, dtype=np.float64) + self._mean = np.mean(p_dataset, axis=0, dtype=np.float64) + self._n = len(p_dataset) + + if self._param_new is None: + self._param_new = np.zeros([2, self._std.shape[-1]]) + else: - raise ParamError("Wrong parameters for update_parameters(). Please either provide a dataset as p_dataset " - "or a new data element as p_data ") + # 2 Update on new data + if p_data_new is not None: + try: + data_new = np.ndarray(p_data_new.get_values()) + except: + data_new = p_data_new + + if self._n == 0: + self._n = 1 + self._mean = data_new.copy() + self._std = np.zeros(shape=data_new.shape) + else: + old_mean = self._mean.copy() + self._mean = (old_mean * self._n + data_new) / (self._n + 1) + + # TO BE REVIEWED + raise Error('To be reviewed!!') + self._std = np.sqrt((np.square(self._std) * self._n + + (data_new - self._mean) * (data_new - old_mean)) / (self._n)) + self._n += 1 + + if self._param_new is None: + self._param_new = np.zeros([2, data_new.shape[-1]]) + + + # 3 Update on obsolete data + if ( p_data_del is not None ) and ( self._n > 0 ): + try: + data_del = np.ndarray(p_data_del.get_values()) + except: + data_del = p_data_del + + # TO BE REVIEWED + raise Error('To be reviewed!!') + self._mean = self._mean - ( data_del / self._n) + + # TO BE REVIEWED + raise Error('To be reviewed!!') + self._std = np.sqrt(np.square(self._std) + ( + ((np.square(data_new) - np.square(data_del)) - self._n * (np.square( + self._mean) - np.square(old_mean)))) / self._n) + + self._n -= 1 + + + # 4 Update of parameters self._param_new[0] = np.divide(1, self._std, out = np.zeros_like(self._std), where = self._std!=0) self._param_new[1] = np.divide(self._mean, self._std, out = np.zeros_like(self._std), where = self._std!=0) - # self._param_new[1 == np.inf] = 0 - if self._param is not None: - self._param_old = self._param.copy() - else: - self._param_old = self._param_new.copy() - self._param = self._param_new.copy() + + if self._param_old is None: + self._param_old = self._param_new + + self._set_parameters( p_param = self._param_new ) + diff --git a/src/mlpro/bf/streams/basics.py b/src/mlpro/bf/streams/basics.py index 75b5a2c66..d0c52a77e 100644 --- a/src/mlpro/bf/streams/basics.py +++ b/src/mlpro/bf/streams/basics.py @@ -64,10 +64,11 @@ ## -- - refactoring of instance handling ## -- Class StreamTask: ## -- - optimization of code for plotting +## -- 2024-05-23 2.0.1 DA Bugfix in method StreamTask.run() ## ------------------------------------------------------------------------------------------------- """ -Ver. 2.0.0 (2024-05-21) +Ver. 2.0.1 (2024-05-23) This module provides classes for standardized stream processing. @@ -898,28 +899,28 @@ def run( self, so : StreamShared = self.get_so() if p_inst is not None: - inst = p_inst + instances = p_inst else: if so is None: raise ImplementationError('Class StreamTask needs instance data as parameters or from a shared object') try: - inst = so.get_instances(p_task_ids=self._predecessor_ids) + instances = so.get_instances(p_task_ids=self._predecessor_ids) except AttributeError: raise ImplementationError('Shared object not compatible to class StreamShared') - if len(inst) == 0: + if len(instances) == 0: self.log(Log.C_LOG_TYPE_S, 'No inputs -> SKIP') if self._duplicate_data: inst_copy : InstDict = {} - for inst_entry in inst_copy.values: - inst_copy[inst_entry[1].id] = ( inst_entry[0], inst_entry[1].copy()) + for inst_id, (inst_type, inst) in instances.items(): + inst_copy[inst_id] = ( inst_type, inst.copy() ) - inst = inst_copy + instances = inst_copy - Task.run(self, p_range=p_range, p_wait=p_wait, p_inst=inst) + Task.run(self, p_range=p_range, p_wait=p_wait, p_inst=instances) ## ------------------------------------------------------------------------------------------------- diff --git a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py index 5da4ad145..30f9967e9 100644 --- a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py +++ b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py @@ -24,10 +24,11 @@ ## -- 2022-12-31 1.1.4 LSB Refactoring ## -- 2023-02-02 1.1.5 DA Methods Window._init_plot_*: removed figure creation ## -- 2024-05-22 1.2.0 DA Refactoring, splitting, and renaming to RingBuffer +## -- 2024-05-23 1.2.1 DA Bugfixes on plotting ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.2.0 (2024-05-22) +Ver. 1.2.1 (2024-05-23) This module provides pool of window objects further used in the context of online adaptivity. """ @@ -223,16 +224,11 @@ def _init_plot_2d(self, p_figure: Figure, p_settings: PlotSettings): """ - if p_settings: - self._plot_settings = p_settings + Plottable._init_plot_2d(self, p_figure=p_figure, p_settings=p_settings) - if not p_settings.axes: - self.axes = Axes(p_figure, [0.05,0.05,0.9,0.9]) - - else: - self.axes = p_settings.axes self._patch_windows: dict = None self._window_patch2D = Rectangle((0, 0),0,0) + p_settings.axes.grid(True) ## ------------------------------------------------------------------------------------------------- @@ -248,14 +244,7 @@ def _init_plot_3d(self, p_figure: Figure, p_settings: PlotSettings): Additional Settings for the plot """ - - if p_settings: - self._plot_settings = p_settings - - if not p_settings.axes: - self.axes = p_figure.add_subplot(projection = '3d') - else: - self.axes = p_settings.axes + Plottable._init_plot_3d(self, p_figure=p_figure, p_settings=p_settings) self._patch_windows: dict = None self._window_patch3D = Poly3DCollection([]) @@ -275,18 +264,7 @@ def _init_plot_nd(self, p_figure: Figure, p_settings: PlotSettings): """ - if p_settings: - self._plot_settings = p_settings - - if not p_settings.axes: - self.axes = p_figure.add_subplot() - p_settings.axes = self.axes - p_settings.axes.set_xlabel(self.C_PLOT_ND_XLABEL_INST) - p_settings.axes.set_ylabel(self.C_PLOT_ND_YLABEL) - p_settings.axes.grid(visible=True) - - else: - self.axes = p_settings.axes + Plottable._init_plot_nd(self, p_figure=p_figure, p_settings=p_settings) self._patch_windows = None @@ -305,17 +283,21 @@ def _update_plot_2d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): Stream instances to be plotted. p_kwargs : dict Further optional plot parameters. - """ - if len(p_inst) == 0 : return - self.axes.grid(True) + # 1 No visualization until the buffer has been filledd + if not self._buffer_full: return + + + # 2 Initialization of the rectangle if self._patch_windows is None: self._patch_windows = {} self._patch_windows['2D'] = Rectangle((0, 0),0,0, ec= 'red', facecolor='none', zorder = -999) self._plot_settings.axes.add_patch(self._patch_windows['2D']) self._patch_windows['2D'].set_visible(True) + + # 3 Update of the rectangle boundaries = self.get_boundaries() x = boundaries[0][0] y = boundaries[1][0] @@ -340,18 +322,23 @@ def _update_plot_3d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): Stream instances to be plotted. p_kwargs : dict Further optional plot parameters. - """ - # 1. Returns if no new instances passed - if len(p_inst) == 0 : return - b = self.get_boundaries() + # 1 No visualization until the buffer has been filledd + if not self._buffer_full: return + + + + # 2 Initialization of the cuboid if self._patch_windows is None: self._patch_windows = {} self._patch_windows['3D'] = Poly3DCollection(verts= [], edgecolors='red', facecolors='red', alpha = 0) self._plot_settings.axes.add_collection(self._patch_windows['3D']) - # 2. Logic for vertices of the cuboid + + # 3 Update of the cuboid + b = self.get_boundaries() + verts = np.asarray([[[b[0][0], b[1][0], b[2][1]], [b[0][1], b[1][0], b[2][1]], [b[0][1], b[1][0], b[2][0]], @@ -382,7 +369,6 @@ def _update_plot_3d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): [b[0][1], b[1][1], b[2][0]], [b[0][0], b[1][1], b[2][0]]]]) - # 3. Setting the vertices for the cuboid self._patch_windows['3D'].set_verts(verts) @@ -402,24 +388,26 @@ def _update_plot_nd(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): Further optional plot parameters. """ - # 1. Check if there is a new instance to be plotted - if len(p_inst) == 0 : return + # 1 No visualization until the buffer has been filledd + if not self._buffer_full: return + - # 2. Check if the rectangle patches are already created + # 2 Check if the rectangle patches are already created if self._patch_windows is None: self._patch_windows = {} - bg = self.axes.get_facecolor() - ec = self.axes.patch.get_edgecolor() + bg = p_settings.axes.get_facecolor() + ec = p_settings.axes.patch.get_edgecolor() obs_window = Rectangle((0,0), 0,0, facecolor = bg, edgecolor=ec, lw = 1, zorder=9999, alpha = 0.75 ) - self._plot_settings.axes.add_patch(obs_window) + p_settings.axes.add_patch(obs_window) self._patch_windows['nD'] = obs_window - # 3. Add the hiding plot around obsolete data - x1 = self._plot_num_inst-self.buffer_size+1 - y1 = self.axes.get_ylim()[0] - w1 = -(x1 - self.axes.get_xlim()[0]) - h1 = self.axes.get_ylim()[1] - y1 - self._patch_windows['nD'].set_bounds(x1, y1, w1, h1) + # 3 Add the hiding plot around obsolete data + inst_oldest = self._buffer[self._buffer_pos] + x = p_settings.axes.get_xlim()[0] + y = p_settings.axes.get_ylim()[0] + w = inst_oldest.tstamp - x + h = p_settings.axes.get_ylim()[1] - y + self._patch_windows['nD'].set_bounds(x, y, w, h) self._patch_windows['nD'].set_visible(True) diff --git a/src/mlpro/oa/streams/tasks/normalizers/ztrans.py b/src/mlpro/oa/streams/tasks/normalizers/ztrans.py index e1e7f6b19..b691fe2eb 100644 --- a/src/mlpro/oa/streams/tasks/normalizers/ztrans.py +++ b/src/mlpro/oa/streams/tasks/normalizers/ztrans.py @@ -17,10 +17,11 @@ ## -- 2023-05-03 1.2.1 DA Bugfix in NormalizerMinMax._update_plot_2d/3d/nd ## -- 2023-05-22 1.2.2 SY Refactoring ## -- 2024-05-22 1.3.0 DA Refactoring and splitting +## -- 2024-05-23 1.3.1 DA Bugfix ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.3.0 (2024-05-22) +Ver. 1.3.1 (2024-05-23) This module provides implementation for adaptive normalizers for ZTransformation """ @@ -94,16 +95,17 @@ def _run(self, p_inst : InstDict): self.adapt( p_inst = p_inst ) # 2 Z-transformation of stream instances - for inst_type, inst in sorted(p_inst.values()): + for inst_id, (inst_type, inst) in sorted(p_inst.items()): + + feature_data = inst.get_feature_data() if self._param is None: if inst_type == InstTypeNew: - self.update_parameters(p_data_new=inst.get_feature_data()) + self.update_parameters( p_data_new = feature_data ) else: - self.update_parameters(p_data_del=inst.get_feature_data()) + self.update_parameters( p_data_del = feature_data ) - normalized_element = self.normalize(inst.get_feature_data()) - inst.get_feature_data().set_values(normalized_element.get_values()) + feature_data.set_values( p_values = self.normalize(feature_data).get_values() ) ## ------------------------------------------------------------------------------------------------- @@ -123,7 +125,7 @@ def _adapt(self, p_inst_new : Instance) -> bool: """ - self.update_parameters(p_data_new=p_inst_new.get_feature_data()) + self.update_parameters( p_data_new = p_inst_new.get_feature_data() ) return True @@ -144,5 +146,5 @@ def _adapt_reverse(self, p_inst_del:Instance) -> bool: """ - self.update_parameters(p_data_del=p_inst_del.get_feature_data()) + self.update_parameters( p_data_del = p_inst_del.get_feature_data() ) return True diff --git a/test/howtos/bf/howto_bf_streams_110_stream_task_window.py b/test/howtos/bf/howto_bf_streams_110_stream_task_window.py index a93bfb546..6cbd44416 100644 --- a/test/howtos/bf/howto_bf_streams_110_stream_task_window.py +++ b/test/howtos/bf/howto_bf_streams_110_stream_task_window.py @@ -70,7 +70,7 @@ def _setup(self, p_mode, p_visualize:bool, p_logging): p_logging=logging) # 2.1 Set up and add a window task - task_window = RingBuffer( p_buffer_size=30, + task_window = RingBuffer( p_buffer_size=50, p_name = 'T1 - Ring Buffer', p_delay = True, p_visualize = p_visualize, diff --git a/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py b/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py index 649dee3ab..7bd43f6e7 100644 --- a/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py +++ b/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py @@ -134,10 +134,10 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): else: # 1.2 Parameters for internal unit test - cycle_limit = 2 + cycle_limit = 60 logging = Log.C_LOG_NOTHING visualize = False - step_rate = 1 + step_rate = 5 diff --git a/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py b/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py index 3b3c4be5e..3398b9397 100644 --- a/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py +++ b/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py @@ -135,10 +135,10 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): else: # 1.2 Parameters for internal unit test - cycle_limit = 2 + cycle_limit = 60 logging = Log.C_LOG_NOTHING visualize = False - step_rate = 1 + step_rate = 5 # 2 Instantiate the stream scenario diff --git a/test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py b/test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py index 83d6d7592..fe523deb1 100644 --- a/test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py +++ b/test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py @@ -45,7 +45,7 @@ ## ------------------------------------------------------------------------------------------------- class MyAdaptiveScenario (OAScenario): - C_NAME = 'Dummy' + C_NAME = 'Demo' ## ------------------------------------------------------------------------------------------------- def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): @@ -61,7 +61,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2 Set up a stream workflow based on a custom stream task # 2.1 Creation of a workflow - workflow = OAWorkflow( p_name='wf1', + workflow = OAWorkflow( p_name='Input Signal', p_range_max=OAWorkflow.C_RANGE_NONE, p_ada=p_ada, p_visualize=p_visualize, @@ -74,7 +74,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = stream.get_feature_space().get_dims() features_new = [ ( 'F', features[1:7] ) ] - task_rearranger = Rearranger( p_name='t1', + task_rearranger = Rearranger( p_name='T1 - Rearranger', p_range_max=Task.C_RANGE_THREAD, p_visualize=p_visualize, p_logging=p_logging, @@ -86,7 +86,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): task_window = RingBuffer( p_buffer_size=50, p_delay=True, p_enable_statistics=True, - p_name='t2', + p_name='T2 - Ring Buffer', p_duplicate_data=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -94,7 +94,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task=task_window, p_pred_tasks=[task_rearranger]) # 2.2.3 Boundary detector - task_bd = BoundaryDetector( p_name='t3', + task_bd = BoundaryDetector( p_name='T3 - Boundary Detector', p_ada=True, p_visualize=True, p_logging=p_logging ) @@ -103,7 +103,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task = task_bd, p_pred_tasks=[task_window]) # # 2.2.4 MinMax-Normalizer - task_norm_minmax = NormalizerMinMax( p_name='t4', + task_norm_minmax = NormalizerMinMax( p_name='T4 - MinMax Normalizer', p_ada=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -156,6 +156,8 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): if __name__ == '__main__': myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_plot_horizon = 100, + p_data_horizon = 150, p_step_rate = step_rate ) ) input('\nPlease arrange all windows and press ENTER to start stream processing...') diff --git a/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py b/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py index 117fda342..3b294c277 100644 --- a/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py +++ b/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py @@ -65,7 +65,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2 Set up a stream workflow - workflow = OAWorkflow( p_name = 'Complex Preprocessing', + workflow = OAWorkflow( p_name = 'Input Signal - "DoubleSpiral2D"', p_range_max = Task.C_RANGE_NONE, p_ada = p_ada, p_visualize = p_visualize, @@ -76,7 +76,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = stream.get_feature_space().get_dims() features_new = [ ( 'F', features[0:1] ) ] - task1_rearranger = Rearranger( p_name = '#1: Rearranger', + task1_rearranger = Rearranger( p_name = '1 - Rearranger', p_range_max = Task.C_RANGE_THREAD, p_visualize = p_visualize, p_logging = p_logging, @@ -89,7 +89,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = task1_rearranger._feature_space.get_dims() derived_feature = features[0] - task2_deriver1 = Deriver( p_name = '#2: Deriver #1', + task2_deriver1 = Deriver( p_name = '2 - Deriver #1', p_range_max = Task.C_RANGE_THREAD, p_visualize = p_visualize, p_logging = p_logging, @@ -107,7 +107,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = task2_deriver1._feature_space.get_dims() derived_feature = features[0] - task3_deriver2 = Deriver( p_name = '#3: Deriver #2', + task3_deriver2 = Deriver( p_name = '3 - Deriver #2', p_range_max = Task.C_RANGE_THREAD, p_visualize = p_visualize, p_logging = p_logging, @@ -123,7 +123,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2.4 Set up and add a Sliding Window task4_window = RingBuffer( p_buffer_size = 100, - p_name = '#4: Ring Buffer', + p_name = '4 - Ring Buffer', p_delay = True, p_visualize = p_visualize, p_enable_statistics = True, @@ -134,7 +134,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2.5 Setup and add a Boundary Detector - task5_bd = BoundaryDetector( p_name = '#5: Boundary Detector', + task5_bd = BoundaryDetector( p_name = '5 - Boundary Detector', p_ada = p_ada, p_visualize = p_visualize, p_logging = p_logging ) @@ -148,7 +148,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2.6 Setup Z Transform-Normalizer in Parallel - task6_norm_ztrans = NormalizerZTransform( p_name = '#6: Normalizer Z-Trans', + task6_norm_ztrans = NormalizerZTransform( p_name = '6 - Normalizer Z-Trans', p_ada = p_ada, p_visualize = p_visualize, p_logging = p_logging ) @@ -158,7 +158,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2.7 Setup MinMax-Normalizer - task7_norm_minmax = NormalizerMinMax( p_name = '#7: Normalizer MinMax', + task7_norm_minmax = NormalizerMinMax( p_name = '7 - Normalizer MinMax', p_ada = p_ada, p_visualize = p_visualize, p_logging = p_logging ) @@ -187,10 +187,10 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): else: # 1.2 Parameters for internal unit test - cycle_limit = 2 + cycle_limit = 102 logging = Log.C_LOG_NOTHING visualize = False - step_rate = 1 + step_rate = 5 From aeb257fefe614727fb8bd3a2fc04412b6fa615f1 Mon Sep 17 00:00:00 2001 From: detlefarend Date: Thu, 23 May 2024 18:53:14 +0200 Subject: [PATCH 03/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- ...o_oa_pp_003_rearranger_window_bd_normminmax_2d.py | 12 +++++++----- ...o_oa_pp_004_rearranger_window_bd_normminmax_3d.py | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py b/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py index 7bd43f6e7..e14347dcf 100644 --- a/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py +++ b/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py @@ -59,7 +59,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2 Set up a stream workflow based on a custom stream task # 2.1 Creation of a workflow - workflow = OAWorkflow( p_name='wf1', + workflow = OAWorkflow( p_name='Input Signal', p_range_max=OAWorkflow.C_RANGE_NONE, p_ada=p_ada, p_visualize=p_visualize, @@ -72,7 +72,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = stream.get_feature_space().get_dims() features_new = [ ( 'F', features[1:3] ) ] - task_rearranger = Rearranger( p_name='t1', + task_rearranger = Rearranger( p_name='T1 - Rearranger', p_range_max=Task.C_RANGE_THREAD, p_visualize=p_visualize, p_logging=p_logging, @@ -84,7 +84,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): task_window = RingBuffer( p_buffer_size=50, p_delay=True, p_enable_statistics=True, - p_name='t2', + p_name='T2 - Ring Buffer', p_duplicate_data=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -92,7 +92,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task=task_window, p_pred_tasks=[task_rearranger]) # 2.2.3 Boundary detector - task_bd = BoundaryDetector( p_name='t3', + task_bd = BoundaryDetector( p_name='T3 - Boundary Detector', p_ada=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -101,7 +101,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task = task_bd, p_pred_tasks=[task_window]) # # 2.2.4 MinMax-Normalizer - task_norm_minmax = NormalizerMinMax( p_name='t4', + task_norm_minmax = NormalizerMinMax( p_name='T4 - MinMax Normalizer', p_ada=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -154,6 +154,8 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): if __name__ == '__main__': myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_plot_horizon = 100, + p_data_horizon = 150, p_step_rate = step_rate ) ) input('\nPlease arrange all windows and press ENTER to start stream processing...') diff --git a/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py b/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py index 3398b9397..3d79ebe67 100644 --- a/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py +++ b/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py @@ -60,7 +60,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2 Set up a stream workflow based on a custom stream task # 2.1 Creation of a workflow - workflow = OAWorkflow( p_name='wf1', + workflow = OAWorkflow( p_name='Input Signal', p_range_max=OAWorkflow.C_RANGE_NONE, p_ada=p_ada, p_visualize=p_visualize, @@ -73,7 +73,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): features = stream.get_feature_space().get_dims() features_new = [ ( 'F', features[1:4] ) ] - task_rearranger = Rearranger( p_name='t1', + task_rearranger = Rearranger( p_name='T1 - Rearranger', p_range_max=Task.C_RANGE_THREAD, p_visualize=p_visualize, p_logging=p_logging, @@ -85,7 +85,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): task_window = RingBuffer( p_buffer_size=50, p_delay=True, p_enable_statistics=True, - p_name='t2', + p_name='T2 - Ring Buffer', p_duplicate_data=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -93,7 +93,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task=task_window, p_pred_tasks=[task_rearranger]) # 2.2.3 Boundary detector - task_bd = BoundaryDetector( p_name='t3', + task_bd = BoundaryDetector( p_name='T3 - Boundary Detector', p_ada=True, p_visualize=True, p_logging=p_logging ) @@ -102,7 +102,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): workflow.add_task(p_task = task_bd, p_pred_tasks=[task_window]) # # 2.2.4 MinMax-Normalizer - task_norm_minmax = NormalizerMinMax( p_name='t4', + task_norm_minmax = NormalizerMinMax( p_name='T4 - MinMax Normalizer', p_ada=True, p_visualize=p_visualize, p_logging=p_logging ) @@ -155,6 +155,8 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): if __name__ == '__main__': myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_plot_horizon = 100, + p_data_horizon = 150, p_step_rate = step_rate ) ) input('\nPlease arrange all windows and press ENTER to start stream processing...') From 384d6ba02a80d9d59a4720fbd6fb95da0bb1b07d Mon Sep 17 00:00:00 2001 From: detlefarend Date: Thu, 23 May 2024 21:45:08 +0200 Subject: [PATCH 04/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- src/mlpro/bf/streams/tasks/windows/ringbuffer.py | 16 ++++++++-------- src/mlpro/oa/streams/tasks/normalizers/minmax.py | 8 ++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py index 30f9967e9..12f58299a 100644 --- a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py +++ b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py @@ -285,8 +285,8 @@ def _update_plot_2d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): Further optional plot parameters. """ - # 1 No visualization until the buffer has been filledd - if not self._buffer_full: return + # 1 No visualization until first data is buffered + if ( len(self._buffer) == 0 ) or ( len(p_inst) == 0 ): return # 2 Initialization of the rectangle @@ -324,9 +324,8 @@ def _update_plot_3d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): Further optional plot parameters. """ - # 1 No visualization until the buffer has been filledd - if not self._buffer_full: return - + # 1 No visualization until first data is buffered + if ( len(self._buffer) == 0 ) or ( len(p_inst) == 0 ): return # 2 Initialization of the cuboid @@ -375,8 +374,9 @@ def _update_plot_3d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): ## ------------------------------------------------------------------------------------------------- def _update_plot_nd(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): """ - Default N-dimensional plotting implementation for window tasks. See class mlpro.bf.plot.Plottable - for more details. + The n-dimensional representation of the ring buffer visualizes the removal of obsolete data + from the buffer by hiding it behind a semi-transparent rectangle. The visualization starts + when the buffer is completely filled and data is removed. Parameters ---------- @@ -389,7 +389,7 @@ def _update_plot_nd(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): """ # 1 No visualization until the buffer has been filledd - if not self._buffer_full: return + if ( not self._buffer_full ) or ( len(p_inst) == 0 ): return # 2 Check if the rectangle patches are already created diff --git a/src/mlpro/oa/streams/tasks/normalizers/minmax.py b/src/mlpro/oa/streams/tasks/normalizers/minmax.py index 1858c3ecf..3153ac175 100644 --- a/src/mlpro/oa/streams/tasks/normalizers/minmax.py +++ b/src/mlpro/oa/streams/tasks/normalizers/minmax.py @@ -163,8 +163,12 @@ def _update_plot_2d( self, plot_data_renormalized = self.renormalize(self._plot_data_2d) - self._plot_2d_xdata = list(j[0] for j in plot_data_renormalized) - self._plot_2d_ydata = list(j[1] for j in plot_data_renormalized) + self._plot_2d_xdata = {} + self._plot_2d_ydata = {} + + for i, data_2d in enumerate(plot_data_renormalized): + self._plot_2d_xdata[i] = data_2d[0] + self._plot_2d_ydata[i] = data_2d[1] self._parameters_updated = False From f710e48f367324f2005d70a5496cfa7f283b4bdc Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 04:00:37 +0200 Subject: [PATCH 05/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- .../streams/howto.bf.streams.111.rst | 13 +- .../streams/howto.bf.streams.112.rst | 10 +- .../streams/howto.bf.streams.113.rst | 10 +- .../streams/howto.bf.streams.121.rst | 29 ++++ ...reams.110.rst => howto.bf.streams.122.rst} | 10 +- .../streams/howto.bf.streams.123.rst | 30 +++++ ...reams.114.rst => howto.bf.streams.131.rst} | 6 +- ...reams_window.gif => streams_window_nd.gif} | Bin .../bf/streams/tasks/windows/ringbuffer.py | 4 +- ...to_bf_streams_111_stream_task_window_2d.py | 125 ++++++++++++++++++ ...to_bf_streams_112_stream_task_window_3d.py | 125 ++++++++++++++++++ ...o_bf_streams_113_stream_task_window_nd.py} | 2 +- ..._streams_121_stream_task_rearranger_2d.py} | 2 +- ..._streams_122_stream_task_rearranger_3d.py} | 2 +- ..._streams_123_stream_task_rearranger_nd.py} | 2 +- ...wto_bf_streams_131_stream_task_deriver.py} | 2 +- 16 files changed, 341 insertions(+), 31 deletions(-) create mode 100644 doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.121.rst rename doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/{howto.bf.streams.110.rst => howto.bf.streams.122.rst} (67%) create mode 100644 doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.123.rst rename doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/{howto.bf.streams.114.rst => howto.bf.streams.131.rst} (82%) rename doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/images/{streams_window.gif => streams_window_nd.gif} (100%) create mode 100644 test/howtos/bf/howto_bf_streams_111_stream_task_window_2d.py create mode 100644 test/howtos/bf/howto_bf_streams_112_stream_task_window_3d.py rename test/howtos/bf/{howto_bf_streams_110_stream_task_window.py => howto_bf_streams_113_stream_task_window_nd.py} (98%) rename test/howtos/bf/{howto_bf_streams_111_stream_task_rearranger_2d.py => howto_bf_streams_121_stream_task_rearranger_2d.py} (98%) rename test/howtos/bf/{howto_bf_streams_112_stream_task_rearranger_3d.py => howto_bf_streams_122_stream_task_rearranger_3d.py} (98%) rename test/howtos/bf/{howto_bf_streams_113_stream_task_rearranger_nd.py => howto_bf_streams_123_stream_task_rearranger_nd.py} (98%) rename test/howtos/bf/{howto_bf_streams_114_stream_task_deriver.py => howto_bf_streams_131_stream_task_deriver.py} (99%) diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.111.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.111.rst index 93fbdb084..b9ae07408 100644 --- a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.111.rst +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.111.rst @@ -1,18 +1,19 @@ -.. _Howto BF STREAMS 111: -Howto BF-STREAMS-111: Rearranger (2D) -===================================== +.. _Howto BF STREAMS 110: +Howto BF-STREAMS-111: Window (2D) +=============================================== **Prerequisites** Please install the following packages to run this example properly: + - `Numpy `_ - `Matplotlib `_ - - `Tkinter `_ + **Executable code** -.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_111_stream_task_rearranger_2d.py +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_111_stream_task_window_2d.py :language: python @@ -20,7 +21,7 @@ Please install the following packages to run this example properly: **Results** .. image:: - images/streams_rearranger.gif + images/streams_window_2d.gif diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.112.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.112.rst index e404938bd..84836ed72 100644 --- a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.112.rst +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.112.rst @@ -1,19 +1,19 @@ -.. _Howto BF STREAMS 112: -Howto BF-STREAMS-112: Rearranger (3D) +.. _Howto BF STREAMS 110: +Howto BF-STREAMS-112: Window (3D) =============================================== **Prerequisites** Please install the following packages to run this example properly: + - `Numpy `_ - `Matplotlib `_ - - `Tkinter `_ **Executable code** -.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_112_stream_task_rearranger_3d.py +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_112_stream_task_window_3d.py :language: python @@ -21,7 +21,7 @@ Please install the following packages to run this example properly: **Results** .. image:: - images/streams_rearranger_3d.gif + images/streams_window_3d.gif diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.113.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.113.rst index 489ce58be..bd49faa5d 100644 --- a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.113.rst +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.113.rst @@ -1,19 +1,19 @@ -.. _Howto BF STREAMS 113: -Howto BF-STREAMS-113: Rearranger (nD) +.. _Howto BF STREAMS 110: +Howto BF-STREAMS-111: Window (nD) =============================================== **Prerequisites** Please install the following packages to run this example properly: + - `Numpy `_ - `Matplotlib `_ - - `Tkinter `_ **Executable code** -.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_113_stream_task_rearranger_nd.py +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_113_stream_task_window_nd.py :language: python @@ -21,7 +21,7 @@ Please install the following packages to run this example properly: **Results** .. image:: - images/streams_rearranger_nd.gif + images/streams_window_nd.gif diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.121.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.121.rst new file mode 100644 index 000000000..66c257e3d --- /dev/null +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.121.rst @@ -0,0 +1,29 @@ +.. _Howto BF STREAMS 121: +Howto BF-STREAMS-121: Rearranger (2D) +===================================== + +**Prerequisites** + +Please install the following packages to run this example properly: + + - `Matplotlib `_ + - `Tkinter `_ + + +**Executable code** + +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_121_stream_task_rearranger_2d.py + :language: python + + + +**Results** + +.. image:: + images/streams_rearranger.gif + + + +**Cross Reference** + + - :ref:`API Reference: Streams ` \ No newline at end of file diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.110.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.122.rst similarity index 67% rename from doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.110.rst rename to doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.122.rst index d4c24de0b..3ed746fbb 100644 --- a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.110.rst +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.122.rst @@ -1,19 +1,19 @@ -.. _Howto BF STREAMS 110: -Howto BF-STREAMS-110: Window +.. _Howto BF STREAMS 122: +Howto BF-STREAMS-122: Rearranger (3D) =============================================== **Prerequisites** Please install the following packages to run this example properly: - - `Numpy `_ - `Matplotlib `_ + - `Tkinter `_ **Executable code** -.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_110_stream_task_window.py +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_122_stream_task_rearranger_3d.py :language: python @@ -21,7 +21,7 @@ Please install the following packages to run this example properly: **Results** .. image:: - images/streams_window.gif + images/streams_rearranger_3d.gif diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.123.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.123.rst new file mode 100644 index 000000000..7f6808515 --- /dev/null +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.123.rst @@ -0,0 +1,30 @@ +.. _Howto BF STREAMS 123: +Howto BF-STREAMS-123: Rearranger (nD) +=============================================== + +**Prerequisites** + +Please install the following packages to run this example properly: + + - `Matplotlib `_ + - `Tkinter `_ + + + +**Executable code** + +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_123_stream_task_rearranger_nd.py + :language: python + + + +**Results** + +.. image:: + images/streams_rearranger_nd.gif + + + +**Cross Reference** + + - :ref:`API Reference: Streams ` \ No newline at end of file diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.114.rst b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.131.rst similarity index 82% rename from doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.114.rst rename to doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.131.rst index faa1d31d0..e90e73d6a 100644 --- a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.114.rst +++ b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/howto.bf.streams.131.rst @@ -1,5 +1,5 @@ -.. _Howto BF STREAMS 114: -Howto BF-STREAMS-114: Deriver +.. _Howto BF STREAMS 131: +Howto BF-STREAMS-131: Deriver ============================= **Prerequisites** @@ -13,7 +13,7 @@ Please install the following packages to run this example properly: **Executable code** -.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_114_stream_task_deriver.py +.. literalinclude:: ../../../../../../../../../test/howtos/bf/howto_bf_streams_131_stream_task_deriver.py :language: python diff --git a/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/images/streams_window.gif b/doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/images/streams_window_nd.gif similarity index 100% rename from doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/images/streams_window.gif rename to doc/rtd/content/99_appendices/appendix1/sub/mlpro_bf/layer3_application_support/streams/images/streams_window_nd.gif diff --git a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py index 12f58299a..1524ef9fc 100644 --- a/src/mlpro/bf/streams/tasks/windows/ringbuffer.py +++ b/src/mlpro/bf/streams/tasks/windows/ringbuffer.py @@ -286,7 +286,7 @@ def _update_plot_2d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): """ # 1 No visualization until first data is buffered - if ( len(self._buffer) == 0 ) or ( len(p_inst) == 0 ): return + if len(self._buffer) == 0: return # 2 Initialization of the rectangle @@ -325,7 +325,7 @@ def _update_plot_3d(self, p_settings:PlotSettings, p_inst:InstDict, **p_kwargs): """ # 1 No visualization until first data is buffered - if ( len(self._buffer) == 0 ) or ( len(p_inst) == 0 ): return + if len(self._buffer) == 0: return # 2 Initialization of the cuboid diff --git a/test/howtos/bf/howto_bf_streams_111_stream_task_window_2d.py b/test/howtos/bf/howto_bf_streams_111_stream_task_window_2d.py new file mode 100644 index 000000000..5b3019065 --- /dev/null +++ b/test/howtos/bf/howto_bf_streams_111_stream_task_window_2d.py @@ -0,0 +1,125 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.bf.examples +## -- Module : howto_bf_streams_111_stream_task_window_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-11-27 1.0.0 LSB Creation +## -- 2022-12-14 1.1.0 DA - Changed the stream provider from OpenML to MLPro +## -- - Added a custom task behind the window task +## -- 2024-05-22 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.1.0 (2024-05-22) + +This module demonstrates the functionality of stream window task in MLPro. + +You will learn: + +1) How to implement an own custom stream task. + +2) How to set up a stream workflow based on stream tasks. + +3) How to set up a stream scenario based on a stream and a processing stream workflow. + +4) How to run a stream scenario dark or with default visualization. +""" + + +from mlpro.bf.streams.streams import * +from mlpro.bf.streams.tasks.windows import RingBuffer + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyTask (StreamTask): + """ + Demo implementation of a stream task with custom method _run(). + """ + + # needed for proper logging (see class mlpro.bf.various.Log) + C_NAME = 'Custom' + +## ------------------------------------------------------------------------------------------------- + def _run(self, p_inst: InstDict ): + pass + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyStreamScenario(StreamScenario): + + C_NAME = 'Demo Window' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_visualize:bool, p_logging): + + # 1 Import a native stream from MLPro + provider_mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = provider_mlpro.get_stream('Clouds2D4C1000Static', p_mode=p_mode, p_logging=p_logging) + + + # 2 Set up a stream workflow + workflow = StreamWorkflow( p_name='Input Signal "Clouds2D4C1000Static"', + p_range_max=StreamWorkflow.C_RANGE_NONE, + p_visualize=p_visualize, + p_logging=logging) + + # 2.1 Set up and add a window task + task_window = RingBuffer( p_buffer_size=50, + p_name = 'T1 - Ring Buffer', + p_delay = True, + p_visualize = p_visualize, + p_enable_statistics = True, + p_logging = p_logging ) + + workflow.add_task(task_window) + + # 2.2 Set up and add an own custom task + task_custom = MyTask( p_name='T2 - My Task', p_visualize=p_visualize, p_logging=logging ) + workflow.add_task( p_task=task_custom, p_pred_tasks=[task_window] ) + + + # 3 Return stream and workflow + return stream, workflow + + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + logging = Log.C_LOG_NOTHING + visualize = False + + +# 2 Instantiate the stream scenario +myscenario = MyStreamScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view=PlotSettings.C_VIEW_ND, + p_plot_horizon=100, + p_data_horizon=150) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/bf/howto_bf_streams_112_stream_task_window_3d.py b/test/howtos/bf/howto_bf_streams_112_stream_task_window_3d.py new file mode 100644 index 000000000..76b483b6b --- /dev/null +++ b/test/howtos/bf/howto_bf_streams_112_stream_task_window_3d.py @@ -0,0 +1,125 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.bf.examples +## -- Module : howto_bf_streams_112_stream_task_window_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-11-27 1.0.0 LSB Creation +## -- 2022-12-14 1.1.0 DA - Changed the stream provider from OpenML to MLPro +## -- - Added a custom task behind the window task +## -- 2024-05-22 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.1.0 (2024-05-22) + +This module demonstrates the functionality of stream window task in MLPro. + +You will learn: + +1) How to implement an own custom stream task. + +2) How to set up a stream workflow based on stream tasks. + +3) How to set up a stream scenario based on a stream and a processing stream workflow. + +4) How to run a stream scenario dark or with default visualization. +""" + + +from mlpro.bf.streams.streams import * +from mlpro.bf.streams.tasks.windows import RingBuffer + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyTask (StreamTask): + """ + Demo implementation of a stream task with custom method _run(). + """ + + # needed for proper logging (see class mlpro.bf.various.Log) + C_NAME = 'Custom' + +## ------------------------------------------------------------------------------------------------- + def _run(self, p_inst: InstDict ): + pass + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyStreamScenario(StreamScenario): + + C_NAME = 'Demo Window' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_visualize:bool, p_logging): + + # 1 Import a native stream from MLPro + provider_mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = provider_mlpro.get_stream('Clouds3D8C2000Static', p_mode=p_mode, p_logging=p_logging) + + + # 2 Set up a stream workflow + workflow = StreamWorkflow( p_name='Input Signal "Clouds3D8C2000Static"', + p_range_max=StreamWorkflow.C_RANGE_NONE, + p_visualize=p_visualize, + p_logging=logging) + + # 2.1 Set up and add a window task + task_window = RingBuffer( p_buffer_size=50, + p_name = 'T1 - Ring Buffer', + p_delay = True, + p_visualize = p_visualize, + p_enable_statistics = True, + p_logging = p_logging ) + + workflow.add_task(task_window) + + # 2.2 Set up and add an own custom task + task_custom = MyTask( p_name='T2 - My Task', p_visualize=p_visualize, p_logging=logging ) + workflow.add_task( p_task=task_custom, p_pred_tasks=[task_window] ) + + + # 3 Return stream and workflow + return stream, workflow + + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + logging = Log.C_LOG_NOTHING + visualize = False + + +# 2 Instantiate the stream scenario +myscenario = MyStreamScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view=PlotSettings.C_VIEW_ND, + p_plot_horizon=100, + p_data_horizon=150) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/bf/howto_bf_streams_110_stream_task_window.py b/test/howtos/bf/howto_bf_streams_113_stream_task_window_nd.py similarity index 98% rename from test/howtos/bf/howto_bf_streams_110_stream_task_window.py rename to test/howtos/bf/howto_bf_streams_113_stream_task_window_nd.py index 6cbd44416..860cf9d36 100644 --- a/test/howtos/bf/howto_bf_streams_110_stream_task_window.py +++ b/test/howtos/bf/howto_bf_streams_113_stream_task_window_nd.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.bf.examples -## -- Module : howto_bf_streams_110_stream_task_window.py +## -- Module : howto_bf_streams_113_stream_task_window_nd.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description diff --git a/test/howtos/bf/howto_bf_streams_111_stream_task_rearranger_2d.py b/test/howtos/bf/howto_bf_streams_121_stream_task_rearranger_2d.py similarity index 98% rename from test/howtos/bf/howto_bf_streams_111_stream_task_rearranger_2d.py rename to test/howtos/bf/howto_bf_streams_121_stream_task_rearranger_2d.py index 9be70ca4a..44097376d 100644 --- a/test/howtos/bf/howto_bf_streams_111_stream_task_rearranger_2d.py +++ b/test/howtos/bf/howto_bf_streams_121_stream_task_rearranger_2d.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.bf.examples -## -- Module : howto_bf_streams_111_stream_task_rearranger_2d.py +## -- Module : howto_bf_streams_121_stream_task_rearranger_2d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description diff --git a/test/howtos/bf/howto_bf_streams_112_stream_task_rearranger_3d.py b/test/howtos/bf/howto_bf_streams_122_stream_task_rearranger_3d.py similarity index 98% rename from test/howtos/bf/howto_bf_streams_112_stream_task_rearranger_3d.py rename to test/howtos/bf/howto_bf_streams_122_stream_task_rearranger_3d.py index 83e424783..7d1682f50 100644 --- a/test/howtos/bf/howto_bf_streams_112_stream_task_rearranger_3d.py +++ b/test/howtos/bf/howto_bf_streams_122_stream_task_rearranger_3d.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.bf.examples -## -- Module : howto_bf_streams_112_stream_task_rearranger_3d.py +## -- Module : howto_bf_streams_122_stream_task_rearranger_3d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description diff --git a/test/howtos/bf/howto_bf_streams_113_stream_task_rearranger_nd.py b/test/howtos/bf/howto_bf_streams_123_stream_task_rearranger_nd.py similarity index 98% rename from test/howtos/bf/howto_bf_streams_113_stream_task_rearranger_nd.py rename to test/howtos/bf/howto_bf_streams_123_stream_task_rearranger_nd.py index 38425228f..de8460a9b 100644 --- a/test/howtos/bf/howto_bf_streams_113_stream_task_rearranger_nd.py +++ b/test/howtos/bf/howto_bf_streams_123_stream_task_rearranger_nd.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.bf.examples -## -- Module : howto_bf_streams_113_stream_task_rearranger_nd.py +## -- Module : howto_bf_streams_123_stream_task_rearranger_nd.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description diff --git a/test/howtos/bf/howto_bf_streams_114_stream_task_deriver.py b/test/howtos/bf/howto_bf_streams_131_stream_task_deriver.py similarity index 99% rename from test/howtos/bf/howto_bf_streams_114_stream_task_deriver.py rename to test/howtos/bf/howto_bf_streams_131_stream_task_deriver.py index fafd92cd4..eaff4c741 100644 --- a/test/howtos/bf/howto_bf_streams_114_stream_task_deriver.py +++ b/test/howtos/bf/howto_bf_streams_131_stream_task_deriver.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.bf.examples -## -- Module : howto_bf_streams_114_stream_task_deriver.py +## -- Module : howto_bf_streams_131_stream_task_deriver.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description From 5487808a308f155c8b3b63c5a09290e5c7b60c5e Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 05:26:35 +0200 Subject: [PATCH 06/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- .../oa/streams/tasks/boundarydetectors.py | 2 - .../oa/streams/tasks/normalizers/minmax.py | 12 +- ...owto_oa_streams_pp_001_bd_normminmax_2d.py | 129 ++++++++++++++++++ ...owto_oa_streams_pp_002_bd_normminmax_3d.py | 129 ++++++++++++++++++ ...wto_oa_streams_pp_003_bd_normminmax_nd.py} | 29 ++-- .../howto_oa_streams_pp_006_normztrans_2d.py | 119 ++++++++++++++++ .../howto_oa_streams_pp_007_normztrans_3d.py | 119 ++++++++++++++++ ... howto_oa_streams_pp_008_normztrans_nd.py} | 39 +++--- ...101_rearranger_window_bd_normminmax_2d.py} | 0 ...102_rearranger_window_bd_normminmax_3d.py} | 0 ...103_rearranger_window_bd_normminmax_nd.py} | 0 ...arranger_window_bd_normminmax_2d_3d_nD.py} | 0 ...normminmax_2d_3d_nD_multithreading.py.off} | 0 ..._111_rearranger_deriver_normalizer.py.off} | 0 ...a_streams_pp_121_complex_preprocessing.py} | 2 +- 15 files changed, 541 insertions(+), 39 deletions(-) create mode 100644 test/howtos/oa/howto_oa_streams_pp_001_bd_normminmax_2d.py create mode 100644 test/howtos/oa/howto_oa_streams_pp_002_bd_normminmax_3d.py rename test/howtos/oa/{howto_oa_pp_001_normalization_of_streamed_data_minmax.py => howto_oa_streams_pp_003_bd_normminmax_nd.py} (80%) create mode 100644 test/howtos/oa/howto_oa_streams_pp_006_normztrans_2d.py create mode 100644 test/howtos/oa/howto_oa_streams_pp_007_normztrans_3d.py rename test/howtos/oa/{howto_oa_pp_002_normalization_of_streamed_data_ztransform.py => howto_oa_streams_pp_008_normztrans_nd.py} (78%) rename test/howtos/oa/{howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py => howto_oa_streams_pp_101_rearranger_window_bd_normminmax_2d.py} (100%) rename test/howtos/oa/{howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py => howto_oa_streams_pp_102_rearranger_window_bd_normminmax_3d.py} (100%) rename test/howtos/oa/{howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py => howto_oa_streams_pp_103_rearranger_window_bd_normminmax_nd.py} (100%) rename test/howtos/oa/{howto_oa_pp_006_rearranger_window_bd_normminmax_2d_3d_nD.py => howto_oa_streams_pp_104_rearranger_window_bd_normminmax_2d_3d_nD.py} (100%) rename test/howtos/oa/{howto_oa_pp_007_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off => howto_oa_streams_pp_105_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off} (100%) rename test/howtos/oa/{howto_oa_pp_008_rearranger_deriver_normalizer.py.off => howto_oa_streams_pp_111_rearranger_deriver_normalizer.py.off} (100%) rename test/howtos/oa/{howto_oa_pp_009_complex_preprocessing.py => howto_oa_streams_pp_121_complex_preprocessing.py} (99%) diff --git a/src/mlpro/oa/streams/tasks/boundarydetectors.py b/src/mlpro/oa/streams/tasks/boundarydetectors.py index 6bf9dd516..41cf73034 100644 --- a/src/mlpro/oa/streams/tasks/boundarydetectors.py +++ b/src/mlpro/oa/streams/tasks/boundarydetectors.py @@ -223,8 +223,6 @@ def _init_plot_nd(self, p_figure: Figure, p_settings: PlotSettings): """ - super()._init_plot_nd( p_figure = p_figure, p_settings=p_settings) - if not p_settings.axes: self.axes = p_figure.add_axes([0.1,0.1,0.7,0.8]) self.axes.set_xlabel(self.C_PLOT_ND_XLABEL_FEATURE) diff --git a/src/mlpro/oa/streams/tasks/normalizers/minmax.py b/src/mlpro/oa/streams/tasks/normalizers/minmax.py index 3153ac175..90a0602cd 100644 --- a/src/mlpro/oa/streams/tasks/normalizers/minmax.py +++ b/src/mlpro/oa/streams/tasks/normalizers/minmax.py @@ -207,12 +207,16 @@ def _update_plot_3d( self, self._plot_data_3d[i][1] = self._plot_3d_ydata[i] self._plot_data_3d[i][2] = self._plot_3d_zdata[i] - plot_data_renormalized = self.renormalize(self._plot_data_3d) - self._plot_3d_xdata = list(j[0] for j in plot_data_renormalized) - self._plot_3d_ydata = list(j[1] for j in plot_data_renormalized) - self._plot_3d_zdata = list(j[2] for j in plot_data_renormalized) + self._plot_3d_xdata = {} + self._plot_3d_ydata = {} + self._plot_3d_zdata = {} + + for i, data_3d in enumerate(plot_data_renormalized): + self._plot_3d_xdata[i] = data_3d[0] + self._plot_3d_ydata[i] = data_3d[1] + self._plot_3d_zdata[i] = data_3d[2] self._parameters_updated = False diff --git a/test/howtos/oa/howto_oa_streams_pp_001_bd_normminmax_2d.py b/test/howtos/oa/howto_oa_streams_pp_001_bd_normminmax_2d.py new file mode 100644 index 000000000..1d728def0 --- /dev/null +++ b/test/howtos/oa/howto_oa_streams_pp_001_bd_normminmax_2d.py @@ -0,0 +1,129 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.examples +## -- Module : howto_oa_streams_pp_001_bd_normminmax_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-12-07 0.0.0 LSB Creation +## -- 2022-12-09 1.0.0 LSB Release +## -- 2022-12-13 1.0.1 LSB Refctoring +## -- 2022-12-31 1.0.2 LSB Using native stream +## -- 2023-02-23 1.0.3 DA Little refactoring +## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario +## -- 2023-05-02 1.1.1 DA Correction in class MyAdaptiveScenario +## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2024-05-24) + +This module is an example of adaptive normalization of streaming data using MinMax Normalizer + +You will learn: + +1. Creating tasks and workflows in MLPro-OA. + +2. Registering Event handlers for events and tasks. + +3. Normalizing streaming data using MinMax Normalizer, with boundary detector as a predecessor task. + +""" + +from mlpro.oa.streams import * +from mlpro.bf.streams.streams import * +from mlpro.oa.streams.tasks.normalizers import NormalizerMinMax +from mlpro.oa.streams.tasks.boundarydetectors import BoundaryDetector + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyAdaptiveScenario(OAScenario): + + C_NAME = 'Demo' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + + # 1 Import a stream from OpenML + mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = mlpro.get_stream( p_name=StreamMLProClouds2D4C1000Static.C_NAME, + p_mode=p_mode, + p_visualize=p_visualize, + p_logging=p_logging) + + # 2 Set up a stream workflow based on a custom stream task + + # 2.1 Creation of a tasks + task_bd = BoundaryDetector( p_name='T1 - Boundary Detector', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + task_norm = NormalizerMinMax( p_name='T2 - MinMax Normalizer', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging) + + # 2.2 Creation of a workflow + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProClouds2D4C1000Static.C_NAME + '"', + p_range_max = OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.3 Addition of the task to the workflow + workflow.add_task(p_task = task_bd) + workflow.add_task(p_task = task_norm, p_pred_tasks=[task_bd]) + + + # 3 Registering event handlers for normalizer on events raised by boundaries + task_bd.register_event_handler(BoundaryDetector.C_EVENT_ADAPTED, task_norm.adapt_on_event) + + + # 4 Return stream and workflow + return stream, workflow + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False + + + +# 2 Instantiate the stream scenario +myscenario = MyAdaptiveScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_streams_pp_002_bd_normminmax_3d.py b/test/howtos/oa/howto_oa_streams_pp_002_bd_normminmax_3d.py new file mode 100644 index 000000000..5251e500f --- /dev/null +++ b/test/howtos/oa/howto_oa_streams_pp_002_bd_normminmax_3d.py @@ -0,0 +1,129 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.examples +## -- Module : howto_oa_streams_pp_002_bd_normminmax_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-12-07 0.0.0 LSB Creation +## -- 2022-12-09 1.0.0 LSB Release +## -- 2022-12-13 1.0.1 LSB Refctoring +## -- 2022-12-31 1.0.2 LSB Using native stream +## -- 2023-02-23 1.0.3 DA Little refactoring +## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario +## -- 2023-05-02 1.1.1 DA Correction in class MyAdaptiveScenario +## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2024-05-24) + +This module is an example of adaptive normalization of streaming data using MinMax Normalizer + +You will learn: + +1. Creating tasks and workflows in MLPro-OA. + +2. Registering Event handlers for events and tasks. + +3. Normalizing streaming data using MinMax Normalizer, with boundary detector as a predecessor task. + +""" + +from mlpro.oa.streams import * +from mlpro.bf.streams.streams import * +from mlpro.oa.streams.tasks.normalizers import NormalizerMinMax +from mlpro.oa.streams.tasks.boundarydetectors import BoundaryDetector + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyAdaptiveScenario(OAScenario): + + C_NAME = 'Demo' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + + # 1 Import a stream from OpenML + mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = mlpro.get_stream( p_name=StreamMLProClouds3D8C2000Static.C_NAME, + p_mode=p_mode, + p_visualize=p_visualize, + p_logging=p_logging) + + # 2 Set up a stream workflow based on a custom stream task + + # 2.1 Creation of a tasks + task_bd = BoundaryDetector( p_name='T1 - Boundary Detector', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + task_norm = NormalizerMinMax( p_name='T2 - MinMax Normalizer', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging) + + # 2.2 Creation of a workflow + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProClouds3D8C2000Static.C_NAME + '"', + p_range_max = OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.3 Addition of the task to the workflow + workflow.add_task(p_task = task_bd) + workflow.add_task(p_task = task_norm, p_pred_tasks=[task_bd]) + + + # 3 Registering event handlers for normalizer on events raised by boundaries + task_bd.register_event_handler(BoundaryDetector.C_EVENT_ADAPTED, task_norm.adapt_on_event) + + + # 4 Return stream and workflow + return stream, workflow + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False + + + +# 2 Instantiate the stream scenario +myscenario = MyAdaptiveScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_pp_001_normalization_of_streamed_data_minmax.py b/test/howtos/oa/howto_oa_streams_pp_003_bd_normminmax_nd.py similarity index 80% rename from test/howtos/oa/howto_oa_pp_001_normalization_of_streamed_data_minmax.py rename to test/howtos/oa/howto_oa_streams_pp_003_bd_normminmax_nd.py index bafe9a59e..24159179d 100644 --- a/test/howtos/oa/howto_oa_pp_001_normalization_of_streamed_data_minmax.py +++ b/test/howtos/oa/howto_oa_streams_pp_003_bd_normminmax_nd.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.oa.examples -## -- Module : howto_oa_pp_001_normalization_of_streamed_data_minmax.py +## -- Module : howto_oa_streams_pp_003_bd_normminmax_nd.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -13,10 +13,11 @@ ## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario ## -- 2023-05-02 1.1.1 DA Correction in class MyAdaptiveScenario ## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.2 (2023-08-23) +Ver. 1.2.0 (2024-05-24) This module is an example of adaptive normalization of streaming data using MinMax Normalizer @@ -30,10 +31,10 @@ """ -from mlpro.oa.streams.tasks.normalizers import * -from mlpro.oa.streams.tasks.boundarydetectors import * from mlpro.oa.streams import * from mlpro.bf.streams.streams import * +from mlpro.oa.streams.tasks.normalizers import NormalizerMinMax +from mlpro.oa.streams.tasks.boundarydetectors import BoundaryDetector @@ -67,7 +68,7 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_logging=p_logging) # 2.2 Creation of a workflow - workflow = OAWorkflow( p_name='Demo', + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProRnd10D.C_NAME + '"', p_range_max = OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, p_ada=p_ada, p_visualize=p_visualize, @@ -89,15 +90,17 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True else: # 1.2 Parameters for internal unit test cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False @@ -113,7 +116,11 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.reset() if __name__ == '__main__': - myscenario.init_plot() + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) input('Press ENTER to start stream processing...') myscenario.run() diff --git a/test/howtos/oa/howto_oa_streams_pp_006_normztrans_2d.py b/test/howtos/oa/howto_oa_streams_pp_006_normztrans_2d.py new file mode 100644 index 000000000..f1618a501 --- /dev/null +++ b/test/howtos/oa/howto_oa_streams_pp_006_normztrans_2d.py @@ -0,0 +1,119 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.examples +## -- Module : howto_oa_pp_002_normalization_of_streamed_data_ztransform.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-12-30 1.0.0 LSB Creation/Release +## -- 2022-12-31 1.0.1 LSB Using native stream +## -- 2023-02-23 1.0.2 DA Little refactoring +## -- 2023-04-10 1.0.3 LSB Adding a window task to validate the _adapt_reverse() method +## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario +## -- 2023-05-02 1.1.1 DA Minor corrections +## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2024-05-24) + +This module is an example of adaptive normalization of streaming data using MinMax Normalizer + +You will learn: + +1. Creating tasks and workflows in MLPro-OA. + +2. Registering Event handlers for events and tasks. + +3. Normalizing streaming data using Z Transformer, with boundary detector as a predecessor task. + +""" + +from mlpro.oa.streams import * +from mlpro.bf.streams.streams import * +from mlpro.oa.streams.tasks.normalizers import NormalizerZTransform + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyAdaptiveScenario (OAScenario): + + C_NAME = 'Demo' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + + # 1 Import a stream from OpenML + mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = mlpro.get_stream(p_name=StreamMLProClouds2D4C1000Static.C_NAME, + p_mode=p_mode, + p_visualize=p_visualize, + p_logging=p_logging) + + + # 2 Set up the stream workflow + + # 2.1 Creation of a tasks + task_norm = NormalizerZTransform( p_name='T2 - Z-transformation', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.2 Creation of a workflow + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProClouds2D4C1000Static.C_NAME + '"', + p_range_max=OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.3 Addition of the Z-transform task to the workflow + workflow.add_task(p_task = task_norm) + + + # 3 Return stream and workflow + return stream, workflow + + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False + + +# 2 Instantiate the stream scenario +myscenario = MyAdaptiveScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_streams_pp_007_normztrans_3d.py b/test/howtos/oa/howto_oa_streams_pp_007_normztrans_3d.py new file mode 100644 index 000000000..5e46aa9fc --- /dev/null +++ b/test/howtos/oa/howto_oa_streams_pp_007_normztrans_3d.py @@ -0,0 +1,119 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.examples +## -- Module : howto_oa_pp_002_normalization_of_streamed_data_ztransform.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2022-12-30 1.0.0 LSB Creation/Release +## -- 2022-12-31 1.0.1 LSB Using native stream +## -- 2023-02-23 1.0.2 DA Little refactoring +## -- 2023-04-10 1.0.3 LSB Adding a window task to validate the _adapt_reverse() method +## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario +## -- 2023-05-02 1.1.1 DA Minor corrections +## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2024-05-24) + +This module is an example of adaptive normalization of streaming data using MinMax Normalizer + +You will learn: + +1. Creating tasks and workflows in MLPro-OA. + +2. Registering Event handlers for events and tasks. + +3. Normalizing streaming data using Z Transformer, with boundary detector as a predecessor task. + +""" + +from mlpro.oa.streams import * +from mlpro.bf.streams.streams import * +from mlpro.oa.streams.tasks.normalizers import NormalizerZTransform + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MyAdaptiveScenario (OAScenario): + + C_NAME = 'Demo' + +## ------------------------------------------------------------------------------------------------- + def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + + # 1 Import a stream from OpenML + mlpro = StreamProviderMLPro(p_logging=p_logging) + stream = mlpro.get_stream(p_name=StreamMLProClouds3D8C2000Static.C_NAME, + p_mode=p_mode, + p_visualize=p_visualize, + p_logging=p_logging) + + + # 2 Set up the stream workflow + + # 2.1 Creation of a tasks + task_norm = NormalizerZTransform( p_name='T2 - Z-transformation', + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.2 Creation of a workflow + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProClouds3D8C2000Static.C_NAME + '"', + p_range_max=OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2.3 Addition of the Z-transform task to the workflow + workflow.add_task(p_task = task_norm) + + + # 3 Return stream and workflow + return stream, workflow + + + + +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 2 + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False + + +# 2 Instantiate the stream scenario +myscenario = MyAdaptiveScenario(p_mode=Mode.C_MODE_REAL, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging) + + + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_pp_002_normalization_of_streamed_data_ztransform.py b/test/howtos/oa/howto_oa_streams_pp_008_normztrans_nd.py similarity index 78% rename from test/howtos/oa/howto_oa_pp_002_normalization_of_streamed_data_ztransform.py rename to test/howtos/oa/howto_oa_streams_pp_008_normztrans_nd.py index ccf625222..1c1ce5727 100644 --- a/test/howtos/oa/howto_oa_pp_002_normalization_of_streamed_data_ztransform.py +++ b/test/howtos/oa/howto_oa_streams_pp_008_normztrans_nd.py @@ -12,10 +12,11 @@ ## -- 2023-04-10 1.1.0 DA Refactoring after changes on class OAScenario ## -- 2023-05-02 1.1.1 DA Minor corrections ## -- 2023-08-23 1.1.2 DA Minor corrections +## -- 2024-05-24 1.2.0 DA Refactoring ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.2 (2023-08-23) +Ver. 1.2.0 (2024-05-24) This module is an example of adaptive normalization of streaming data using MinMax Normalizer @@ -29,11 +30,9 @@ """ -from mlpro.oa.streams.tasks.normalizers import * -from mlpro.oa.streams.tasks.boundarydetectors import * from mlpro.oa.streams import * from mlpro.bf.streams.streams import * -from mlpro.bf.streams.tasks.windows import RingBuffer +from mlpro.oa.streams.tasks.normalizers import NormalizerZTransform @@ -57,28 +56,20 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 2 Set up the stream workflow # 2.1 Creation of a tasks - task_window = RingBuffer( p_name = 'T1 - Window', - p_buffer_size=10, - p_visualize=p_visualize, - p_logging=p_logging ) - task_norm = NormalizerZTransform( p_name='T2 - Z-transformation', p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) # 2.2 Creation of a workflow - workflow = OAWorkflow( p_name='Demo', + workflow = OAWorkflow( p_name='Input Signal "' + StreamMLProRnd10D.C_NAME + '"', p_range_max=OAWorkflow.C_RANGE_NONE, # StreamWorkflow.C_RANGE_THREAD, p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - # 2.3 Add a window task of size 2 - workflow.add_task(p_task=task_window) - - # 2.4 Addition of the Z-transform task to the workflow - workflow.add_task(p_task = task_norm, p_pred_tasks=[task_window]) + # 2.3 Addition of the Z-transform task to the workflow + workflow.add_task(p_task = task_norm) # 3 Return stream and workflow @@ -89,15 +80,17 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True + cycle_limit = 200 + step_rate = 2 + logging = Log.C_LOG_ALL + visualize = True else: # 1.2 Parameters for internal unit test cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False + step_rate = 1 + logging = Log.C_LOG_NOTHING + visualize = False # 2 Instantiate the stream scenario @@ -113,7 +106,11 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.reset() if __name__ == '__main__': - myscenario.init_plot() + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate, + p_plot_horizon = 100, + p_data_horizon = 200 ) ) input('Press ENTER to start stream processing...') myscenario.run() diff --git a/test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py b/test/howtos/oa/howto_oa_streams_pp_101_rearranger_window_bd_normminmax_2d.py similarity index 100% rename from test/howtos/oa/howto_oa_pp_003_rearranger_window_bd_normminmax_2d.py rename to test/howtos/oa/howto_oa_streams_pp_101_rearranger_window_bd_normminmax_2d.py diff --git a/test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py b/test/howtos/oa/howto_oa_streams_pp_102_rearranger_window_bd_normminmax_3d.py similarity index 100% rename from test/howtos/oa/howto_oa_pp_004_rearranger_window_bd_normminmax_3d.py rename to test/howtos/oa/howto_oa_streams_pp_102_rearranger_window_bd_normminmax_3d.py diff --git a/test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py b/test/howtos/oa/howto_oa_streams_pp_103_rearranger_window_bd_normminmax_nd.py similarity index 100% rename from test/howtos/oa/howto_oa_pp_005_rearranger_window_bd_normminmax_nd.py rename to test/howtos/oa/howto_oa_streams_pp_103_rearranger_window_bd_normminmax_nd.py diff --git a/test/howtos/oa/howto_oa_pp_006_rearranger_window_bd_normminmax_2d_3d_nD.py b/test/howtos/oa/howto_oa_streams_pp_104_rearranger_window_bd_normminmax_2d_3d_nD.py similarity index 100% rename from test/howtos/oa/howto_oa_pp_006_rearranger_window_bd_normminmax_2d_3d_nD.py rename to test/howtos/oa/howto_oa_streams_pp_104_rearranger_window_bd_normminmax_2d_3d_nD.py diff --git a/test/howtos/oa/howto_oa_pp_007_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off b/test/howtos/oa/howto_oa_streams_pp_105_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off similarity index 100% rename from test/howtos/oa/howto_oa_pp_007_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off rename to test/howtos/oa/howto_oa_streams_pp_105_rearranger_window_bd_normminmax_2d_3d_nD_multithreading.py.off diff --git a/test/howtos/oa/howto_oa_pp_008_rearranger_deriver_normalizer.py.off b/test/howtos/oa/howto_oa_streams_pp_111_rearranger_deriver_normalizer.py.off similarity index 100% rename from test/howtos/oa/howto_oa_pp_008_rearranger_deriver_normalizer.py.off rename to test/howtos/oa/howto_oa_streams_pp_111_rearranger_deriver_normalizer.py.off diff --git a/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py b/test/howtos/oa/howto_oa_streams_pp_121_complex_preprocessing.py similarity index 99% rename from test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py rename to test/howtos/oa/howto_oa_streams_pp_121_complex_preprocessing.py index 3b294c277..1bc91e18f 100644 --- a/test/howtos/oa/howto_oa_pp_009_complex_preprocessing.py +++ b/test/howtos/oa/howto_oa_streams_pp_121_complex_preprocessing.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro.oa.examples -## -- Module : howto_oa_pp_009_complex_preprocessing.py +## -- Module : howto_oa_streams_pp_121_complex_preprocessing.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description From d09c7de64e3300f1505194bfcb42b3832b6717c8 Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 06:03:35 +0200 Subject: [PATCH 07/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- src/mlpro/bf/systems/basics.py | 27 +++++------------------- src/mlpro/oa/systems/basics.py | 38 ++++++++++++---------------------- 2 files changed, 18 insertions(+), 47 deletions(-) diff --git a/src/mlpro/bf/systems/basics.py b/src/mlpro/bf/systems/basics.py index 68bcb4be1..444d1623d 100644 --- a/src/mlpro/bf/systems/basics.py +++ b/src/mlpro/bf/systems/basics.py @@ -44,10 +44,11 @@ ## -- 2023-06-06 1.14.0 LSB New functions to fetch the functions of a system ## -- 2023-05-01 2.0.0 LSB New class MultiSystem ## -- 2024-05-14 2.0.1 SY Migration from MLPro to MLPro-Int-MuJoCo +## -- 2024-05-24 2.1.0 DA Class State: removed parent class TStamp ## ------------------------------------------------------------------------------------------------- """ -Ver. 2.0.1 (2024-05-14) +Ver. 2.1.0 (2024-05-24) This module provides models and templates for state based systems. """ @@ -70,12 +71,9 @@ - - - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- -class State(Instance, Element, TStamp): +class State(Instance, Element): """ State of a system as an element of a given state space. Additionally, the state can be labeled with various properties. @@ -106,7 +104,6 @@ def __init__(self, p_timeout: bool = False, **p_kwargs): - TStamp.__init__(self) Element.__init__(self, p_state_space) Instance.__init__(self, p_feature_data=self, **p_kwargs) self.set_initial(p_initial) @@ -214,6 +211,7 @@ def copy(self): + ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class ActionElement (Element): @@ -758,7 +756,6 @@ def _set_actuator_value(self, p_id, p_value) -> bool: - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class SystemShared(Shared): @@ -794,7 +791,6 @@ class SystemShared(Shared): C_NAME = 'System Shared' - ## ------------------------------------------------------------------------------------------------- def __init__(self, p_range: int = Range.C_RANGE_NONE): @@ -835,7 +831,6 @@ def reset(self, p_seed: int = None): self._actions[system] = np.zeros(len(self._spaces[system][1].get_dims())) - ## ------------------------------------------------------------------------------------------------- def update_state(self, p_sys_id, p_state: State) -> bool: """ @@ -918,7 +913,6 @@ def _map_values(self, p_state: State = None, p_action:Action = None): self._actions[output_sys][action_space.get_dim_ids().index(output_dim)] = value - ## ------------------------------------------------------------------------------------------------- def get_actions(self): @@ -1064,8 +1058,6 @@ def register_system(self, raise Error("Registration of the system failed. Possible reason maybe false provision of mappings") - - @@ -1988,10 +1980,8 @@ class MultiSystem(Workflow, System): p_kwargs """ - C_TYPE = 'Multi-System' - ## ------------------------------------------------------------------------------------------------- def __init__(self, p_name: str = None, @@ -2013,7 +2003,6 @@ def __init__(self, p_visualize: bool = False, p_logging=Log.C_LOG_ALL, **p_kwargs): - System.__init__( self, p_name=p_name, @@ -2035,7 +2024,6 @@ def __init__(self, p_visualize=p_visualize, p_logging=p_logging ) - Workflow.__init__(self, p_name = p_name, p_range_max = p_range_max, @@ -2231,7 +2219,6 @@ def compute_success(self, p_state: State) -> bool: - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class DemoScenario(ScenarioBase): @@ -2265,8 +2252,6 @@ class DemoScenario(ScenarioBase): C_ACTION_RANDOM = 'random' C_ACTION_CONSTANT = 'constant' - - ## ------------------------------------------------------------------------------------------------- def __init__(self, p_system : System, @@ -2283,7 +2268,6 @@ def __init__(self, self._system = p_system self._action_pattern = p_action_pattern self._action = p_action - ScenarioBase.__init__(self, p_mode = p_mode, @@ -2293,8 +2277,6 @@ def __init__(self, p_visualize = p_visualize, p_logging = p_logging) - - self._action_length = len(self._system.get_action_space().get_dims()) if (self._action_pattern == DemoScenario.C_ACTION_CONSTANT): @@ -2384,6 +2366,7 @@ def _get_next_action(self): return Action(p_action_space=action_space, p_values=action) + ## ------------------------------------------------------------------------------------------------- def update_plot(self, **p_kwargs): diff --git a/src/mlpro/oa/systems/basics.py b/src/mlpro/oa/systems/basics.py index 726756fe3..7a9dc3717 100644 --- a/src/mlpro/oa/systems/basics.py +++ b/src/mlpro/oa/systems/basics.py @@ -10,15 +10,19 @@ ## -- 2023-05-31 0.1.1 LSB cleaning ## -- 2023-05-31 0.1.2 LSB Visualization bug fixed ## -- 2023-06-06 0.1.3 LSB Renaming _wf and run methods with *_strans +## -- 2024-05-24 0.2.0 DA Refactoring class PseudoTask +## -- - constructor: changes on parameter p_wrap_method +## -- - method _run(): changes on parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 0.1.2 (2023-05-31) +Ver. 0.2.0 (2024-05-24) This module provides modules and template classes for adaptive systems and adaptive functions. """ -import copy + +import copy from mlpro.bf.ml.systems import * from mlpro.bf.systems import * from mlpro.bf.ml import Model @@ -28,13 +32,9 @@ - - - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class PseudoTask(OATask): - """ A template class PseudoTask, only to be used by the OASystem. This functions runs a wrapped method as it's run method. @@ -50,12 +50,9 @@ class PseudoTask(OATask): p_kwargs """ - ## ------------------------------------------------------------------------------------------------- def __init__(self, - p_wrap_method:Callable[[List[State], - List[State]], - None], + p_wrap_method:Callable[[InstDict],None], p_name='PseudoTask', p_range_max=Range.C_RANGE_NONE, p_duplicate_data=True, @@ -77,13 +74,9 @@ def __init__(self, ## ------------------------------------------------------------------------------------------------- - def _run( self, - p_inst_new : list, - p_inst_del : list ): - - self._host_task(p_inst_new = p_inst_new, - p_inst_del = p_inst_del) + def _run( self, p_inst : InstDict ): + self._host_task( p_inst = p_inst ) @@ -119,7 +112,6 @@ class OAFctSTrans(FctSTrans, Model): p_kwargs """ - ## ------------------------------------------------------------------------------------------------- def __init__(self, p_id = None, @@ -209,7 +201,6 @@ def simulate_reaction(self, p_state: State, p_action: Action, p_t_step : timedel """ - self._state_obj = p_state.copy() self._action_obj = copy.deepcopy(p_action) self.log(Log.C_LOG_TYPE_I, 'Reaction Simulation Started...') @@ -231,6 +222,7 @@ def simulate_reaction(self, p_state: State, p_action: Action, p_t_step : timedel return state + ## ------------------------------------------------------------------------------------------------- def _adapt(self, **p_kwargs) -> bool: """ @@ -261,6 +253,7 @@ def _adapt(self, **p_kwargs) -> bool: return adapted + ## ------------------------------------------------------------------------------------------------- def _adapt_on_event(self, p_event_id:str, p_event_object:Event) -> bool: """ @@ -345,8 +338,6 @@ def _setup_oafct_strans(self): - - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class OAFctSuccess(FctSuccess, Model): @@ -377,7 +368,6 @@ class OAFctSuccess(FctSuccess, Model): p_kwargs """ - ## ------------------------------------------------------------------------------------------------- def __init__(self, p_id = None, @@ -440,6 +430,7 @@ def __init__(self, self._setup_wf_success = False + ## ------------------------------------------------------------------------------------------------- def compute_success(self, p_state: State) -> bool: """ @@ -582,8 +573,6 @@ def _setup_oafct_success(self): - - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class OAFctBroken(FctBroken, Model): @@ -678,6 +667,7 @@ def __init__(self, self._setup_wf_broken = False + ## ------------------------------------------------------------------------------------------------- def compute_broken(self, p_state: State) -> bool: """ @@ -819,8 +809,6 @@ def _setup_oafct_broken(self): - - ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class OASystem(OAFctBroken, OAFctSTrans, OAFctSuccess, ASystem): From fdcd4d291f7099b0ea6d78567e07fc5d31352c3c Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 11:55:12 +0200 Subject: [PATCH 08/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- .../tasks/clusteranalyzers/clusters/properties/basics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py index f4d42ce44..4637b988b 100644 --- a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py +++ b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py @@ -24,7 +24,7 @@ # Typical cluster properties to be reused in your own cluster analyzers # -# Size with 0,1,2 order derivatives +# Size (=number of associated instances) with 0,1,2 order derivatives cprop_size : PropertyDefinition = ( 'size', 0, Property ) cprop_size1 : PropertyDefinition = ( 'size', 1, Property ) cprop_size2 : PropertyDefinition = ( 'size', 2, Property ) From b5700de8f066c7b1867155ce17400fd87e9e1d34 Mon Sep 17 00:00:00 2001 From: Laxmikant Date: Fri, 24 May 2024 12:56:02 +0200 Subject: [PATCH 09/15] Z-trensform fix --- src/mlpro/bf/math/normalizers/ztrans.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mlpro/bf/math/normalizers/ztrans.py b/src/mlpro/bf/math/normalizers/ztrans.py index deceba856..db61736b0 100644 --- a/src/mlpro/bf/math/normalizers/ztrans.py +++ b/src/mlpro/bf/math/normalizers/ztrans.py @@ -23,10 +23,11 @@ ## -- 2023-02-13 1.0.14 LSB BugFix: Changed the direct reference to p_param to a copy object ## -- 2024-04-30 1.1.0 DA Refactoring/separation ## -- 2024-05-23 1.2.0 DA Refactoring (not yet finished) +## -- 2024-05-24 1.2.1 LSB Bug fix for Parameter update using only p_data_del in Z-transform ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.2.0 (2024-05-23) +Ver. 1.2.1 (2024-05-24) This module provides a class for Z transformation. """ @@ -73,7 +74,7 @@ def update_parameters(self, """ # 2024-0523/DA - Needs to be reviewed...!!! - raise Error('To be reviewed!!') + # raise Error('To be reviewed!!') # 0 Backup current parameters @@ -108,9 +109,9 @@ def update_parameters(self, self._mean = (old_mean * self._n + data_new) / (self._n + 1) # TO BE REVIEWED - raise Error('To be reviewed!!') + # raise Error('To be reviewed!!') self._std = np.sqrt((np.square(self._std) * self._n - + (data_new - self._mean) * (data_new - old_mean)) / (self._n)) + + (data_new - self._mean) * (data_new - old_mean)) / (self._n+1)) self._n += 1 if self._param_new is None: @@ -125,14 +126,13 @@ def update_parameters(self, data_del = p_data_del # TO BE REVIEWED - raise Error('To be reviewed!!') - self._mean = self._mean - ( data_del / self._n) + # raise Error('To be reviewed!!') + old_mean = self._mean.copy() + self._mean = (old_mean * self._n - data_del) / (self._n-1) # TO BE REVIEWED - raise Error('To be reviewed!!') - self._std = np.sqrt(np.square(self._std) + ( - ((np.square(data_new) - np.square(data_del)) - self._n * (np.square( - self._mean) - np.square(old_mean)))) / self._n) + # raise Error('To be reviewed!!') + self._std = np.sqrt((np.square(self._std)*self._n - (data_del - old_mean)*(data_del - self._mean)) / (self._n-1)) self._n -= 1 From 78dc7388e969c2805fef16c4eee180256bff10bd Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 14:26:26 +0200 Subject: [PATCH 10/15] Bug: Normalizer Z-Trans needs to be reviewed #993 --- src/mlpro/bf/math/normalizers/ztrans.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/mlpro/bf/math/normalizers/ztrans.py b/src/mlpro/bf/math/normalizers/ztrans.py index db61736b0..61e6777e1 100644 --- a/src/mlpro/bf/math/normalizers/ztrans.py +++ b/src/mlpro/bf/math/normalizers/ztrans.py @@ -73,10 +73,6 @@ def update_parameters(self, """ - # 2024-0523/DA - Needs to be reviewed...!!! - # raise Error('To be reviewed!!') - - # 0 Backup current parameters if self._param_new is not None: self._param_old = self._param_new.copy() @@ -96,7 +92,7 @@ def update_parameters(self, # 2 Update on new data if p_data_new is not None: try: - data_new = np.ndarray(p_data_new.get_values()) + data_new = np.array(p_data_new.get_values()) except: data_new = p_data_new @@ -108,8 +104,6 @@ def update_parameters(self, old_mean = self._mean.copy() self._mean = (old_mean * self._n + data_new) / (self._n + 1) - # TO BE REVIEWED - # raise Error('To be reviewed!!') self._std = np.sqrt((np.square(self._std) * self._n + (data_new - self._mean) * (data_new - old_mean)) / (self._n+1)) self._n += 1 @@ -121,17 +115,13 @@ def update_parameters(self, # 3 Update on obsolete data if ( p_data_del is not None ) and ( self._n > 0 ): try: - data_del = np.ndarray(p_data_del.get_values()) + data_del = np.array(p_data_del.get_values()) except: data_del = p_data_del - # TO BE REVIEWED - # raise Error('To be reviewed!!') old_mean = self._mean.copy() self._mean = (old_mean * self._n - data_del) / (self._n-1) - # TO BE REVIEWED - # raise Error('To be reviewed!!') self._std = np.sqrt((np.square(self._std)*self._n - (data_del - old_mean)*(data_del - self._mean)) / (self._n-1)) self._n -= 1 From 56f4d5232db27ddf038c046ba2a2353ff48c55db Mon Sep 17 00:00:00 2001 From: detlefarend Date: Fri, 24 May 2024 15:29:42 +0200 Subject: [PATCH 11/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- src/mlpro/bf/math/geometry.py | 58 +++++++++++++++-------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/src/mlpro/bf/math/geometry.py b/src/mlpro/bf/math/geometry.py index 8d101fd89..70d28bfd6 100644 --- a/src/mlpro/bf/math/geometry.py +++ b/src/mlpro/bf/math/geometry.py @@ -12,10 +12,11 @@ ## -- 2024-04-30 1.3.0 DA Class Point: re-normalization added ## -- 2024-05-06 1.4.0 DA Class Point: refactoring ## -- 2024-05-07 1.4.1 DA Bugfix in method Point.renormalize() +## -- 2024-05-24 1.4.2 DA Bugfix in method _update_plot_2d() ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.4.1 (2024-05-07) +Ver. 1.4.2 (2024-05-24) This module provides class for geometric objects like points, etc. @@ -59,43 +60,32 @@ def _update_plot_2d(self, p_settings: PlotSettings, **p_kwargs): point_pos = self.value - if self._plot_pos is None: - self._plot_pos, = p_settings.axes.plot( point_pos[0], - point_pos[1], - marker='+', - color='red', - linestyle='', - markersize=3 ) + if self._plot_pos is not None: + self._plot_pos.remove() + + self._plot_pos, = p_settings.axes.plot( point_pos[0], + point_pos[1], + marker='+', + color='red', + linestyle='', + markersize=3 ) - else: - self._plot_pos.set_xdata( point_pos[0] ) - self._plot_pos.set_ydata( point_pos[1] ) + if self._plot_vel is not None: + self._plot_vel.remove() - if self._plot_vel is not None: - self._plot_vel.remove() + try: + point_vel = self.derivatives[1] + except: + return - try: - point_vel = self.derivatives[1] - except: - return - - if point_vel is not None: - self._plot_vel = p_settings.axes.arrow( point_pos[0], - point_pos[1], - point_vel[0], - point_vel[1], - color='red' ) + if point_vel is not None: + self._plot_vel = p_settings.axes.arrow( point_pos[0], + point_pos[1], + point_vel[0], + point_vel[1], + color='red' ) - # self._plot_vel = p_settings.axes.quiver( np.array([point_pos[0]]), - # np.array([point_pos[1]]), - # np.array([point_vel[0]]), - # np.array([point_vel[1]]), - # #scale = 1, - # units = 'dots', - # width = 2, - # color='red' ) - - + ## ------------------------------------------------------------------------------------------------- def _update_plot_3d(self, p_settings: PlotSettings, **p_kwargs): From fcb9a2233e6f28250dbbccf9657cb2b2adf4d6fb Mon Sep 17 00:00:00 2001 From: syam47 Date: Fri, 24 May 2024 15:33:10 +0200 Subject: [PATCH 12/15] OA: Addition of further cluster properties #994 --- .../clusters/properties/basics.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py index 4637b988b..fe5690dbf 100644 --- a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py +++ b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py @@ -7,10 +7,11 @@ ## -- yyyy-mm-dd Ver. Auth. Description ## -- 2024-05-05 0.1.0 DA Creation ## -- 2024-05-07 0.2.0 DA Extensions +## -- 2024-05-24 0.3.0 SK Addition of further properties ## ------------------------------------------------------------------------------------------------- """ -Ver. 0.2.0 (2024-05-07) +Ver. 0.3.0 (2024-05-24) This module provides typical cluster properties to be reused in own cluster analyzers. """ @@ -25,17 +26,37 @@ # # Size (=number of associated instances) with 0,1,2 order derivatives -cprop_size : PropertyDefinition = ( 'size', 0, Property ) -cprop_size1 : PropertyDefinition = ( 'size', 1, Property ) -cprop_size2 : PropertyDefinition = ( 'size', 2, Property ) +cprop_size : PropertyDefinition = ( 'size', 0, Property ) +cprop_size1 : PropertyDefinition = ( 'size', 1, Property ) +cprop_size2 : PropertyDefinition = ( 'size', 2, Property ) # Age with 0,1,2 order derivatives -cprop_age : PropertyDefinition = ( 'age', 0, Property ) -cprop_age1 : PropertyDefinition = ( 'age', 1, Property ) -cprop_age2 : PropertyDefinition = ( 'age', 2, Property ) +cprop_age : PropertyDefinition = ( 'age', 0, Property ) +cprop_age1 : PropertyDefinition = ( 'age', 1, Property ) +cprop_age2 : PropertyDefinition = ( 'age', 2, Property ) # Centroid with 0,1,2 order derivatives and plot functionality -cprop_centroid : PropertyDefinition = ( 'centroid', 0, Centroid ) -cprop_centroid1 : PropertyDefinition = ( 'centroid', 1, Centroid ) -cprop_centroid2 : PropertyDefinition = ( 'centroid', 2, Centroid ) +cprop_centroid : PropertyDefinition = ( 'centroid', 0, Centroid ) +cprop_centroid1 : PropertyDefinition = ( 'centroid', 1, Centroid ) +cprop_centroid2 : PropertyDefinition = ( 'centroid', 2, Centroid ) + +# Center with 0,1,2 order derivatives and plot functionality +cprop_center : PropertyDefinition = ( 'center', 0, Centroid ) +cprop_center1 : PropertyDefinition = ( 'center', 1, Centroid ) +cprop_center2 : PropertyDefinition = ( 'center', 2, Centroid ) + +# Density with 0,1,2 order derivatives +cprop_density : PropertyDefinition = ( 'density', 0, Property ) +cprop_density1 : PropertyDefinition = ( 'density', 1, Property ) +cprop_density2 : PropertyDefinition = ( 'density', 2, Property ) + +# Geometric size with 0,1,2 order derivatives +cprop_geometric_size : PropertyDefinition = ( 'geometric size', 0, Property ) +cprop_geometric_size1 : PropertyDefinition = ( 'geometric size', 1, Property ) +cprop_geometric_size2 : PropertyDefinition = ( 'geometric size', 2, Property ) + +# Compactness with 0,1,2 order derivatives +cprop_compactness : PropertyDefinition = ( 'compactness', 0, Property ) +cprop_compactness1 : PropertyDefinition = ( 'compactness', 1, Property ) +cprop_compactness2 : PropertyDefinition = ( 'compactness', 2, Property ) From 060ee3c8602b9b5f3af2ff1ae6223603663c5ea5 Mon Sep 17 00:00:00 2001 From: syam47 Date: Fri, 24 May 2024 16:41:18 +0200 Subject: [PATCH 13/15] OA: Addition of further cluster properties #994 --- .../clusters/properties/basics.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py index fe5690dbf..4d20f3a8a 100644 --- a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py +++ b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/properties/basics.py @@ -26,37 +26,37 @@ # # Size (=number of associated instances) with 0,1,2 order derivatives -cprop_size : PropertyDefinition = ( 'size', 0, Property ) -cprop_size1 : PropertyDefinition = ( 'size', 1, Property ) -cprop_size2 : PropertyDefinition = ( 'size', 2, Property ) +cprop_size : PropertyDefinition = ( 'size', 0, Property ) +cprop_size1 : PropertyDefinition = ( 'size', 1, Property ) +cprop_size2 : PropertyDefinition = ( 'size', 2, Property ) # Age with 0,1,2 order derivatives -cprop_age : PropertyDefinition = ( 'age', 0, Property ) -cprop_age1 : PropertyDefinition = ( 'age', 1, Property ) -cprop_age2 : PropertyDefinition = ( 'age', 2, Property ) +cprop_age : PropertyDefinition = ( 'age', 0, Property ) +cprop_age1 : PropertyDefinition = ( 'age', 1, Property ) +cprop_age2 : PropertyDefinition = ( 'age', 2, Property ) # Centroid with 0,1,2 order derivatives and plot functionality -cprop_centroid : PropertyDefinition = ( 'centroid', 0, Centroid ) -cprop_centroid1 : PropertyDefinition = ( 'centroid', 1, Centroid ) -cprop_centroid2 : PropertyDefinition = ( 'centroid', 2, Centroid ) +cprop_centroid : PropertyDefinition = ( 'centroid', 0, Centroid ) +cprop_centroid1 : PropertyDefinition = ( 'centroid', 1, Centroid ) +cprop_centroid2 : PropertyDefinition = ( 'centroid', 2, Centroid ) -# Center with 0,1,2 order derivatives and plot functionality -cprop_center : PropertyDefinition = ( 'center', 0, Centroid ) -cprop_center1 : PropertyDefinition = ( 'center', 1, Centroid ) -cprop_center2 : PropertyDefinition = ( 'center', 2, Centroid ) +# Geometric center with 0,1,2 order derivatives and plot functionality +cprop_center_geo : PropertyDefinition = ( 'center_geo', 0, Centroid ) +cprop_center_geo1 : PropertyDefinition = ( 'center_geo', 1, Centroid ) +cprop_center_geo2 : PropertyDefinition = ( 'center_geo', 2, Centroid ) # Density with 0,1,2 order derivatives -cprop_density : PropertyDefinition = ( 'density', 0, Property ) -cprop_density1 : PropertyDefinition = ( 'density', 1, Property ) -cprop_density2 : PropertyDefinition = ( 'density', 2, Property ) +cprop_density : PropertyDefinition = ( 'density', 0, Property ) +cprop_density1 : PropertyDefinition = ( 'density', 1, Property ) +cprop_density2 : PropertyDefinition = ( 'density', 2, Property ) # Geometric size with 0,1,2 order derivatives -cprop_geometric_size : PropertyDefinition = ( 'geometric size', 0, Property ) -cprop_geometric_size1 : PropertyDefinition = ( 'geometric size', 1, Property ) -cprop_geometric_size2 : PropertyDefinition = ( 'geometric size', 2, Property ) +cprop_size_geo : PropertyDefinition = ( 'size_geo', 0, Property ) +cprop_size_geo1 : PropertyDefinition = ( 'size_geo', 1, Property ) +cprop_size_geo2 : PropertyDefinition = ( 'size_geo', 2, Property ) # Compactness with 0,1,2 order derivatives -cprop_compactness : PropertyDefinition = ( 'compactness', 0, Property ) -cprop_compactness1 : PropertyDefinition = ( 'compactness', 1, Property ) -cprop_compactness2 : PropertyDefinition = ( 'compactness', 2, Property ) +cprop_compactness : PropertyDefinition = ( 'compactness', 0, Property ) +cprop_compactness1 : PropertyDefinition = ( 'compactness', 1, Property ) +cprop_compactness2 : PropertyDefinition = ( 'compactness', 2, Property ) From d8aeb94fd24581246d88d645e6275516978e0e79 Mon Sep 17 00:00:00 2001 From: syam47 Date: Fri, 24 May 2024 18:17:45 +0200 Subject: [PATCH 14/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- .../tasks/anomalydetectors/anomalies/basics.py | 8 ++++---- .../anomalydetectors/anomalies/contextual.py | 4 ++-- .../tasks/anomalydetectors/anomalies/drift.py | 4 ++-- .../tasks/anomalydetectors/anomalies/group.py | 17 ++++++----------- .../tasks/anomalydetectors/anomalies/point.py | 15 ++++++--------- .../oa/streams/tasks/anomalydetectors/basics.py | 4 +--- .../tasks/anomalydetectors/paga_detectors.py | 8 +++----- 7 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/basics.py b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/basics.py index aaec46e4f..f2583754d 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/basics.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/basics.py @@ -25,7 +25,7 @@ from mlpro.bf.various import Id from mlpro.bf.plot import Plottable, PlotSettings from mlpro.bf.events import Event -from mlpro.bf.streams import Instance, InstDict +from mlpro.bf.streams import Instance @@ -63,7 +63,7 @@ class Anomaly (Id, Event, Plottable): ## ------------------------------------------------------------------------------------------------- def __init__(self, - p_instances: list[InstDict] = None, + p_instances: list[Instance] = None, p_ano_scores : list = None, p_visualize : bool = False, p_raising_object : object = None, @@ -75,12 +75,12 @@ def __init__(self, p_tstamp=p_det_time, **p_kwargs) Plottable.__init__( self, p_visualize = p_visualize ) - self.instances : list[InstDict] = p_instances + self.instances : list[Instance] = p_instances self.ano_scores = p_ano_scores ## ------------------------------------------------------------------------------------------------- - def get_instances(self) -> list[InstDict]: + def get_instances(self) -> list[Instance]: return self.instances diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/contextual.py b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/contextual.py index 4f8533e17..6cbe3aaa8 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/contextual.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/contextual.py @@ -19,7 +19,7 @@ This module provides templates for anomaly detection to be used in the context of online adaptivity. """ -from mlpro.bf.streams import Instance, InstDict +from mlpro.bf.streams import Instance from mlpro.oa.streams.tasks.anomalydetectors.anomalies.basics import Anomaly @@ -36,7 +36,7 @@ class ContextualAnomaly (Anomaly): # ------------------------------------------------------------------------- def __init__(self, - p_instances : InstDict = None, + p_instances : Instance = None, p_ano_scores : list = None, p_visualize : bool = False, p_raising_object : object = None, diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/drift.py b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/drift.py index a06841d50..133fda7a2 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/drift.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/drift.py @@ -19,7 +19,7 @@ This module provides templates for anomaly detection to be used in the context of online adaptivity. """ -from mlpro.bf.streams import Instance, InstDict +from mlpro.bf.streams import Instance from mlpro.oa.streams.tasks.anomalydetectors.anomalies.basics import Anomaly @@ -36,7 +36,7 @@ class DriftAnomaly (Anomaly): ## ------------------------------------------------------------------------------------------------- def __init__(self, - p_instances : InstDict = None, + p_instances : Instance = None, p_ano_scores : list = None, p_visualize : bool = False, p_raising_object : object = None, diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/group.py b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/group.py index 37cc07ecd..375af8fa2 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/group.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/group.py @@ -21,7 +21,7 @@ """ from mlpro.bf.plot import PlotSettings -from mlpro.bf.streams import Instance, InstDict +from mlpro.bf.streams import Instance from mlpro.oa.streams.tasks.anomalydetectors.anomalies.basics import Anomaly from matplotlib.figure import Figure from matplotlib.text import Text @@ -41,7 +41,7 @@ class GroupAnomaly (Anomaly): ## ------------------------------------------------------------------------------------------------- def __init__(self, - p_instances : list[InstDict] = None, + p_instances : list[Instance] = None, p_ano_scores : list = None, p_visualize : bool = False, p_raising_object : object = None, @@ -53,7 +53,7 @@ def __init__(self, p_visualize=p_visualize, p_raising_object=p_raising_object, p_det_time=p_det_time, **p_kwargs) - self.instances : list[InstDict] = p_instances + self.instances : list[Instance] = p_instances p_ano_scores = p_ano_scores self.plot_update = True @@ -87,20 +87,15 @@ def _update_plot_nd(self, p_settings: PlotSettings, **p_kwargs): label = self.C_NAME[0] - - x1 : Instance = None - x2 : Instance = None - (inst_type, x1) = self.get_instances()[0].values()[-1] - (inst_type, x2) = self.get_instances()[-1].values()[-1] + x1 = self.get_instances()[0] + x2 = self.get_instances()[-1] x1 = x1.get_id() x2 = x2.get_id() a=[] b=[] for instance in self.get_instances(): - inst : Instance = None - (inst_type, inst) = instance.values()[-1] - a.append(inst.get_feature_data().get_values()) + a.append(instance.get_feature_data().get_values()) for x in a: b.extend(x) y1 = min(b) diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/point.py b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/point.py index 4682ee46e..a95c384b8 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/point.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/anomalies/point.py @@ -22,7 +22,7 @@ """ from mlpro.bf.plot import PlotSettings -from mlpro.bf.streams import Instance, InstDict +from mlpro.bf.streams import Instance from mlpro.oa.streams.tasks.anomalydetectors.anomalies.basics import Anomaly from matplotlib.figure import Figure from matplotlib.text import Text @@ -44,7 +44,7 @@ class PointAnomaly (Anomaly): ## ------------------------------------------------------------------------------------------------- def __init__(self, - p_instances : list[InstDict] = None, + p_instances : list[Instance] = None, p_ano_scores : list = None, p_visualize : bool = False, p_raising_object : object = None, @@ -59,7 +59,7 @@ def __init__(self, p_det_time=p_det_time, **p_kwargs ) - self.instances : list[InstDict] = p_instances + self.instances : list[Instance] = p_instances self.ano_scores = p_ano_scores @@ -90,8 +90,7 @@ def _update_plot_2d(self, p_settings: PlotSettings, p_axlimits_changed: bool, p_ if ( self._plot_line_x1 is not None ) and not p_axlimits_changed: return - inst : Instance = None - (inst_type, inst) = self.get_instances()[-1].values()[-1] + inst = self.get_instances()[-1] feature_values = inst.get_feature_data().get_values() len_x = ( p_xlim[1] - p_xlim[0] ) * self.C_PLOT_CH_SIZE / 2 @@ -133,8 +132,7 @@ def _update_plot_3d(self, p_settings: PlotSettings, p_axlimits_changed: bool, p_ if ( self._plot_line_x1 is not None ) and not p_axlimits_changed: return - inst : Instance = None - (inst_type, inst) = self.get_instances()[-1].values()[-1] + inst = self.get_instances()[-1] feature_values = inst.get_feature_data().get_values() len_x = ( p_xlim[1] - p_xlim[0] ) * self.C_PLOT_CH_SIZE / 2 @@ -195,8 +193,7 @@ def _update_plot_nd(self, p_settings: PlotSettings, p_axlimits_changed: bool, p_ if ( self._plot_line is not None ) and not p_axlimits_changed: return - inst : Instance = None - (inst_type, inst) = self.get_instances()[-1].values()[-1] + inst = self.get_instances()[-1] inst_id = inst.get_id() xpos = [inst_id, inst_id] diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/basics.py b/src/mlpro/oa/streams/tasks/anomalydetectors/basics.py index e3b1cc8fc..c6ab57efe 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/basics.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/basics.py @@ -211,10 +211,8 @@ def _renormalize(self, p_normalizer: Normalizer): anomaly : Anomaly = None for anomaly in self._anomalies.values(): - instances : list[InstDict] = None instances = anomaly.get_instances() - for item in instances: - (inst_id, inst) = item.values()[-1] + for inst in instances: inst.remormalize( p_normalizer=p_normalizer) diff --git a/src/mlpro/oa/streams/tasks/anomalydetectors/paga_detectors.py b/src/mlpro/oa/streams/tasks/anomalydetectors/paga_detectors.py index 4fffe48dc..fe83c1b0a 100644 --- a/src/mlpro/oa/streams/tasks/anomalydetectors/paga_detectors.py +++ b/src/mlpro/oa/streams/tasks/anomalydetectors/paga_detectors.py @@ -88,12 +88,10 @@ def _buffer_anomaly(self, p_anomaly): if len(self.group_anomalies_instances) > 1: - inst_1 : Instance = None - inst_2 : Instance = None - (inst_type, inst_2) = self.group_anomalies_instances[-1].values()[-1] + inst_2 = self.group_anomalies_instances[-1] second = inst_2.get_id() - (inst_type, inst_1) = self.group_anomalies_instances[-2].values()[-1] - first = inst_1.get_id() + inst_1 = self.group_anomalies_instances[-2] + first = inst_1.get_id() if int(second) - 1 == int(first): From 96a16c29093acad16f386011114b60f523ae44e1 Mon Sep 17 00:00:00 2001 From: detlefarend Date: Sat, 25 May 2024 12:58:54 +0200 Subject: [PATCH 15/15] Refact: StreamTask, OATask - order of processing/adaption on new/obsolete instances #988 --- .../oa/streams/tasks/clusteranalyzers/basics.py | 15 +++++++++------ .../tasks/clusteranalyzers/clusters/basics.py | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/mlpro/oa/streams/tasks/clusteranalyzers/basics.py b/src/mlpro/oa/streams/tasks/clusteranalyzers/basics.py index 2a085e468..5b3c275a4 100644 --- a/src/mlpro/oa/streams/tasks/clusteranalyzers/basics.py +++ b/src/mlpro/oa/streams/tasks/clusteranalyzers/basics.py @@ -27,11 +27,11 @@ ## -- 2024-02-24 0.8.2 DA Class ClusterCentroid: redefined method remove_plot() ## -- 2024-04-10 0.8.3 DA Refactoring ## -- 2024-05-04 0.9.0 DA Introduction of cluster properties -## -- 2024-05-22 1.0.0 DA Initial design finished +## -- 2024-05-25 1.0.0 DA Initial design finished ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.0.0 (2024-05-22) +Ver. 1.0.0 (2024-05-25) This module provides templates for cluster analysis to be used in the context of online adaptivity. """ @@ -44,11 +44,14 @@ from mlpro.bf.plot import * from mlpro.oa.streams import OATask from mlpro.bf.math.normalizers import Normalizer -from mlpro.oa.streams.tasks.clusteranalyzers.clusters import Cluster +from mlpro.oa.streams.tasks.clusteranalyzers.clusters import Cluster, ClusterId, MembershipValue from typing import List, Tuple +MembershipItem = Tuple[ClusterId, MembershipValue, object] + + ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- @@ -255,7 +258,7 @@ def _remove_cluster(self, p_cluster:Cluster): ## ------------------------------------------------------------------------------------------------- def get_cluster_memberships( self, p_inst : Instance, - p_scope : int = C_MS_SCOPE_MAX ) -> List[Tuple[str, float, Cluster]]: + p_scope : int = C_MS_SCOPE_MAX ) -> List[MembershipItem]: """ Method to determine the membership of the given instance to each cluster as a value in percent. @@ -270,8 +273,8 @@ def get_cluster_memberships( self, Returns ------- - membership : List[Tuple[str, float, Cluster]] - List of membership tuples. A tuple consists of a cluster id, a relative membership + membership : List[MembershipItem] + List of membership items which are tuples of a cluster id, a relative membership value in [0,1] and a reference to the cluster object. """ diff --git a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/basics.py b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/basics.py index 35e17e6fe..aaa56d51a 100644 --- a/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/basics.py +++ b/src/mlpro/oa/streams/tasks/clusteranalyzers/clusters/basics.py @@ -33,10 +33,11 @@ ## -- 2024-05-04 1.3.0 DA Class Cluster: generic property systematics ## -- 2024-05-06 1.4.0 DA Plot functionality ## -- 2024-05-22 1.5.0 DA Refactoring +## -- 2024-05-25 1.6.0 DA Aliases ClusterId, MembershipValue ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.5.0 (2024-05-22) +Ver. 1.6.0 (2024-05-25) This module provides templates for clusters to be used in cluster analyzer algorithms. @@ -50,6 +51,8 @@ from mlpro.bf.math.normalizers import Renormalizable +ClusterId = int +MembershipValue = float @@ -61,8 +64,8 @@ class Cluster (Id, Plottable, Properties, Renormalizable): Parameters ---------- - p_id - Optional external id. + p_id : ClusterId + Unique cluster id. p_properties : PropertyDefinitions List of property definitions. p_visualize : bool @@ -80,7 +83,7 @@ class Cluster (Id, Plottable, Properties, Renormalizable): ## ------------------------------------------------------------------------------------------------- def __init__( self, - p_id = None, + p_id : ClusterId, p_properties : PropertyDefinitions = [], p_visualize : bool = False ):