From abecf89688a5b3638cd18beca432796f3f051666 Mon Sep 17 00:00:00 2001 From: Katherine Klise Date: Mon, 11 Oct 2021 10:27:25 -0700 Subject: [PATCH 1/2] Update to custom static and streaming methods to use a specific column --- pecos/monitoring.py | 6 +++--- pecos/tests/test_monitoring.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pecos/monitoring.py b/pecos/monitoring.py index 31aa6ca..d09df99 100644 --- a/pecos/monitoring.py +++ b/pecos/monitoring.py @@ -788,7 +788,7 @@ def check_custom_static(self, quality_control_func, key=None, min_failures=1, # Function that operates on the entire dataset and returns a mask and # metadata for the entire dataset - mask, metadata = quality_control_func(self.df) + mask, metadata = quality_control_func(df) assert isinstance(mask, pd.DataFrame), 'mask returned by quality_control_func must be of type pd.DataFrame' assert isinstance(metadata, pd.DataFrame), 'metadata returned by quality_control_func must be of type pd.DataFrame' @@ -852,7 +852,7 @@ def check_custom_streaming(self, quality_control_func, window, key=None, # The streaming framework uses numpy arrays to improve performance but # still expects pandas DataFrames and Series in the user defined quality # control function to keep data types consitent on the user side. - np_mask = pd.DataFrame(True, index=self.df.index, columns=self.df.columns).values + np_mask = pd.DataFrame(True, index=df.index, columns=df.columns).values np_data = df.values.astype('float64') ti = df.index.get_loc(df.index[0]+history_window) @@ -881,7 +881,7 @@ def check_custom_streaming(self, quality_control_func, window, key=None, np_data[t][check_rebase] = df.iloc[t][check_rebase] rebase_count = rebase_count + sum(check_rebase) - mask = pd.DataFrame(np_mask, index=self.df.index, columns=self.df.columns) + mask = pd.DataFrame(np_mask, index=df.index, columns=df.columns) self._append_test_results(mask, error_message, min_failures) # Convert metadata to a dataframe diff --git a/pecos/tests/test_monitoring.py b/pecos/tests/test_monitoring.py index 784e141..df8acd6 100644 --- a/pecos/tests/test_monitoring.py +++ b/pecos/tests/test_monitoring.py @@ -590,11 +590,15 @@ def custom_func(data): percent = 1-self.pm.test_results['Timesteps'].sum()/N assert_almost_equal(percent, 0.95, 2) # 95% within 2 std + # using a specific key + metadata_A = self.pm.check_custom_static(custom_func, key='A', error_message='Static') + assert_frame_equal(metadata[['A']], metadata_A) + # Functional tests results = pecos.monitoring.check_custom_static(self.pm.data, custom_func, error_message='Static') percent = 1-results['test_results']['Timesteps'].sum()/N assert_almost_equal(percent, 0.95, 2) # 95% within 2 std - + def test_custom_streaming(self): def custom_func(data_pt, history): @@ -607,6 +611,10 @@ def custom_func(data_pt, history): percent = 1-self.pm.test_results['Timesteps'].sum()/N assert_almost_equal(percent, 0.95, 2) # 95% within 2 std + # using a specific key + metadata_A = self.pm.check_custom_streaming(custom_func, 50, key='A', error_message='Streaming') + assert_frame_equal(metadata[['A']], metadata_A) + # Functional tests results = pecos.monitoring.check_custom_streaming(self.pm.data, custom_func, 50, error_message='Streaming') percent = 1-results['test_results']['Timesteps'].sum()/N From 301a5e2bdce897cdbd02efa19b8122ace504b61a Mon Sep 17 00:00:00 2001 From: Katherine Klise Date: Mon, 11 Oct 2021 10:34:50 -0700 Subject: [PATCH 2/2] Updated release notes --- documentation/whatsnew/v0.2.1.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/whatsnew/v0.2.1.rst b/documentation/whatsnew/v0.2.1.rst index 127170b..d0b722e 100644 --- a/documentation/whatsnew/v0.2.1.rst +++ b/documentation/whatsnew/v0.2.1.rst @@ -3,6 +3,7 @@ v0.2.1 (main) -------------------------- +* Bug fix in custom static and streaming quality control tests to use a specific column * Minor updates for testing and documentation * Added GitHub Actions and Python 3.9 tests