feat(engine): dataset type detection, update flatline logic #22
Conversation
| @@ -64,6 +66,13 @@ def dtypes(self, dtypes: dict): | |||
| dtypes[col] = dtype | |||
| self._dtypes = dtypes | |||
|
|
|||
| @property | |||
There was a problem hiding this comment.
Same logic as dtypes except it has no explicit setter method (will make sense to include if we wish to add a new argument dataset_type p.e.).
In init the private attribute _df_types is stored as None, the first time the df_type property is called, the getter method will use the default inference to set the private attribute
| @@ -189,31 +187,6 @@ def predict_missingness(df: pd.DataFrame, feature: str): | |||
| # 5. Return the area under the roc curve | |||
| return roc_auc_score(y_test, y_pred) | |||
|
|
|||
There was a problem hiding this comment.
makes no sense to keep this in modelling (smart inference could be a different story). Moved to auxiliary
| @@ -19,7 +20,10 @@ def __init__(self, df: pd.DataFrame, vmv_extensions: Optional[list]=[]): | |||
| vmv_extensions: A list of user provided Value Missing Values to append to defaults. | |||
| """ | |||
| super().__init__(df=df) | |||
| self._tests = ["flatlines", "predefined_valued_missing_values"] | |||
| if self.df_type == DataFrameType.TIMESERIES: | |||
There was a problem hiding this comment.
This impacts the evaluate of the legacy engines
| @@ -78,10 +82,13 @@ def flatlines(self, th: int=5, skip: list=[]): | |||
| th: Defines the minimum length required for a flatline event to be reported. | |||
| skip: List of columns that will not be target of search for flatlines. | |||
| Pass '__index' inside skip list to skip looking for flatlines at the index.""" | |||
| if self.df_type == DataFrameType.TABULAR: | |||
There was a problem hiding this comment.
This impacts the explicit method execution of legacy engines
| @@ -40,12 +41,6 @@ def tdf(self): | |||
| def __get_missing_labels(df: pd.DataFrame, label: str): | |||
| return df[df[label].isna()] | |||
|
|
|||
| def _get_data_types(self): | |||
There was a problem hiding this comment.
This was unused
| @@ -40,3 +41,40 @@ def random_split(df: Union[pd.DataFrame, pd.Series], split_size: float, shuffle: | |||
| split = sample.iloc[:split_len] | |||
| remainder = sample.iloc[split_len:] | |||
| return split, remainder | |||
|
|
|||
| def infer_dtypes(df: Union[pd.DataFrame, pd.Series], skip: Union[list, set] = []): | |||
There was a problem hiding this comment.
This method made more sense to be in auxiliary than modelling
UrbanoFonseca
force-pushed
the
fix/smarter_flatline_detection
branch
from
010e05e
to
5203eb5
Compare
A series of changes are introduced:
Important changes
Minor changes