From bacedc53ad034555dcae578d7c8e78bb245656ba Mon Sep 17 00:00:00 2001 From: brunalopes Date: Wed, 5 Jun 2024 21:56:26 -0300 Subject: [PATCH 01/39] Creating new splitters and base evaluation --- moabb/evaluations/splitters.py | 115 +++++++++++++++++ moabb/evaluations/unified_eval.py | 200 ++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 moabb/evaluations/splitters.py create mode 100644 moabb/evaluations/unified_eval.py diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py new file mode 100644 index 000000000..fde6b05af --- /dev/null +++ b/moabb/evaluations/splitters.py @@ -0,0 +1,115 @@ +import numpy as np +from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold + + +class WithinSubjectSplitter(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + sessions_subjects = len(metadata.groupby(['subject', 'session']).first()) + return self.n_folds * sessions_subjects + + def split(self, X, y, metadata, **kwargs): + + subjects = metadata.subject.values + + split = IndividualWithinSubjectSplitter(self.n_folds) + + for subject in np.unique(subjects): + + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + + yield split.split(X_, y_, meta_) + + +class IndividualWithinSubjectSplitter(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + return self.n_folds + + def split(self, X, y, metadata, **kwargs): + + sessions = metadata.subject.values + + cv = StratifiedKFold(self.n_folds, **kwargs) + + for session in np.unique(sessions): + X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] + + for ix_train, idx_test in cv.split(X_, y_): + X_train, y_train = X_[ix_train], y_[ix_train] + X_test, y_test = X_[ix_train], y_[ix_train] + + yield X_train, X_test, y_train, y_test + + +class CrossSessionSplitter(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + sessions_subjects = len(metadata.groupby(['subject', 'session']).first()) + return sessions_subjects + + def split(self, X, y, metadata, **kwargs): + + subjects = metadata.subject.values + split = IndividualCrossSessionSplitter(self.n_folds) + + for subject in np.unique(subjects): + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + + yield split.split(X_, y_, meta_) + + +class IndividualCrossSessionSplitter(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + sessions = metadata.session.values + return np.unique(sessions) + + def split(self, X, y, metadata, **kwargs): + + cv = LeaveOneGroupOut() + + sessions = metadata.session.values + + for ix_train, idx_test in cv.split(X, y, groups=sessions): + X_train, y_train = X[ix_train], y[ix_train] + X_test, y_test = X[ix_train], y[ix_train] + + yield X_train, X_test, y_train, y_test + + +class CrossSubjectSplitter(BaseCrossValidator): + + def __init__(self, n_groups=None): + self.n_groups = n_groups + + def get_n_splits(self, dataset=None): + return self.n_groups + + def split(self, X, y, metadata, **kwargs): + + groups = metadata.subject.values + + # Define split + if self.n_groups is None: + cv = LeaveOneGroupOut() + else: + cv = GroupKFold(n_splits=self.n_groups) + + for ix_train, idx_test in cv.split(metadata, groups=groups): + X_train, y_train = X[ix_train], y[ix_train] + X_test, y_test = X[ix_train], y[ix_train] + + yield X_train, X_test, y_train, y_test diff --git a/moabb/evaluations/unified_eval.py b/moabb/evaluations/unified_eval.py new file mode 100644 index 000000000..be66e115b --- /dev/null +++ b/moabb/evaluations/unified_eval.py @@ -0,0 +1,200 @@ +import numpy as np + +from typing import Optional, Union + +from moabb.evaluations.base import BaseEvaluation +from moabb.evaluations.splitters import CrossSubjectSplitter + +Vector = Union[list, tuple, np.ndarray] + + +class GroupEvaluation(BaseEvaluation): + """Perform specific evaluation based on a given data splitter. + + Possible modes: + Within-session evaluation uses k-fold cross_validation to determine train + and test sets on separate session for each subject, it is possible to + estimate the performance on a subset of training examples to obtain + learning curves. + + Cross-session evaluation uses a Leave-One-Group-Out cross-validation to + evaluate performance across sessions, but for a single subject. + + Cross-subject evaluation also uses Leave-One-Subject-Out to evaluate performance + on a pipeline trained in all subjects but one. + + It's also possible to determine how test data is being used + .... meta splitters + + Parameters + ---------- + n_perms : + Number of permutations to perform. If an array + is passed it has to be equal in size to the data_size array. + Values in this array must be monotonically decreasing (performing + more permutations for more data is not useful to reduce standard + error of the mean). + Default: None + data_size : + If None is passed, it performs conventional WithinSession evaluation. + Contains the policy to pick the datasizes to + evaluate, as well as the actual values. The dict has the + key 'policy' with either 'ratio' or 'per_class', and the key + 'value' with the actual values as an numpy array. This array should be + sorted, such that values in data_size are strictly monotonically increasing. + Default: None + paradigm : Paradigm instance + The paradigm to use. + datasets : List of Dataset instance + The list of dataset to run the evaluation. If none, the list of + compatible dataset will be retrieved from the paradigm instance. + random_state: int, RandomState instance, default=None + If not None, can guarantee same seed for shuffling examples. + n_jobs: int, default=1 + Number of jobs for fitting of pipeline. + n_jobs_evaluation: int, default=1 + Number of jobs for evaluation, processing in parallel the within session, + cross-session or cross-subject. + overwrite: bool, default=False + If true, overwrite the results. + error_score: "raise" or numeric, default="raise" + Value to assign to the score if an error occurs in estimator fitting. If set to + 'raise', the error is raised. + suffix: str + Suffix for the results file. + hdf5_path: str + Specific path for storing the results and models. + additional_columns: None + Adding information to results. + return_epochs: bool, default=False + use MNE epoch to train pipelines. + return_raws: bool, default=False + use MNE raw to train pipelines. + mne_labels: bool, default=False + if returning MNE epoch, use original dataset label if True + """ + + VALID_POLICIES = ["per_class", "ratio"] + SPLITTERS = {'cross_subject': CrossSubjectSplitter, + } + META_SPLITTERS = {} + + def __init__( + self, + split_method, + meta_split_method, + n_folds=None, + n_perms: Optional[Union[int, Vector]] = None, + data_size: Optional[dict] = None, + calib_size: Optional[int] = None, + **kwargs, + ): + self.data_size = data_size + self.n_perms = n_perms + self.split_method = split_method + self.meta_split_method = meta_split_method + self.n_folds = n_folds + self.calib_size = calib_size + + # Check if splitters are valid + if self.split_method not in self.SPLITTERS: + raise ValueError(f"{self.split_method} does not correspond to a valid data splitter." + f"Please use one of {self.SPLITTERS.keys()}") + + if self.meta_split not in self.META_SPLITTERS: + raise ValueError(f"{self.meta_split} does not correspond to a valid evaluation split." + f"Please use one of {self.META_SPLITTERS.keys()}") + + # Initialize splitter + self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) + + # If SamplerSplit + if self.meta_split_method == 'sampler': + + # Check if data_size is defined + if self.data_size is None: + raise ValueError( + "Please pass data_size parameter with the policy and values for the evaluation" + "split." + ) + + # Check correct n_perms parameter + if self.n_perms is None: + raise ValueError( + "When passing data_size, please also indicate number of permutations" + ) + + if isinstance(n_perms, int): + self.n_perms = np.full_like(self.data_size["value"], n_perms, dtype=int) + elif len(self.n_perms) != len(self.data_size["value"]): + raise ValueError( + "Number of elements in n_perms must be equal " + "to number of elements in data_size['value']" + ) + elif not np.all(np.diff(n_perms) <= 0): + raise ValueError( + "If n_perms is passed as an array, it has to be monotonically decreasing" + ) + + # Check correct data size parameter + if not np.all(np.diff(self.data_size["value"]) > 0): + raise ValueError( + "data_size['value'] must be sorted in strictly monotonically increasing order." + ) + + if data_size["policy"] not in self.VALID_POLICIES: + raise ValueError( + f"{data_size['policy']} is not valid. Please use one of" + f"{self.VALID_POLICIES}" + ) + + self.test_size = 0.2 # Roughly similar to 5-fold CV + + # TODO: Initialize Meta Splitter + # self.meta_splitter ...... + + add_cols = ["data_size", "permutation"] + super().__init__(additional_columns=add_cols, **kwargs) + + else: + + # Initialize Meta Splitter + # self.meta_splitter ...... + + super().__init__(**kwargs) + + def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None): + + if not self.is_valid(dataset): + raise AssertionError("Dataset is not appropriate for evaluation") + # this is a bit akward, but we need to check if at least one pipe + # have to be run before loading the data. If at least one pipeline + # need to be run, we have to load all the data. + # we might need a better granularity, if we query the DB + run_pipes = {} + for subject in dataset.subject_list: + run_pipes.update( + self.results.not_yet_computed( + pipelines, dataset, subject, process_pipeline + ) + ) + if len(run_pipes) == 0: + return + + + # Take into account the split + # for .... + + # Inside, for inference, take into account the meta split + + return + + def is_valid(self, dataset): + + if self.split_method == 'within_subject': + return True + elif self.split_method == 'cross_session': + return dataset.n_sessions > 1 + elif self.split_method == 'cross_subject': + return len(dataset.subject_list) > 1 + From 419b2caaab1315f327770e3bdc2bc169e62384a4 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Fri, 7 Jun 2024 18:05:02 -0300 Subject: [PATCH 02/39] Adding metasplitters --- moabb/evaluations/metasplitters.py | 163 +++++++++++ moabb/evaluations/splitters.py | 18 +- moabb/evaluations/unified_eval.py | 437 ++++++++++++++++++++++++++++- 3 files changed, 601 insertions(+), 17 deletions(-) create mode 100644 moabb/evaluations/metasplitters.py diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py new file mode 100644 index 000000000..88c6db1a1 --- /dev/null +++ b/moabb/evaluations/metasplitters.py @@ -0,0 +1,163 @@ +import numpy as np +from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold, \ + StratifiedShuffleSplit + + +class OfflineSplit(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + subjects = len(metadata.subject.unique()) + sessions = len(metadata.session.unique()) + return subjects * sessions + + def split(self, X, y, metadata, **kwargs): + + subjects = metadata.subject + + for subject in subjects.unique(): + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + sessions = meta_.session.values + + for session in sessions: + ix_test = np.nonzero(sessions == session)[0] + + yield ix_test + + +class TimeSeriesSplit(BaseCrossValidator): + + def __init__(self, n_folds): + self.n_folds = n_folds + + def get_n_splits(self, metadata): + + runs = metadata.run.unique() + + if len(runs) > 1: + splits = len(runs) + else: + splits = self.n_folds + + return splits + + def split(self, X, y, metadata, **kwargs): + + runs = metadata.run.unique() + + # If runs.unique != 1 + if len(runs) > 1: + cv = LeaveOneGroupOut() + else: + cv = StratifiedKFold(n_splits=self.n_folds, shuffle=False) + # Else, do a k-fold? + + subjects = metadata.subject + + for subject in subjects.unique(): + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + sessions = meta_.session.values + + for session in sessions.unique(): + + X_s, y_s, meta_s = X[sessions == session], y[subjects == session], metadata[subjects == session] + runs = meta_s.run.values + + if len(runs) > 1: + param = (X_s, y_s, runs) + else: + param = (X_s, y_s) + + for ix_test, ix_calib in cv.split(*param): + yield ix_test, ix_calib + break + + +class SamplerSplit(BaseCrossValidator): + + def __init__(self, test_size, n_perms, data_size=None): + self.data_size = data_size + self.test_size = test_size + self.n_perms = n_perms + + self.split = IndividualSamplerSplit(self.test_size, self.n_perms, data_size=self.data_size) + + def get_n_splits(self, y=None): + return self.n_perms[0] * len(self.split.get_data_size_subsets(y)) + + def split(self, X, y, metadata, **kwargs): + subjects = metadata.subject.values + split = self.split + + for subject in np.unique(subjects): + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + + yield split.split(X_, y_, meta_) + + +class IndividualSamplerSplit(BaseCrossValidator): + + def __init__(self, test_size, n_perms, data_size=None): + self.data_size = data_size + self.test_size = test_size + self.n_perms = n_perms + + def get_n_splits(self, y=None): + return self.n_perms[0] * len(self.get_data_size_subsets(y)) + + def get_data_size_subsets(self, y): + if self.data_size is None: + raise ValueError( + "Cannot create data subsets without valid policy for data_size." + ) + if self.data_size["policy"] == "ratio": + vals = np.array(self.data_size["value"]) + if np.any(vals < 0) or np.any(vals > 1): + raise ValueError("Data subset ratios must be in range [0, 1]") + upto = np.ceil(vals * len(y)).astype(int) + indices = [np.array(range(i)) for i in upto] + elif self.data_size["policy"] == "per_class": + classwise_indices = dict() + n_smallest_class = np.inf + for cl in np.unique(y): + cl_i = np.where(cl == y)[0] + classwise_indices[cl] = cl_i + n_smallest_class = ( + len(cl_i) if len(cl_i) < n_smallest_class else n_smallest_class + ) + indices = [] + for ds in self.data_size["value"]: + if ds > n_smallest_class: + raise ValueError( + f"Smallest class has {n_smallest_class} samples. " + f"Desired samples per class {ds} is too large." + ) + indices.append( + np.concatenate( + [classwise_indices[cl][:ds] for cl in classwise_indices] + ) + ) + else: + raise ValueError(f"Unknown policy {self.data_size['policy']}") + return indices + + def split(self, X, y, metadata, **kwargs): + + sessions = metadata.session.unique() + + cv = StratifiedShuffleSplit( + n_splits=self.n_perms[0], test_size=self.test_size + ) + + for session in np.unique(sessions): + X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] + + for ix_train, ix_test in cv.split(X_, y_): + + y_split = y_[ix_train] + data_size_steps = self.get_data_size_subsets(y_split) + for subset_indices in data_size_steps: + ix_train = ix_train[subset_indices] + yield ix_train, ix_test diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index fde6b05af..6b6a2f24a 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -41,11 +41,9 @@ def split(self, X, y, metadata, **kwargs): for session in np.unique(sessions): X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] - for ix_train, idx_test in cv.split(X_, y_): - X_train, y_train = X_[ix_train], y_[ix_train] - X_test, y_test = X_[ix_train], y_[ix_train] + for ix_train, ix_test in cv.split(X_, y_): - yield X_train, X_test, y_train, y_test + yield ix_train, ix_test class CrossSessionSplitter(BaseCrossValidator): @@ -83,11 +81,9 @@ def split(self, X, y, metadata, **kwargs): sessions = metadata.session.values - for ix_train, idx_test in cv.split(X, y, groups=sessions): - X_train, y_train = X[ix_train], y[ix_train] - X_test, y_test = X[ix_train], y[ix_train] + for ix_train, ix_test in cv.split(X, y, groups=sessions): - yield X_train, X_test, y_train, y_test + yield ix_train, ix_test class CrossSubjectSplitter(BaseCrossValidator): @@ -108,8 +104,6 @@ def split(self, X, y, metadata, **kwargs): else: cv = GroupKFold(n_splits=self.n_groups) - for ix_train, idx_test in cv.split(metadata, groups=groups): - X_train, y_train = X[ix_train], y[ix_train] - X_test, y_test = X[ix_train], y[ix_train] + for ix_train, ix_test in cv.split(metadata, groups=groups): - yield X_train, X_test, y_train, y_test + yield ix_train, ix_test diff --git a/moabb/evaluations/unified_eval.py b/moabb/evaluations/unified_eval.py index be66e115b..a4b17fd52 100644 --- a/moabb/evaluations/unified_eval.py +++ b/moabb/evaluations/unified_eval.py @@ -1,10 +1,28 @@ +from copy import deepcopy +from time import time + import numpy as np from typing import Optional, Union +from mne import BaseEpochs +from sklearn.metrics import get_scorer +from sklearn.model_selection import StratifiedKFold +from sklearn.model_selection._validation import _score +from sklearn.preprocessing import LabelEncoder +from tqdm import tqdm + +from moabb.evaluations import create_save_path, save_model_cv from moabb.evaluations.base import BaseEvaluation from moabb.evaluations.splitters import CrossSubjectSplitter +try: + from codecarbon import EmissionsTracker + + _carbonfootprint = True +except ImportError: + _carbonfootprint = False + Vector = Union[list, tuple, np.ndarray] @@ -75,7 +93,7 @@ class GroupEvaluation(BaseEvaluation): """ VALID_POLICIES = ["per_class", "ratio"] - SPLITTERS = {'cross_subject': CrossSubjectSplitter, + SPLITTERS = {'CrossSubject': CrossSubjectSplitter, } META_SPLITTERS = {} @@ -181,13 +199,422 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess if len(run_pipes) == 0: return + # Get data spliter type + splitter = self.data_splitter + + # Like this, I will need to lead all data before looping through subjects + X, y, metadata = self.paradigm.get_data( + dataset=dataset, + return_epochs=self.return_epochs, + return_raws=self.return_raws, + cache_config=self.cache_config, + postprocess_pipeline=postprocess_pipeline, + ) + + # encode labels + le = LabelEncoder() + y = y if self.mne_labels else le.fit_transform(y) + + # extract metadata + groups = metadata.subject.values + sessions = metadata.session.values + n_subjects = len(dataset.subject_list) + + scorer = get_scorer(self.paradigm.scoring) + + # Define inner cv + inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=self.random_state) + + if _carbonfootprint: + # Initialise CodeCarbon + tracker = EmissionsTracker(save_to_file=False, log_level="error") + + # Progressbar at subject level + for cv_ind, (train, test) in enumerate( + tqdm( + splitter.split(X, y, groups), + total=n_subjects, + desc=f"{dataset.code}-{self.split_method}", + ) + ): + + subject = groups[test[0]] + + # now we can check if this subject has results + run_pipes = self.results.not_yet_computed( + pipelines, dataset, subject, process_pipeline + ) + if len(run_pipes) == 0: + print(f"Subject {subject} already processed") + return [] + + # iterate over pipelines + for name, clf in run_pipes.items(): + if _carbonfootprint: + tracker.start() + + t_start = time() + clf = self._grid_search( + param_grid=param_grid, name=name, grid_clf=clf, inner_cv=inner_cv + ) + model = deepcopy(clf).fit(X[train], y[train]) + + if _carbonfootprint: + emissions = tracker.stop() + if emissions is None: + emissions = 0 + duration = time() - t_start + + if self.hdf5_path is not None and self.save_model: + model_save_path = create_save_path( + hdf5_path=self.hdf5_path, + code=dataset.code, + subject=subject, + session="", + name=name, + grid=False, + eval_type=f"{self.split_method}", + ) + + save_model_cv( + model=model, save_path=model_save_path, cv_index=str(cv_ind) + ) + + + # Now, evaluation: iterate over meta splitters + + """# we eval on each session + for session in np.unique(sessions[test]): + ix = sessions[test] == session + score = _score(model, X[test[ix]], y[test[ix]], scorer) + + nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] + res = { + "time": duration, + "dataset": dataset, + "subject": subject, + "session": session, + "score": score, + "n_samples": len(train), + "n_channels": nchan, + "pipeline": name, + } + + if _carbonfootprint: + res["carbon_emission"] = (1000 * emissions,) + yield res""" + + + def is_valid(self, dataset): + + if self.split_method == 'within_subject': + return True + elif self.split_method == 'cross_session': + return dataset.n_sessions > 1 + elif self.split_method == 'cross_subject': + return len(dataset.subject_list) > 1 - # Take into account the split - # for .... - # Inside, for inference, take into account the meta split +class LazyEvaluation(BaseEvaluation): + """Perform specific evaluation based on a given data splitter. + + Possible modes: + Within-session evaluation uses k-fold cross_validation to determine train + and test sets on separate session for each subject, it is possible to + estimate the performance on a subset of training examples to obtain + learning curves. - return + Cross-session evaluation uses a Leave-One-Group-Out cross-validation to + evaluate performance across sessions, but for a single subject. + + Cross-subject evaluation also uses Leave-One-Subject-Out to evaluate performance + on a pipeline trained in all subjects but one. + + It's also possible to determine how test data is being used + .... meta splitters + + Parameters + ---------- + n_perms : + Number of permutations to perform. If an array + is passed it has to be equal in size to the data_size array. + Values in this array must be monotonically decreasing (performing + more permutations for more data is not useful to reduce standard + error of the mean). + Default: None + data_size : + If None is passed, it performs conventional WithinSession evaluation. + Contains the policy to pick the datasizes to + evaluate, as well as the actual values. The dict has the + key 'policy' with either 'ratio' or 'per_class', and the key + 'value' with the actual values as an numpy array. This array should be + sorted, such that values in data_size are strictly monotonically increasing. + Default: None + paradigm : Paradigm instance + The paradigm to use. + datasets : List of Dataset instance + The list of dataset to run the evaluation. If none, the list of + compatible dataset will be retrieved from the paradigm instance. + random_state: int, RandomState instance, default=None + If not None, can guarantee same seed for shuffling examples. + n_jobs: int, default=1 + Number of jobs for fitting of pipeline. + n_jobs_evaluation: int, default=1 + Number of jobs for evaluation, processing in parallel the within session, + cross-session or cross-subject. + overwrite: bool, default=False + If true, overwrite the results. + error_score: "raise" or numeric, default="raise" + Value to assign to the score if an error occurs in estimator fitting. If set to + 'raise', the error is raised. + suffix: str + Suffix for the results file. + hdf5_path: str + Specific path for storing the results and models. + additional_columns: None + Adding information to results. + return_epochs: bool, default=False + use MNE epoch to train pipelines. + return_raws: bool, default=False + use MNE raw to train pipelines. + mne_labels: bool, default=False + if returning MNE epoch, use original dataset label if True + """ + + VALID_POLICIES = ["per_class", "ratio"] + SPLITTERS = {'CrossSubject': CrossSubjectSplitter, + } + META_SPLITTERS = {} + + def __init__( + self, + split_method, + meta_split_method, + n_folds=None, + n_perms: Optional[Union[int, Vector]] = None, + data_size: Optional[dict] = None, + calib_size: Optional[int] = None, + **kwargs, + ): + self.data_size = data_size + self.n_perms = n_perms + self.split_method = split_method + self.meta_split_method = meta_split_method + self.n_folds = n_folds + self.calib_size = calib_size + + # Check if splitters are valid + if self.split_method not in self.SPLITTERS: + raise ValueError(f"{self.split_method} does not correspond to a valid data splitter." + f"Please use one of {self.SPLITTERS.keys()}") + + if self.meta_split_method not in self.META_SPLITTERS: + raise ValueError(f"{self.meta_split_method} does not correspond to a valid evaluation split." + f"Please use one of {self.META_SPLITTERS.keys()}") + + # Initialize splitter + self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) + self.meta_splitter = self.META_SPLITTERS[self.split_method](self.n_folds) + + # If SamplerSplit + if self.meta_split_method == 'sampler': + + # Check if data_size is defined + if self.data_size is None: + raise ValueError( + "Please pass data_size parameter with the policy and values for the evaluation" + "split." + ) + + # Check correct n_perms parameter + if self.n_perms is None: + raise ValueError( + "When passing data_size, please also indicate number of permutations" + ) + + if isinstance(n_perms, int): + self.n_perms = np.full_like(self.data_size["value"], n_perms, dtype=int) + elif len(self.n_perms) != len(self.data_size["value"]): + raise ValueError( + "Number of elements in n_perms must be equal " + "to number of elements in data_size['value']" + ) + elif not np.all(np.diff(n_perms) <= 0): + raise ValueError( + "If n_perms is passed as an array, it has to be monotonically decreasing" + ) + + # Check correct data size parameter + if not np.all(np.diff(self.data_size["value"]) > 0): + raise ValueError( + "data_size['value'] must be sorted in strictly monotonically increasing order." + ) + + if data_size["policy"] not in self.VALID_POLICIES: + raise ValueError( + f"{data_size['policy']} is not valid. Please use one of" + f"{self.VALID_POLICIES}" + ) + + self.test_size = 0.2 # Roughly similar to 5-fold CV + + # self.meta_splitter ...... + + add_cols = ["data_size", "permutation"] + super().__init__(additional_columns=add_cols, **kwargs) + + else: + + # Initialize Meta Splitter + # self.meta_splitter ...... + + super().__init__(**kwargs) + + def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None): + + if not self.is_valid(dataset): + raise AssertionError("Dataset is not appropriate for evaluation") + # this is a bit akward, but we need to check if at least one pipe + # have to be run before loading the data. If at least one pipeline + # need to be run, we have to load all the data. + # we might need a better granularity, if we query the DB + run_pipes = {} + for subject in dataset.subject_list: + run_pipes.update( + self.results.not_yet_computed( + pipelines, dataset, subject, process_pipeline + ) + ) + if len(run_pipes) == 0: + return + + if _carbonfootprint: + # Initialise CodeCarbon + tracker = EmissionsTracker(save_to_file=False, log_level="error") + + self.scorer = get_scorer(self.paradigm.scoring) + + # Define inner cv + self.inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=self.random_state) + + subjects_list = dataset.subject_list + # Avoid downloading everything at once if possible + if self.split_method == 'CrossSubject': + subjects = subjects_list + self._evaluate(dataset, pipelines, param_grid, process_pipeline, + subjects=subjects, tracker=None, postprocess_pipeline=None) + else: + + for subject in tqdm(subjects_list, desc=f"{dataset.code}-{self.split_method}"): + subjects = [subject] + run_pipes = self.results.not_yet_computed( + pipelines, dataset, subject, process_pipeline + ) + self._evaluate(dataset, pipelines, param_grid, process_pipeline, + subjects=subjects, tracker=None, postprocess_pipeline=None) + + def _evaluate(self, dataset, pipelines, param_grid, process_pipeline, subjects, tracker=None, postprocess_pipeline=None): + + # Get data spliter type + splitter = self.data_splitter + + X, y, metadata = self.paradigm.get_data( + dataset=dataset, + subjects=subjects, + return_epochs=self.return_epochs, + return_raws=self.return_raws, + cache_config=self.cache_config, + postprocess_pipeline=postprocess_pipeline, + ) + + # encode labels + le = LabelEncoder() + y = y if self.mne_labels else le.fit_transform(y) + + # extract metadata + groups = metadata.subject.values + sessions = metadata.session.values + n_subjects = len(groups) + + for cv_ind, (train, test) in enumerate( + tqdm( + splitter.split(X, y, metadata), + total=self.split_method.get_n_splits(metadata), + desc=f"{dataset.code}-{self.split_method}", + ) + ): + + if self.split_method == 'CrossSubject': + subject = groups[test[0]] + else: + subject = subjects[0] + + # now we can check if this subject has results + run_pipes = self.results.not_yet_computed( + pipelines, dataset, subject, process_pipeline + ) + if len(run_pipes) == 0: + print(f"Subject {subject} already processed") + return [] + + # iterate over pipelines + for name, clf in run_pipes.items(): + + # Start tracker + if _carbonfootprint: + tracker.start() + + t_start = time() + clf = self._grid_search( + param_grid=param_grid, name=name, grid_clf=clf, inner_cv=self.inner_cv + ) + model = deepcopy(clf).fit(X[train], y[train]) + + # Check carbon emissions + if _carbonfootprint: + emissions = tracker.stop() + if emissions is None: + emissions = 0 + duration = time() - t_start + + if self.hdf5_path is not None and self.save_model: + model_save_path = create_save_path( + hdf5_path=self.hdf5_path, + code=dataset.code, + subject=subject, + session="", + name=name, + grid=False, + eval_type=f"{self.split_method}", + ) + + save_model_cv( + model=model, save_path=model_save_path, cv_index=str(cv_ind) + ) + + # Remove and use the meta splitters + # Now, for evaluation, we will need to use the new metasplitters Offline or TimeSeries + X_test, y_test, meta_test = X[test], y[test], metadata[test] + + meta_splitter = self.meta_split_method + for test_split in meta_splitter.split(X_test, y_test, meta_test): + score = _score(model, X_test[test_split], y_test[test_split], self.scorer) + + nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] + res = { + "time": duration, + "dataset": dataset, + "subject": subject, + "session": meta_test[test_split].session_name, + "score": score, + "n_samples": len(train), + "n_channels": nchan, + "pipeline": name, + } + + if _carbonfootprint: + res["carbon_emission"] = (1000 * emissions,) + yield res def is_valid(self, dataset): From d6e795d6740987b3be1959b61da1270643c03754 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 10 Jun 2024 10:19:24 -0300 Subject: [PATCH 03/39] Fixing LazyEvaluation --- moabb/evaluations/unified_eval.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/moabb/evaluations/unified_eval.py b/moabb/evaluations/unified_eval.py index a4b17fd52..01c220f0b 100644 --- a/moabb/evaluations/unified_eval.py +++ b/moabb/evaluations/unified_eval.py @@ -280,20 +280,20 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess model=model, save_path=model_save_path, cv_index=str(cv_ind) ) + # Remove and use the meta splitters + # Now, for evaluation, we will need to use the new metasplitters Offline or TimeSeries + X_test, y_test, meta_test = X[test], y[test], metadata[test] - # Now, evaluation: iterate over meta splitters - - """# we eval on each session - for session in np.unique(sessions[test]): - ix = sessions[test] == session - score = _score(model, X[test[ix]], y[test[ix]], scorer) + meta_splitter = self.meta_split_method + for test_split in meta_splitter.split(X_test, y_test, meta_test): + score = _score(model, X_test[test_split], y_test[test_split], self.scorer) nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] res = { "time": duration, "dataset": dataset, "subject": subject, - "session": session, + "session": meta_test[test_split].session_name, "score": score, "n_samples": len(train), "n_channels": nchan, @@ -302,7 +302,7 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess if _carbonfootprint: res["carbon_emission"] = (1000 * emissions,) - yield res""" + yield res def is_valid(self, dataset): @@ -389,8 +389,9 @@ class LazyEvaluation(BaseEvaluation): def __init__( self, split_method, - meta_split_method, + eval_split_method, n_folds=None, + meta_split_method=None, n_perms: Optional[Union[int, Vector]] = None, data_size: Optional[dict] = None, calib_size: Optional[int] = None, From d724674483e6d938ba53f399e39bc44fd20a3bed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:39:57 +0000 Subject: [PATCH 04/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 46 ++++++-- moabb/evaluations/splitters.py | 29 +++-- moabb/evaluations/unified_eval.py | 166 ++++++++++++++++++----------- 3 files changed, 163 insertions(+), 78 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 88c6db1a1..ce34b1bde 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -1,6 +1,10 @@ import numpy as np -from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold, \ - StratifiedShuffleSplit +from sklearn.model_selection import ( + BaseCrossValidator, + LeaveOneGroupOut, + StratifiedKFold, + StratifiedShuffleSplit, +) class OfflineSplit(BaseCrossValidator): @@ -18,7 +22,11 @@ def split(self, X, y, metadata, **kwargs): subjects = metadata.subject for subject in subjects.unique(): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) sessions = meta_.session.values for session in sessions: @@ -57,12 +65,20 @@ def split(self, X, y, metadata, **kwargs): subjects = metadata.subject for subject in subjects.unique(): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) sessions = meta_.session.values for session in sessions.unique(): - X_s, y_s, meta_s = X[sessions == session], y[subjects == session], metadata[subjects == session] + X_s, y_s, meta_s = ( + X[sessions == session], + y[subjects == session], + metadata[subjects == session], + ) runs = meta_s.run.values if len(runs) > 1: @@ -82,7 +98,9 @@ def __init__(self, test_size, n_perms, data_size=None): self.test_size = test_size self.n_perms = n_perms - self.split = IndividualSamplerSplit(self.test_size, self.n_perms, data_size=self.data_size) + self.split = IndividualSamplerSplit( + self.test_size, self.n_perms, data_size=self.data_size + ) def get_n_splits(self, y=None): return self.n_perms[0] * len(self.split.get_data_size_subsets(y)) @@ -92,7 +110,11 @@ def split(self, X, y, metadata, **kwargs): split = self.split for subject in np.unique(subjects): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) yield split.split(X_, y_, meta_) @@ -147,12 +169,14 @@ def split(self, X, y, metadata, **kwargs): sessions = metadata.session.unique() - cv = StratifiedShuffleSplit( - n_splits=self.n_perms[0], test_size=self.test_size - ) + cv = StratifiedShuffleSplit(n_splits=self.n_perms[0], test_size=self.test_size) for session in np.unique(sessions): - X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] + X_, y_, meta_ = ( + X[sessions == session], + y[sessions == session], + metadata[sessions == session], + ) for ix_train, ix_test in cv.split(X_, y_): diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 6b6a2f24a..2748b0c00 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,5 +1,10 @@ import numpy as np -from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold +from sklearn.model_selection import ( + BaseCrossValidator, + GroupKFold, + LeaveOneGroupOut, + StratifiedKFold, +) class WithinSubjectSplitter(BaseCrossValidator): @@ -8,7 +13,7 @@ def __init__(self, n_folds): self.n_folds = n_folds def get_n_splits(self, metadata): - sessions_subjects = len(metadata.groupby(['subject', 'session']).first()) + sessions_subjects = len(metadata.groupby(["subject", "session"]).first()) return self.n_folds * sessions_subjects def split(self, X, y, metadata, **kwargs): @@ -19,7 +24,11 @@ def split(self, X, y, metadata, **kwargs): for subject in np.unique(subjects): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) yield split.split(X_, y_, meta_) @@ -39,7 +48,11 @@ def split(self, X, y, metadata, **kwargs): cv = StratifiedKFold(self.n_folds, **kwargs) for session in np.unique(sessions): - X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] + X_, y_, meta_ = ( + X[sessions == session], + y[sessions == session], + metadata[sessions == session], + ) for ix_train, ix_test in cv.split(X_, y_): @@ -52,7 +65,7 @@ def __init__(self, n_folds): self.n_folds = n_folds def get_n_splits(self, metadata): - sessions_subjects = len(metadata.groupby(['subject', 'session']).first()) + sessions_subjects = len(metadata.groupby(["subject", "session"]).first()) return sessions_subjects def split(self, X, y, metadata, **kwargs): @@ -61,7 +74,11 @@ def split(self, X, y, metadata, **kwargs): split = IndividualCrossSessionSplitter(self.n_folds) for subject in np.unique(subjects): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) yield split.split(X_, y_, meta_) diff --git a/moabb/evaluations/unified_eval.py b/moabb/evaluations/unified_eval.py index 01c220f0b..268632214 100644 --- a/moabb/evaluations/unified_eval.py +++ b/moabb/evaluations/unified_eval.py @@ -1,10 +1,8 @@ from copy import deepcopy from time import time - -import numpy as np - from typing import Optional, Union +import numpy as np from mne import BaseEpochs from sklearn.metrics import get_scorer from sklearn.model_selection import StratifiedKFold @@ -16,6 +14,7 @@ from moabb.evaluations.base import BaseEvaluation from moabb.evaluations.splitters import CrossSubjectSplitter + try: from codecarbon import EmissionsTracker @@ -93,19 +92,20 @@ class GroupEvaluation(BaseEvaluation): """ VALID_POLICIES = ["per_class", "ratio"] - SPLITTERS = {'CrossSubject': CrossSubjectSplitter, - } + SPLITTERS = { + "CrossSubject": CrossSubjectSplitter, + } META_SPLITTERS = {} def __init__( - self, - split_method, - meta_split_method, - n_folds=None, - n_perms: Optional[Union[int, Vector]] = None, - data_size: Optional[dict] = None, - calib_size: Optional[int] = None, - **kwargs, + self, + split_method, + meta_split_method, + n_folds=None, + n_perms: Optional[Union[int, Vector]] = None, + data_size: Optional[dict] = None, + calib_size: Optional[int] = None, + **kwargs, ): self.data_size = data_size self.n_perms = n_perms @@ -116,18 +116,22 @@ def __init__( # Check if splitters are valid if self.split_method not in self.SPLITTERS: - raise ValueError(f"{self.split_method} does not correspond to a valid data splitter." - f"Please use one of {self.SPLITTERS.keys()}") + raise ValueError( + f"{self.split_method} does not correspond to a valid data splitter." + f"Please use one of {self.SPLITTERS.keys()}" + ) if self.meta_split not in self.META_SPLITTERS: - raise ValueError(f"{self.meta_split} does not correspond to a valid evaluation split." - f"Please use one of {self.META_SPLITTERS.keys()}") + raise ValueError( + f"{self.meta_split} does not correspond to a valid evaluation split." + f"Please use one of {self.META_SPLITTERS.keys()}" + ) # Initialize splitter self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) # If SamplerSplit - if self.meta_split_method == 'sampler': + if self.meta_split_method == "sampler": # Check if data_size is defined if self.data_size is None: @@ -181,7 +185,9 @@ def __init__( super().__init__(**kwargs) - def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None): + def evaluate( + self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None + ): if not self.is_valid(dataset): raise AssertionError("Dataset is not appropriate for evaluation") @@ -223,7 +229,9 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess scorer = get_scorer(self.paradigm.scoring) # Define inner cv - inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=self.random_state) + inner_cv = StratifiedKFold( + n_splits=3, shuffle=True, random_state=self.random_state + ) if _carbonfootprint: # Initialise CodeCarbon @@ -286,7 +294,9 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess meta_splitter = self.meta_split_method for test_split in meta_splitter.split(X_test, y_test, meta_test): - score = _score(model, X_test[test_split], y_test[test_split], self.scorer) + score = _score( + model, X_test[test_split], y_test[test_split], self.scorer + ) nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] res = { @@ -304,14 +314,13 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess res["carbon_emission"] = (1000 * emissions,) yield res - def is_valid(self, dataset): - if self.split_method == 'within_subject': + if self.split_method == "within_subject": return True - elif self.split_method == 'cross_session': + elif self.split_method == "cross_session": return dataset.n_sessions > 1 - elif self.split_method == 'cross_subject': + elif self.split_method == "cross_subject": return len(dataset.subject_list) > 1 @@ -382,20 +391,21 @@ class LazyEvaluation(BaseEvaluation): """ VALID_POLICIES = ["per_class", "ratio"] - SPLITTERS = {'CrossSubject': CrossSubjectSplitter, - } + SPLITTERS = { + "CrossSubject": CrossSubjectSplitter, + } META_SPLITTERS = {} def __init__( - self, - split_method, - eval_split_method, - n_folds=None, - meta_split_method=None, - n_perms: Optional[Union[int, Vector]] = None, - data_size: Optional[dict] = None, - calib_size: Optional[int] = None, - **kwargs, + self, + split_method, + eval_split_method, + n_folds=None, + meta_split_method=None, + n_perms: Optional[Union[int, Vector]] = None, + data_size: Optional[dict] = None, + calib_size: Optional[int] = None, + **kwargs, ): self.data_size = data_size self.n_perms = n_perms @@ -406,19 +416,23 @@ def __init__( # Check if splitters are valid if self.split_method not in self.SPLITTERS: - raise ValueError(f"{self.split_method} does not correspond to a valid data splitter." - f"Please use one of {self.SPLITTERS.keys()}") + raise ValueError( + f"{self.split_method} does not correspond to a valid data splitter." + f"Please use one of {self.SPLITTERS.keys()}" + ) if self.meta_split_method not in self.META_SPLITTERS: - raise ValueError(f"{self.meta_split_method} does not correspond to a valid evaluation split." - f"Please use one of {self.META_SPLITTERS.keys()}") + raise ValueError( + f"{self.meta_split_method} does not correspond to a valid evaluation split." + f"Please use one of {self.META_SPLITTERS.keys()}" + ) # Initialize splitter self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) self.meta_splitter = self.META_SPLITTERS[self.split_method](self.n_folds) # If SamplerSplit - if self.meta_split_method == 'sampler': + if self.meta_split_method == "sampler": # Check if data_size is defined if self.data_size is None: @@ -471,7 +485,9 @@ def __init__( super().__init__(**kwargs) - def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None): + def evaluate( + self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None + ): if not self.is_valid(dataset): raise AssertionError("Dataset is not appropriate for evaluation") @@ -496,25 +512,52 @@ def evaluate(self, dataset, pipelines, param_grid, process_pipeline, postprocess self.scorer = get_scorer(self.paradigm.scoring) # Define inner cv - self.inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=self.random_state) + self.inner_cv = StratifiedKFold( + n_splits=3, shuffle=True, random_state=self.random_state + ) subjects_list = dataset.subject_list # Avoid downloading everything at once if possible - if self.split_method == 'CrossSubject': + if self.split_method == "CrossSubject": subjects = subjects_list - self._evaluate(dataset, pipelines, param_grid, process_pipeline, - subjects=subjects, tracker=None, postprocess_pipeline=None) + self._evaluate( + dataset, + pipelines, + param_grid, + process_pipeline, + subjects=subjects, + tracker=None, + postprocess_pipeline=None, + ) else: - for subject in tqdm(subjects_list, desc=f"{dataset.code}-{self.split_method}"): + for subject in tqdm( + subjects_list, desc=f"{dataset.code}-{self.split_method}" + ): subjects = [subject] run_pipes = self.results.not_yet_computed( pipelines, dataset, subject, process_pipeline ) - self._evaluate(dataset, pipelines, param_grid, process_pipeline, - subjects=subjects, tracker=None, postprocess_pipeline=None) + self._evaluate( + dataset, + pipelines, + param_grid, + process_pipeline, + subjects=subjects, + tracker=None, + postprocess_pipeline=None, + ) - def _evaluate(self, dataset, pipelines, param_grid, process_pipeline, subjects, tracker=None, postprocess_pipeline=None): + def _evaluate( + self, + dataset, + pipelines, + param_grid, + process_pipeline, + subjects, + tracker=None, + postprocess_pipeline=None, + ): # Get data spliter type splitter = self.data_splitter @@ -538,14 +581,14 @@ def _evaluate(self, dataset, pipelines, param_grid, process_pipeline, subjects, n_subjects = len(groups) for cv_ind, (train, test) in enumerate( - tqdm( - splitter.split(X, y, metadata), - total=self.split_method.get_n_splits(metadata), - desc=f"{dataset.code}-{self.split_method}", - ) + tqdm( + splitter.split(X, y, metadata), + total=self.split_method.get_n_splits(metadata), + desc=f"{dataset.code}-{self.split_method}", + ) ): - if self.split_method == 'CrossSubject': + if self.split_method == "CrossSubject": subject = groups[test[0]] else: subject = subjects[0] @@ -599,7 +642,9 @@ def _evaluate(self, dataset, pipelines, param_grid, process_pipeline, subjects, meta_splitter = self.meta_split_method for test_split in meta_splitter.split(X_test, y_test, meta_test): - score = _score(model, X_test[test_split], y_test[test_split], self.scorer) + score = _score( + model, X_test[test_split], y_test[test_split], self.scorer + ) nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] res = { @@ -619,10 +664,9 @@ def _evaluate(self, dataset, pipelines, param_grid, process_pipeline, subjects, def is_valid(self, dataset): - if self.split_method == 'within_subject': + if self.split_method == "within_subject": return True - elif self.split_method == 'cross_session': + elif self.split_method == "cross_session": return dataset.n_sessions > 1 - elif self.split_method == 'cross_subject': + elif self.split_method == "cross_subject": return len(dataset.subject_list) > 1 - From a278026b98284ef16a6f389539e2f0af418a6509 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 10 Jun 2024 11:11:51 -0300 Subject: [PATCH 05/39] More optimized version of TimeSeriesSplit --- moabb/evaluations/metasplitters.py | 97 +++++++++++------------------- 1 file changed, 34 insertions(+), 63 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index ce34b1bde..57849fea8 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -1,10 +1,6 @@ import numpy as np -from sklearn.model_selection import ( - BaseCrossValidator, - LeaveOneGroupOut, - StratifiedKFold, - StratifiedShuffleSplit, -) +from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold, \ + StratifiedShuffleSplit, KFold class OfflineSplit(BaseCrossValidator): @@ -22,11 +18,7 @@ def split(self, X, y, metadata, **kwargs): subjects = metadata.subject for subject in subjects.unique(): - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] sessions = meta_.session.values for session in sessions: @@ -37,58 +29,45 @@ def split(self, X, y, metadata, **kwargs): class TimeSeriesSplit(BaseCrossValidator): - def __init__(self, n_folds): - self.n_folds = n_folds + def __init__(self, calib_size): + self.calib_size = calib_size def get_n_splits(self, metadata): + sessions = metadata.session.unique() + subjects = metadata.subject.unique() - runs = metadata.run.unique() - - if len(runs) > 1: - splits = len(runs) - else: - splits = self.n_folds - + splits = len(sessions) * len(subjects) return splits def split(self, X, y, metadata, **kwargs): runs = metadata.run.unique() + sessions = metadata.session.unique() + subjects = metadata.subject.unique() - # If runs.unique != 1 if len(runs) > 1: - cv = LeaveOneGroupOut() + for subject in subjects: + for session in sessions: + # Index of specific session of this subejct + session_indices = metadata[(metadata.subject == subject) & (metadata.session == session)].index + + for run in runs: + test_ix = session_indices[metadata.loc[session_indices].run != run] + calib_ix = session_indices[metadata.loc[session_indices].run == run] + yield test_ix, calib_ix + break # Take the fist run as calibration else: - cv = StratifiedKFold(n_splits=self.n_folds, shuffle=False) - # Else, do a k-fold? - - subjects = metadata.subject - - for subject in subjects.unique(): - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) - sessions = meta_.session.values + for subject in subjects: + for session in sessions: + session_indices = metadata[(metadata.subject == subject) & (metadata.session == session)].index + calib_size = self.calib_size - for session in sessions.unique(): + indices = session_indices.to_numpy() + calib_ix = indices[:calib_size] + test_ix = indices[calib_size:] - X_s, y_s, meta_s = ( - X[sessions == session], - y[subjects == session], - metadata[subjects == session], - ) - runs = meta_s.run.values + yield test_ix, calib_ix # Take first #calib_size samples as calibration - if len(runs) > 1: - param = (X_s, y_s, runs) - else: - param = (X_s, y_s) - - for ix_test, ix_calib in cv.split(*param): - yield ix_test, ix_calib - break class SamplerSplit(BaseCrossValidator): @@ -98,9 +77,7 @@ def __init__(self, test_size, n_perms, data_size=None): self.test_size = test_size self.n_perms = n_perms - self.split = IndividualSamplerSplit( - self.test_size, self.n_perms, data_size=self.data_size - ) + self.split = IndividualSamplerSplit(self.test_size, self.n_perms, data_size=self.data_size) def get_n_splits(self, y=None): return self.n_perms[0] * len(self.split.get_data_size_subsets(y)) @@ -110,11 +87,7 @@ def split(self, X, y, metadata, **kwargs): split = self.split for subject in np.unique(subjects): - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] yield split.split(X_, y_, meta_) @@ -169,14 +142,12 @@ def split(self, X, y, metadata, **kwargs): sessions = metadata.session.unique() - cv = StratifiedShuffleSplit(n_splits=self.n_perms[0], test_size=self.test_size) + cv = StratifiedShuffleSplit( + n_splits=self.n_perms[0], test_size=self.test_size + ) for session in np.unique(sessions): - X_, y_, meta_ = ( - X[sessions == session], - y[sessions == session], - metadata[sessions == session], - ) + X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] for ix_train, ix_test in cv.split(X_, y_): From 300a6b90199e3e68c53c6055581e82e11e1521fb Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 10 Jun 2024 11:13:45 -0300 Subject: [PATCH 06/39] More optimized version of TimeSeriesSplit --- moabb/evaluations/metasplitters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 57849fea8..f55070146 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -69,7 +69,6 @@ def split(self, X, y, metadata, **kwargs): yield test_ix, calib_ix # Take first #calib_size samples as calibration - class SamplerSplit(BaseCrossValidator): def __init__(self, test_size, n_perms, data_size=None): From 7cb79f68af62f4530fd0c8ec141a8fed9213eb4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:15:19 +0000 Subject: [PATCH 07/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 48 ++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index f55070146..6a3bdfef3 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -1,6 +1,8 @@ import numpy as np -from sklearn.model_selection import BaseCrossValidator, GroupKFold, LeaveOneGroupOut, StratifiedKFold, \ - StratifiedShuffleSplit, KFold +from sklearn.model_selection import ( + BaseCrossValidator, + StratifiedShuffleSplit, +) class OfflineSplit(BaseCrossValidator): @@ -18,7 +20,11 @@ def split(self, X, y, metadata, **kwargs): subjects = metadata.subject for subject in subjects.unique(): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) sessions = meta_.session.values for session in sessions: @@ -49,17 +55,25 @@ def split(self, X, y, metadata, **kwargs): for subject in subjects: for session in sessions: # Index of specific session of this subejct - session_indices = metadata[(metadata.subject == subject) & (metadata.session == session)].index + session_indices = metadata[ + (metadata.subject == subject) & (metadata.session == session) + ].index for run in runs: - test_ix = session_indices[metadata.loc[session_indices].run != run] - calib_ix = session_indices[metadata.loc[session_indices].run == run] + test_ix = session_indices[ + metadata.loc[session_indices].run != run + ] + calib_ix = session_indices[ + metadata.loc[session_indices].run == run + ] yield test_ix, calib_ix break # Take the fist run as calibration else: for subject in subjects: for session in sessions: - session_indices = metadata[(metadata.subject == subject) & (metadata.session == session)].index + session_indices = metadata[ + (metadata.subject == subject) & (metadata.session == session) + ].index calib_size = self.calib_size indices = session_indices.to_numpy() @@ -76,7 +90,9 @@ def __init__(self, test_size, n_perms, data_size=None): self.test_size = test_size self.n_perms = n_perms - self.split = IndividualSamplerSplit(self.test_size, self.n_perms, data_size=self.data_size) + self.split = IndividualSamplerSplit( + self.test_size, self.n_perms, data_size=self.data_size + ) def get_n_splits(self, y=None): return self.n_perms[0] * len(self.split.get_data_size_subsets(y)) @@ -86,7 +102,11 @@ def split(self, X, y, metadata, **kwargs): split = self.split for subject in np.unique(subjects): - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) yield split.split(X_, y_, meta_) @@ -141,12 +161,14 @@ def split(self, X, y, metadata, **kwargs): sessions = metadata.session.unique() - cv = StratifiedShuffleSplit( - n_splits=self.n_perms[0], test_size=self.test_size - ) + cv = StratifiedShuffleSplit(n_splits=self.n_perms[0], test_size=self.test_size) for session in np.unique(sessions): - X_, y_, meta_ = X[sessions == session], y[sessions == session], metadata[sessions == session] + X_, y_, meta_ = ( + X[sessions == session], + y[sessions == session], + metadata[sessions == session], + ) for ix_train, ix_test in cv.split(X_, y_): From 55db70f4d4c6778bf492e449ed974c5cde939275 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 10 Jun 2024 19:02:30 -0300 Subject: [PATCH 08/39] Addressing some comments: documentation, types, inconsistencies --- moabb/evaluations/metasplitters.py | 176 +++++++++++++++++------------ moabb/evaluations/splitters.py | 93 +++++++++++++-- 2 files changed, 186 insertions(+), 83 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 6a3bdfef3..c492b1f11 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -6,8 +6,20 @@ class OfflineSplit(BaseCrossValidator): + """ Offline split for evaluation test data. - def __init__(self, n_folds): + Assumes that, per session, all test trials are available for inference. It can be used when + no filtering or data alignment is needed. + + Parameters + ---------- + n_folds: int + Not used in this case, just so it can be initialized in the same way as + TimeSeriesSplit. + + """ + + def __init__(self, n_folds: int): self.n_folds = n_folds def get_n_splits(self, metadata): @@ -15,27 +27,40 @@ def get_n_splits(self, metadata): sessions = len(metadata.session.unique()) return subjects * sessions - def split(self, X, y, metadata, **kwargs): + def split(self, X, y, metadata): - subjects = metadata.subject + subjects = metadata.subject.unique() - for subject in subjects.unique(): - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) - sessions = meta_.session.values + for subject in subjects: + X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + sessions = meta_.session.unique() for session in sessions: - ix_test = np.nonzero(sessions == session)[0] + ix_test = meta_[meta_['session'] == session].index yield ix_test class TimeSeriesSplit(BaseCrossValidator): + """ Pseudo-online split for evaluation test data. + + It takes into account the time sequence for obtaining the test data, and uses first run, + or first #calib_size trial as calibration data, and the rest as evaluation data. + Calibration data is important in the context where data alignment or filtering is used on + training data. - def __init__(self, calib_size): + OBS: Be careful! Since this inference split is based on time disposition of obtained data, + if your data is not organized by time, but by other parameter, such as class, you may want to + be extra careful when using this split. + + Parameters + ---------- + calib_size: int + Size of calibration set, used if there is just one run. + + """ + + def __init__(self, calib_size:int): self.calib_size = calib_size def get_n_splits(self, metadata): @@ -45,7 +70,7 @@ def get_n_splits(self, metadata): splits = len(sessions) * len(subjects) return splits - def split(self, X, y, metadata, **kwargs): + def split(self, X, y, metadata): runs = metadata.run.unique() sessions = metadata.session.unique() @@ -54,72 +79,95 @@ def split(self, X, y, metadata, **kwargs): if len(runs) > 1: for subject in subjects: for session in sessions: - # Index of specific session of this subejct - session_indices = metadata[ - (metadata.subject == subject) & (metadata.session == session) - ].index + # Index of specific session of this subject + session_indices = metadata[(metadata['subject'] == subject) & + (metadata['session'] == session)].index for run in runs: - test_ix = session_indices[ - metadata.loc[session_indices].run != run - ] - calib_ix = session_indices[ - metadata.loc[session_indices].run == run - ] + test_ix = session_indices[metadata.loc[session_indices]['run'] != run] + calib_ix = session_indices[metadata.loc[session_indices]['run'] == run] yield test_ix, calib_ix break # Take the fist run as calibration else: for subject in subjects: for session in sessions: - session_indices = metadata[ - (metadata.subject == subject) & (metadata.session == session) - ].index + session_indices = metadata[(metadata['subject'] == subject) & + (metadata['session'] == session)].index calib_size = self.calib_size - indices = session_indices.to_numpy() - calib_ix = indices[:calib_size] - test_ix = indices[calib_size:] + calib_ix = session_indices[:calib_size] + test_ix = session_indices[calib_size:] yield test_ix, calib_ix # Take first #calib_size samples as calibration class SamplerSplit(BaseCrossValidator): - - def __init__(self, test_size, n_perms, data_size=None): + """ Return subsets of the training data with different number of samples. + + Util for estimating the performance of a model when using different number of + training samples and plotting the learning curve. You must define the data + evaluation type (WithinSubject, CrossSession, CrossSubject) so the training set + can be sampled. + + Parameters + ---------- + data_eval: BaseCrossValidator object + Evaluation splitter already initialized. It can be WithinSubject, CrossSession, + or CrossSubject Splitters. + data_size : dict + Contains the policy to pick the datasizes to + evaluate, as well as the actual values. The dict has the + key 'policy' with either 'ratio' or 'per_class', and the key + 'value' with the actual values as a numpy array. This array should be + sorted, such that values in data_size are strictly monotonically increasing. + + """ + + def __init__(self, data_eval, data_size): + self.data_eval = data_eval self.data_size = data_size - self.test_size = test_size - self.n_perms = n_perms - self.split = IndividualSamplerSplit( - self.test_size, self.n_perms, data_size=self.data_size - ) + self.sampler = IndividualSamplerSplit(self.data_size) - def get_n_splits(self, y=None): - return self.n_perms[0] * len(self.split.get_data_size_subsets(y)) + def get_n_splits(self, y, metadata): + return self.data_eval.get_n_splits(metadata) * len(self.sampler.get_data_size_subsets(y)) def split(self, X, y, metadata, **kwargs): - subjects = metadata.subject.values - split = self.split - - for subject in np.unique(subjects): - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) + cv = self.data_eval + sampler = self.sampler - yield split.split(X_, y_, meta_) + for ix_train, _ in cv.split(X, y, metadata, **kwargs): + X_train, y_train, meta_train = X[ix_train], y[ix_train], metadata[ix_train] + yield sampler.split(X_train, y_train, meta_train) class IndividualSamplerSplit(BaseCrossValidator): + """ Return subsets of the training data with different number of samples. + + Util for estimating the performance of a model when using different number of + training samples and plotting the learning curve. It must be used after already splitting + data using one of the other evaluation data splitters (WithinSubject, CrossSession, CrossSubject) + since it corresponds to a subsampling of the training data. + + This 'Individual' Sampler Split assumes that data and metadata being passed is training, and was + already split by WithinSubject, CrossSession, or CrossSubject splitters. - def __init__(self, test_size, n_perms, data_size=None): + Parameters + ---------- + data_size : dict + Contains the policy to pick the datasizes to + evaluate, as well as the actual values. The dict has the + key 'policy' with either 'ratio' or 'per_class', and the key + 'value' with the actual values as a numpy array. This array should be + sorted, such that values in data_size are strictly monotonically increasing. + + """ + + def __init__(self, data_size): self.data_size = data_size - self.test_size = test_size - self.n_perms = n_perms def get_n_splits(self, y=None): - return self.n_perms[0] * len(self.get_data_size_subsets(y)) + return len(self.get_data_size_subsets(y)) def get_data_size_subsets(self, y): if self.data_size is None: @@ -157,23 +205,9 @@ def get_data_size_subsets(self, y): raise ValueError(f"Unknown policy {self.data_size['policy']}") return indices - def split(self, X, y, metadata, **kwargs): - - sessions = metadata.session.unique() - - cv = StratifiedShuffleSplit(n_splits=self.n_perms[0], test_size=self.test_size) - - for session in np.unique(sessions): - X_, y_, meta_ = ( - X[sessions == session], - y[sessions == session], - metadata[sessions == session], - ) - - for ix_train, ix_test in cv.split(X_, y_): + def split(self, X, y, metadata): - y_split = y_[ix_train] - data_size_steps = self.get_data_size_subsets(y_split) - for subset_indices in data_size_steps: - ix_train = ix_train[subset_indices] - yield ix_train, ix_test + data_size_steps = self.get_data_size_subsets(y) + for subset_indices in data_size_steps: + ix_train = subset_indices + yield ix_train diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 2748b0c00..242ff39e3 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -8,12 +8,24 @@ class WithinSubjectSplitter(BaseCrossValidator): + """ Data splitter for within session evaluation. - def __init__(self, n_folds): + Within-session evaluation uses k-fold cross_validation to determine train + and test sets on separate session for each subject. This splitter assumes that + all data from all subjects is already known and loaded. + + Parameters + ---------- + n_folds : int + Number of folds. Must be at least 2. + + """ + + def __init__(self, n_folds: int): self.n_folds = n_folds def get_n_splits(self, metadata): - sessions_subjects = len(metadata.groupby(["subject", "session"]).first()) + sessions_subjects = metadata.groupby(["subject", "session"]).ngroups return self.n_folds * sessions_subjects def split(self, X, y, metadata, **kwargs): @@ -30,12 +42,27 @@ def split(self, X, y, metadata, **kwargs): metadata[subjects == subject], ) - yield split.split(X_, y_, meta_) + yield split.split(X_, y_, meta_, **kwargs) class IndividualWithinSubjectSplitter(BaseCrossValidator): + """ Data splitter for within session evaluation. - def __init__(self, n_folds): + Within-session evaluation uses k-fold cross_validation to determine train + and test sets on separate session for each subject. This splitter does not assume + that all data and metadata from all subjects is already loaded. If X, y and metadata + are from a single subject, it returns data split for this subject only. + + It can be used as basis for WithinSessionSplitter or to avoid downloading all data at + once when it is not needed, + + Parameters + ---------- + n_folds : int + Number of folds. Must be at least 2. + + """ + def __init__(self, n_folds: int): self.n_folds = n_folds def get_n_splits(self, metadata): @@ -45,7 +72,7 @@ def split(self, X, y, metadata, **kwargs): sessions = metadata.subject.values - cv = StratifiedKFold(self.n_folds, **kwargs) + cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) for session in np.unique(sessions): X_, y_, meta_ = ( @@ -60,18 +87,31 @@ def split(self, X, y, metadata, **kwargs): class CrossSessionSplitter(BaseCrossValidator): + """ Data splitter for cross session evaluation. + + Cross-session evaluation uses a Leave-One-Group-Out cross-validation to + evaluate performance across sessions, but for a single subject. This splitter + assumes that all data from all subjects is already known and loaded. + + Parameters + ---------- + n_folds : + Not used. For compatibility with other cross-validation splitters. + Default:None - def __init__(self, n_folds): + """ + + def __init__(self, n_folds=None): self.n_folds = n_folds def get_n_splits(self, metadata): sessions_subjects = len(metadata.groupby(["subject", "session"]).first()) return sessions_subjects - def split(self, X, y, metadata, **kwargs): + def split(self, X, y, metadata): subjects = metadata.subject.values - split = IndividualCrossSessionSplitter(self.n_folds) + split = IndividualCrossSessionSplitter() for subject in np.unique(subjects): X_, y_, meta_ = ( @@ -84,15 +124,31 @@ def split(self, X, y, metadata, **kwargs): class IndividualCrossSessionSplitter(BaseCrossValidator): + """ Data splitter for cross session evaluation. + + Cross-session evaluation uses a Leave-One-Group-Out cross-validation to + evaluate performance across sessions, but for a single subject. This splitter does + not assumethat all data and metadata from all subjects is already loaded. If X, y + and metadata are from a single subject, it returns data split for this subject only. + + It can be used as basis for CrossSessionSplitter or to avoid downloading all data at + once when it is not needed, + + Parameters + ---------- + n_folds : + Not used. For compatibility with other cross-validation splitters. + Default:None - def __init__(self, n_folds): + """ + def __init__(self, n_folds=None): self.n_folds = n_folds def get_n_splits(self, metadata): sessions = metadata.session.values return np.unique(sessions) - def split(self, X, y, metadata, **kwargs): + def split(self, X, y, metadata): cv = LeaveOneGroupOut() @@ -104,14 +160,27 @@ def split(self, X, y, metadata, **kwargs): class CrossSubjectSplitter(BaseCrossValidator): + """ Data splitter for cross session evaluation. + + Cross-session evaluation uses a Leave-One-Group-Out cross-validation to + evaluate performance across sessions, but for a single subject. This splitter + assumes that all data from all subjects is already known and loaded. + + Parameters + ---------- + n_groups : int or None + If None, Leave-One-Subject-Out is performed. + If int, Leave-k-Subjects-Out is performed. - def __init__(self, n_groups=None): + + """ + def __init__(self, n_groups): self.n_groups = n_groups def get_n_splits(self, dataset=None): return self.n_groups - def split(self, X, y, metadata, **kwargs): + def split(self, X, y, metadata): groups = metadata.subject.values From 2851a1508dd9d8da2ba4f95686a57244ede96f70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:05:44 +0000 Subject: [PATCH 09/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 47 ++++++++++++++++++------------ moabb/evaluations/splitters.py | 13 +++++---- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index c492b1f11..6f22402e0 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -1,12 +1,9 @@ import numpy as np -from sklearn.model_selection import ( - BaseCrossValidator, - StratifiedShuffleSplit, -) +from sklearn.model_selection import BaseCrossValidator class OfflineSplit(BaseCrossValidator): - """ Offline split for evaluation test data. + """Offline split for evaluation test data. Assumes that, per session, all test trials are available for inference. It can be used when no filtering or data alignment is needed. @@ -32,17 +29,21 @@ def split(self, X, y, metadata): subjects = metadata.subject.unique() for subject in subjects: - X_, y_, meta_ = X[subjects == subject], y[subjects == subject], metadata[subjects == subject] + X_, y_, meta_ = ( + X[subjects == subject], + y[subjects == subject], + metadata[subjects == subject], + ) sessions = meta_.session.unique() for session in sessions: - ix_test = meta_[meta_['session'] == session].index + ix_test = meta_[meta_["session"] == session].index yield ix_test class TimeSeriesSplit(BaseCrossValidator): - """ Pseudo-online split for evaluation test data. + """Pseudo-online split for evaluation test data. It takes into account the time sequence for obtaining the test data, and uses first run, or first #calib_size trial as calibration data, and the rest as evaluation data. @@ -60,7 +61,7 @@ class TimeSeriesSplit(BaseCrossValidator): """ - def __init__(self, calib_size:int): + def __init__(self, calib_size: int): self.calib_size = calib_size def get_n_splits(self, metadata): @@ -80,19 +81,27 @@ def split(self, X, y, metadata): for subject in subjects: for session in sessions: # Index of specific session of this subject - session_indices = metadata[(metadata['subject'] == subject) & - (metadata['session'] == session)].index + session_indices = metadata[ + (metadata["subject"] == subject) + & (metadata["session"] == session) + ].index for run in runs: - test_ix = session_indices[metadata.loc[session_indices]['run'] != run] - calib_ix = session_indices[metadata.loc[session_indices]['run'] == run] + test_ix = session_indices[ + metadata.loc[session_indices]["run"] != run + ] + calib_ix = session_indices[ + metadata.loc[session_indices]["run"] == run + ] yield test_ix, calib_ix break # Take the fist run as calibration else: for subject in subjects: for session in sessions: - session_indices = metadata[(metadata['subject'] == subject) & - (metadata['session'] == session)].index + session_indices = metadata[ + (metadata["subject"] == subject) + & (metadata["session"] == session) + ].index calib_size = self.calib_size calib_ix = session_indices[:calib_size] @@ -102,7 +111,7 @@ def split(self, X, y, metadata): class SamplerSplit(BaseCrossValidator): - """ Return subsets of the training data with different number of samples. + """Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. You must define the data @@ -130,7 +139,9 @@ def __init__(self, data_eval, data_size): self.sampler = IndividualSamplerSplit(self.data_size) def get_n_splits(self, y, metadata): - return self.data_eval.get_n_splits(metadata) * len(self.sampler.get_data_size_subsets(y)) + return self.data_eval.get_n_splits(metadata) * len( + self.sampler.get_data_size_subsets(y) + ) def split(self, X, y, metadata, **kwargs): cv = self.data_eval @@ -142,7 +153,7 @@ def split(self, X, y, metadata, **kwargs): class IndividualSamplerSplit(BaseCrossValidator): - """ Return subsets of the training data with different number of samples. + """Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. It must be used after already splitting diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 242ff39e3..6187f70dc 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -8,7 +8,7 @@ class WithinSubjectSplitter(BaseCrossValidator): - """ Data splitter for within session evaluation. + """Data splitter for within session evaluation. Within-session evaluation uses k-fold cross_validation to determine train and test sets on separate session for each subject. This splitter assumes that @@ -46,7 +46,7 @@ def split(self, X, y, metadata, **kwargs): class IndividualWithinSubjectSplitter(BaseCrossValidator): - """ Data splitter for within session evaluation. + """Data splitter for within session evaluation. Within-session evaluation uses k-fold cross_validation to determine train and test sets on separate session for each subject. This splitter does not assume @@ -62,6 +62,7 @@ class IndividualWithinSubjectSplitter(BaseCrossValidator): Number of folds. Must be at least 2. """ + def __init__(self, n_folds: int): self.n_folds = n_folds @@ -87,7 +88,7 @@ def split(self, X, y, metadata, **kwargs): class CrossSessionSplitter(BaseCrossValidator): - """ Data splitter for cross session evaluation. + """Data splitter for cross session evaluation. Cross-session evaluation uses a Leave-One-Group-Out cross-validation to evaluate performance across sessions, but for a single subject. This splitter @@ -124,7 +125,7 @@ def split(self, X, y, metadata): class IndividualCrossSessionSplitter(BaseCrossValidator): - """ Data splitter for cross session evaluation. + """Data splitter for cross session evaluation. Cross-session evaluation uses a Leave-One-Group-Out cross-validation to evaluate performance across sessions, but for a single subject. This splitter does @@ -141,6 +142,7 @@ class IndividualCrossSessionSplitter(BaseCrossValidator): Default:None """ + def __init__(self, n_folds=None): self.n_folds = n_folds @@ -160,7 +162,7 @@ def split(self, X, y, metadata): class CrossSubjectSplitter(BaseCrossValidator): - """ Data splitter for cross session evaluation. + """Data splitter for cross session evaluation. Cross-session evaluation uses a Leave-One-Group-Out cross-validation to evaluate performance across sessions, but for a single subject. This splitter @@ -174,6 +176,7 @@ class CrossSubjectSplitter(BaseCrossValidator): """ + def __init__(self, n_groups): self.n_groups = n_groups From c73dd1aaecf189ffdb9daa3e2d0eb7fbf695eacd Mon Sep 17 00:00:00 2001 From: brunalopes Date: Wed, 12 Jun 2024 11:36:54 -0300 Subject: [PATCH 10/39] Addressing some comments: optimizing code, adjusts Deleting unified_eval, so it can be addressed on another pr. Working on tests. --- moabb/evaluations/metasplitters.py | 101 ++--- moabb/evaluations/splitters.py | 30 +- moabb/evaluations/unified_eval.py | 672 ----------------------------- moabb/evaluations/utils.py | 51 ++- 4 files changed, 93 insertions(+), 761 deletions(-) delete mode 100644 moabb/evaluations/unified_eval.py diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 6f22402e0..50c57f6fb 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -1,5 +1,24 @@ +""" +The data splitters defined in this file are not directly related to a evaluation method +the way that WithinSession, CrossSession and CrossSubject splitters are. + +OfflineSplit and TimeSeriesSplit split the test data, indicating weather model inference +will be computed using a Offline or a Pseudo-Online validation. Pseudo-online evaluation +is important when training data is pre-processed with some data-dependent transformation. +One part of the test data is separated as a calibration to compute the transformation. + +SamplerSplit is an optional subsplit done on the training set to generate subsets with +different numbers of samples. It can be used to estimate the performance of the model +on different training sizes and plot learning curves. + +""" + import numpy as np -from sklearn.model_selection import BaseCrossValidator +from sklearn.model_selection import ( + BaseCrossValidator, +) + +from moabb.evaluations.utils import sort_group class OfflineSplit(BaseCrossValidator): @@ -20,30 +39,25 @@ def __init__(self, n_folds: int): self.n_folds = n_folds def get_n_splits(self, metadata): - subjects = len(metadata.subject.unique()) - sessions = len(metadata.session.unique()) - return subjects * sessions + return metadata.groupby(['subject', 'session']).ngroups def split(self, X, y, metadata): subjects = metadata.subject.unique() for subject in subjects: - X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], - ) + mask = subjects == subject + X_, y_, meta_ = X[mask], y[mask], metadata[mask] sessions = meta_.session.unique() for session in sessions: - ix_test = meta_[meta_["session"] == session].index + ix_test = meta_[meta_['session'] == session].index yield ix_test class TimeSeriesSplit(BaseCrossValidator): - """Pseudo-online split for evaluation test data. + """ Pseudo-online split for evaluation test data. It takes into account the time sequence for obtaining the test data, and uses first run, or first #calib_size trial as calibration data, and the rest as evaluation data. @@ -61,57 +75,34 @@ class TimeSeriesSplit(BaseCrossValidator): """ - def __init__(self, calib_size: int): + def __init__(self, calib_size: int = None): self.calib_size = calib_size def get_n_splits(self, metadata): - sessions = metadata.session.unique() - subjects = metadata.subject.unique() - - splits = len(sessions) * len(subjects) - return splits + return metadata.groupby(['subject', 'session']) def split(self, X, y, metadata): - runs = metadata.run.unique() - sessions = metadata.session.unique() - subjects = metadata.subject.unique() - - if len(runs) > 1: - for subject in subjects: - for session in sessions: - # Index of specific session of this subject - session_indices = metadata[ - (metadata["subject"] == subject) - & (metadata["session"] == session) - ].index - - for run in runs: - test_ix = session_indices[ - metadata.loc[session_indices]["run"] != run - ] - calib_ix = session_indices[ - metadata.loc[session_indices]["run"] == run - ] - yield test_ix, calib_ix - break # Take the fist run as calibration - else: - for subject in subjects: - for session in sessions: - session_indices = metadata[ - (metadata["subject"] == subject) - & (metadata["session"] == session) - ].index - calib_size = self.calib_size - - calib_ix = session_indices[:calib_size] - test_ix = session_indices[calib_size:] + for _, group in metadata.groupby(['subject', 'session']): + runs = group.run.unique() + if len(runs) > 1: + # To guarantee that the runs are on the right order + runs = sort_group(runs) + for run in runs: + test_ix = group[group['run'] != run].index + calib_ix = group[group['run'] == run].index + yield test_ix, calib_ix + break # Take the fist run as calibration + else: + calib_size = self.calib_size + calib_ix = group[:calib_size] + test_ix = group[calib_size:] - yield test_ix, calib_ix # Take first #calib_size samples as calibration + yield test_ix, calib_ix # Take first #calib_size samples as calibration class SamplerSplit(BaseCrossValidator): - """Return subsets of the training data with different number of samples. + """ Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. You must define the data @@ -139,9 +130,7 @@ def __init__(self, data_eval, data_size): self.sampler = IndividualSamplerSplit(self.data_size) def get_n_splits(self, y, metadata): - return self.data_eval.get_n_splits(metadata) * len( - self.sampler.get_data_size_subsets(y) - ) + return self.data_eval.get_n_splits(metadata) * len(self.sampler.get_data_size_subsets(y)) def split(self, X, y, metadata, **kwargs): cv = self.data_eval @@ -153,7 +142,7 @@ def split(self, X, y, metadata, **kwargs): class IndividualSamplerSplit(BaseCrossValidator): - """Return subsets of the training data with different number of samples. + """ Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. It must be used after already splitting diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 6187f70dc..c96f5fabe 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -35,11 +35,11 @@ def split(self, X, y, metadata, **kwargs): split = IndividualWithinSubjectSplitter(self.n_folds) for subject in np.unique(subjects): - + mask = subjects == subject X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], + X[mask], + y[mask], + metadata[mask], ) yield split.split(X_, y_, meta_, **kwargs) @@ -71,19 +71,20 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata, **kwargs): - sessions = metadata.subject.values + assert len(np.unique(metadata.subject)) == 1 + sessions = metadata.subject.values cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) for session in np.unique(sessions): + mask = sessions == session X_, y_, meta_ = ( - X[sessions == session], - y[sessions == session], - metadata[sessions == session], + X[mask], + y[mask], + metadata[mask], ) for ix_train, ix_test in cv.split(X_, y_): - yield ix_train, ix_test @@ -115,10 +116,11 @@ def split(self, X, y, metadata): split = IndividualCrossSessionSplitter() for subject in np.unique(subjects): + mask = subjects == subject X_, y_, meta_ = ( - X[subjects == subject], - y[subjects == subject], - metadata[subjects == subject], + X[mask], + y[mask], + metadata[mask], ) yield split.split(X_, y_, meta_) @@ -152,12 +154,12 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata): - cv = LeaveOneGroupOut() + assert len(np.unique(metadata.subject)) == 1 + cv = LeaveOneGroupOut() sessions = metadata.session.values for ix_train, ix_test in cv.split(X, y, groups=sessions): - yield ix_train, ix_test diff --git a/moabb/evaluations/unified_eval.py b/moabb/evaluations/unified_eval.py deleted file mode 100644 index 268632214..000000000 --- a/moabb/evaluations/unified_eval.py +++ /dev/null @@ -1,672 +0,0 @@ -from copy import deepcopy -from time import time -from typing import Optional, Union - -import numpy as np -from mne import BaseEpochs -from sklearn.metrics import get_scorer -from sklearn.model_selection import StratifiedKFold -from sklearn.model_selection._validation import _score -from sklearn.preprocessing import LabelEncoder -from tqdm import tqdm - -from moabb.evaluations import create_save_path, save_model_cv -from moabb.evaluations.base import BaseEvaluation -from moabb.evaluations.splitters import CrossSubjectSplitter - - -try: - from codecarbon import EmissionsTracker - - _carbonfootprint = True -except ImportError: - _carbonfootprint = False - -Vector = Union[list, tuple, np.ndarray] - - -class GroupEvaluation(BaseEvaluation): - """Perform specific evaluation based on a given data splitter. - - Possible modes: - Within-session evaluation uses k-fold cross_validation to determine train - and test sets on separate session for each subject, it is possible to - estimate the performance on a subset of training examples to obtain - learning curves. - - Cross-session evaluation uses a Leave-One-Group-Out cross-validation to - evaluate performance across sessions, but for a single subject. - - Cross-subject evaluation also uses Leave-One-Subject-Out to evaluate performance - on a pipeline trained in all subjects but one. - - It's also possible to determine how test data is being used - .... meta splitters - - Parameters - ---------- - n_perms : - Number of permutations to perform. If an array - is passed it has to be equal in size to the data_size array. - Values in this array must be monotonically decreasing (performing - more permutations for more data is not useful to reduce standard - error of the mean). - Default: None - data_size : - If None is passed, it performs conventional WithinSession evaluation. - Contains the policy to pick the datasizes to - evaluate, as well as the actual values. The dict has the - key 'policy' with either 'ratio' or 'per_class', and the key - 'value' with the actual values as an numpy array. This array should be - sorted, such that values in data_size are strictly monotonically increasing. - Default: None - paradigm : Paradigm instance - The paradigm to use. - datasets : List of Dataset instance - The list of dataset to run the evaluation. If none, the list of - compatible dataset will be retrieved from the paradigm instance. - random_state: int, RandomState instance, default=None - If not None, can guarantee same seed for shuffling examples. - n_jobs: int, default=1 - Number of jobs for fitting of pipeline. - n_jobs_evaluation: int, default=1 - Number of jobs for evaluation, processing in parallel the within session, - cross-session or cross-subject. - overwrite: bool, default=False - If true, overwrite the results. - error_score: "raise" or numeric, default="raise" - Value to assign to the score if an error occurs in estimator fitting. If set to - 'raise', the error is raised. - suffix: str - Suffix for the results file. - hdf5_path: str - Specific path for storing the results and models. - additional_columns: None - Adding information to results. - return_epochs: bool, default=False - use MNE epoch to train pipelines. - return_raws: bool, default=False - use MNE raw to train pipelines. - mne_labels: bool, default=False - if returning MNE epoch, use original dataset label if True - """ - - VALID_POLICIES = ["per_class", "ratio"] - SPLITTERS = { - "CrossSubject": CrossSubjectSplitter, - } - META_SPLITTERS = {} - - def __init__( - self, - split_method, - meta_split_method, - n_folds=None, - n_perms: Optional[Union[int, Vector]] = None, - data_size: Optional[dict] = None, - calib_size: Optional[int] = None, - **kwargs, - ): - self.data_size = data_size - self.n_perms = n_perms - self.split_method = split_method - self.meta_split_method = meta_split_method - self.n_folds = n_folds - self.calib_size = calib_size - - # Check if splitters are valid - if self.split_method not in self.SPLITTERS: - raise ValueError( - f"{self.split_method} does not correspond to a valid data splitter." - f"Please use one of {self.SPLITTERS.keys()}" - ) - - if self.meta_split not in self.META_SPLITTERS: - raise ValueError( - f"{self.meta_split} does not correspond to a valid evaluation split." - f"Please use one of {self.META_SPLITTERS.keys()}" - ) - - # Initialize splitter - self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) - - # If SamplerSplit - if self.meta_split_method == "sampler": - - # Check if data_size is defined - if self.data_size is None: - raise ValueError( - "Please pass data_size parameter with the policy and values for the evaluation" - "split." - ) - - # Check correct n_perms parameter - if self.n_perms is None: - raise ValueError( - "When passing data_size, please also indicate number of permutations" - ) - - if isinstance(n_perms, int): - self.n_perms = np.full_like(self.data_size["value"], n_perms, dtype=int) - elif len(self.n_perms) != len(self.data_size["value"]): - raise ValueError( - "Number of elements in n_perms must be equal " - "to number of elements in data_size['value']" - ) - elif not np.all(np.diff(n_perms) <= 0): - raise ValueError( - "If n_perms is passed as an array, it has to be monotonically decreasing" - ) - - # Check correct data size parameter - if not np.all(np.diff(self.data_size["value"]) > 0): - raise ValueError( - "data_size['value'] must be sorted in strictly monotonically increasing order." - ) - - if data_size["policy"] not in self.VALID_POLICIES: - raise ValueError( - f"{data_size['policy']} is not valid. Please use one of" - f"{self.VALID_POLICIES}" - ) - - self.test_size = 0.2 # Roughly similar to 5-fold CV - - # TODO: Initialize Meta Splitter - # self.meta_splitter ...... - - add_cols = ["data_size", "permutation"] - super().__init__(additional_columns=add_cols, **kwargs) - - else: - - # Initialize Meta Splitter - # self.meta_splitter ...... - - super().__init__(**kwargs) - - def evaluate( - self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None - ): - - if not self.is_valid(dataset): - raise AssertionError("Dataset is not appropriate for evaluation") - # this is a bit akward, but we need to check if at least one pipe - # have to be run before loading the data. If at least one pipeline - # need to be run, we have to load all the data. - # we might need a better granularity, if we query the DB - run_pipes = {} - for subject in dataset.subject_list: - run_pipes.update( - self.results.not_yet_computed( - pipelines, dataset, subject, process_pipeline - ) - ) - if len(run_pipes) == 0: - return - - # Get data spliter type - splitter = self.data_splitter - - # Like this, I will need to lead all data before looping through subjects - X, y, metadata = self.paradigm.get_data( - dataset=dataset, - return_epochs=self.return_epochs, - return_raws=self.return_raws, - cache_config=self.cache_config, - postprocess_pipeline=postprocess_pipeline, - ) - - # encode labels - le = LabelEncoder() - y = y if self.mne_labels else le.fit_transform(y) - - # extract metadata - groups = metadata.subject.values - sessions = metadata.session.values - n_subjects = len(dataset.subject_list) - - scorer = get_scorer(self.paradigm.scoring) - - # Define inner cv - inner_cv = StratifiedKFold( - n_splits=3, shuffle=True, random_state=self.random_state - ) - - if _carbonfootprint: - # Initialise CodeCarbon - tracker = EmissionsTracker(save_to_file=False, log_level="error") - - # Progressbar at subject level - for cv_ind, (train, test) in enumerate( - tqdm( - splitter.split(X, y, groups), - total=n_subjects, - desc=f"{dataset.code}-{self.split_method}", - ) - ): - - subject = groups[test[0]] - - # now we can check if this subject has results - run_pipes = self.results.not_yet_computed( - pipelines, dataset, subject, process_pipeline - ) - if len(run_pipes) == 0: - print(f"Subject {subject} already processed") - return [] - - # iterate over pipelines - for name, clf in run_pipes.items(): - if _carbonfootprint: - tracker.start() - - t_start = time() - clf = self._grid_search( - param_grid=param_grid, name=name, grid_clf=clf, inner_cv=inner_cv - ) - model = deepcopy(clf).fit(X[train], y[train]) - - if _carbonfootprint: - emissions = tracker.stop() - if emissions is None: - emissions = 0 - duration = time() - t_start - - if self.hdf5_path is not None and self.save_model: - model_save_path = create_save_path( - hdf5_path=self.hdf5_path, - code=dataset.code, - subject=subject, - session="", - name=name, - grid=False, - eval_type=f"{self.split_method}", - ) - - save_model_cv( - model=model, save_path=model_save_path, cv_index=str(cv_ind) - ) - - # Remove and use the meta splitters - # Now, for evaluation, we will need to use the new metasplitters Offline or TimeSeries - X_test, y_test, meta_test = X[test], y[test], metadata[test] - - meta_splitter = self.meta_split_method - for test_split in meta_splitter.split(X_test, y_test, meta_test): - score = _score( - model, X_test[test_split], y_test[test_split], self.scorer - ) - - nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] - res = { - "time": duration, - "dataset": dataset, - "subject": subject, - "session": meta_test[test_split].session_name, - "score": score, - "n_samples": len(train), - "n_channels": nchan, - "pipeline": name, - } - - if _carbonfootprint: - res["carbon_emission"] = (1000 * emissions,) - yield res - - def is_valid(self, dataset): - - if self.split_method == "within_subject": - return True - elif self.split_method == "cross_session": - return dataset.n_sessions > 1 - elif self.split_method == "cross_subject": - return len(dataset.subject_list) > 1 - - -class LazyEvaluation(BaseEvaluation): - """Perform specific evaluation based on a given data splitter. - - Possible modes: - Within-session evaluation uses k-fold cross_validation to determine train - and test sets on separate session for each subject, it is possible to - estimate the performance on a subset of training examples to obtain - learning curves. - - Cross-session evaluation uses a Leave-One-Group-Out cross-validation to - evaluate performance across sessions, but for a single subject. - - Cross-subject evaluation also uses Leave-One-Subject-Out to evaluate performance - on a pipeline trained in all subjects but one. - - It's also possible to determine how test data is being used - .... meta splitters - - Parameters - ---------- - n_perms : - Number of permutations to perform. If an array - is passed it has to be equal in size to the data_size array. - Values in this array must be monotonically decreasing (performing - more permutations for more data is not useful to reduce standard - error of the mean). - Default: None - data_size : - If None is passed, it performs conventional WithinSession evaluation. - Contains the policy to pick the datasizes to - evaluate, as well as the actual values. The dict has the - key 'policy' with either 'ratio' or 'per_class', and the key - 'value' with the actual values as an numpy array. This array should be - sorted, such that values in data_size are strictly monotonically increasing. - Default: None - paradigm : Paradigm instance - The paradigm to use. - datasets : List of Dataset instance - The list of dataset to run the evaluation. If none, the list of - compatible dataset will be retrieved from the paradigm instance. - random_state: int, RandomState instance, default=None - If not None, can guarantee same seed for shuffling examples. - n_jobs: int, default=1 - Number of jobs for fitting of pipeline. - n_jobs_evaluation: int, default=1 - Number of jobs for evaluation, processing in parallel the within session, - cross-session or cross-subject. - overwrite: bool, default=False - If true, overwrite the results. - error_score: "raise" or numeric, default="raise" - Value to assign to the score if an error occurs in estimator fitting. If set to - 'raise', the error is raised. - suffix: str - Suffix for the results file. - hdf5_path: str - Specific path for storing the results and models. - additional_columns: None - Adding information to results. - return_epochs: bool, default=False - use MNE epoch to train pipelines. - return_raws: bool, default=False - use MNE raw to train pipelines. - mne_labels: bool, default=False - if returning MNE epoch, use original dataset label if True - """ - - VALID_POLICIES = ["per_class", "ratio"] - SPLITTERS = { - "CrossSubject": CrossSubjectSplitter, - } - META_SPLITTERS = {} - - def __init__( - self, - split_method, - eval_split_method, - n_folds=None, - meta_split_method=None, - n_perms: Optional[Union[int, Vector]] = None, - data_size: Optional[dict] = None, - calib_size: Optional[int] = None, - **kwargs, - ): - self.data_size = data_size - self.n_perms = n_perms - self.split_method = split_method - self.meta_split_method = meta_split_method - self.n_folds = n_folds - self.calib_size = calib_size - - # Check if splitters are valid - if self.split_method not in self.SPLITTERS: - raise ValueError( - f"{self.split_method} does not correspond to a valid data splitter." - f"Please use one of {self.SPLITTERS.keys()}" - ) - - if self.meta_split_method not in self.META_SPLITTERS: - raise ValueError( - f"{self.meta_split_method} does not correspond to a valid evaluation split." - f"Please use one of {self.META_SPLITTERS.keys()}" - ) - - # Initialize splitter - self.data_splitter = self.SPLITTERS[self.split_method](self.n_folds) - self.meta_splitter = self.META_SPLITTERS[self.split_method](self.n_folds) - - # If SamplerSplit - if self.meta_split_method == "sampler": - - # Check if data_size is defined - if self.data_size is None: - raise ValueError( - "Please pass data_size parameter with the policy and values for the evaluation" - "split." - ) - - # Check correct n_perms parameter - if self.n_perms is None: - raise ValueError( - "When passing data_size, please also indicate number of permutations" - ) - - if isinstance(n_perms, int): - self.n_perms = np.full_like(self.data_size["value"], n_perms, dtype=int) - elif len(self.n_perms) != len(self.data_size["value"]): - raise ValueError( - "Number of elements in n_perms must be equal " - "to number of elements in data_size['value']" - ) - elif not np.all(np.diff(n_perms) <= 0): - raise ValueError( - "If n_perms is passed as an array, it has to be monotonically decreasing" - ) - - # Check correct data size parameter - if not np.all(np.diff(self.data_size["value"]) > 0): - raise ValueError( - "data_size['value'] must be sorted in strictly monotonically increasing order." - ) - - if data_size["policy"] not in self.VALID_POLICIES: - raise ValueError( - f"{data_size['policy']} is not valid. Please use one of" - f"{self.VALID_POLICIES}" - ) - - self.test_size = 0.2 # Roughly similar to 5-fold CV - - # self.meta_splitter ...... - - add_cols = ["data_size", "permutation"] - super().__init__(additional_columns=add_cols, **kwargs) - - else: - - # Initialize Meta Splitter - # self.meta_splitter ...... - - super().__init__(**kwargs) - - def evaluate( - self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None - ): - - if not self.is_valid(dataset): - raise AssertionError("Dataset is not appropriate for evaluation") - # this is a bit akward, but we need to check if at least one pipe - # have to be run before loading the data. If at least one pipeline - # need to be run, we have to load all the data. - # we might need a better granularity, if we query the DB - run_pipes = {} - for subject in dataset.subject_list: - run_pipes.update( - self.results.not_yet_computed( - pipelines, dataset, subject, process_pipeline - ) - ) - if len(run_pipes) == 0: - return - - if _carbonfootprint: - # Initialise CodeCarbon - tracker = EmissionsTracker(save_to_file=False, log_level="error") - - self.scorer = get_scorer(self.paradigm.scoring) - - # Define inner cv - self.inner_cv = StratifiedKFold( - n_splits=3, shuffle=True, random_state=self.random_state - ) - - subjects_list = dataset.subject_list - # Avoid downloading everything at once if possible - if self.split_method == "CrossSubject": - subjects = subjects_list - self._evaluate( - dataset, - pipelines, - param_grid, - process_pipeline, - subjects=subjects, - tracker=None, - postprocess_pipeline=None, - ) - else: - - for subject in tqdm( - subjects_list, desc=f"{dataset.code}-{self.split_method}" - ): - subjects = [subject] - run_pipes = self.results.not_yet_computed( - pipelines, dataset, subject, process_pipeline - ) - self._evaluate( - dataset, - pipelines, - param_grid, - process_pipeline, - subjects=subjects, - tracker=None, - postprocess_pipeline=None, - ) - - def _evaluate( - self, - dataset, - pipelines, - param_grid, - process_pipeline, - subjects, - tracker=None, - postprocess_pipeline=None, - ): - - # Get data spliter type - splitter = self.data_splitter - - X, y, metadata = self.paradigm.get_data( - dataset=dataset, - subjects=subjects, - return_epochs=self.return_epochs, - return_raws=self.return_raws, - cache_config=self.cache_config, - postprocess_pipeline=postprocess_pipeline, - ) - - # encode labels - le = LabelEncoder() - y = y if self.mne_labels else le.fit_transform(y) - - # extract metadata - groups = metadata.subject.values - sessions = metadata.session.values - n_subjects = len(groups) - - for cv_ind, (train, test) in enumerate( - tqdm( - splitter.split(X, y, metadata), - total=self.split_method.get_n_splits(metadata), - desc=f"{dataset.code}-{self.split_method}", - ) - ): - - if self.split_method == "CrossSubject": - subject = groups[test[0]] - else: - subject = subjects[0] - - # now we can check if this subject has results - run_pipes = self.results.not_yet_computed( - pipelines, dataset, subject, process_pipeline - ) - if len(run_pipes) == 0: - print(f"Subject {subject} already processed") - return [] - - # iterate over pipelines - for name, clf in run_pipes.items(): - - # Start tracker - if _carbonfootprint: - tracker.start() - - t_start = time() - clf = self._grid_search( - param_grid=param_grid, name=name, grid_clf=clf, inner_cv=self.inner_cv - ) - model = deepcopy(clf).fit(X[train], y[train]) - - # Check carbon emissions - if _carbonfootprint: - emissions = tracker.stop() - if emissions is None: - emissions = 0 - duration = time() - t_start - - if self.hdf5_path is not None and self.save_model: - model_save_path = create_save_path( - hdf5_path=self.hdf5_path, - code=dataset.code, - subject=subject, - session="", - name=name, - grid=False, - eval_type=f"{self.split_method}", - ) - - save_model_cv( - model=model, save_path=model_save_path, cv_index=str(cv_ind) - ) - - # Remove and use the meta splitters - # Now, for evaluation, we will need to use the new metasplitters Offline or TimeSeries - X_test, y_test, meta_test = X[test], y[test], metadata[test] - - meta_splitter = self.meta_split_method - for test_split in meta_splitter.split(X_test, y_test, meta_test): - score = _score( - model, X_test[test_split], y_test[test_split], self.scorer - ) - - nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1] - res = { - "time": duration, - "dataset": dataset, - "subject": subject, - "session": meta_test[test_split].session_name, - "score": score, - "n_samples": len(train), - "n_channels": nchan, - "pipeline": name, - } - - if _carbonfootprint: - res["carbon_emission"] = (1000 * emissions,) - yield res - - def is_valid(self, dataset): - - if self.split_method == "within_subject": - return True - elif self.split_method == "cross_session": - return dataset.n_sessions > 1 - elif self.split_method == "cross_subject": - return len(dataset.subject_list) > 1 diff --git a/moabb/evaluations/utils.py b/moabb/evaluations/utils.py index fad15e064..6c5054041 100644 --- a/moabb/evaluations/utils.py +++ b/moabb/evaluations/utils.py @@ -1,9 +1,11 @@ from __future__ import annotations +import re from pathlib import Path from pickle import HIGHEST_PROTOCOL, dump from typing import Sequence +import numpy as np from numpy import argmax from sklearn.pipeline import Pipeline @@ -154,13 +156,13 @@ def save_model_list(model_list: list | Pipeline, score_list: Sequence, save_path def create_save_path( - hdf5_path, - code: str, - subject: int | str, - session: str, - name: str, - grid=False, - eval_type="WithinSession", + hdf5_path, + code: str, + subject: int | str, + session: str, + name: str, + grid=False, + eval_type="WithinSession", ): """Create a save path based on evaluation parameters. @@ -192,23 +194,34 @@ def create_save_path( if grid: path_save = ( - Path(hdf5_path) - / f"GridSearch_{eval_type}" - / code - / f"{str(subject)}" - / str(session) - / str(name) + Path(hdf5_path) + / f"GridSearch_{eval_type}" + / code + / f"{str(subject)}" + / str(session) + / str(name) ) else: path_save = ( - Path(hdf5_path) - / f"Models_{eval_type}" - / code - / f"{str(subject)}" - / str(session) - / str(name) + Path(hdf5_path) + / f"Models_{eval_type}" + / code + / f"{str(subject)}" + / str(session) + / str(name) ) return str(path_save) else: print("No hdf5_path provided, models will not be saved.") + + +def sort_group(groups): + runs_sort = [] + pattern = r"([0-9]+)(|[a-zA-Z]+[a-zA-Z0-9]*)" + for i, group in enumerate(groups): + index, description = re.fullmatch(pattern, group).groups() + index = int(index) + runs_sort.append(index) + sorted_ix = np.argsort(runs_sort) + return groups[sorted_ix] From 2b0e7359539cb71c984fb5c1d10b7efd8ba444c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:38:59 +0000 Subject: [PATCH 11/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 26 ++++++++++---------- moabb/evaluations/utils.py | 38 +++++++++++++++--------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 50c57f6fb..b85fe73bc 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -14,9 +14,7 @@ """ import numpy as np -from sklearn.model_selection import ( - BaseCrossValidator, -) +from sklearn.model_selection import BaseCrossValidator from moabb.evaluations.utils import sort_group @@ -39,7 +37,7 @@ def __init__(self, n_folds: int): self.n_folds = n_folds def get_n_splits(self, metadata): - return metadata.groupby(['subject', 'session']).ngroups + return metadata.groupby(["subject", "session"]).ngroups def split(self, X, y, metadata): @@ -51,13 +49,13 @@ def split(self, X, y, metadata): sessions = meta_.session.unique() for session in sessions: - ix_test = meta_[meta_['session'] == session].index + ix_test = meta_[meta_["session"] == session].index yield ix_test class TimeSeriesSplit(BaseCrossValidator): - """ Pseudo-online split for evaluation test data. + """Pseudo-online split for evaluation test data. It takes into account the time sequence for obtaining the test data, and uses first run, or first #calib_size trial as calibration data, and the rest as evaluation data. @@ -79,18 +77,18 @@ def __init__(self, calib_size: int = None): self.calib_size = calib_size def get_n_splits(self, metadata): - return metadata.groupby(['subject', 'session']) + return metadata.groupby(["subject", "session"]) def split(self, X, y, metadata): - for _, group in metadata.groupby(['subject', 'session']): + for _, group in metadata.groupby(["subject", "session"]): runs = group.run.unique() if len(runs) > 1: # To guarantee that the runs are on the right order runs = sort_group(runs) for run in runs: - test_ix = group[group['run'] != run].index - calib_ix = group[group['run'] == run].index + test_ix = group[group["run"] != run].index + calib_ix = group[group["run"] == run].index yield test_ix, calib_ix break # Take the fist run as calibration else: @@ -102,7 +100,7 @@ def split(self, X, y, metadata): class SamplerSplit(BaseCrossValidator): - """ Return subsets of the training data with different number of samples. + """Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. You must define the data @@ -130,7 +128,9 @@ def __init__(self, data_eval, data_size): self.sampler = IndividualSamplerSplit(self.data_size) def get_n_splits(self, y, metadata): - return self.data_eval.get_n_splits(metadata) * len(self.sampler.get_data_size_subsets(y)) + return self.data_eval.get_n_splits(metadata) * len( + self.sampler.get_data_size_subsets(y) + ) def split(self, X, y, metadata, **kwargs): cv = self.data_eval @@ -142,7 +142,7 @@ def split(self, X, y, metadata, **kwargs): class IndividualSamplerSplit(BaseCrossValidator): - """ Return subsets of the training data with different number of samples. + """Return subsets of the training data with different number of samples. Util for estimating the performance of a model when using different number of training samples and plotting the learning curve. It must be used after already splitting diff --git a/moabb/evaluations/utils.py b/moabb/evaluations/utils.py index 6c5054041..0542fa160 100644 --- a/moabb/evaluations/utils.py +++ b/moabb/evaluations/utils.py @@ -156,13 +156,13 @@ def save_model_list(model_list: list | Pipeline, score_list: Sequence, save_path def create_save_path( - hdf5_path, - code: str, - subject: int | str, - session: str, - name: str, - grid=False, - eval_type="WithinSession", + hdf5_path, + code: str, + subject: int | str, + session: str, + name: str, + grid=False, + eval_type="WithinSession", ): """Create a save path based on evaluation parameters. @@ -194,21 +194,21 @@ def create_save_path( if grid: path_save = ( - Path(hdf5_path) - / f"GridSearch_{eval_type}" - / code - / f"{str(subject)}" - / str(session) - / str(name) + Path(hdf5_path) + / f"GridSearch_{eval_type}" + / code + / f"{str(subject)}" + / str(session) + / str(name) ) else: path_save = ( - Path(hdf5_path) - / f"Models_{eval_type}" - / code - / f"{str(subject)}" - / str(session) - / str(name) + Path(hdf5_path) + / f"Models_{eval_type}" + / code + / f"{str(subject)}" + / str(session) + / str(name) ) return str(path_save) From cf4b7091471b0dc58ef250c8feeceee67107b875 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Wed, 26 Jun 2024 18:59:22 -0300 Subject: [PATCH 12/39] Adding examples --- moabb/evaluations/splitters.py | 125 ++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 11 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index c96f5fabe..59490f83e 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -7,7 +7,7 @@ ) -class WithinSubjectSplitter(BaseCrossValidator): +class WithinSessionSplitter(BaseCrossValidator): """Data splitter for within session evaluation. Within-session evaluation uses k-fold cross_validation to determine train @@ -19,6 +19,37 @@ class WithinSubjectSplitter(BaseCrossValidator): n_folds : int Number of folds. Must be at least 2. + Examples + ----------- + + >>> import pandas as pd + >>> import numpy as np + >>> from moabb.evaluations.splitters import WithinSessionSplitter + >>> X = np.array([[1, 2], [3, 4], [5, 6], [1,4], [7, 4], [5, 8], [0,3], [2,4]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 1, 1, 1, 1, 1, 1]) + >>> sessions = np.array(['T', 'T', 'E', 'E', 'T', 'T', 'E', 'E']) + >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) + >>> csess = WithinSessionSplitter(2) + >>> csess.get_n_splits(metadata) + >>> for i, (train_index, test_index) in enumerate(csess.split(X, y, metadata)): + ... print(f"Fold {i}:") + ... print(f" Train: index={train_index}, group={subjects[train_index]}, session={sessions[train_index]}") + ... print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") + Fold 0: + Train: index=[2 7], group=[1 1], session=['E' 'E'] + Test: index=[3 6], group=[1 1], sessions=['E' 'E'] + Fold 1: + Train: index=[3 6], group=[1 1], session=['E' 'E'] + Test: index=[2 7], group=[1 1], sessions=['E' 'E'] + Fold 2: + Train: index=[4 5], group=[1 1], session=['T' 'T'] + Test: index=[0 1], group=[1 1], sessions=['T' 'T'] + Fold 3: + Train: index=[0 1], group=[1 1], session=['T' 'T'] + Test: index=[4 5], group=[1 1], sessions=['T' 'T'] + + """ def __init__(self, n_folds: int): @@ -31,8 +62,7 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata, **kwargs): subjects = metadata.subject.values - - split = IndividualWithinSubjectSplitter(self.n_folds) + cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) for subject in np.unique(subjects): mask = subjects == subject @@ -42,10 +72,23 @@ def split(self, X, y, metadata, **kwargs): metadata[mask], ) - yield split.split(X_, y_, meta_, **kwargs) + sessions = meta_.session.values + + for session in np.unique(sessions): + mask_s = sessions == session + X_s, y_s, meta_s = ( + X_[mask_s], + y_[mask_s], + meta_[mask_s], + ) + + for ix_train, ix_test in cv.split(X_s, y_s): + ix_train_global = np.where(mask)[0][np.where(mask_s)[0][ix_train]] + ix_test_global = np.where(mask)[0][np.where(mask_s)[0][ix_test]] + yield ix_train_global, ix_test_global -class IndividualWithinSubjectSplitter(BaseCrossValidator): +class IndividualWithinSessionSplitter(BaseCrossValidator): """Data splitter for within session evaluation. Within-session evaluation uses k-fold cross_validation to determine train @@ -101,6 +144,36 @@ class CrossSessionSplitter(BaseCrossValidator): Not used. For compatibility with other cross-validation splitters. Default:None + Examples + ---------- + >>> import numpy as np + >>> import pandas as pd + >>> from moabb.evaluations.splitters import CrossSessionSplitter + >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2]) + >>> sessions = np.array(['T', 'T', 'E', 'E', 'T', 'T', 'E', 'E']) + >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) + >>> csess = CrossSessionSplitter() + >>> csess.get_n_splits(metadata) + 4 + >>> for i, (train_index, test_index) in enumerate(csess.split(X, y, metadata)): + ... print(f"Fold {i}:") + ... print(f" Train: index={train_index}, group={subjects[train_index]}, session={sessions[train_index]}") + ... print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") + Fold 0: + Train: index=[0 1], group=[1 1], session=['T' 'T'] + Test: index=[2 3], group=[1 1], sessions=['E' 'E'] + Fold 1: + Train: index=[2 3], group=[1 1], session=['E' 'E'] + Test: index=[0 1], group=[1 1], sessions=['T' 'T'] + Fold 2: + Train: index=[4 5], group=[2 2], session=['T' 'T'] + Test: index=[6 7], group=[2 2], sessions=['E' 'E'] + Fold 3: + Train: index=[6 7], group=[2 2], session=['E' 'E'] + Test: index=[4 5], group=[2 2], sessions=['T' 'T'] + """ def __init__(self, n_folds=None): @@ -123,7 +196,10 @@ def split(self, X, y, metadata): metadata[mask], ) - yield split.split(X_, y_, meta_) + for ix_train, ix_test in split.split(X_, y_, meta_): + ix_train = np.where(mask)[0][ix_train] + ix_test = np.where(mask)[0][ix_test] + yield ix_train, ix_test class IndividualCrossSessionSplitter(BaseCrossValidator): @@ -153,7 +229,6 @@ def get_n_splits(self, metadata): return np.unique(sessions) def split(self, X, y, metadata): - assert len(np.unique(metadata.subject)) == 1 cv = LeaveOneGroupOut() @@ -176,14 +251,43 @@ class CrossSubjectSplitter(BaseCrossValidator): If None, Leave-One-Subject-Out is performed. If int, Leave-k-Subjects-Out is performed. + Examples + -------- + >>> import numpy as np + >>> import pandas as pd + >>> from moabb.evaluations.splitters import CrossSubjectSplitter + >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8],[8,9],[5,4],[2,5],[1,7]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> metadata = pd.DataFrame(data={'subject': subjects}) + >>> csubj = CrossSubjectSplitter() + >>> csubj.get_n_splits(metadata) + 4 + >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): + ... print(f"Fold {i}:") + ... print(f" Train: index={train_index}, group={subjects[train_index]}") + ... print(f" Test: index={test_index}, group={subjects[test_index]}") + Fold 0: + Train: index=[2 3 4 5 6 7], group=[2 2 3 3 4 4] + Test: index=[0 1], group=[1 1] + Fold 1: + Train: index=[0 1 4 5 6 7], group=[1 1 3 3 4 4] + Test: index=[2 3], group=[2 2] + Fold 2: + Train: index=[0 1 2 3 6 7], group=[1 1 2 2 4 4] + Test: index=[4 5], group=[3 3] + Fold 3: + Train: index=[0 1 2 3 4 5], group=[1 1 2 2 3 3] + Test: index=[6 7], group=[4 4] + """ - def __init__(self, n_groups): + def __init__(self, n_groups=None): self.n_groups = n_groups - def get_n_splits(self, dataset=None): - return self.n_groups + def get_n_splits(self, X, y, metadata): + return len(metadata.subject.unique()) def split(self, X, y, metadata): @@ -196,5 +300,4 @@ def split(self, X, y, metadata): cv = GroupKFold(n_splits=self.n_groups) for ix_train, ix_test in cv.split(metadata, groups=groups): - yield ix_train, ix_test From 177bf65d4060a35e0d2c03c71e3eabdc8e88d3d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:59:58 +0000 Subject: [PATCH 13/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/splitters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 59490f83e..811d9d3ea 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -88,6 +88,7 @@ def split(self, X, y, metadata, **kwargs): ix_test_global = np.where(mask)[0][np.where(mask_s)[0][ix_test]] yield ix_train_global, ix_test_global + class IndividualWithinSessionSplitter(BaseCrossValidator): """Data splitter for within session evaluation. From a6b57723ee9deb79a8e1396ce52e45e0c5c7e404 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Thu, 15 Aug 2024 10:59:40 -0300 Subject: [PATCH 14/39] Adding: Pytests for evaluation splitters, and examples for meta splitters --- moabb/evaluations/metasplitters.py | 132 ++++++++++++++++++++++++++--- moabb/evaluations/splitters.py | 2 +- moabb/tests/splits.py | 87 +++++++++++++++++++ 3 files changed, 209 insertions(+), 12 deletions(-) create mode 100644 moabb/tests/splits.py diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index b85fe73bc..ffcf3186a 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -22,6 +22,8 @@ class OfflineSplit(BaseCrossValidator): """Offline split for evaluation test data. + It can be used for further splitting of the test data based on sessions or runs as needed. + Assumes that, per session, all test trials are available for inference. It can be used when no filtering or data alignment is needed. @@ -31,9 +33,41 @@ class OfflineSplit(BaseCrossValidator): Not used in this case, just so it can be initialized in the same way as TimeSeriesSplit. + Examples + -------- + >>> import numpy as np + >>> import pandas as pd + >>> from moabb.evaluations.splitters import CrossSubjectSplitter + >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8],[8,9],[5,4],[2,5],[1,7]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 2, 2, 3, 3, 4, 4]) + >>> sessions = np.array([0, 0, 1, 1, 0, 0, 1, 1]) + >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) + >>> csubj = CrossSubjectSplitter() + >>> csubj.get_n_splits(metadata) + 2 + >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): + >>> print(f"Fold {i}:") + >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}") + >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[train_index]}") + >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] + >>> for j, test_session in enumerate(off.split(X_test, y_test, meta_test)): + >>> print(f" By session - Test: index={test_session}, group={subjects[test_session]}, sessions={sessions[test_session]}") + + Fold 0: + Train: index=[2 3 4 5 6 7], group=[2 2 3 3 4 4] + Test: index=[0 1], group=[1 1] + By session - Test: index=[0, 1], group=[1 1], sessions=[0 0] + By session - Test: index=[2, 3], group=[1 1], sessions=[1 1] + Fold 1: + Train: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1] + Test: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1] + By session - Test: index=[4, 5], group=[2 2], sessions=[0 0] + By session - Test: index=[6, 7], group=[2 2], sessions=[1 1] + """ - def __init__(self, n_folds: int): + def __init__(self, n_folds = None): self.n_folds = n_folds def get_n_splits(self, metadata): @@ -41,9 +75,9 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata): - subjects = metadata.subject.unique() + subjects = metadata["subject"] - for subject in subjects: + for subject in subjects.unique(): mask = subjects == subject X_, y_, meta_ = X[mask], y[mask], metadata[mask] sessions = meta_.session.unique() @@ -51,7 +85,7 @@ def split(self, X, y, metadata): for session in sessions: ix_test = meta_[meta_["session"] == session].index - yield ix_test + yield list(ix_test) class TimeSeriesSplit(BaseCrossValidator): @@ -71,13 +105,53 @@ class TimeSeriesSplit(BaseCrossValidator): calib_size: int Size of calibration set, used if there is just one run. + Examples + -------- + >>> import numpy as np + >>> import pandas as pd + >>> from moabb.evaluations.splitters import CrossSubjectSplitter + >>> from moabb.evaluations.metasplitters import TimeSeriesSplit + >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2]) + >>> sessions = np.array([0, 0, 1, 1, 0, 0, 1, 1]) + >>> runs = np.array(['0', '1', '0', '1', '0', '1', '0', '1']) + >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) + >>> csubj = CrossSubjectSplitter() + >>> tssplit = TimeSeriesSplit() + >>> tssplit.get_n_splits(metadata) + 4 + >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): + >>> print(f"Fold {i}:") + >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}, runs={runs[train_index]}") + >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}, runs={runs[test_index]}") + >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] + >>> for j, (test_ix, calib_ix) in enumerate(tssplit.split(X_test, y_test, meta_test)): + >>> print(f" Evaluation: index={test_ix}, group={subjects[test_ix]}, sessions={sessions[test_ix]}, runs={runs[test_ix]}") + >>> print(f" Calibration: index={calib_ix}, group={subjects[calib_ix]}, sessions={sessions[calib_ix]}, runs={runs[calib_ix]}") + + Fold 0: + Train: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] + Test: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] + Evaluation: index=[1], group=[1], sessions=[0], runs=['1'] + Calibration: index=[0], group=[1], sessions=[0], runs=['0'] + Evaluation: index=[3], group=[1], sessions=[1], runs=['1'] + Calibration: index=[2], group=[1], sessions=[1], runs=['0'] + Fold 1: + Train: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] + Test: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] + Evaluation: index=[5], group=[2], sessions=[0], runs=['1'] + Calibration: index=[4], group=[2], sessions=[0], runs=['0'] + Evaluation: index=[7], group=[2], sessions=[1], runs=['1'] + Calibration: index=[6], group=[2], sessions=[1], runs=['0'] + """ def __init__(self, calib_size: int = None): self.calib_size = calib_size def get_n_splits(self, metadata): - return metadata.groupby(["subject", "session"]) + return len(metadata.groupby(["subject", "session"])) def split(self, X, y, metadata): @@ -93,10 +167,10 @@ def split(self, X, y, metadata): break # Take the fist run as calibration else: calib_size = self.calib_size - calib_ix = group[:calib_size] - test_ix = group[calib_size:] + calib_ix = group[:calib_size].index + test_ix = group[calib_size:].index - yield test_ix, calib_ix # Take first #calib_size samples as calibration + yield list(test_ix), list(calib_ix) # Take first #calib_size samples as calibration class SamplerSplit(BaseCrossValidator): @@ -119,6 +193,40 @@ class SamplerSplit(BaseCrossValidator): 'value' with the actual values as a numpy array. This array should be sorted, such that values in data_size are strictly monotonically increasing. + Examples + -------- + + >>> import pandas as pd + >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7], [8, 9], [5, 4], [2, 5], [1, 7]]) + >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]) + >>> subjects = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) + >>> sessions = np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) + >>> runs = np.array(['0', '0', '1', '2', '0', '0', '1', '2', '0', '0', '1', '2']) + >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) + >>> from moabb.evaluations.metasplitters import SamplerSplit + >>> from moabb.evaluations.splitters import CrossSessionSplitter + >>> data_size = dict(policy="per_class", value=np.array([2,3])) + >>> data_eval = CrossSessionSplitter() + >>> sampler = SamplerSplit(data_eval, data_size) + >>> for i, (train_index, test_index) in enumerate(sampler.split(X, y, metadata)): + >>> print(f"Fold {i}:") + >>> print(f" Train: index={train_index}, sessions={sessions[train_index]}") + >>> print(f" Test: index={test_index}, sessions={sessions[test_index]}") + + Fold 0: + Train: index=[6 8 7 9], sessions=[1 1 1 1] + Test: index=[0 1 2 3 4 5], sessions=[0 0 0 0 0 0] + Fold 1: + Train: index=[ 6 8 10 7 9 11], sessions=[1 1 1 1 1 1] + Test: index=[0 1 2 3 4 5], sessions=[0 0 0 0 0 0] + Fold 2: + Train: index=[0 2 1 3], sessions=[0 0 0 0] + Test: index=[ 6 7 8 9 10 11], sessions=[1 1 1 1 1 1] + Fold 3: + Train: index=[0 2 4 1 3 5], sessions=[0 0 0 0 0 0] + Test: index=[ 6 7 8 9 10 11], sessions=[1 1 1 1 1 1] + + """ def __init__(self, data_eval, data_size): @@ -136,9 +244,11 @@ def split(self, X, y, metadata, **kwargs): cv = self.data_eval sampler = self.sampler - for ix_train, _ in cv.split(X, y, metadata, **kwargs): - X_train, y_train, meta_train = X[ix_train], y[ix_train], metadata[ix_train] - yield sampler.split(X_train, y_train, meta_train) + for ix_train, ix_test in cv.split(X, y, metadata, **kwargs): + X_train, y_train, meta_train = X[ix_train], y[ix_train], metadata.iloc[ix_train] + for ix_train_sample in sampler.split(X_train, y_train, meta_train): + ix_train_sample = ix_train[ix_train_sample] + yield ix_train_sample, ix_test class IndividualSamplerSplit(BaseCrossValidator): diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 811d9d3ea..90f4c5472 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -287,7 +287,7 @@ class CrossSubjectSplitter(BaseCrossValidator): def __init__(self, n_groups=None): self.n_groups = n_groups - def get_n_splits(self, X, y, metadata): + def get_n_splits(self, metadata): return len(metadata.subject.unique()) def split(self, X, y, metadata): diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py new file mode 100644 index 000000000..2c125bd53 --- /dev/null +++ b/moabb/tests/splits.py @@ -0,0 +1,87 @@ +import os +import os.path as osp + +import numpy as np +import pytest +import torch +from sklearn.model_selection import StratifiedKFold, LeaveOneGroupOut + +from moabb.evaluations.splitters import (CrossSessionSplitter, CrossSubjectSplitter, WithinSessionSplitter) +from moabb.datasets.fake import FakeDataset +from moabb.paradigms.motor_imagery import FakeImageryParadigm + +dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) +paradigm = FakeImageryParadigm() + + +# Split done for the Within Session evaluation +def eval_split_within_session(): + for subject in dataset.subject_list: + X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) + sessions = metadata.session + for session in np.unique(sessions): + ix = sessions == session + cv = StratifiedKFold(5, shuffle=True, random_state=42) + X_, y_ = X[ix], y[ix] + for train, test in cv.split(X_, y_): + yield X_[train], X_[test] + +# Split done for the Cross Session evaluation +def eval_split_cross_session(): + for subject in dataset.subject_list: + X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) + groups = metadata.session.values + cv = LeaveOneGroupOut() + for train, test in cv.split(X, y, groups): + yield X[train], X[test] + +# Split done for the Cross Subject evaluation +def eval_split_cross_subject(): + X, y, metadata = paradigm.get_data(dataset=dataset) + groups = metadata.subject.values + cv = LeaveOneGroupOut() + for train, test in cv.split(X, y, groups): + yield X[train], X[test] + + +def test_within_session(): + X, y, metadata = paradigm.get_data(dataset=dataset) + + split = WithinSessionSplitter(n_folds=5) + + for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( + zip(eval_split_within_session(), split.split(X, y, metadata, random_state=42))): + X_train, X_test = X[train], X[test] + + # Check if the output is the same as the input + assert np.array_equal(X_train, X_train_t) + assert np.array_equal(X_test, X_test_t) + + +def test_cross_session(): + X, y, metadata = paradigm.get_data(dataset=dataset) + + split = CrossSessionSplitter() + + for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( + zip(eval_split_cross_session(), split.split(X, y, metadata))): + X_train, X_test = X[train], X[test] + + # Check if the output is the same as the input + assert np.array_equal(X_train, X_train_t) + assert np.array_equal(X_test, X_test_t) + + +def test_cross_subject(): + X, y, metadata = paradigm.get_data(dataset=dataset) + + split = CrossSubjectSplitter() + + for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( + zip(eval_split_cross_subject(), split.split(X, y, metadata))): + X_train, X_test = X[train], X[test] + + # Check if the output is the same as the input + assert np.array_equal(X_train, X_train_t) + assert np.array_equal(X_test, X_test_t) + From 26b13d56501a0d93d802b345001ab45cf69da389 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 30 Sep 2024 08:27:03 +0200 Subject: [PATCH 15/39] Changing: name of TimeSeriesSplit to PseudoOnlineSplit Adding: figures for documentation --- docs/source/images/crosssess.pdf | Bin 0 -> 24887 bytes docs/source/images/crosssubj.pdf | Bin 0 -> 33857 bytes docs/source/images/withinsess.pdf | Bin 0 -> 34008 bytes docs/source/whats_new.rst | 1 + moabb/evaluations/splitters.py | 12 ++++++++++++ 5 files changed, 13 insertions(+) create mode 100644 docs/source/images/crosssess.pdf create mode 100644 docs/source/images/crosssubj.pdf create mode 100644 docs/source/images/withinsess.pdf diff --git a/docs/source/images/crosssess.pdf b/docs/source/images/crosssess.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7a39c96002972a508148490b2f9698f6c2ab7175 GIT binary patch literal 24887 zcmaHRV|1iJ({`}2oouq<#s5;{JLw$YHdWgEc{dp+h4`mWlojNtuJ7bUaRclqwoV0W zmD@Z%8+>*E0b5U)U=QhVPsH9&@AqT5eZ9@oXrmHKr`P4fQd+3bms-!9Q7!u6c|p=ZZV4KvhQ9= zmQ@KJp~;ltT)8pHqnMPekGcV!1ugqM+%gue)Hx`b{BoVql>LR`<(pgA(5@TPY%ckQ ze>9iaTss#^TFxbMVHa6+QV`axHXB@?H4z?w4MU=C);G9KD7X>rViT-;QaWSMPb z;k&6nQ1=%@aViQ;EC!U8PBRRjJ3`>M`VBJmA4rkEM9`>y>Vd|bR7BOVo8W7)oBS=Y zTGGYG$QlZ`Qa4QI`wQk>^0!^~r@`Fx9|U&;p|mMOydoA7Lg)vUUG^qCmktRJSE)2r?Uwtr{1Ht;!OQO?W2` zPyGit(#7B%kpoM#JjhF+d=)Phsxr zLqgnuUYBF1;X*|(;Tsn7pbsYBS7F~w*_(xMGkYyh-Sy8fCp7f8F+d`lfE=JwNL5XM zc8l)2zc+bvAPRFREFn~lR%|EK54%s@H%{%J^W2PMTh;)EO+WNUnP#(E#I36(cAlGp zjbydQWVI-wzLW4`t@r_N{|r1IQ}6y=XXfsFzjek9bh_^Py!d!N3d6Z0ArZ9l#1uH@ z{lvvmZC@qj@!Hra8A?!GSE^w8qQ6zDNK%}pB$cI1b=}|3-F)Hi-tF;iQ>XgQDK1;t z?Ns*mq4e;ftl_PYvdUfDZ6zM-7hCdvrt`<} z>a$9xqPvmzuO;rS3pax}FOO!!^M!>GY9tKbs4}e!{3B!X_&d$kc;kwzmnV`Yi|%r{CI}P6OVhB+R`By z7SqHDSwO+QM-Lu!4DZ_x>T7gk=_|W832|3#i+OYmd1S!q*R|#MvB58Z&+~U|5@Bq1 zj#0=5vOR~0-lPLQPEP`NoRhHDLTtv*F=BS(qYfe#&q2AN2nCG>b1Tm+@!w8RXXPJc46rqEN+(|^vRvxzEWsap- zJ$-sr7gHH^vreRU^Ni9)@!C?!8(rR&2saAayP=_`;O%U?;?tY90$`$y*E_aGaT`mm z@rHFmH#Qc|7^x7>0*OP?RxUcyNiHYSO~NL^TjmA%z4%tmu=mNdesCv05+Z$8N31pl zalofYO(rv1*ISC7y{Si&zRBm@tN3}@rTTf!CyihDpJ@)kjwq~<}} za3u7{d^;21P}L<-Pck0}Rv_MxxV0Y$fTxZQQh;61c*HECU@H#-?I<3#l?QTo%XZvb ztU=djx9rrXYuktKi4fYo<=5>>kLY>jgn4xgd6f!)S_=dY`e&rNmy~KJ4)u^yKlVikp?yK9n zZ==C4)?9C%_fEdkS*!`TT(PjV+V?(o?CBnrv-*2k&y%=;!QBC^SB65~?|c#ObK&r} zNU_hzs^9c)Nryg8G^;C4I?2l0~-s&|J?&IvU6}U{jVnIsvE{jd3l}xi}!RZ zojY;;#yCCQIAJPVnrzcp&o3yxod`xtz-3IH7{(x(LIR{xmDE-x5KiTCL9`CDnT|dY zgez}(se+a`Kx&&RAgr@|sbCp)EB91dAkTZZXY#h!i_dO0=rrpw>y*dLbLNHSWd;OH zfXMHMbhd`-y2*A9XdH)M3um;F<_scDCs-LbEQj4SM{RfH_ACv^Ge}#}VZgHG3NhfP zyfaarc5{){x%;VTH#6w5CLv`QhzPHc#?m-j2 z$FdvL1lXziekM4^%(+!qq2S@HdChjW(%A>~G8AWu4AAdPdqlj~>OOmHvRB6H)MgoF zAoh7Ff*Y|K^TIRW_!yRfi8JgsMym@YhWjg$FpEP@{HF7?jBhMqDVxhPEZhei9*QSl zk;hsPcGQ4hsSXc~_7^ro=c4jisn1Y>%80ni(wVgD^XQw(h`FFiZlA6AcGJrj9veH0P}Ez#%XnhQ); z4^CDOSf=0+zTjVZqh>^!RPc|NXM~%q!@k&EmKFt9Er8MqXfb-W*OJUHz|THI=SnrA zOfhJtun3`uZ+!=&86hUJ5Keey2aT*p<{kW-P9@1f&?uY2igVotOmEDY1>*DrNO8g3Zsdg-U@A65Ot z^O4~$_dV)#8w+o@D`w$mNEk--cyRNSZPx|tvUWy7vlII%6$MGGXM&;Tx8d>cJ=pX! z8i9}X;!imd%jBcR#HQgp8gviY^Z?hUv0im?nh|$(3J=ftybJqi*|Ca%UdgpdN9Sz3 z;oFV=MYTd5!dt4O2#cZBi#N>1x2V5<+=chK(?)^kP1yPj;zB5B>7?Ou8{Z|ihv^;B zrI#m+T8az4QmtMYwj_kOa%U+f0tbwt%TxlVQCScCi#ANTWV4Ofvhy~39m&3Bo5oy; zP9@B_<(t2At6DA8TNh2=p*rMd-*SPTWFyKf<6~O!2lmh_zACN59+p!-emiWVelwHb z=d*!WL_8olYtekOZkrO0(?l||5|0mrZa}*!D*m{1Hfzoqf8&OEw;%;ojkHi_pS3<9 zbzfW%am~KD`p3*^#ue(QDdD$mDf!enW-ZGEs$u&ATy%n0iEXCF(|eum4lFCLPTkB{ zdDb}RB57%sDwaS!dAJXIB$)<{XD2hYnmQnA5r;vxSOk{1^69xqYco^hP_O(h={-tDbcEY~g<8qM?H>}v z&@sXd__p1@u2To2PuWoNioh>f$7}Y2v3XgwpLL4;X@&Z`C)YO+*XRPF`pO4_~KC#$~ zEb*MY)Cg%Bdi21kGyt;>sY*F32zm4;FxxLR^&iF+w)$T|FadwXW)^%)IxGGysT0X- zo}qo_5HYe>naIvbwD`UR9shI9|Tu7n#u5LMMiN_^K{g>ERCVCE~_MC9GeuA4(AsP1blSq+Y*k}0vZ~b zx^d3&Td7$|*ULGs*eXfKtrMal!xaJ!O8VN^SW{Lp6Cc3P$dc~d@^qq)n{x-uBg?NX zt+&pZ7@K9n%1fmzfz~)qf~JrnH>CtRYm~i2{RXQINPx_Bq~T&oD+{Z~=a;#JLllei z56lr{Y+R?HAn2bWjEqLY(wM_#lkQe^ve)$ZSDkz{`UYBK9TatC-ab-M&41i+X5^=S znMbwL^Buj>X)5AL*_yYXEGg4pT0PLHxi{6`{$y=&@R%$w_mM4BkOO9b9BqD%ja85}&{29SKU;XJaYTswc&06aDe3TZz38G=M#|_E zf{nLi_Uf&g+gBP<*^i|jdSntC|9m;Q(l(nyTo;w7&%(4n#a2 zrvx6K+-WV9^fv8x*=D(DQ^X1WGTCs5pu#ZzjFRY2W8`3*JJR4Y(uFgd)1N}d!!p<0 zVI#UYb)rpKm>cL~$$(a%N2@R#M|hO8Jc-AoUL}Qy9{9oA z`+GQal(jlmvX1OM>a7f8E{N3U2-HfOB~uuof^ zvBb#>VrzFYoWy`gK^y`vaD6l z1gz3M!%~U}F*Jm3mw&FbX}q;2lh`6*YIEi+Tq)c!sldRp*)U+buQbT&8z@SfDPEEM zUa+kYm?`K`WY3F@!UN<3x0&eXn&^dsU$x!}RV4;uk$O7l$BLk~zr2FJb%-1zkVkp* zyX0`ZiN!*U7%`*j_4TaJ&*x3Z3lCp=Thh>JWTfNLlE|$W3Ak5rFQYMZnd^p(0sAz2r^psle`hNROny1fIyyctoWCHfsM z5?okK$0Bha$sA|nur6UCHn1LT#!!3UG_ur*mU~2Q?QM2S{P;I|IK%=A=!4xz@WL)aCH=M5Sb%7 z_NNh1d_5SR1_QK?_gwMrTbYuO*BIG<#j zLCabz8Er&i_NJby8ab~pD3U$M9cgrvL(;@Ycl-%2r%YK)IQSxLciAXc^6{Z5b{~1^ zVnjXOya)WAGR&P1jfWt3N5d7u=3`^gpL!TxFV#|yr&r7!LClZ#4nnx%y)wHM3>ZKZ zG`J-i*b&C!{*nDFK#;)r*4o0dV(H*2n>Jr@9|;{@)%Fwy949@w*YtKSG!7}3Z|omZ zTNm;1wST=)C^(52DVSD-B-P6*d420WjwfzX_w>I)!gwnwL5t&k58O?p0lL6Ip08Ak`bSslwSvB-T;Q1YjC%E3iE;T{ z*gHjEaYGLIOY%IXfZXsX8%^uT;&iR2|}7sG&O({zfDKG{~uXn?oK^d-%rae%(P0Ve51Z zXY01t^M&g~V0WSiUMXIgqzWxSt&@@)(@Bga)kz^6rRvA&1ax1aic^?X=GCiwC?IP= z1Xl-4sQT}NXax@j5WK4z##ImK_V-?Bl|S!F+8ioI+^<7x(b|M z|5u*4$*(b!UuCAh%#DA=sgDZSrrSYr|Rl-B@3VFs61@ANuM$PJ|r zmsba|HaXJ|+5!SeF$J2;DL;d^Hi^g4hxaiO$0;(=WXEYU1Q;q@T!RmXdO0{w3f%wg;b?cyYg@!3XGi$q4l(-+ z8td&M+nCakGWA*@?jb`OpUwd+Jxum>>{aBX#NRNdG9C+=90A-PRxj)5QIk&=ER8JU z5#fgjNh2~c6%z__@;_nAapth??N(Rr3+Y%MU{CF+4i>v})s|4OC|1>5dhY>qTi-WR;ZT#!NUe3E>&~lF^q? zO~}5?q|0UYvHWtxaU$yZSqnpv)JbU+WJD%gbsme*35 z!5tD!H>FYVlFMhK(e(gWvz$4QfFM9gQ^wj04$quznwWMVpBN~gGl?A(j zHS@ubpx$83galm8jjpdV_0$Dcv8niZpIMWgU#MfBbwLG;R{GXcnAp?_k{41J?g>BQ zjZd*!1|@PP#ooKQ$36!oB!M^%b|f;7%p|VdSG8ud|DdsmOt88)0D+tuQcZG@T(I$z~|1FK92=A0V?Z(KoR-5s;i&RPJBiUk;(fE)g&A z8=ErBI$;9Lq`x7cK_-EI4-Nq9m083(Ad0KWay)*Q!=Qtsi`zrlMbyP+io)oogmrsCAD!7Rs&tJ=_fMKR>N3@YIYdusCZ6^ zPDwMFYgjc5!-0zSjE0H|jfPxSw_S=_jItS_Gpcpyb|{?h+G*1*u}Ptmstu;ZTvoK5 zr&*+d%$2S)oz76PTB+LiZZeHlQ!+LxoK09dGisVQV`Bxly@(212_euvbDrswes~M1 z2B513fKUr!Q1?r#f`R+I+nkxG2Doa#d<{qv?oZeq+rFvZ2G7n9FWK@qD+JR_>Y1pKN(ovoMb8mGEv ztQ^`^Cxx#{P=P(RLUm&3fUOkz-DHX&ls{)Fs~W7Dh%7`jU}N@NArJjQkXuu!>NJnc z>5Um!r5UJFIoH&MQmGoRQVHRTG20UIPQF#KEF=XTf(trm{0g5KPbvF%ZWr`P*zarq zUIL&n2>T-@w~|2r17~Y+@cUJ`cC!w&okN=H4jqya=A{6C(Vlw52#IYlCUT5U96Ar> zg+$yd4;zY|aZk3(|Lv^blfw^;<j68hlOE%xZ*GtDpe(3>MZw)qt$!GSJkG4`QVr5^buUTW;m zT_M6{k!87@p8xz>2!EWZH$=%E@!;i&&Dl__J(5GBpm*;VXigk7rV(vTUvnE8ICfES zRezmA*QGP&j2JiOA(UHSQ5 zDC8rHM;!k*eF8s@E|#5IP5#<&7km_ER{IKUnO=5-;}WRvtgOFAW-Xm-393a zS?J*t2qvj4%J1LTK7HG~7E}ya5zN*@CVgDTAs~>kF2`$Mutp5RrGxSHqER z7D3rvvbCYM9-ZehGUpa0nL!k<3G4AT&aChDxV1aJ1a)+}=xRxcU^Slep zA(057&zbR2JW-;|C5Bw4hA1ohy9|CD2W}X8$m9wV5ICYUWXxlVj%e-CL*uwd2$n@Y z8#4T+j4hGfq|pr6Ph7tThvmMZ_32Xb(Ngy{x1wa)2xj%kB}m+hf|3%|fD|dN47LR^ z52S73T~oEj%%Z{&0(^e^kHo{>Gw-5;bRh#8X*43~|F(+r_z zMPeLWx6(cBp=!tQZM{|}Z%0MDAv^GdXTO5~+gHLPBJ@#|O-XbbcRxaPWNnY%62JM4 z6u%p1HYWRk_?F;njyRnU;rt_d4w5T|u2^lc^1Rxa-aFZ6j7bAZYtlo5zB7_{U?=W% zQl}$-LEa)wU)`*2woQsIj(iMGW|rU=m#Q<3$JO;AM~=gX=cpa`SLPwXnbT9G0I~>- zqytP%f2=+E%?O$G0Qg$epF5hIQLBQQTr^4eAsL?E0ukbj<%B3qInuw;#?hIwrO6w| zt(j7%NFK$xHN;cmO^j*JN!8-0$5oCXAB^tX1GgmD;`QSHjFB8EGz4A|`Vzj6zaJSr za6&+sEXf42xp5`Wl)}uDO06P6xFf((WKI$ypGpR(oOAQe1(?@F$w8}m&55Eu{^XsY#-r{%8VxX z89O27p3x=WBjO{vA|uEXwYv{$&3(=2mhOP0VIrPPSM+|*?53W&|C0$<5c`~FW!%=7 z9w#MkP@2>(fv+X2dhXWX@J!8`G+j9NH|LaDnURz|uk!<;cPM^7fw+Y+=4TkXI6rah zLUNUVL~AZQy_Cg5atIPA;OE6zv2h&VV2!cbMXy-)K3&7EJ(f30g|lCYK0nUy)wxa3 zE`ij=zvl532)p{6>3@7o+TyeYN9L>~P&`FD;BM_IHkB_fk;lTnK^wuN4Cn&3km8kj&^!czPo$2k-=s4?3RM zR(r&eeH~?DaDxx%NBdUgD37wVFxwA!er`U{j90RLIw1$j3kJwj!weSC;5He4tX3mV zM17{NweVxWwfUTHG!7scz(q56yK-toX?pQe2#s|Hj_q#+ufR2moKSJk=8{m#2S#}nX^4ur&+`FuYXF8TvV7lY z(aRU}cb|CBM|$;k(fc>d8#!W9WAbIufBEhj8WVfqtXwz}1| z+~OMFf3jn}TmSuhBW1(Uwe9;u;oJ?x3Zdsi4;%XGg-9wJ{SY%mw^Rt7hd!i<>Q|2% zvFc35kA%xase9gVw5ph$W1U*_X}vwMAWEf$=^&-K3N)rW*+3kRDd3E2%kR}=C+St1mM?2}_k zd?A)ZLLQC=z^HDp9NSM;_KWQk8?WyCPkvuML9o)CF$F-ONMOEKkSYtnE-u{gg`h3I z-0P294%poj%=lt28m#_CzzOi!pYIhTV|uEMCpBm50<7KVtTIK`;+p8;X6{} z8w{fSNqr=={2f8~0nm#N#rv!s!)gL)2c%Mn$bX}XjxF|zK)(SkG5;?JN%=Bge?|0j zy*%kT$Dk*YGJ$%ZAITK+v3ZhnmPzNnf(R4Oz4}3UBntW85QXyPzA>80=edQ5tbkK& zT61iYr&$L~u`}Nh?5L&hC}^>B-;fZl&)Kp+**@4tjGFvWkU`Ib=0h3)1D1W7Pv$6L zk#G3J{PSFcfIKvX+HaUd2Gwu0jQKpEHedD)qcm59v~L}?7|MWg_)H{?=FCNQmTL$} zcAjfklGiv_qmDU-1P~kXGUw;GS0F8+8lAMZo+NJ0Fn_80CO! ze2T3DX?%{I3*iL(F(1|bB%vA7{uFl36a7YxWRL@jd)`lBn*Nt3q&oCR;NL9Zs{v5z zbZ71AdA+e1=8LcPbTqAE;h zgp~78d(f22&^zy{x!vL50No(jPaW;3$0yf&@T^*bA<_Wd2)a)~I^k9yWUKk=-ay@_ zv)et#s)gNt40WJMT`$J(S84C>1EO~L4gmc?#=3yAN1$7JIn>lHx*h%d>fb})9Z)k! z3fOGJPra}^gw!tSjmCxOhUo_IRgE44_&%W3&fe`C^IG>Ly$NiGRRCJAj5RB}12(Vu zUzoO|tUx08>c#u(oAhpI?jDynfltI=APau=(r*7WU?&*&skIK2{ldp8VvY3{U6#uujZF6;|6XS5I?wHDp>g{s@< z@+o@Nj|dbPPog{}PF;&!@r-L zUNP)Xr*~Lg<+np99}2tS^gljP?asOH)w-&`@%0RPzdXHY&vxwvzIl84xdFHK%5z3; zYTv(N4&%PK&)npE|LLWD_2Mn^qojR>_dUfzl=#vxd>Z+m6TnG-i|YgafGzT)rhUZ& zp&uo_zZpJBfDnxIx4b?e6bPZGedYH(Aps#mhEH%H1TF2Wrw@1sTinG;`|9a?dVwhE zC1?2L^*_Ibk@l6-2N3z6hyExbKuz-jM(+grVWfT4^gX#kln|h&`2fQT`kzAqNhA!P zzknoQyg&K?P~aSpgq-FhpbzlzKYxcPxy4HB0cHXkc5u?ZTKa&^ZvsQWufeph5ns4P zJz(~ZzNb}SAUcN6zy9a7Kqno2fEC!H9$MO01`zsD(o4qhsRD#xrF~WO0hM5j-@r{^ z04IP+MfCyHKwr$XuZF&-RA3ljRY?8M>4BS&&<98XA>h^v`kt_W5U?tuKnNH{K_3ta zw)g@oy-UsDUEBB64K$#ocS#t$WBZZZ1!vDMzxDm{>9^mpH!4_R% zq<2Xfyi@z0bpG2o1^S?-^+@Rh=zxSDCH%m8a{8ao0+%AI50Ldg=LR~Ur}Zf518%>u z3_Q+&CrU5K;QiRk@c9iPUHajg1fPBY#cJ$4QiIt1r1XI84JJ1B zG!BYg#Nlm7Eu1g~z6re`BQ}`$1oP#oU4(4f5Re5fC;rCdMdQ`t_?zcg9XtN6zRr=w zi$+`F<~kHTRc~_vhIsUN6~GO{W0Kz>?=HG@`@|jSQ<`fm$x+eJUzh>9E07HNmiRUZ z!u&I(KNwV(oL1$r$(_vu(e1Yz3sN@-|BvR?(zxRyeA9;b?~yp@%rUVaAP*RL#+=$k z_=LGuE(iAV%%qTw8>n zDJSNLf?LT^UC35Z&1ieDxg>j_gQ+IwB|EgbwZaA2$cNYa{bd_5a>UD+6m?N#U{pDk zbm&|&dQIW09aK+Gk0Bh728|Q=NqN3GY;cby69RGi>lbcszD_c76xLa=i!?B>7F%s; z6&t+?KcqIV1P7Un6A2<$^oTrZnr-DyG?wvcAFD2$W%1VO)bjCC$R~OeqB8)z7CGmh zH|{wYCCf9*uTRx?aM3ch*vO}_Oc0G6Od-pl>d}zI|Wu-_o6g&DeQ*I<(&D@O} zHxKHKRZ3&!dS|>7sXP}b|C$wPEk?7Y$RdcvZYHNtm5R4;Xq2Q+YgVu>SdpD@R4!CS z5FM5{HxPL{amnoVSM-e`XhAodj*cT_AdCbmDPa~>No!;h7Y~J372nY#nF=f~=_WUq zPShiaOx5nsUhV`X6(eb&oEUv#A{!YZN z5Sc*7-_6yp+}IzfyY|NWGT+W;P8~@+;xmg{xw1HqrC-%x7^P@yNhJr3tw=4}iI)oZ7`YH zXt;|eG58|LD`jJiF;4iSkW^5))8m*>y;J{*!XC%hwtoV#AivN#rQe9dlHPq>lzE>Z z-+q6_Zm4s*X@fW9_>zf>B6a<^c<@|xw)6d${4ACFWw?1iSeO)nYWv;|;7;hT11mhs zHhfx4^wpNe8xo|2M=S>k4w+MCO3ujs>%b(t$!)k427i)_>4sTEKc>FN`qLVcI|A~u zXgOuMW%(Z*p3b^XPfbJ3!=*HTleSUkh)?oi#@y$^r;21l^@OTT<#@$J#Y_byiAj@) zVxx+lfq}#_HdBaL(Zkcjm7`w!chyCsCD|Go`gFro{j|yn`V5T8TUe||r5EcCN+VC> zf|-d3^{Lu)ZH^wZ?@(8~i)lT2iF#KsXrv~4M_pG9c{}-W#}@Yy_rwiNZ?ART9=qsl z2;?Q=YH^Ou&*Ndqi*4zD$dW6lLN0_e8LV2pYJ4&A#6+;x8o4TCql&QS4qJ(L2{_qT@ zto^*WNoQJe3d+=*teBoP+D;bQtNR4sCj z&&8yyuwx(3M%=0`OF7nvo>?h>Hdaa8PU)^9LdUK~R6e8$5!uXK$=Au{sKndo7GfM+MF-S?5du!mO}N5sGM~}vcps8Ow4NV19RH)p2M=( z@sEv4dtpj18uqg#B`Yh&`8h_Cs8cN;N}6g+s-ChNJB{4(<4qRvII)YUgrsqw!oA7G!_2c8P(i_c)UZhAtMb?R*8tJ4%; z?I1xMFX(I80mIv&v%C~zA~H)WZ^D%y(qbJB>OqshBtv>MC8cxA`h~r{W#jz%30~_Y z*2GC|--bJ)G4(oJ1G}lzI=4BfHlQ>^RN>584HC;`_ z9l`p_c_G^{r7HPwl6tmK@_4g}iWi6yn7daW*o<;(Wd+r=o{|cuqewJ!90_i;BcW)= zX1tliJ#F@L5xbq`di`$F?D*g9#=Ja_gsf6*G6G*qRom?`yBO^YYFNn1W&CkVx$`W! z5kJEMzM&eUOoi~qwT>YlA+LO`X!EXpl;{_&3^h803>r)>ZVCV$w3UiBf0af;;`tR7 z7FK!0rJddJFI9}ySaXZ6+{z$pm1-KV_M)VhW=q+UMybj;S`5roH7(Z0;F%Un4UH_W z#B*yaZC}yJ6Cu^grWIkHfJ+oqe9|=9&`8oG1ksQj#9+-ttI5}(4IP)LTHM+om&ArS z+kRLC#2dKa997+Q9R++?_vYl3sUE5f9;Y7?s@~1r@$b{Izw3|WABx+WE<)AKG}Mfy zBwdy%<9EW0XR@r*tgRI@9m!TL;x8MdC0sN_XG9jVSmv4POWcixw}%=}G*dKd*z}T} zlZP26{}x9Ah~bEvrj>&)>@5r=o{fg65!VA_)2yl?QZ%XUJjHaP4YN&)GyY~YPfsZC z>*yy}8OUk->QFZ=$v)HERXx=)&1xb&Yo91V+c~(a!!6p^FR3hO2}xPPMou1DXPqR` z6bF(P714%KWiD&#=m1M+tAc)bRWbUi7LI&Gx?zR#1mPKY8JK{{gICloh^pKsf5*51 zf@1;vMZT7eRyM_QeWzgqR-CU^q%@(rXBwMi^AYuu~67Nd-- z$JE)hFOJD^DNJF>oQ*WSY(BT}MBXU6ST&fLniOrE5;2!H@#u#bt|c?$gmQI4hNYBa zNrJ=`rzBZ9a|ko12pNc~fDMJRva<8racUy|LbUC|cHl}#ioZCMMRdXnvd7PpR_p4y ztL9>d&ptomu0L1JBr2ZOKUewj>Kt$VOzW3Cl%=Y*nwLx+t!C?%$S~9`|6tnlm%A7X zD%vRex4RhoYY04*(?zV5xn_=%m}pfVK7?*^Rj)i)lhXONH8X3`(MM%kYjID~DstE| zAvsH?7_6Oeryz&kJVcn!QQ(4ey0b2m0Pj-D7PFkU>m>`$Vh1Ztk!@e<=nOWDL zCWVZ&%PJ}oqq}k32UQUKV{}{3=UeIV`uMbya4ExV+Ft+8rjCBM3fGqeccv&W#cxKI zvd!SIzEwPCovUSS%Fny)XKh`k937#rCabEZPHE_!;o^1xUT3KPluoqVm2LjCBdXqi zI%wH|fqO-2XS)df9k?&{Dlgw+>8^HVNd+@u)&c^Y(dreG+< z??S>Rz+sFH`VaUOf^<(a@?$p$+=x0#QWj`t7-=+hF+riNq>VIvqZuBaI%wFi9$BXg zpG8g;l{xOA;=~(Sr#hyr&{Wl=FcvwCYK$IfP~giuw-RR(EO1i3v zzS`7LxY>?j^0uFRyG6JlIS+coP?Y-uC8j0DwtA9!pyKqv&q2d zkVzpQ6QRXTXW_GuUHG-&Es8sN)ZZ{jR8b-S$^8UgcLvo|Qpcsj`$s2Xsa&_pwh!_d z)V=e=I-YbkDZ#Eb+04|86_5v@RdmQWW7T7%r2EGhiG#z(Zt5Gv>6kpmQ6-sVJ47Hl@z(ul zALqU^WJjX)*XuoT#{H4ksoN&D`Z(_g<4^vRJ^_g2-UXCsCjsm!~l z2RustmSnjXYSOm7h^U%iOc%6G^gNDk_`OT>DfBZAn_iNwUGXhy`p`5t<6=@aFb%ry z;k;k#l3}K$lVj5M#f02+ut(mUfsyy$jhJ!S^ZCk(d~unTl@)a^rl!XOn4Fad?uMeC zNr*FR<4U#ZN=MU_CPA%=;$V?wVr~Bl*8sV6eV%5W8D_GoXY~0=N|KW{+}0z{?qnPc zGpQ}so=^GIi) zm+?i)i@NCoJJgG|H;+A9^ma)`4GKNVlA`U@(Q2%u?QHIX8yigs7VnEiO)R&QsPPAY z9secGaC%nCEl-`yIPJ-Ix1Q8Hl=xiscG#|FL@bSS&C8DQVeLPPN}e)BTTMG6lQQh- z|1#4I7o=@U=#-|M`;1w5_2m-nf3;GpR*|M|vc9u$2kO*Uy11;#UUiY3gh(v^3%Lq8 z8G^sLIZt=2;;zE8y0*PWq!tEV2R_oLXq@P<&m6$ZH=~^?x=Bp5{-Yn*U4hbJ#|NRkp@$4ZX*`E5JmIoFWHrsX+*A(qZV61<4%3}w(w?T%o3*F$T!Odp z#?cz07u=+$7!>B(qQYFg?vBHOrOO?nqiDIXu1o)plr4pg;}0HY41Fxznm=JHxdE{K zsP*_M=3W}qDDRebVH7STjO%dW;iD1_yfwa6q>te?a+q;Mn1Y@0^uDGp@={F4kd51Z zg>xw6JO0LnFedT&f+h?Np5i!?G#-KPLV**D6OA8_EW58c>i=2>i5>`LyEIkv8QC`x zl!#E!(y-sANRz<*%0@0TM5he^eHyb(yPQC7|7MXPl5|%vXjn8)NJOW<9~qCLWlxSv;P`a(Lk~U>nYV@r z{=L4hZ0o`yPRIU0Yh$xKP+KYW8ZSODTaDavGV!EV+lt1=in-Q#75J~)rI5GV4T_n` z{dIuQU`i=uU;?GS5rF`P`nDDS0_C*T{#9L$!*NPuQ*)#i1D9?gA@Hrdrb}N_cTOcP z{9|cLnyIVez8Ic&yL!H?bvGL~m4A6EdwpG&eNXw|fjrh3lPelBQ3LN)S|S`=Po<1$ zfLkMLy_fqUdhbtnMn%R2is2>s%&oio+m>pyg@ua6(azLfTKece49qAQ0&Mg3mF}+M z8m>(p*74Ix>SZQ=lf>BV%}k}NOs%=LxS34ko||yiXwt$4tB=#-B4rxyzgIuxzak%% z*|wnJ9M2|ht6aGqp9N_nGmRT$)+(myT}*wD*yVjALemccxh)Zof3^ARRPbuvEJeREO`$eR|AGCT>> zfmYL>Vl2s{d9b8d>gZHiU*+mLl*;;Wp_b#vg{36&(QtEIHB(A#NBE!0zB;ImXX};( zcL>2DxVs$OPH-o<93Z&c!5xCTTd)w^gS)#2hv4pdaQ(>r>fO4x>R0d8tLZ;__ujj^ zX7yA}t?B92kFYx)^<3QO@P|iO@0h%|rM9fcq`oej=d~({T<;CiwYuwDrU?`-W)S`g z3p3310x$H!Y%=jPD(Gc3zU-)Hb@Iksa>mebvkF+bgnSMY+_X&;zd4CMRv?OY1BmJ| zd3!TkjGPSXX1B22+E)O_GLvNi3gD)Db#--QUsc_17COZ{{~zadQ&*9eiUuyjc|jH|)8%wWQN5bT+nPIx+j-Zfh7~46`h=E^g}med>!4Xn*@hjDTBUNy z3>C>4waVLAi%(O#2bX2H)>lL~cmmM={+@TwAw@SB*#wfGb8>oXLs71I5<<9rd*nCu zLu^iLsRSpBT=^pl!X*Kuk}d^#NCArYTz69a7!B9M2I&>WpW?_}>@8jh46b_`F7AQK znw%a^U%t3qnH*qW4-gW_w3+hqQ`B@SD5G*;2HJm_+~0aJI?VVpy1X2DZawv~We8Y_ zTY~L=g@-+4T@K~E{9T-UGS?xe=~6N)wQWF|+*pp8J1b*`Wd(W#EtCN&d|Q)h%4j0z zhvhSt-pr1+sJ$uzpE0!VUbN2=Nc=@w%E|x^jV!n{boZi5N+p`I4U6d6!VLLKl|b); za3e_>`d3VlJ6De%^S+Ywol zNMp>qyu!r^;DEk>6P3D4{oV6>%h<|}-r4HD&~b8k(Bfqtk#efdD0<%R(xFPV3gNw? zH(`PZ*k;ZE5Ec>jeRB4w-1vJ^tyW9fQ8{6GNIBe*-_geaL=stn)cbKUZRrLP83Qt% z>94W!-R#?W35uqJ2L<-sAADl<1Yg`JmA@O@y0ar?^jb7tq(AWHR`>qmI4Rt^A14lc zjpcY`Sc;^l&a5VAvEpnWPL6=K2>RiY(1X2xWJebRTMb5*@5Yh)-%ZGVFv=|sSh3|`(rZntMh9GTa{{-C?@g_|jC zk5-I&jA<{`yyExWyT+eWH!%DNLBwKc{Gp_ZL(bdk!}xuwy949Z>?Y>>(~3#nWIhwh z{MI&PRfgyxO(wL842sC$LuMd<;b`AUxNDSyOAtZ?k}BmDKUlt-;BPVX{R$2XsFM8! zq8Bx4&=~c~ zDAh+QD>Q1{(Wb{CZ~*J1Px9RXq%65AHo$h~Uu4aMFCb z)TkIy0rNE6^rr;l@M(Rk)$WqAWpe$E+V1-9iqFLj_TC0%u{la~c|nFFnTI9J)}`&P z<&E67C#M^P2S$h_m{l#IDB|7YfBtM(I2bB!NSjH}rH(JB^?tp=W53J#rf+p23BQt8 zM_8Um;~-O!NG#1sVOwpvt3ba;%au$RSreecPb}T!MK1pNba(^8zJcD}TOP;HR(c1g zIa(?b?L`l1DOMN%e%9X9q~g5Dgl!ZPqcB+#pZ7qw*3oHLX)Ywh=L`z!Nr_84BP}N5 zyHbT@oKp?3d_R(%7#$m%aFnB3>!^JU-_k0tV6?M6y%RT{D?r32iUv;Rkr*y5n)1Yo z%POR$rxmN`cB)q0*M-;AsBPd*%%Ox$aAgfbYa^M-e3k>l%i~ZX7`!5S5w9Y7xk7P4 zoS0L(I6RxMO{I#vB~;48JC($tiYQO(+?C?1%jbH5b6Pq$%8k7AEqyD|3zu9}w*_&J zpqR!iGX60|vo-@RYTsBf>eO_Q<8%e}e0rs0>qe5dwn-Ev+Al({?Zw(md{szXSOHd@ zbaG=_U_4V?@rOyu>Xc~<<@6b<@)X1PFAp;%R669}odc4z6|+|Eh#7^Wg1#HUzWo+B zT`+OfV5)e0D~)9~YIP6jaNZT}9q|2S_r@b8gOdm1y6DNslGJ?iytW0Y`0`N1LkN3G z9OBBhe+b6FaX{j~^$QeIsEOkYDTR$>lyp;=6;bvT6!ngLOv=+*rk_H!RmvVGn%j3i2bzBSc{qscw_sh-TwGHXhE9#G!>M>eU zyDX0c70N6xJa1dGcBo6Qm>3?|RDSVgI?|N`{wlaTg&z$umG2a6i`rGuZiU|6wm15y z)*t|Xvc@nxv$QvS%=a=NhtMhS|@K(ecmxnUalDBtTEBZv&ur59Ki1@T?XQyPYg;L7Kaj2<Aum8Eu{E~}5}6n@{ZSP2=szN=for5? z1+Moz<$P%Fh*vL;AS&Fr2^zYb;LpysUv0JDcEh@d+Z?BEY3AylT0RRyNzFjB|zGmnwprY)-xpdxU!Pc7VMa(&?kj|jTOa?3-%RUmIlhJSrE0* zViwYpD}3qQ7-QYh;T2vxTOT0L+FjRF8=5A@uBKbtNQoDP1IPJ05rmRyN({apY(9u? z6>Pe-5AB?KNPh=`oAkg^`n6JVC57R%1 z(LQb#fD8{l>lXo{sLu>Hux}7#YjlHtt7GYL>T_Gv$nHW_p0j8S;8TSs`3Np6v?E#L z!!K1_RkCeHsat>Do#>Ib4qk6Ff*ztYttKKE>Fmu&G?-Vixc8^mWB zA6n{G(j4_yWevsqt>9r6M4=O4?mZQda2N!H4H#2oOAw+8esqOw5&`rtfv&khfyHAq@@*>c(hW}9 z1GshfJnfCr$u;=?$e~<_Wr)H``QV(GpXUXKc9z=jt=y~J@a-7Azre+^Y^@IxA&fD}?d)oYYL1Qc-DYJ7a7ZzE7hzN11 zv&GEI1!#!wFnNaTeuvUS{HX^OL^PQY$VCAB}A3)LPW+>l@1 ziq>qZtKOZZ)3WJ?R;`ji3X2OxR4yN@Xx0ZUCXv$c9o5^G;3}(ZN zu9<1akv$%Fd5qlOYdB|;r!g8a#ab{Et-plx0q-zX!4!jJ<)|0&bQ()c%lBjHj?!Y^ zf~%Pfx!z=-)<8n#q%=9TyYV`XVeIcUBx)Hb2I4P#ZJU3FW(0=fNsCZvbYd+#nY_D* z_al-OQ6QK3^K7``E!kc-)1y6wZQh)BbfK&-74cD1xYo(C3?p7%xR#Tf+VNIPP@5XP zTkk`{0D*KM9l7;I=XZp9H{+8iICKyC6>syi8?*B94Rn^#j*BZz?GubCi>(^(rJGj! z+lH*HGzw@%)t~XB95km{!Y`NOSs~#2spv_TXoYl4Vwcw27Zvj*hzfeHAiVGa#uC=u zcJFTX)z@{Rq&!DL{NITT3|Aa<-2q?$e_(3~RB3F^XPLZB?pLyW)h+;biZbjODbO4V z*QM8u(XUt$>6@yv!S&nriTo;cVqZZ9Z~gY$@&ad!k61Kq%?24)=_4@S`$y1YcFZj> zPBD(1VbV(Iz3Ufvu}e*GFg?+ORQ^<~!BnxqRM$yr+N8`<>N51DZ-Y@OfxamB!%2x= z#T*`*Oz<->Nuk*H%3qi!%Lj#>M%{qPGfA7^hQMMD_0hCt4gep$m-?skW{op z_d)B^7b+C&e}u(A__er@fHZ^sa5fPhu$v?>0^R&QEQKkqQblT~U2x>me1{1OY$rs684QlUooDrrr-N zg?|t0my%EqEtrp59QXGhgHAWtn=MTOJ;W7d7TYnQVhb%iX)8(!*`J;&E@^aB2*yvG z5SF9FOtIu|!GAhheIp@lBu=!*k(BW^7p(U&79kS3sMKX4rPiL((!R)s?#G+FV5d4T`oSZgNU1t&)VB{hKrO z?)VWg`OS+qT1)ru&TW%;{t0iu{|j$bcY7037DYo#6(<`c76nqkzl40ij!vYU96bM) zFy!C_0RC$}-X2rOrABO@+saOzghh#NVq@+Ee4T183qwWfo73^yC3tbuR!i$iiXr(a z{Wa+Kq`+|Qp~JWJTQ1R17$G8}j1l-yWrXff@&Z-^1Rv}q7zgiYm=E@*fuzL~^q-N& zGgLnIXo=xrqQFY?e&ds)`-w5BgxNY{P9IK54uxaz9ySM=ZRTBLC~f!;p$OU#2F#DC zetdpJ4h0yn+fq;jRLvM&*do!!jF@jGUE|L(t%VUu}C!H`m_|TVG;Bo zTarC-tWhihn@Dt%AA=te@-qwRz=xi<+@xfc{r7{ToZ!5x{T~>POuU93ve1l4B^7{tE%e&l%blxPP)LbnQ&B^$LNk>8Xi@6u( zAed5_&Ex*k+>KX%jdFj{I;}#z)VPiS#8jxid$4|bI1S0&Fi(@4KlmWXFZ;^h(UgYP z2bH^)^~|Ra%GX!lfBIl*=jSVC54s$k{wH_Hc&wj`qdFbrr#LR*^}2P)Ri2s+eQ3+b zP@u+Rh^&I?lMZ7e?s12aM1c>Qw~vuf3Kn(n>;&|&nhx088KM=-5qb-kGFU&V@QCUb zA4SUReoC8n?`W=62>`Zihv{nxj!*ZwHN%<2yh^a`@D#9x@#Jj6N4~!IW&K@qRnk5z zyuRD_Jm5=HQ>XGC)*l~XI1N)(T*9x+h{^Alvj{@)7-c}MG>MK@Oxr~yN^t3bL780Y3!r^11rMLX@9rBu zo8+p7IP)p>(F~sdT&}Nvhv|dgOK)!meuH-}Hlw1eEm|O9P4aynYx5ybt+m|Z1?#lp zN-AeCmsD3&R_zlnPU-$~)QR!*r^bYc#m9tTHntyQEqtcS!JmHM>4Dbb!k(cF7s*fZ7}WKqv-w3NI_ucFKHey-hILx2B?shS zdas{0KH7Wr-^j`ojoIq@+N&Kl(Li6;O@#(r64+|8!vrO6oI#Ywnf)j64Jj$clH*o= zH5WsD!ug*_(;54lk6KG)m{;H>ObMB(V{SrAk@y|NN-??VpK@7zBt|os=ra=9)tBlo z+Gbz-T~=F+Z7;A)Ag0tTEY@ZWW!`-Q(T!<)H`3z1uM_Q${b2Rpqo~vsY!aQ_fjxp4 z(^f$*j&v)3e%?Ek>C7L&Yf>YE*L-AK^H@EPfQh*#jxA23p0sNKF@IBp;OM1eIDb=0 zym;%_rG@OMO$6KUWmH9D<{k_GEiZb>eqOV>vWxY-;o@$D08Y)Ak6MuLy&JQ{NENpv zs`p-m?Ytr=vSEg~VgcsbY0EHO7usO}T>>YYYd>$zvyL(Jnyk|wuf_tS}ArM_gr z=&|+deG-65I$nz0i2gnPMWG~rw3Uh%sBw72@y8XVv?8(*UwhEnKQ)pG_G&~xw$!7GVb7XIDG2gRN??nN<0iECo!`hF(^;7F<-6OgBRb~B12jldA043)yJyw?4`;8C?*Scq8SE;1 zq`QfpY}(jDcSXC2b?4l%9LxANJ0Q_nhgNR|6`-5KrARngvlw zG=wK|%H-8Ag2qOg%B#U7%YqCA!}A<%e1tTOZnlMz*;Kk!Ss9Jcj-rZN-{9>8M88yR zG2Vy=>nCqAt{BKo^|ql3)_m@$o8Wh^c3Ko0@=ZU^1x8DxpYWqI_ao<)M!7HSK9k=* zecQ@NGV93DZK-N{rXVpg(jThuB;qg@W&4{!A-XSV*;KftD^%VPn!{uNf@uO+CBMs5Gn zM#S&PsvgJtuTTnjD;=}IiOryipGHJN5ie#VzGWdoEtmQOt{ZR7#1giMyyyPBv2{yd z2K`rf?OELA{s?TL?JVvKU1~-`Ch}UIk*OK(HX!?C@KeK>;ID)Jie&Kh_tJy@AsuJ} zeoy$|b^jd^GrKbJYy)DvzkXhQ6khvVBE0q##_dkY6zG-zlf|`Dt?RgsHeoE-yqCuM z0ueTwyTbphH&}f^q~-?4AP_LS9E|m+gEfp9U2e6#_YOb@t;?8w!A+p zdBbouA2K9*)BOGnsr9J%yaM`eJmjwnsg>bx%*%GG?0U$F>td7L7MH)Px1=W@=c*co z;!Bwopy5O`y8c3Av#>rtGDN=Bt*KP+@WHt$V4?Nka=I}$CYdcUYYSVRr!rcvjqhLY zuL|zYen3n1HK{c*Io=!Ja4}blFP|3AL=RNMrBC*?Bki|w9>;q59`iykH?qZUr|0I zk$=70QAIWUFn~#(ie`w}6LabYbCS^K5kwB{Q>C{$F?Uuta!ei^XDt4a-+E=aLIVtVJ~s=p-~-eDzkvZ`szvX#){*wuI9wF(hLC(JTxf#h+mR(xRp;4IEqtmqX8v+Vw+1%7)xWEeHgr&<5>5QN#9seiCL zQ_nmLr~x<0#iLswU3Q2Errxtp=bmK*gtf`HTNG`QZ}!s$+x737w9^B2FS{o&0Iu%MFZL!M{YT(GOmRj7M4u@q!@eV?XaqU-I=+X#JMfiiScI!tV zd_KXCdV&H*s0bW03Y&=e!SU`z1dmRr=0*ITmP5{pl&lM%@K7Y$@VEGn8$S%u!#PFz zbWjpD+xe*K_;!pXGpFF@w|=IO7wtJEa*H_Ab_4H~0d6r1{Z6!Jr7GH`DnQp+i+~iw zMMcak-^rp+GA?f+bm%2rp4(gnqPgvSCUtyC#2b}g3d_1iRE7CXvV5z8H7g$$=^!6) zt@b?5&>>nGZPAFd-`Q$l6}v|iSE4N05`?fsLbkBUYVIg&ecT) z!dAx9l(M&wRv}5u(%d{m7T^4y1s^Pczz|BHcrNDs8{F2*GpX=F(SNSE2wKG3py?{;4hC;`(xwe;RO^8SSd}eO%_~Z}~oV#)V zd_u3o1KwM7kvrRMfLJG~6g*+))aLD}Xaa52IeR-kLsl$*%C)U@{}|0CZedX|7-$?R zn~U3He|Gvq?Xd7N=RmcZghmZT?ir7&dAX>X0Y=J5T zu+iR$9IX@|c2tj1@-h@xkcu-1bm5rIcvIv?4hFSAyoHTa8oYvyjH=OB%sGjo)f^jO zkh4`IIMgYwk>~`Z#1ulchqbe?i4(NLy7BYN;+F2$VmIKa@tpfHQnuvINdL3M^z{{W zn-KfsdzTIqH!fWtpPIYGhm9^hcJ2g*nvI+Wui+M$d+>9L`z4iDyFPXHhjaS|QIk%t zQ)1vRos|{2j;QIy9WU>MkLUR(eo^w+ZJ5;=Yvt4Vi-}V&YxSAzO=)_Sq#7tO*KorG z;%xYS%of`>s&ng#;7)r=L&4hZ8(hjEC=MV@ErI-*9u4GZZC*--kMt|a?{m~5dm(;3 zP{6`MsDojBryc@6YO0_1K5P$|l-CWakqtM1HkxNYybqe}SGd#h6e@)NV zB_T2V?VpGq*Z(ikWBV)E`j_Vc{>}2(xmf>;pH8 zJf?cGi1i?q(k>;Pes#K+N11Oz?8sfqo~FAptfaq|$Uv0|@1D#hLpIaXcWh2+%*-%m zIH?qnKpTC z8wIXo9!KfBvo(sG0zLMPF<7Hzwxu$4vpAaV`m}cMn%NSL%T0&id!fUe+#+PJAB$!I z>5|MPr0;Ov(Iaq@HgX5XO4HMBk*aLQ$vN6dpD>MU{cMV#LJnU=h^z|4$e;cyt%U!3 z4O-jn3jHgetg^K7J-WpYAdj#=DccVB47C2pKOxQiA4scMc$km^-gvUIo!uLftZy!5 zBo=8~Q#;bXLb-4CfBeYtzoc7J*~IkU*t2oG&4UK1&RZV)+ok)r7h@B*w+7a?0p9HY zJV1bILK>$_1PX4WLnja~AT=bl5KrTn*bmDMZ5Ia(PnRX#PAxq?u|S<05k+&*AKv{u zs+*Bc1oJ&D5T9Nth+u{+5RNkk7PB@7T6Z5v0>>JN8ENpIP0j$$QM2=d`mbPAYXf*s zsdXqxo-TZ5>htRI(pg>hiIA;p$JUPM7?$UEDzexD|A397lL6St4g7Xn?3@4|b|fk) IaYc#$0bUAf^Z)<= literal 0 HcmV?d00001 diff --git a/docs/source/images/crosssubj.pdf b/docs/source/images/crosssubj.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e5c65135d492d3fdbeda81338beb2967f9f58a1b GIT binary patch literal 33857 zcmaI7V{~NE7Ovf~ZFFqgwr$%+$LNY}bjLO-X21t(-$f{)RV(Vo`#>>gZg(xC|=;`HQZSIWdn|onwIP-PPJy5jIq5#U5`w2-5zR`MO zJnTCk)sAf#@GEpd{J4AQ>uX_vO_zfYn&=O&O0|9sKu3kwBhUOMcSX024I1#QV9MF$ z--rrnP<8iH?^93j$NSBYcVOOUWgqUC`ST zTHl1|2RO%>`1N5XsUaAvUGyW%1PE??F*#re@qfL%=^jH%TDy1Je*D?K6Z@X$SS}dR zyR_v2?~fLU=&y$~a-^=T~~#c123gC+we6 zBK){g6S$S<$>Ic9%nT&fbE0=9f8Ny1f;$RAbq@GI=Q^IFT?-@KK4JQFO1TGcZziMk2qQKFq|f5tfRuh?mdbR9t>`Kc=I7Dz}UFAQ-n_#k9;vVe2an_TsfNZ+;-TWSJJ63%e#LKdVA}jEe{C}8m zc2MMP-LM&rDa?F@0XSx&)|B(>}$97$FL(*t)B)-Y@qD*?k`w zD@&TBflfC|h9pP|#pL}xFxkV1X$E<&erd{uo?#Q2gOi?JrekU|mUKtmHR3Y8^U#e6 zD@oO${`Xj|gkLWkFL{SychhM0PuYPgS;2$Lb5nlF``c|H>)_i-YBai604mLWCA{Fh zMzrd}7vkTQKqgR;ItF3Y3xBMJF8Pi|1qDYMgc~}orFai*GbDR53=3-29pt=0g7bLo zfWiT!lGy=Ep@0XI+urQ}R#0jmz<#1ad?K!ETlD?yref?;<6)g7Lj8%V1eyAsvvRV) z^{DU1hs}G=%m1K&`>rqOd}8F+MOjzV1@&<4WC%o3-;1QtB&+P?>+O#{Rbo|4KD80s z+y+q09ZXuJ+)m*4(h1ab)vK(I#3tdUSylP^;_?4X8FJx{`5(M;41TkjW~KZE zce%z<*-y&fN9XuFnrjN%i++e#R%DQrYntyC>^Jyg-B>8<5v)FFHOd;MyP6b>>Bi3< zOkkAvcZtTP~z`Vl8rk#Q&2>1roa z-1Rt1(iM{6nXKHRUEe1H>A{o>-5!L=M@F`&0X8b%bSU+|fk15|3wN}^ z@!T!}nT?cN?7@4c{rjq~IUv{KzL<9!@c-ag?A}$IaVLCZ>^r7Eh6Ebsh{k%entwyL zP+D-HUyrdw7-*2i)y(t{m>sStYM=k?gZjmd?@Ahu_^$rLRyVLFH45#N;MPRJgE=n0 z=$t!RJGz+MY2TFKqSg|gG5F!~X0gx7bt~{;-Sz!tIcNRXJ6N~x{pe=V$NSDX;He^t z)Bz1)vD@k(V0A0#!*|~(U2Qv8akcZ_@i>(jNhi;9mcQWF*4EqD{`z+C-Sgw-3D5hO z3zowhNr^Az&$O0=iv-~Pv|bHLqZ`*zUr2J7bX`m(7jsc+;_ z;>=F*=2?QLNXYZ_%@8ZDC!Tw3drkSJ5$bf=Rjy))Y)KiKsUR9eJhHY)=i_dtDRThNCb``K(9ah;j*8chLJ{#nJz5`CsNKI0TcXki)^ z^r9L03R&8mjhZ}g8W?HD*g$7Tk0_sAZ5XR;S9^MAFXR|2{q^8^j=7A!(#=xM%Ssd` zhspq~lvch%yN%e!Q4LUWSn{&ERPW=bGt$TfGO|^gV3lG9A1)ktX|W;tUcn=c}z;icB%VFFd1cgepp9Suqbs9_yq>k`#4E zg+;HAw?-9~^VMR}A?Of8QeU*_JOGf8ve0F+QT)n;DD2nmxp;QKpEyW~$P==ifaeY< zi!SB~!6wMlI-BwIVc?Qv9lkWZBS?Xd&w3 zwsoYkj3P*<7V~Qc*?svI9bKo_RczC0$C=Z=xkekN*{d z_p_4;m*L`rQ}>!eYy*i&S$T1fx!}VT3&q1Zec`m+3Ka*Kn{}8`Wb z4^HextJdAWw%^!&rbKjudVWM=D{Z)G{m~iK_^Xl27wpB^Nw3FRO_i}=<*e<}J9qGN zMawXyxek{?U#%5q05(oaR9U)Eo_DsUDn5-nuV8G#g2Cf<{ND_8yvJ-jU0#Guo1tV+Bg~*4u=Y!L51GKL z7xUOH@zN+2hFr)(-jhq#Rbvi+YL+p}X%@{{^Onh711pomQE6eNOm9cJA)X>!q}AJm zKCQbdN3k_q6;}O}DO-66BcI6|cPTSw>UQGb(LTOJf#;Ohs(EkI07Sl@DKJ$k-~knb zZ$L%mL$92}*<^{pW1nfH_FGEf`k-3+^5^~4(@ozkcCh`irvDpV~KVwF`2bn=cTW)GJ#U%IQe?}hn1(|GFn&ig?L&No-l=%p@WL{7^8^D^SK0$UQgR-KX|gn7bMhe}R7yfv^*BNN{J4Gm zzwzE^>IsYl)d}qcA3V{JJ=e#aEm!}(lehAx|G}T~Csi!JK)22Kl5ny<3RUzP-S1KK ztdWDoSKMuX12;AvFSpxxB>^cRQUG-m494)#o2y+pCCDtC-a$k8LiX!;U zD+ok3S$AEZWItogHRZ6+nA7%zjU4EB#6+^#w`uyHSm@gnRsZ-FqWyp3F@s02G9+L{ z64VO>w;LW|Va}3F-pMJgG3Ox8+PBv-9#jQUPoti0ldzs{yC9f?ECQNBB_Y@H8&0=H zZf$-*cM6MYemth|{E@s`mmPEps@8lXlFUEd#=n(*V_dG}qv`_{~Di0oYDRQDaGd-4M3TBu#O@or-aA1q|?(opE3Lh;gI=5Yb} z)*zPE*~{J)_p002{Hn|5_s>811JOT!{d4@7{tWDHx=6*!f!k11qG{Em*|^7IHG^okE}F*_b^trWJcb0BeSjr|p8#aC~?$j~LI zU2~nSQ`h99-^$=`U9GLYuLq1JoQ>6P?pz1vn{5IHq5RQc5brs&ILt8U{QEuBh`%Se z6O6=TJY?E8(*5qQ?F@v16DxvC1OSr#uyqr6{43c9q{|SwbKCFfP5w0Nlrf3OYzuTu z1$?FS>`wG}D$}7Z?h=9h_h<(=)T~ozbgR;i@oLC(&<2!!rfeafX?Ej((*L++FV!VUEUY;aIbrWP>>b!j+^^Y$}|q z6`o%H25lM>^d=qJP^kNxo=s82K6WxOORry|rAgQtT}sxaXV3mK5Q?G*dWw=SO2Yyh zq{1!CF3$-MgM{@slEy1pftV>eNxCvuH@`ta_8DRs!k@!c@b`lHLBgg4y~mU#PZsek zMOg&K72v~3a>_sE;rFEalT0|vidAlE>0w6wt|ki`A=ifyH1-uW9HzS&My*1De(rL&z=PA*?onM5|ud+!H8Z{ zS=gP)otbDVR`uDB+ivTXj&vwKiCtUd#W26(DgllZ@7J3cRp;~=f0t^=E^inqC5qkA z)b1UiNBO0utM}U z34V{>^$n}t&3z;&-9nPvB+i%HgoH{Fh(M?CM&gk=#Z41C#qNq;U>?V)GGwFDsA!Zg zb%Dr~+uUPI5s-GG@Q%BXIvs_OIvvK9I*rqbo=*sjo{uSsUPzewjECq2@ucAsnUdj? zP3OXW)j8n}r{Lz7sUzN;?f-%)eon~J3He*=33)I)JfM#Kg&qP^zbaTdD`zxZ*hwHPCTJxOR`ZJ=C6N;Da6WIgR`irR+gtL}?0}TCQ)k#j zYa2J((bBDqTee}`TQFM&CJ~hR9Q74>_{onImvI|+3quN}r-QHYZ0y)MUsqq%*D(?9 zl>F|fl;lQ46nR{6{;`__7+p8Rs~T4h5j1UpU(=N;zjhCtEEsH}B$h9&y=!o_F7Xr> zSGBg#6MyEHI$W*aE)Zlj9qm>5liBn9bR*sdeob>$Ng}}ir-$5?d>U4rv>j$iE3vYV z6w-W;+!^9eQF~(Yoq9Rm7iTh!Mf6KS&NP~Et27I{KT z=xdJ1xoR@oLQ3pgkG!~ymvS{}=eu57Jnwh|(WVpaTj3G2b^h!^%|wfZ`G^Y*8hRc` z{L_n;lRBlCO;a(yjC_7R~?!qVBl8Q>hQ130cd%{;PnSvvrevMkL_(Cft6h0{9U#wu51#uGX&SyeqR*}rA3#%Fi%nf4e351rFj$DP$#UuP=g9nD_~ z{g(H;Odh@TQsG(674deR{Zc3$ahamRmxj9JYx0Cy&mvF<43t28Ol#pRM-O>s$a;u~ zGGW{$W+p9N*yreHY6B)fGaTfcTDgeD>lrj*NVYNS@mKN+?ZE)P<~HmZ8iMQWr{Q?fC_g%3}Pe=|7(@(or@2CD_rJTLJ%sev0(X{nhctw&|D8U4OXC5YdaN)!qo2!r)q zDeacTP^=TpjG+Xug)skyMFuZ{;Ta~pRIACz6Vphv3u{QU|2C6oH-3g46WuOwQ2uZ2 z0z7lNMu`R;mnOjb&_G9*el;Ju3h))C3SjgZ`7l)qud?FR63mNXg-ZRMwJq{9*n%hO zj{5#u{Dl5mfoQ4ie!XEN-doS5vaPh?YpARi{#rfkXXC;jcInZM7RW4#={3Am?|2IJ4=0onvn;4CWvK z-{phK-_*_NkK35U8rmUmNh$Y~4q>L20l%r>dR3BNR!4vs=aRI-^5C!_Y zIe+w zko_}94b$xW^~mr4sCA}(@z}Msj=lxkpHc=zWX(t^?M!^T|FmKMdp%Okq~7c&?2$oT z9pjlmop^4>zochTI`F3wILrmdN(UScA3#!IVs99T%h20P!&D@upABRJ4r|6xUMg$Mslnqm?>?kk-$4^H+56y*)=rvX4uhf2P8PBa}(o*9>%uwU%gIWJ$uq^JZbrkR<4*p@1d%LhC zl|Iu}F144lmdkOlB$zJWZc>^n_`qv@{HJvs5~(;)R4D-w>d?UQ`-MhTkx&h=|7)eM zxr?MVY6sq;Gn55CZGl$IM8k?ubGMm}ML0=w5Y(7=*FyurI3 zI84i(Ng%zPZS$xS&sF}T*G~NhIA@PnKT_JiiOdo@d#yAvuolRaNIuWa07$^HP2t4v zPMQDgg}bNdA!VuKu>#_~zN2+nbH{pBzUCd+h7S2Jgq%Qs_;KPMKE#h0k+!@^62xr_ zzvCGmg?AxZyIB3NB>74IDP$t@as6K<&Hq*}*~vILd3ibh|0|pvZ0y{8|4)hYyaypr zOS+5aU2E|~Wb=eOuP#kxu44Wq`M}CFG(0mmIh^_2kuZiTT)V^y!q!V`ud)Od!)Vl6 z24+nB>ugcl7aq`J{C;I9JWB~ES_f__r_LYkFtPq zi){~J9mn#1vwQQ>2XW92F+cPNmzNb>t1SqT%<^R@i`j64n0IvnHTmTWN0xEV?O56} zVdfVlsPW<@9OkLWx##UK_cUJ&SzX@UbIv)Pq2w689Ht({;puMA8X?N{!LD4l;p8zs zC^13GC0De2Gr62Gc0S;eS$}ccfQ5v`vOq@)s~>{c;wA4w5${1Jz*ez)CNO)}17pLg zvB6jT>zEGNfhYT?vSeP5wO#*ugmVldvHydGijSkw`kWMx*$ujni#kVYuwhFzAo5>> zX+Yu0dIqhFhl$xw)c!yND9b}DVPrOfU~@VcIkMB{1U(7kVX>+d0xBqL%q-#Nas4z$ zY&*5aS-1HKV5!z)3N079@iN4>Zgb2l0;(^ClKlGvQdKv}f2jg#@u}$OMwpleBJ4&G zYX@cOArNEPv8MDAb&LS0(~H}p$Md&g!F-3OA87@fR(L@#Ay<@jY}juBq<*m{6-RhX z-fbt3%&+@Oad|j`f4CR9xkb3m$v>#J(w$U(?5tYUqPNiyzxfb8n7U)SX+hpnnH_yV zcz?Y4dlf7Cla_qcysQ`~B^QSLS+b~wUTA8~mLeTk&^&2@6gMbpM~;}0Y?;zQs1#(r zgLh&=iL-;*E3<3=ojq2k#sS6kRTU)yVGig{r(`}>CutrDtA!d-#%YD@MrldsBn4Z5 ztd|%E#LvqHj67r5n8C2faIy?=O%7X44iYoTzzG0}8Ql=DGe(sw?065>2Gmiz{PzvF<05t%+~m${S=q-U=v*y)AvBlr+7 z&5g)$MX!Z6hz^aAU+u732pFT*G`_$r+DX^^?IczNA~Q)x6)%Us49eE{2Te_uX`8Mp%+rD8X}B-g(YP& z?dfd1PK!AKc03;Qk${2QZ1TBOZ9m6s`#umcPKs4#edM1-r=8+ipJ4%Yu_$)@fCrOQ z%??l87GHOQx-es;L3Ol-P7xGyac_mprL08{pDhl{0L5$oqE0?Z8bUU;zisEZPYh^V zo#MlS9Egu+Sp>R}1~5o+3sBd<2Uj#^9oh*B$8NWWRWuMwY^h*JR}C+pyx=y!BI}hunL*&_)*Jq^nu!6Fxu9&Fj>vkSezOT# zPDVa*$N{I{Q$;2Jom3$0h1gG@C%0(5-&^pb$R%yUi{*^iR>ZJfo@b(G3v=0-aPF&P z^BdIWQFHSNgnz5l`n9<8$u9ZJR+)9z zR8-M#tEv#eSTl{ZYxAZHvC*;i9C)9<(EF1+$;)@_a+y%pL=tfEqL zkzf0Fa#fByYdv!rFKfH_E>r^I8%dh&Qb#Pl^=GBtqIr#tj=pnlBG`@++nydhH0ezN z;Y-VIj4%0FSXeHNmiSpjPA0SG554&7B)j&FdeSWRlF#lw53(aXJls?NxT7?^`4sa? z!EN)SG#)&KxB_v@pFqx`AtLcDm--v|7J@R}*(YontJ z_4y8sL=ZoU80iFSa5}$lgu=;rP$KYmidCnkhrR5N{igu{00=4WZrPjEQxR7YPsEeE zA%hWW;N>j)5?8JS5p3p0O-=nF^)!Ar9eRZaCEh~TxGUerT~vvTc6!y`1-bKE_JVX5b!F*%}X1gr$aVhu&dc~ zVqcY?Zsnt45tz%kbkew9n~C2n&qGKhgK&HLYMGPnrlYC%`!G8 z)YS45_$IcOBvWOuFbd_pGla2e41-OK`9@3)@l+m{ZwbTo3+w&Veb??Iy9|;#mr}R*!GWZHIMC&+mD^%%M4r2p`+|GvHB{>CKfK;nR-PtcwIiXO8!Wx@n5d&4cqSp+wkYsh)^tq_X8yc;zYZ=k-op{h3*3sQ1AfEU%ppLAA9o+HCsGCjW|- ziYGgNW(z*6m~m7)?aGM+w*@ch1MR@$^Y)VDn9A_;VtR<-3EvUk1)gpR?g;t9U4mXRG0STbvZF`yoqMYm|k}hWO7EJ4Jb(sF=7o9z`%` zRMwtu^!T5uO)N;(k7YNgz`0U+SK;COxw3iBR(ob`>eM!(jLAUM(Zt7RQs3V4%`&H} zdkmZb+*ID2!svA@GIa-c`!vEX{oT}A_HuZp1W$A4U$5jynA^^CX#6NRKBnK?jPvh@ zY$hE1SrHZ8AN`ZN=vs^W0F71aO(bM!Pc7I)mAdwqzB2cVpD?zi|LZ~Ii7;WO`B6{g zR~4~Na~(-m>XveQrApeg4YpqMC`dbiB7YXApVThKLK>I=rFVhqKvvnjFT}~z_D2_meSyB(z zVxtp=%PCQ_AY)%*kMnSkK7IPNNc)nipAPVAwlvn)zsqh4c_1;lK41N(ZPZ^bOSvYb z68K~9yd!D4junDfi6T2m*X*|-B%2vW5K#id9tIaSV#03XzV%dXB&RJP6i);RI$D?E0!UU;`K7-y-V7haOYzxZt#$slQ8giF?!am=h+VB>lI=O zXE@&_!r|t$dWD4*9j?uHHYBg8to*OW-vk6QKX@a`#KgS-;blzbIlrzW^4%Qegqi1v zFAXZa7*#^3+Yb~{{D?nX5s4u82aTDh0uza#unbG*hp9Gu4IAs7WkY*gV$a@|GI~$ zurnboGpMH&R+4deY?OEM$r(4w!8v$$&f}QBko1X-$ulnPuH2mS1^QSQdia!}idoB; zs3m~OX_+p+ChVOwh9bqB+G`A}KrSmWGPGpYt&{c|Zz3aA@!G>`yaBHwTL@UbOk512 ztPFOjZUc?vWbaLbcs0Ta9yE7 zvGe?n2_Uh4rZ@7G3>@!vX0YXPho{fJV)^RKR6)eql*m(#EMWh!|2$(uH`p#F=|x3} zbw&3BRkOpNzx4>fHm@Atvw!}SexHQJ2+u-!B%JTTc+@BT^Prp{Uzu^#v z_s1oilMSAe(Q!R~?(x%^h^$zRBXiN(dL z@_U%G1o?V6juX}rP&t_`P8bpp$7hY!jFbYw+8wi*2<^K{_$}u;WXeSnvFBX({9EbJ zpG(_zgpeV%*R1nzDerOQ@0;h}s0ROB65d`}wGTkCc86`UycVF1uv*th{mPJhmswiP zx%AelXkIUDTvr=*feIURk;Ul#O4yyZP1Uq6zQSFroxU(+pnjXjZAAR=5d6)$VTN@S z^!r!oL2AMx`?*5+Y6i$w*Fv-#Gj(gGo{!p7T8pN@_&ZxSF7wUYwM{p!u`CG3 zKzy#R%+(coC|6^?R9;m z;6Vd1j2VOE2?MoJ1GPT}WWda=Nr@{J9vHlw!qwiI6Nfef1Lx#ehnO>a9e_PeKN8*_ z*}vfzTQK4rdtC{O%fN_0)iK<`3Ebif!4P3Xcm&y1NOZb$NIEBo*cHgno2tP96D(g= zAo*4y&u#I{LYYG$RE|OiiRmm*&uvkW+2H!vtx!`qE(BGuJPV!a7JUT{wMbY((ks~D zA|WOdDQw_IzBKRCp|K0n1lzu~UDa;G{&vETe7om^&%Niv_9vbPxw8!r6^^|;&+b>u z$41WOfZXw=!Rq9m49fd{C+^IydLgI?$H@u&Rzaw<0PRI;0~u}am_LVIE}NyyY5Dmr zuBk0%bFuO@v_K4u-o6H1W(JI4nb3dh{%M*DGJMstv&>|AV(n&7n*d2KB>WyPB*YV3 ztyyM0@piFd!(WO|b7N|9q>?y6$eJgXdSawu5K$n@!0EX$DgQ3)% zlHQCm%^KQySzxG0EXut)8}^*6x-==y-0WZ60knBpcCu&9r~_WaUr&NDnts{iLqN+d z^oJ(DN{hS3&?!lx95|>)U?<=3k;j`8OOF%F+KQ2rxhFZ$&xs`G&(Yi%$nsttO!sHr zX#~N>g?F+?9j?wv8HoTDt|p7g9Xj)p&)|e6L1lhWb`5+a52Yk#V}7S zF$DoL>BD>c1EADz<2#|w0in1{jYc5{d7Wzd1lw?Qi-r#O0@HJ#JI?0*@{e8lU|oy` z{=HZ7|85R{N?`gmk~6qNc3=~uiU7;uLgKzY!%)kD;a;o<0NQo}2;^Z@G%!+MT# zYsp<$)LvJUui$o+#$YtNShqV77?eRk5j*H1{7nUze!!T)V!3!Y%q@rCNf@+aN& z`Waq=QkZF_T!g9CT3G1y#V|R8t7PQ0a_@UhFNVbiA>a9%;WfgRTcA8~?~~tlNU9Q< z=gvrgmBf|J)gM7$p9lY^Luom)?kDOGtSwP1neki`*ASCoDgR#8V`;k8ql*clbCfGG zEV|Xp&D8dVf7zy`F1NCDvrJnk1A$kpZ+AUDa1`nH{teRssk%^jSa3f|^m~eWX1C-A zrnZ3J;mLbVnDNOWm`4pyuC8^Uc>`o+#>0P2ZS76MHz=r+N$|Qp9odjP+PqL&P3yS6 zoUc-ToAVgA%E+W`NXPr_d2n(|*Qqs~g~P9`lCiJMk~VL}tFO%d8bFd}RZNq;M+pus zzpc>NOHjGIaUFrD!uY14Vt-$Rqo9P1g^_6a&=P!vkNIt}BzP(N8opJ8;*4{UauM8U zzxIM69KJz}n4~py9|=ED8wo$le8=T!#`N7u3-Q^MI6bt;_j_ zfyo(Uva&&$pvwy^ICp4#%p=+Ar-d1AS`4bq_5QeK566`wwdNG#nlM5L!+yU2^=1 zV^y8NL4aCPxwR-695TmDa%YN86Ex+@>ae@)r~KYE@)_mVKNYh&U({7k^zR@43Y+T8 z4`y0#xS9|LulIHE^o*O<+;g?Rh8^JMZ=5$=h%0W^M&{!>yF0r!FBu)X)}uTKcXXA% z*EYEb8S6})h7>w+>*@^s@o#)5rCPo_fb-LT=&AKLSqfVVE3O3-SjxS%M3P{QhC3X| zzKJ-*xG!>X`2`lS$4s})S|rBbb~fCer;Cw9N~1*7y|bI=gv87JZkTR3lE;)v%`;s= zrS5DXJiI%nLPMkGf3q9}zO$?C)*G4xfE${dJ*{bJYjmpe@)&jNJ&=wRT1C!t^hrtn zTx9`v&5ZRNt5GmSNIn~&tX;#r!+eKXMdFcXd}V}WY|?c~EeBV?mfNUqQfa@`Kx(gQ z=afo>m>05jyQLjG1VC*$UD4QQxE1|`oqF+dGfl3CuScy%a)Eb2AS`xTo+tRn0RqJ1 z5Y%ukSv!rdq?#Wf+!3`o4xkVe*fwgDYg2z7k&(VGV|90ReOq&VTbitH{T|AR7E#M2 zjb~2hpUGUC5hJUm$@$4h94BtWvFt3ICI+4iX0yoXt!+l#Oqu&&=UQNsSFd#mwbM!a zc{~2#Xz+gUe&|i*Y$1MJ{7?6vz=;)`bB=mm7p|YnM9Y6#CgyBTn%F9&f!O+7f4JfX z7_n*%TB|)5tF?QQbn0&HR95xQT89_>3F;nxDz}uH!%*+c;aY+`CvO82lm!14sPrk{%TI$t6j#vh^hk#$g73!)<@W~pneX|*tx2O zhSeEV2*1^y<;Opjs!&+at|_cORZtOF%#ezx#&i*66x7CfEl^#a&M?xVwws^> z4-lh$VF6AcQurNICo!7DR12Z7V5Q-(%2BS3ME8{_=Ei-FCOFL3%8?;>bXK}iiWq!1 z-mT$4uN`>U#>iwU}!t5JLl@k1x{7#;q3Xf~a9UM#K#_K<;ilmodUgEoSBBUhF ziMvnhn>L(cQGPOX&ko+q21H%sRnHw~$Lm#Qhr;N^{R6@@?6du%!qrf@q|TYJx#f6i z-VOB~^uS?}#k(8{^My~cfTO1w;fkU8etU(A4+*X+ZcVGf|V#GhaB_YwBM#om5#p ziVejz51rs4Fvfc&d=GxiNp=)Y2e#;7!`~#^Yo=dh+Zvp`v6pG@W*wiKocnNCtjv*5 z*>UFCvEmwG`=nVZqIbLlm;Vv>0$au(!*TJnMi2c6IY3<+J> zKpAf2DFyGlV@6DpjP-*k9XL7HZ117QRNt*q0yU&eXkAkqKy(I*jweuFq(M=}V@vX5 zD58S=ZnyJk+ozk!>T&LN{R~2Yq37JsGVMV!l02Fzjq z?H^s$p6fUb+elq=ecMgqi=J_d4}F%y&{tuh!b@Wu_dc-k;Oxv1QV5oP7yFjtGokw` z$!P~WRX${oQ&(hpqT&CwH*rhlX@2=cmMNpPaj`be8~AVK+hh4K>E#>>btLAH?-1dT z2i>klA09laZHQ_1pALJ*3e0KkM>(SLwqy!&=c6?C|lB|SVF2AC{h}WOB zIgq}{Gy51$f@hA`68j=^_R(3vN@%+z9r`)D{v1RQ_$N$Dzg=t^@H3N3l*nCET-tJt zOGW!f=NeWKIif_AIkK14CNg(uKvtYVStKQ-3J6xXZ`oFg?a5;&h&qxZ^Qw`$WzZ*t zL$Y6jpfc_i=2SoW>&=%zACU?}K?ljZB47?|`_R!{^b^|Uc|nLCsjPW!yX4=<+ZOyk zWP>fAKX=a&D7%t%o^zjw5Ss^sbv@EIzW)JcDW!9qz_b9IMn!VTPVlKN>?lQ}Wb(=2 z#oc-T2rT9^{BIFhDtqeg`?ZQUYIeq%VO9_~$YxrxOJ?l*K#v zC$eqBnl7oQ*T_Gwv1$hhnItIl2ZTEoJEpD~%2TPtG*q^t@NRfm`t^E}FiXjz4cXA1 z6#XhA0SIlf31r#wA4={Iu-8hGterb1-QedRqP$pR z1~__gG;gSQCSaXp@a9l*(EABLZ=wy}UOD8;A7MYc8$^>4B44Bs zu#4*>DysPxC|^?2Xdfbi-RXY(qVRs_=gGOG24w*ywi8-&nfPE#c7TD|tZF}EO)w;n z_(kIh?M|q&C@L*6*nJrB6rd@Z$jA5Kijk!EW=Pdv77`~le6~%2a0A}nl>xl_P*KW% z2x>q-^w))iX_bPu$&~+k`QE$kNLRn3V3(en4n=XjIlh1Ge&>EwyKhKAkp3Woygmt$ z#O1i}GtVwu9bf2^K!pVBj!S30<7gqz_eqd6J+P)E5x*&7lj(k7MgVZd7n6$c{lM($ zWTv8#8sdM{DnD=z>E=FScL9lFz<1qvshD>>twOC2BnI{R510~Y6pM~`yxgc=Fdlva z1{kk$Q0@)Mu5zJIjGN)11V2n2tiC{2Nh-}eJ)cyJdVEq~$y9wm!;bp%oqrTb0Adkb zz>Z;gFIqyv{YeO|pF-dbCM!3q6^X9GuRyc-Svb#b@`23Brv51~c@h3^oRgzeZ<><> zR-b~?0-^b!iZRIb$wOXmSh=jAz{$U&;*>V01v`+FbE2b=lS`s9SVthz5(3YMBp~{y z`~|?iQ8En*eah$ySXK{k_kae({*NFhfTQ$SKoNR{A)#@9OGEm%rRPO=t}ahVcQBb2 zlCIlV?D9i6 zQEgvR!4Bq|un{>Z!)|_iNZW3Cd&r^X?5Y^)E4aIZFb~}QOs{i#wMSSx2j|AjvIyt) zHMkUQ$MzX05_)!H`G)S&Zw)5jh%Nv{{bdFb8Wr`0<+zo-qb}_+ynWqZ0ttAXzyV2g z&*0pVf~!e~>i}Dkj`v|5Uzbl^_m4YNLWq2JY{BatSj%TuJ1iT8qzA{GZ)CyWi5|k9 zsh2=-?r1J^+oQdJt*BP;`bucdLpuL2GTyXuxW$`22bYJI<*=f)j zZ0L*Le&)JMzERW{5#=8BP7@sVxgX{R{&H@6*k+@yFG3b32)S=zd)U=I>7Awxo^Z7H z^DmbJL12h`i?HA?K`WJ{{W=>Br2Rav3L#MIH=*yaBzRl^(=%5yIFbd^F%|KGeo63| zFDBYq!~XU)(OZ(Q!ruy@{s$%art)3(6*cLkVV6KWBx(D!VOKx=Vfh*^{6Y7I*#BPl zMhN^VkZq?87FmVeK;;mafJWmG7}h=aH=J{6sZTPGdgpjFulN)sFh~Ah{w?9EtFani zh3%we=r_vzrKf%G`?G)7u)!5qLlENI!&eVMzDP5tth=g!UkRch)?KpvdDB5bUz6=P zNni2pIEeGPr#<8IrKdgN^I6m3e*luvIB*;vAnP9Q=6fNjMZqsPQuDq~*|!ct6T!IG z?5VyQwe?lrP7r@p-%b!k0`r?J`|Q(g{q@;LBK{OsU(M}=zCGq3yu@8qFiRjkNY~eJ zJECv*>l;j9f+QH}S>|t_rC12W_MY*ZA2HrN^wx60m^|@PK~y3o*z;rX{$n@$y{i9S z6{b9dg6$3K`$+-3(ub58cnor06gT@F*ngk&X)&?AS$;nuMN<0EGXu+hTA11IDW4W} zc?ccbo8+g3sPv&`28R2z(6isa_ur#^T4Zc*(f#+7kn>6KN}`{mH}>BTLC&KgDT#iL z7xew)3P~x%)C~L&a(){t`#rz^{t3FgkCyGt_|rmE3ehzKcSFwqe}ugSaOFVHAUL7P zgqfL{ITL1PW@cv2gqfL{nVA`0GGXQyrU`R$`EG0X{@wcPw%#jSa(7GWwk=y~w*(1Z zittN@05P5Kn_PVVz3?{$C>egz{}*xpZa~8Lk@FRm?ze(i+V`?Ecn|hn;hW!g?LYnh z1%V4Vwf`mKFKqCN^Ud#D$j<)z6^PP4dd3gefKz|e^v}%i1K;$>89&$pPV2q}P&Ih1 z0$tEYOaF}QzgPb*`tAc-z$wGG*qR2fQ{P2M>7R}L_i5i`zI$XEaQX&OdX16Z``szO zZv?#b8YR6~-r$w*yAhO(zofyd^mn80u}kc~?*UzSf|l8#Vf+vcIQ9Onij>|fZt#l# zUy-QqI=^IgNEm(N`tNx`7u;cFcIX&=+xqXfz7Z&y9WqAWwElbDZ=%lx!KY_^j34^x zpZq@;u0OvAg5#5a@Sg5J=a_&o=XOVlx%f7$m#B!%CujYy@C3>|6(-R_nYUp)@qI-N zEE6+=4T%Ih7M%P-*|WW40&e<*C=j$=^;mrXWq@kNcwmAeupBz~7k3<_Ja|+T4{tp2 zEsk#up_7PTQCUjAZaI3T=m?R1H3R!@)I)*_)EL^bzYfc;Hl-5(_i}m4{^(Fs2@!_6 zB=owDtWBIPFgqwafGoqvtia`xqX}0C2 z)Ui~!smWXOaq$TeeI18S$#qs{CNcP8@KK4`ov+IWJdy=VLV<-yhBEt|+^S-4!Fb4e zz<0MO_%Uj}DlS@8NIR5gX`HPqj#J2-_a)?IrW1m0!CyEU6_sbwjxuKCGmm@6o0;88 z&he*u9=nVQT6nB>OjlFx1K$9}99v+M9P=EKP*9|@QtTeci6}`)^P7!a6Z@^jDP(cd zy-b!l#9c{LvQm<*jmP8a2B5l4LOxpU9dGC;D18MjNq>*v>T@6@#b|eM}!;CZLc?7us5homxfGFVF|< zC(c~+GlUC-uI$QO*(sk_xSyOX>snXR<}?G;h=B|rE^hX|jm=l3<88yJ;4xukU=RiN zMvyASx_9(DpnGg3V`AaR*DP}MG2V-FIYotNbXrwWFh?<=MxDG)w$uW4EokL8oI0Pi ze`|YqvpZ{H0>3`Q(p@{?NJhR)&UrYwv1Q3Je*#Ha%AN*ISi7W?8rql!Ix12)_+>KS zwvpC9RPhmkKu>GXdxBDVO`X~2`B$swzzxW}O!6DOW-}2@eD`%)7CgGF&%dlW6T|b( z2C89)+l?Ps!dI_=C4^e5otsVT^ECR8rEgDd= zmf;b?3PcUs(otu6L+UH6WnKml7(G2VnK}8G0+;+jCnknf!Cv>MulG;lD0Vsrq6-@S z1p(qy<)7r}7|1!evRv@yVp`6nNHG4@hQ3H7akUK&3nz-CQLxQwYH3A?sU?!q!g2GV z3BO<}5x&`LuUST^3zbU`X2UOiWJd=~Z|Z>ZyxlE&EfB+_EigR zUfHVZ5P9Q179+3{s9+aU6GoljgnPSypA)rt$4HQl3NS&!lVY)g zIZ~xc&NimO(M#Xg4;N2}XEDST!~~2Ix<~ZStby3gPbte1(-PZ~4<&JJudKG3`kE85 zg<0B_joLQ-^L9O74X^ga)5etbNt;8<5=&Fd!;s?mBWbD5eZa&71D>qd0(Avt2W8K= zHt=`dMfdGZ>rF=!t+S1Dje(qA$AOj3LAp}6c?<{-Nfs6TvMY4z7Cu1)K*l5K_FG$X|PJ_+RyQ&;*+ z(*fXKv9QEE8EGFttckS0k{!|@KQ=GiAQG{K>`j?EB6ji+=S?81o`bs*u777oNhpuu zqnD3S4{EE^p;lkFdj}c{T9RL$-zPxrkr*usfhD~uT)J7_>MnzS!8y8Dejp+)TEHD% zIZ!!}?;Ib5xCiv|4zTK*f2qcqnT>@*KHhB*M6x2Cc^ohn9?jJx9Wyo8yqzu?cQi>! zoNljHBF4y&MoO}$3mM5QgLghkMGoVo;Ml!Q8&t8NB^zAN8K8{7{#a5^5_4f)X0hlD zk!nWIE(T<1ZAEYL@E7^)Ze!cV%^_4(ZBV3*=8Xw&_yj$hWh5V_AR{kX2=*xA7pC|6q$%iR4O0IInzT4p8ey-#}oxbDw@wAWAgk7ZsBQedsr{r|q zlT7+{-oNj*RI{yDEBAs9sKJdLKlnbOo=ATrcgVvWHzS>u=7-b~u9xhl*je#c{V?mS zYs#VQ*l;KaKO?dEUZig9bc)fN7d}tl?QO3_E^Gp$;6p1rqx^5;g(^K{cX}vYJYvC` zR^)_DbW~9jKrJjtIV)y{-JF!nYsn>c-7OdFbmku76lbC2K8F=2;?+EJO&wsVO&9oHRQ< zS&e&h*+A3N5g?E;Po#P2SaUaFp&;`YaxKONM#=$S8)ZvB65~CEI)fj@FHvtBe zJSt=cKD-yEUkiIK&I&y(YKC4GNL+&SkyFz>Bt`PdL>KVnToF%K6Cc8GjPb8mZJM6D zDmOrmKpO z)i%bMcGiikp85%EO=~?FV_7>aWv;58Vj`Dgl5SG1e8xrnm!uL+0-@9A67PCz7*ZgG(}6m6(pqe1n|+$3WulZl-)$&>+R z&atJ53^ex?3Fq*5*eJN>D&TDVUfhLZJBjw|wSp}?Ta6fsnHhMj9Ee4UxwNu@NK;%I zJo&6m(}cnn4r-w9eW*e#=dSfpsoX*2FarKdShQdtNiUyED=<+o?kEuww9(R`)^UWd zqf0S)7RRDiVwzVejlX4%qFKt2y1<%IQL=3IH5x!^{h=&&jR-{;RR@ zSNo7ShG@RLu*l4BXxRv$r-(QQ>6J6MVONJ)uu790{;61(;tiHLqO_+*izXRaLY$)Q z{k$iHdljav@lf5XYAg)r7BX=Ws-b{ zTo1IBj-;8$@l>`xj?Q+u9BG)~@2M|se;jjWw%18aNKn^QP)g-D_lfM^@q9X9XF&RXQl zfF=x@hC^ir86*}}jo$v?R;}tc6f)bG=&HAtO}AO=+}aJ)Kk#XE{qRYn!&>fgu{1lT z1%%3vnPcdso_9u?X_+Y}iU37t9Uo8+{9KMLgK_A7+m^NHrmf@+{<@hJZO={YxvzIl zbIlsf01Xf-NY$3Sut)fmwloZN9pD_jvsr>ZP8&|!M4ktkuv$!wV|!vtq%QSGv@Qeh zIC~b*k-36(1ffZFtYmDZGpRbBC&J-%{4^8Q8}1ygdE&3>s*ypcd`-lk-wRf*>!oY! zHs`iVX|_FdHpcZ_wvjdC=&cVMj~gc1BG>*r8g|yuJjx8E#Cd)L6Ei42XE_!Z;a8U2 zl}1U_!&n+JwHGEKnn2o+5*kvn(VR$=wvkLLDfMvmTPo7;I2EFqb43E(Q^;F||1P1{ zUuZ2!@e1P07(f;=n8<8xOB&Zu5+J>`0c95{b<$<<>Vuz?!;)Sa-_{H}Qzr<9o3xnj zQ&dacyA|NbGV6#F)!nK#XZSEpPsj1I=ZT)0VYXEN@_GNznE$%6+GOJjD{ue4EoYqX z#!d1RMn3EdfPxU%@M_4OU>gu;=0J1KCJ9`m7FCW`v*QqJXv7(jGM0hmoOVj81Bm;+ ze>mDX_cn)rt}nNHaQ~(&yxyK!scAL}9TZgcRkN-3a#O%u*f#G#>v+el{_T7>wR=pG zZIWf3U*XtPoDrHas_7I|Mx?4|)xP*QgnmbDmLU0K@<%M@G|4z8^BC`M*MPo>tz@r? zE|QiFSLe;w)rTv{iMmsCe}Wf0Pdz`r7vqJ!MfqgoVcTL9U1weEZtGefItCi{^K9lD~n3vBxN2 zW?XcI1Sc_dySeQ1stIb-w7nQTYRpTWGg37na;SpBDC_n3hWExl%x3aEidhabQMu_t zY$S-Nkc^ge#ACq4%9vi&(g!q>6q|y!k+)Rbo8|FsE-d)xwmsT)({n4;;NV-MAolC6 zEwD$YWvnI7WsKWOko=D7DCx1eoJg@G*Ixp9!SE_44Kd~1LnsOfw(!*r9k^+J?Z|O) zeLg;M7S&5FDug$C7He@9=P*Ua;0h1r@N!v22`Pw0nSqnsG72 zMd%`q4GsobE&8aoeC5AZH|eK4IVN~PBbvAG?}LmCV$No+r;0Tu0bDCMJj2;lbDDcL^s8Z4uS?Gc`s4wRDO7?rhxq&7#;0V4XQ1U ztPH}9bI(_}zQ*CGt&W#3XR;F1RNh0IM+>#9sB(}t^e?DJb3;=?B|joBZ-Q%*ZCdZ-J{A=mGWZGiyL9Y>M7C~VUnF5wxC8KfCvW&|*rFAQ& z0<}x@C3}z$L?=pzJ@|~9+SkU$ycbXi$w8}qy#6h7M>kzYWSh2mFU)T9lzTtjeFz^k z#l%=elv^7cljNamHChLBF?B%6-qf)OHzsN&6c?+JT2-Lw=~bm?MwpT_=zDYzqBc!| zdz;w_W{joV*oAiyIttG7JiSmc8=4i_uxr9Wp+s}cXGO1Qcjc^<@CtBrwT<+M?YMLw z@`!y+?(4`pB~{2?CEBxo$@vX9s<2CnwCYM{WXw_&@#V|n+Tlx&Wt4fD&uJcoGzV>C zx*OkaIrg!Cag!UZo57Fq$nzJ2EgLi4lzuP=bnBO1ZXWx#>1|VnnHKq!pvDyrgX=AE zs)tw$4xR_b&_>h>nttqE;4*`GaNFPF_(hJ2F6{Jb(Jq7YN{VV@=h^72}L0>nO1AL>5AgnC3KOSk2Ix0Cu$!4&>^jG!1C%WUWF1qnZ@ah-1*2 zt;6Jm!~|>0nehe@4zv{OwD}SB@jnbSUP$-o zn$0J#o-2pmC~i9~rwa_NU;7;>D@kE+Gr-h6mb!u>J)Mrm!?7EVUg`0yJY>&Vk`%ZM zl9s9$ycv~0x_0eQ)HQryXrM1fywfYsjj0S*$UM}J>oLP7vqa?h6!hLAUn0&o5w`A7 zGexbLWDU z+q9F=JwQrX_Oa%)RGVdd*@bIRCUYgPm+0fvB@;gjOM>`}mBJDI@TlD=hMa5klqmOC z##|eH9#H&5y@ymi2C>X+YKDkjy!!6j_B3%Kxwf{VM)5N>U978&e9agcTtQa$H9fx! zmXCpWWcTj#?c1`DpV^p*mcj2NjlsSH4&>jZa{?gaAaV#;@vjiw)dqGWVhxreRwGA+ zK9{rrrkOO1w0ul7@$qO9({dyA&(kW*&6PJLuy~g--|tTpR4l~)r5V}_3&u%*G|WXl zUA(R3s^<)tqonDTW$qKJROl-9b7-%q`ueu9XX@G}rKhIkwr=lINS_|rZLIaT6*NGg~cve#q(lRo1yW(eX2)b`7R9jw1 z7N&eL#zxQ>e0`@q6u?2KaqL1Nuv|u-mb<_*y!ui{W#>&%TCMIPax!KCaElnnW=G+l z=8lQekV8f-$e0f#+l`3G5?~3TOp8D66^K+pO&Gg~HX9A1o%$StOOs zw2xk=V=}~9;G~Y4WhxyLCjf25#WK$%G(~=Xii=N>!F6b)Wyu}R^9ST+t4{N}C?CCs z6H8K^b<3GHG1~T|qtJvy)MFh+%UKZ}fsQ^kRL8kT<<#aFnl-H^+NIjcG@JW&b{?i{ zUK|-mPeQ_nqP3Nkojv^Tg4G( zejBi&@z;W2IoarRGd{Ny1=UGm0bBW8`1YQ-9E~8;c4;uw`0r!x>wip4;IUD2aB!Yg z0{737*w`TOo3e8gGi@nuLbE)D7&+V*C3m1qKoTf*?IxD|;l% zJQFL;bWHA@j9ZnBN#q7N0w@-%W+ARGiT`#t-!2VOa$il3_NDetb6*W z>Pg+Pw~JiVc)*DB#>dR1o(`@*zWTo6R8UGp;GV2>ZfnJ6ja6z1a*&pBP+G&Ic9aUt zwfeiMhmk?}awBdgFlE3CnSLPFbXIFo{s}pfx$iFgmD)C>qUZdh*@9@jz@By%iYNCH zXWh%bwhvARtg%3;3(mM>a9D(hpJ~qP?!q3wvbeeYS%F$d$&Tm}bvuVa#^x;V3=*ZTDYQCWwSa3JuEM1`rzLPC5i?K|*!FSmP(y!cy8$g|Lxi3Odoy+c%p6 ziGu=UR+E`{?S%A6vP=wM;@7?A0^U_ggm!gyKZiNj}9I5<&`>pjCyF>Cl!nAM>P7!^Do(*x&s_Mu@E+=`Uo{Zgo@MyM_MGH_Vxm~AF`_xY&*XY%7R^5ZH*>Ep zzO8)v_A9(3s4fm&s`e{sXWJ#OhujS+VNDNeA4WORERiX@ze%sdhf-4*9}!s8i0k?B z+K^A-u^gmMBQx@FWmVip1v<5c@>V6vJo$E&(Sqz)93ldESY~ON(=a1X5izpMRikiT zirH)dT6gG6jqA&aqM*|CzhEdl1Wl(i@!qzfbTUL`x&s!vimsOa7_E(Z84EREZkx`< z84MxKOPdlGV+Y~euND>jLZ8?v=j{8D1tU@9!j?e|c>7f|OPCAhF3L_A#gpkI2)-(OO5ha4u3 zkiM7w)yCc<-Qz3uXGab(P>Nq21%c*9?T$`eSfpi>Z>b=gl-51Qjj-k1Bo=qjmIA4C zD%|X2Y)O9GU!1?k!1qXzzVK#~ynKs+gx3{t)xG~-4ipZ&+-RHK^gX%$Bc=ud5oq!t zyf<{xzu{`=Tly3b9|1mEB10%`Vx*}*nuYd~55KRlP{543D44$NLiq-PCnhNvMx?eR zI9znbi{*1pP-~4(hl)<9M2dTZ={WtCd|lD6JdZNj@UH=LA=|Q(FR>to3`ODj<@n(|6-pRVc3Lz|B1NNg=sAhEYbH-u8>P$IKT`stT6p5Dd)-@ zKI*qn04z;HAK8ikrpW{wg+ss3G>>7TSuDIWDI9jy;piK{)$m~d%h(}V@LqY*k4Hu;qMnOZ3+mzAa$%983Cfw`UxO!8IZQk>Nmd!1LwpvI zq(GSJjv<3f_g`(PBSZXfG3Ja#LuvbS@V^kgl6~d+%e=4E4?Q(fz1qAyUZw-G=D`m| z=dy~fbQuPOHC&xP6uA4UgGtgK2YYc>41s-K$?YywKVKyid01-}6}~B# z1wmYUW#r=>S|-yADj6EjKS_k_I|2DpSfc5LgaarKQV0$2!o+%$r_?s{ZOHL zQ)_5s0x^*!Sy)RH*D^e$c{^mI`S+sPxVXndpJf}{-sMWq7cI!M8IE_rP1}b(q*`>! zbYJNX+cu3}No{EzHUYY3g`y3*zdvwopUNYWQ?ylxt_IC^L=Tfo^_HZsC=PeOn@Rr$ zu-7&(6Ri@cBoiTCZE zcXI{mW11iN465Gr$WTShRti)_45=RCfeM|;ubyY>RLo06LGZt4hcx|i?e9o;0uJPc z7l}(q!7}y>q09lEO{R&RTvo_OI|6tr1)pasU!X#@Lbx_IgiGyNfdH&7pQkhgI(H(Q z8A9EcNdOdxHXKDT4Vc?d7XlX_Ne=tF&EISDx1%Af72j2!SFW$USHMH=@6KX`moWju zpYo&)*(r+!;DkYcRyKeh?m*5KKW$szbt^l%QcnP)<@|2c08!)?u`$JwFHWtu_dmA$u75}*F| zKu}^d6dW``jfGXVN%Sw*=K`CgV`u62SK-;-f1@+3{Y|hLOt>hdFEF;_XwD}>jFN{{oufzuy+pJp+EEM!bHD0 zcl$kKi?HVU!NsCDCXYxat1N}ay+7s6eZO?)?5QQ?a#A(rjO?A2G2fzSRD-)gx#Z`` z#TzY?>^l@>2(LvM;YRKKKGk*o?LFxGcnmSt&y)^P@ux=fKBVb_=WBi$7$46l8@sNF z2pmp0X5aBSt$Gu0J+7Y5CGmf~I1QE$@|Np;hVWhqy8L|+1mjiSJ^v!9AEh~49 zr%5@Gb|i);ET)V@%1$+u4i!KLlD!oAXnwk{(aete{S#vF&Rcbmb&>VC3wA*9JY}!V zm(tLus0La{OhKR>M)^4^PUA;0{y z&V8{M^NivlfT|+^)oye6w%sJ(Dn7EX-C=a>?T=d50mtGTLMV z&hW!aUx)Sy4+*ScQ*fP*C^0&2Y0aU3ynHWu1uroKcKMG_?ayFUBHf|L_|PF9Kv)xY zgkdB1X!(^Z9I_@gloNd%jdb{9S_G*b2n}k3j7~payKx|x!g(5c(WmYz0wl4#DTf@<5w|jR&#f~#H$uqEs}(UeIhL~V`XLB@y|NR7^?!l9m$g^? zcY*OZ{!3uI|G!*c%v@~$TO2%H#VM;HM#S!On#Xk9K%jWy&uAh_w&%Z)u6H1@&2A7a zv?hnUeGGaXRQH5*ESYP%mrk1cR@R{Z-1#eig1L-Z#VvR|LGOk3G|);4;-IOCv|($F zwO2C$cpz}CHe#wNe>_7znnC-80yN_|Hkm(<*u0FrS9MIAOuIXNKgEEp{|HPWv$()a zW2o`Ag#?#t)vcNPiE^~1%I+UIyGB$ToU|H)%_di?dZN8SLKBQw7+hFcS!rVN(D%gJ zX_xbA#aq#Y3{9k@NKaY!3y;)OeQ9syjxqsADh-d?7~H)U8H@U;DTA5vLUj}~sUvoa zJb{w%M*2E0!6)uSFhK!5=3-03-uP~}HfZc}cE!6cP*vzi7sQNCu;Egqtuyrk(j1xe z{ND}u|24Uqil>7qA%nb;m9mR1G=m%=8{>as6goM(5VEpx{;%9?ES#MG`w6+!wRPTL zNB+Fh53o;@sY4g|0qzMf8BMq-P2)_8*gBooy$-T!;?>2YA(>dY_xqsx^(UAz&|b-A zbej~`5?Y7W4dabRBaxXTS%g%M^Eh}&k6JQ#yNoKKK91yB|9T;ua+L-v3H4Eu`d5)d z{B1C!*nYB;7-qN>md>Dn8n&Vo=C2@fCPP9Bv<70>;Swd0;DBz{&@W9;Na{d#LN3{I zVx%QVNfBZ?t{8}jd~Yc<3ZyWwQhsCy#1RLvK@dnbfxtr0&RA4jiFQ({UBXfa@+C-W zqU^zdL`IFDkc>QoEkKoitoUgG);%GDVuzokP2oaW8U&=@5DR?)oP?PW4jd^9FD^); zChQqhDZ+ekzvKw&32n0pFKz)@S$@-x2_y(2P*QPN7j#0dKz@-w!0@0M;w8xZCh&=b zA30|T~i)<+sYLGGH{E`<2v0>d{>0Fmn6^l=FZva_H;q^QTcHyYs8GWgFDpl z>-_?7K6TgTPsS0o!*c8E_V9*`-8*x%!m;MX`|)J;!jx4nfv5bDrZerIQ-}U&q;T+I z<>~95QZpXaw9v%jKv=?xUO$p7#L^sH!KXPq($iPZvyujOUC#`eq+4&&p6JtACSsp90kgvdUQ z)EDTNLH_a7`78O?9|VSe-nFGsEF#blE=uZlMHMo!fNtZOKh3rRP;vmmJ3$AQlYOtf zmLNK_WrIcRA-MDm5Sk70>!c0#vQmE4gqEeg9=*1O%Pbc}ma6jzj@*iby>%!|nzT}X zx}9rjIax%BRZbS|IdxHny=+&jK!J_9FP*RHho2u4EKSFQj|plfo~_P&@pjW%$Lgkk zRmTck4<=*7vi#0Qi%n1dBoD4ip6ieAqh)n9^sL7~?%nS{-Zk~7x91Qla#8tcc6?5^ zK7A-|y&3PZ;~U}FC(Jda$x{2NG{g*y51aB`u^g-@WFKbL-|F(y{*CM1j@fu!zxF>~ z1pq={nhvWLZhZ`QuyaaM>P(%_?G24L$=E2QDM)4p*{3$xr-d&mKpKG&FFbJ_ncTc?3%J6vzE=H3d7RnYh|p zXQLw{#ks(Hw$WC=b=--FtQ+sh?8;hFo9j8LN!Q4cnAscPojmfM-Ntg~+#4?g_j7VQd+HdG z&9(*MzLVsb$5A>qd@4K3*h+ZYc9Lb%DYkV}*C74De;|a~t)ez98BjA4xMn1l5cgu& zS2;M~<4G?+#xg4(&>I!fc1}Sz*5#_J6%jSw6fcahces%ieWqvr1@5rs^bjm)O!15*Hz=#g_(cq;(Zdv#+p!Q{;#uru7<0dB9-JPA2Y0rV-RLI<})f|Jzdxq$|Va3)Ptc@k8b9{*F#yei*iV(l%~~5ku{G+uK^EwQ#$UOw`-CsCG?L z7lCdP%4QRnt1Fzo9UmV~lIqL+z7tZFPZz>oE{^iXaZ5}jrg^hZXqj-HcI!SKmt)j4 z-fw^ue)k=ZC|T^CtEG`oPl|aFh;gLs_C^Cs*$7y4!;<-V2BdsiYx-(tU%~UQ6Qt<1 zT0aLaJ#6jE=Ym;Zk|)}rgY1;!ejS~zh7RmR2II0;lNDaMR*Q3EwWD~Ss#xAa$T}K$ zU2Rq0t*S?iRqm0W%h#`aO9f}{OP{X}ox(ePQkkZPTXq`LgcwC{&JCrmvmhPmx6YrI zC3(U(>806vH=Yfnk(1`Lcb(BIwXfZtE3<96EOmRfu0B<}MexdHDY$5nn%A9p)jmJ2 zxJA;a>7|@^u7HBX&fKLb6 zbYQ(tRf8MdrW1#IG0*|$#MsFXDLwc(>Ta~r@Jx6;JW9^jv5|~X#1{(ZOTk_;;@uG2 zb<<8vuoL0S^E~!@wF&NP-3hE0uWT*KP^vM+bRBG-hj|< z>)~!`V`&(Mj+J%c`HKy~>?%ooCwibxBgwJVl>T@}ytArn;(vBT$3ioQFC=MB#ESdSdr=ayspL7R1Z&CgrLv2SQMm&U7q-`EP1Gj?DqXAI5Vcl4gShgPEe zr<>Q#fA=q-`|tMTDjL8T5pEHHU7z;AywelSuCP5Iz0()ixnVE-rp)O(NK`lRt++<7 zoY|?ps~l9U=M%oJ;R|}d5m@C;y_S*Gt{q#S2)bhe#<=iruRhY^ZPyLHsj_i%UAXk2 z6d+IxdaKEYq3rQiQH*w6Jbwq@39DQqx)$1IQ`=rbJ@A-4^SjbKHGb0}{@h;wLp$7e z^*q+&r{zb0)ms*<5w2lj!oH7bM8ww@pz++?@Gri_V_0Z-kja<`;SJ~f2KD?#BW}#A z`#{RgMk7uf9vx`#*^M>W;~{$N+m5k(kl-GZ;vVvha!U`?9q)>F{}1ECjRWaBdLMXD z&y|7$@91vn(2tlWKW>8?VSjlVw#%(+{pr58mcR$^EsjF7t9IPfYvFFx8++Vm3X8X< zL)quDyGW}J{Ogx!P;Er8(@#tf`RgmA%YrYD`bP&syO0*hYF@&p*W1mhSbNsZszXip z_u6Gc$Dr=0tJ$u^`o4z8zp*3Fn^gNdC;#ks)`dlGJe7z0%Yp)AuQ3fTL97upy!+RK z4kW%iXDr5SM7w)zq4ij=Qi-Z$E~J1-u=kC;rGVQT0OE;*7JFL=D6Qhl#06k z=hPi%3}^KIe9X9yxbu4+h|!I}3LPy-`+aR2YSRzB3FN%(+#av!8C$j^0A1E9hzveJ z&JhAS!k_+JVR_>R`hSOxNPosyCW`ltN%8)k`CBIHjpYU}d4{Lm-=&9dH307gi~82! z7VCKL-h}m^|4z;ko^h#1u!A=N>Z-9}Y~d8+J&(RWL3aW~w_Q)3@ky=+-32wOH9|)6|u|`d|iAvuMg7Pba`gAJ}okY)jqey zO1!&0Ih-oOV2CA8^Y@z^2Y$S_U0B=_X)|1_sjp?8$ZyxBs$;B$zN7Hv<|Is*K6(n1 z^ZNvs)?N5nJ@Lo;vN4ohjlrY?QeUzwrhs( zxT0Kb>Qm<8;S zlYca=AvXp47y~(?tbILA>b9+{yPY~F8xf|+z5^~Wn!THD7V&#R4m^;*DMwMuCkAJ_ z@lQ|3&D9c%J)g=xBEbriHyti*jmc=AUgC10d8i1&pDmMACS8oD&A(%(PslF8;DeI{6@L zLTLo6Y_TPB7I}kZp$rNcSSE!p>-Pr&AOVWvRnYP(xQI(esuWUQg(?Cnb%vuQ{kM^K z6|s9e;Sbv{s9yPmE^RDwVe70lE4fj2p%PMxNO0j+Ou;BAUWGE?q5+n}3;0Db7U~O) zS_{yRqTEquzpbGT^9e{4?WyPJ@=5en3^FIsg*g?sS-)3p%82>ePQ`%?V^s`321I)>{5t%nJOOwNmc0g$hc0C2fe&hS~*i14G$e%>9PK<*-ZM-t&*P1Lm?$e`5hH71Hx<6CD9SuH%~N_gzL~S37#BXaIl+V%>va4lGu4celhA&gjslR>?hc3w5yK)| zF#tj-Wa)%H#v1QTS~Au_Q2`noP&`VUmdMIY#_&(Ya$Gu5Zm(Rz%Mhwff&koe7Xlaz zZVIh%Edg~f?yaOm6!gb9@-8_UJ=P(LnYx6~5ghZDRmihAoa6*TTI^I{Ac5(J_Z@l0 zCJ%#irjEVZ&J{-=|B?IzFq3>?zt6-;Qms%xeF6x!<&^V}e~T=si3$&ZfzYuVw`iE$ z?VHsNU4pG(hM@B5Tpl&j9c=^h0Po;F_Ch6;#!{K<<`+pJ0kZgPDTe=#bQ2v*_W&hdy*}G5TDXjLh)m_aB{m)>n=)kr1ea8C&KVM@124Z!XizFs)8vw zTJk!WDMJ1!aJR5>yRcIo+uC38$%d7`x`b{AvW#O=N-W2DA*-@FE=@40)zJgYkY%s+}C8%~i zQ5Y7R;p)J2ALIqDW>TY|#azcY6+K0`6EO|>eBCOEK##> zP)h8?DG8a4Mb{2_p=7%;hSI&|)G29m>AT2DJu4i?y(OmIQGMh81ise8ah~eFi~c9T zWrpM!WLWF#x$(A)-gT*X-mt=+nG$v3F}j}FdZ~ZOyun_RTVz|Oid7M7Te+pV0kR}k zb*XYTu-S@}l6Jxwww~WCD{#rSZuFgTrFCD;Y5hCCXwe3@#+0pjYoCNT85BVzWbU0{ z>rhE~wprGNBU2E*8Oylf;l)hTQlI%O*(_q)Rivc{i>TPE*Z`YZcLfN^ioS31DlC3!aPtM`iX>utn`1+el| zWz?Zrop{Mrg)J-Bnzv&IEwL&3`C)aF3}C36wXZ*7%BqV~L3htFo8rGSdYt{aRXVCvjLnf-5IpiR;S<^{nlakt9z{_8 z(%@=XO{X^3fNpmId&K?4X*@;*_s$Y8H@0j<*+hI7a~(CNqe$5YDQc4hZ~j?frhVbQ z#(gZ!U*z|3IaTU&Y_6;~RZqRwlr=(NzHnzFnbZWSlE2DhKeRGy{m=h2sLDsJnxo2` zo7+>vXF_}eShpnHOa1(?_!RTZG(|Rzsjg(Bzk6JxXy|-0CP*`oB`ClGv2}CNIa2Oq zJWX^Gmjo)^&g|YTYR}a6z=;`G<2&|NkD^fXXW47|mHb8N#d>85MX_vS6clfssT6B- z%x(g&FYwen`qXy)3+&Iztj>R<+_@P4i*i@C^fD#nVuEH+vbT33+JYz*T0=3k4|M z?>yrBUU5Kq-iIFXJ+gR?VNYs literal 0 HcmV?d00001 diff --git a/docs/source/images/withinsess.pdf b/docs/source/images/withinsess.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3e29af0b6fbafdfb34d12fcbefacca52b0324042 GIT binary patch literal 34008 zcmagFby%BU@GnRy6nFRH?(XhZT!Xtqa4qic?oc3jad!{yR@_T*hs*c=cK5lv|7@Oo z&dg-yoV@ROl97`M%=E>T1{?`Syx$_0|Vo0MAcRRh@DJ+p8Y0 z3Ku>)H`luYZW3F5%Z7$K zz(;MAK`r68Yrd_=%Yct?51*^)kK5-9;gAR6w;u1o^P3k~Jn5&2v*}vlp7-Z*t+%s* zfZcTx=~oJp?Q+|Z{nb}$!}<{R<(q%G%K0t;%JK)t{u5!+16Lzl5}fJlcZs9zd|a{K z2rfOMjmSQt@Bot4t^hBu%hRg-t5Xa)8oWzkFz?ZJ-NBiFSYzm!!1fkNK6%HA-3Z#- z-TMi&ZA^u$?UnS48QeQhC1b1h?z+#%RfcH>IOAI7V< z6JzN1XDe5&9SRU@m8)oW0i@S9r>NQm5D9M`s8=>&PGZ^>8pX1Q>3wkd@8bDZmYw&f zzHFDd@Z}u5wx-mo3wyQs4%zTe{{dTcfb|vAnU7XKT1&`x6R!49CX*j704`&p?@Wu+ zWi1|-fZgXxX7>0mdU_W8nI84EF8n$KSRiflkHcf*0lEou@ASG+2YriNWZe4E3kv(C z&Iz|fluW~wx^X|}HL>0m>`a7j1lYXebKV|{T7~>lKuGPE?Tp6WrTERw$}EPshjgO}_sQJgMP8fej)!N!(i;-0vifuj>0_w`Mt1gWO!Qopujw$kRw+MFjyGW7C zUB2)l&u~?~;fV7>=*T(A-`JK#`dgZjovg2tf+V2D`Gy$8;nO5hrX8nJ?@LZY#>s!c zm?5geUVSoR*KfuW$G4%UKqOnF28L)z@}eu62Ji3*PN!)BMvln{XHeo3RE*5Ga-5?; zG%s>R2lv(xsJ)YTo`A|o8o*&|J$_R-jZ+;2CDu&6Q2^o5Q zg&~rv9?+v;kV8K_BUcBSv}*#?_Fj(CeG_(=!_C4H=55&$3)wD4C0T{XpseVU#~MmT zWX)|-PL>(yb3&gCHlmDBQ3w&o-XJ#hV!3vgN|Hi;LMrr!{z~p8 z!;nT~dQ9=8w5BYJ+zH_ASf_s<*rmqX8;zjut)x|S_Y!11o_-$}I2J80Z%iGsym|K)%k{RKG z0FjS;xLe@`>TiU9whYt@EFm_-Z~XKRFamFrglF(i*Ygd9V+%-sAR9aaljqr_4N}-$oCi$`8#{N^6Ynu*N(4?=u3^2@T#e4!`bzTI^|`rqj4t3tpeR>#P1tr$5KDpb zoOIeg&Fs8q0)Xi#@w=c>NHxd$-jy2=t##d9h+WdU7qOs66(5YP@(tB~-;$G25Qzd3 zVI@*td2I}@gj5I;3%ojkvCWDq0DWnk8-vBz8k3{2F^i5_10%GlcyWK;hFWyf zbmFODh<|oyezDWUp32Ta=AA<{QewkZD{{wT|HPsmV-C)i?au3!0fMEjgp(dx<$*rW z_^G~$#-9^J%(L)$PnEk4k5JHafJcAMfs!atvn`*+mrtpMxc#Ps!Gji;qBFP8DT#n2 zhJTC9*msu+ez$qkN|d36fRS2ezB_miBO?Ufm^W-S($CSu6t~T*4~sK$Ny>yv$bR_esMO6e_HTjGA8`Cb zss1U$U(nNTM$ULB{HFL{P;Z)k4Y&MoEY=D*W6swQdg+fz&*e%ta5;QG%EOzvVJ+^s z+=q^(-3hv@!>c{J*R^n4#k$2r4X~Fj(y}KjhD)8t79RV=$nN5QdhhG-1^Mwc}1>0yv68mcfuGNuqRu=A$*zlD-57VY-U++*C*5ey~smi1FL_Eac&%DkqHVMNw<^Fktj`b%80Wt?D z^%Lk$xx7ZlLJh3X=x76F9h^Yye<2FpI40&4e&rER7LkCVtUDf$;^Gq|9sTp2fiViI zGKOm|T?I~vT$i8^tuOXZU%r@Aryd-!?{RlA^-bZk_JZgR_{aD%hv#M0blMYJ*Con> zi;$D)ubh#Jo5T1rdF7k4c%(md)etDH0QEgrUENC?2^=@bOPhU5H|$GqHm!0#xq!5_ zMOS#^Dh~|AlAL z*YioW<~mk6-Fn-^TXrVzaaAsU1xr(8%9B-n<&#-WN%V^9ynQ{rBU<)sX&Geh=|p`; zrn1K}_9Wr~^HA2qb&OaB_z*p8#(r*us>l%RpFs);Dx$X1P9L29VSB#I`nSVFc^9I? zi=&2nD19VB!cYOQ@2Ib&kK+G6oPq*wHm?${|9I?ic9sn0Zd3k!Wjc{k40qF*H`ZIY z{c#-!sv4(~-jOpWg6EVz)~-#!kQ&F73eVoWC2npP(z} zZ@KDN)AS!`y?${WDXAqAm+=@uom;0@JKY#|f{;eeA~-6?N>l z)PTNmsFi%74I?|=te@0rZ(Ppd$n;%=MGMUdF<-Se;|wlxQZiKga1vuJY#HGzBTq;w zwiLWZGB&p>on!*bRg4;L@8VvKQoEh@<-c9i8 zL7ga)=(X74Af16!HnJ|H{c-W{Z8!H{_$EpRiM#N*|H@n4i}6Fi_l_Q?Wr9=!MfMT; z#Yw*l4<6gYMK0UV?^~8v6H^wiUA385h?Jp}b0|$xx2s2?8M59D&ite;v}El$DlJE= z0Tfx1wH5wU#jcIh8!6o4q-~Ro8%|}@&hZvmrGpiEY(582TfWZ}wg_Wc4yQGmX1q^N zB^GX)^!Ah>v0VHZEJnl+MSqOe$B!6VCbw&B#eM-GEb@XE=TJ6HJix7BbV*4ry;0&& zKBCJi2VzXAnPu&nm8{fLC#MagPu!N7aq8)ZbEIjf(Z^Dzg(|Mn4G%)AZfV@4cq|H- zS7{a7<<$VVl$V*Z%bnkn|3@(8!;?8_5EG1Sb-jZ@pk!0CZug=S=Tb0Ku@KKrmq_FxQ0cNFVVN9CSzRo$BBd*iW#igBKeW_%4}# zjeFsrDI5q$RzYDN2*jFW|Ig4lGRKfpUO!2e!_F0t-KfYTO<3i`qypY!Mfo?F#mXa5 zhHU_^mH=q9pi@EyW+{@YksRqJQ)q>rq*RT7Cft@@72=Jk%8yv~=0{SHZl%NqbJ9SC z9tbJ8-%4->LQ_nQXi@Njps0gWeqMZk=iYen=XrN}-FY0i3g1H+A&CosA^d2z@O4M& z1a><6%*`HfEZp%B!vExRk_+vWP{ZWKNFg>K|E1#MP+{%@r*N~-RKzo`Q5rG6SiVu| zFJ_!ZZ2rth6wg=NO5^6r$L{9(ED-7%8%J8g9wE`tY*gILs6tEDI^>u-#@N5!!4q;j ze)V3-+(01>m>0&-GWOsNbpro zbaaNC4KGK+R{CFy$7=Nd$7@~&{V{w*GT;|g4_f|;}YsHUbb>4Z_i+_0q$f{o`1Uu3&+AI@S*2 z6q6%vxZ_@JQ7%e|DquK{lnHhD?t?6CleE%CD5hwJ3WJTlg71C+MB=$+Nfz>00HREB zeYA=^eh2J!NquzUkKMrtS~N<#)D?Dk`Ixdfz7F-IWzQOlbTHE+6o1D*-1GAyOH*5% zxaevCkAJiKEWNoc4j4|a!)bSAO|m?r3($&te$g*!>WBj{xOXokuWLad3hM$aG_Guz zC+K^3;Et8HL>K(WjDIqPIb8r57wZcXiu5I>YeoJ}X^iX{FQ)wD%X4>uCrx26zu#NS zm+?!e4%8<><8*9vL4e=5p}Q zK{R2II>QO1*SPxd(qW{m<06y}{AoRp9j}GY;&u`u4H}&# z=wG?aIG1dD8`5+8vYx5>iXowzOJQs|tKBSFY8bWEs^4%IX)gek`GTKQ0v74@JrS3B zSH^z-o%}Xbi66H)4)#pcb+<_Ft7He`Re^arR7uRLdilthu+L&!LAsd|Fh)oiM4?Se z1EponRhxIej}Qb?BsFi%8Apni8=azO`yiQz6V-!Q|7f!f_e!Yfyb0+~fKdml&lwKM z6b=75*gvD+G<{y2XI}S%y>)TWRB29ejL@NemwUScZX&XBWHYprMJ7t&ni(F(TJKA+w}qU7!_k@pD5K@$8IB zhNYW`W~Y7T``g|9B6-yXY|Y4yn${ZA-c$j*SG>bK@goeq?6%#1a5J?%87GB1ZKl+7 zNaU?^q^N|zEoCw_h{?2C3B2jH)vK8XeF93C#@*b^^Y9$)#b_Z0@-rDe^oh7IW;lDH zF`s@F2`^}$DmcZLi16Ly>M{5VW)EeUBt{rFfaTrj^`-Zbdns{*>(2voZHB6}XUL(t zSf)}Ux_&jc#fiA=)3=?v8rX0o2Vz(Jg5UN?Mn-__+F^B@2ud9YI}^NqUXP!FZWj<3 zXTw?S{ig2Ru;Ff!Ja!2E#*k)lL&EXZFLZYD4;mS37{)*|LP2R8L?ZfusuRBXHmHMo zQ2`5K=+;Zrk*aT3@8haN^`beJYEkQT9P&iAD7Xe?syR2JF~F^eE7k$|VK@5WHlwV* zO{C!yC^2Xtc42s1s>$$%vXhwu-3nWH9)x8LGsSOIG!tDGT3yD|kUs3()VY~^f{_7E z9^wWUf&JV)(oH9UlC1QgIJ%NgVarKCWS!w@4i3&zF~@)!naoLU0ue?a%oU)M4yl(F zw&XpMhRC1w_j@?U#@b*OPDtHF5ac@8d2|Sfpqa6R4XWETBCNcQy87c0Gtg|>>5$oO z)M+z^FdR&Ri{fvzAw;d>Fp?}_WSPen7OUbC4TvuJ-f>E+v--+X6|0l7Vc95XE~|$# zR&M%4akj>%{In_{#@6N)XXXp6kK8?s35j?D)miX7Kv70R_N7Zk8-e?6Ad8?23g*Ss zQP)*ohvcrq_ff`3l7uM&WZn~ANuNB#uN}8-J4^bmDx((!{!A;XhudQ<7RP%KJ~$t> z#%KQ`_G?F5GzLiC9pBJZ4lH2KMD7V)5lXc$*WK-Aw#NHg{F0OgoL@8!=C@H^ZR8k1y6G;V zwCjJ?d8GmkG{c|ORrmARjjqbJP%@5eHkTZ%{Ks@NTJt(IQ!+Yir~PjA)0^A<`zdgy z1JWJjbvK5uY}R>@&&IEA=w7-@hGP9>qX}M;){aUZ8io`6gB(JZ8kj4fNq5FrD9vDEdfKNS2d={pP$o#ygc$;~-|JpqjGwr- zDM`AFP4GVvMXt3XepOe^Y4zjj9-l*dr)RAfxw%P^esD1F@i9a=Mo?Ur7J_3UPqx|& zqWQic8ym-U2uLR$eR4WGO4QZAy*aoogsTnbhAy0%4 zW94S$LR5^d$fQj-Yi0vNS1gah8r1L=@KtzY^o4if4x!mmN(jp+xB{Ut8F0iC66jtd1uJfMR0$oR7Oly- zQ>uk%_x8l?t?-Y#$!ew=c-2)>FLVHlF_T(?jnv$@wVI&HfxE2AvBI}njaj{hzp4|B zZM&4o-!klkk#Fo8^LfLJS-x>spD8W#XG$yjnbP7Lz_TX-8q1iJkdq0I62=l5tF4pI z`k@H!yIC9NvKme>6#m{4iS_0vSc^JRXdO50$G5dOMH!0}uSvAgE~Kgg7WX09A5bcW$A1UAJzb))!~+dI20t zFT?sTT=aF-v4ARjXucR*)8(7FmGjFrOX-OW`8~b3J`2|6%_TZE2KVT;`)ZASu_isl z@{mmv9^g6l{E_Dg_Q0Z13vLa5$B>iXr%Etz)8v@-99w*Q&Vlt-%sslWr9ltzO}BlB z1!9FUEU@2!6_NG=TkU>RgF;}Hu}q<(#1Dwl`ae~OFk!&|lqC;+SIn=(myCaG2Qp?S zINLg%37b>*jtyr?b`t{!+Fb~_cv`uDlG%ucCYgtbQY9neR5+VFn$<+j0NdIw0bLSQ zJ?$0Egkq!!VZWU0rvd!89rfKn#&TV6Q?M3)!f%-7K}Fk^q|p7kYKq!iC9b4_cAJYj zMvB_9*!`w4IX)$<#Y%g$?KxJy_9d;y7kNGFbiP zT;Z%Kkz>cuYvC-3`~d4%i7PzW>%4 zsA)cODsDcitGc+Wg1Zck^{=_pm|s1k-%>&7=m#e&J{N1$an&9G>22#)d{U>FZ?|98$h? zYJmEhH9_XW_hB<5H^%nU_y0uwSC*LAR@1YY*y?HkW-iAs)!G$b7<~HhLAHJPb{AvR z*u48r8mDU$0@O!%T zn)dR^k%VHkdIMFOEyL6GB4s3(3u0ORyQSV!Qlo2x1`!&yS5Zygv1$X?oJyn<%RH;9 zmV+9`qUc)jq4N5cjLt>Ub+}P^14XpY8R}Y@^bN4l#1kfr&&_FWtnk_p!&b(HxO;MR z#e5!POU9P zZ@K1NK^3;PDbm~3?D@97rb3l^Q|6Df)HFtKXPd0G()`?{-oLbu)%@QJKFWC7%Xw^d z)pxdw&5N~t?Mt<#4dIq_h0C8SdB5QcQ^)4KTm?0IJ6H;(mCbNToe$>I-eA{(hWyaPi`VyTMuXv8TzYbiBwX-CvR2v(b?UC8R z%6=n^m?h$Ep)sy(Eh|~xd3u;~9;l}L4Quj?Mq4t}N zAYJs<;PwxpAYFjiU)tJIgM2;ne{STci#Yrh2;3kv6mj@M_PynYYJ%P*+$bX#{ixc> zcWFZI1WUUIo9`nXSo^#wMEfi?-^1;py^xo;Zm@=%j%<;=&YnI)ztOkmrUQCAriECm zgKRA^(DQF!X?fcTt&F92wmLT~?H;0GIvm>oyp`|t2!qBpoO=CExE(+25yO#uZ|5I2 zR%9y?!<$&Odie6W4eL!Uc?>wiZh+6Jf)0pBnMWjFOkzcA{AD41k!44X5ZA3d=m1?Yf}H@t!mM zq_!TpUX}}g%|Rs!DZ*?<3C*Egz*>!8sf>6-#s*?WfND(yBbwocc zU>^TzaL^0b#Dm9svJ7pk}B{W*L4T!IE@g7#D6qK#Z{u@??k9wz<`KC zG$|qVV9GR-)z>v^G@DLJ^{dy?T$cPG51`id5_g`T!G)1yBw8NSA*o(}~IT z7xNEyd9-)D=3D!lQmaI#HYOIcaQQwt*-t~G_3=d`{fWiOu#S`}jcN~ittzoA{hDv{ zx;2_ur`7B9zt2nc8JBUh{xkgqpkusu37zzF!bOBylWtANPw=7*@5Z1d&-R|wz^d)i zJ}}0EINqxdTm={M)vR(Sl!fvtiPOv3CF77MTp>KiUNJ`>r%YQ<$ik$i@zBm2s*ztX z9Fqk{O@9Ul4jZU*9p24YV1k-p#i@PR--M)bz8u%q=;MPLr9)sLK9=4Hzp__vs9b4uI!uz>7^IFB5wvt+j=F6;9uvIJq|?=kY1jlv(Bc^NoOPZ^T6PXFIX!s-&cIaK{e8|Vez`=< z9`p4+!??3<1d&|cXHwsXTRT$YHlt07dj)?v^$Zny%&k+;XHo>X;WOFWexDI&zgZ03 zf7mQ7KQDPCs9R83qaSX+#}`AtT$R^xKC5cJ9ebw1X>f1#pJD0gqDD@x?->IifkZ#N z^z3TbZ6%?;f6z@^t7f3trE3klW*`iNA9{AOpj7t;c(J4Q=4-+L5e8@Wf=g+GgH0TG zAEfGBGP6KDqzblN_6q(vhKkAha8&9wxEPL|XJ6?Jdd-v^jpw*qRQYB;M$w*Wy-blA z^^5i}j1Wtqi z>iF+GQ~QfAb-QUtjE3uo0YTMqoFFgPEK!FqsK|$C@Habx!V8$8t7in%_R?Vc_pznp zka(k+9l>q_%+N$TCUw2mYnHlk#mzoudzl=&UsppBZcpmP`I+U~Yhfp~z!qE|hR+!^ z8dzPJQ6XKf98V=z3_1iC^@HoP`@zxbp_>-L@IKkUE7ooEx9AzzjxhD?`<#;3u^?0(Cp6rLeV%fP0^T;g-I{2po zY8aVAHYqR9nNH$1DKE~MtSvnvxzB4h1+d-IftK9&B`YfcaPFp+c*F5o6JQMN;L4wQ z=a!-A*VU}Edh5nvb`onAvpEzm#AfDk^`cq9Qd4bUtf$wdxxRLaHLPK%?DmmxknMIA z{AUOy=mxmY6E7WiGYId3ocR5y6m{QyYTQ`~nHS7UG0M9R?^1&x>M$W42h2^~yM=Yh z$rF#uj;0+siQeb&OC#hy!j5By9d-VOzzeQZjbn!xa}J~7-4cCW+}{817n1*{HW|3O z4Biv&e1CmDn+UkZlIOWu@7m11^nD#n+Su9dfurUFwrX#E_M<&Ndc2S%2pee4!6dw+ z2zUFxY;X0vY&w2i#edvgZvB0G+5XtMe1Ckt*fzY}^Zs~DUsh4R$^JSry@TkP)l>2i z-_z-Tq%Q-`#TgvE|G0?=&x8o`duMXf6DoE<)Dt z>})KoZ2v!SLQjKY{Y&Tg`Ke> zg?J=Ih(@svM3p9`CL=z#`stEp9ixrp1jQGk85^4lY_z^o2P{!2EKV#&-RguFZ!kr~ z-1hdv!$;5Mho5kMz~zR(QO=RT5%1ZJAGiy62?h020s@J}R7upi+*}%4qzFP6T%Etf zvDvd*-m8uCXJy9S*p_;|?Bv@ogx;{`Xsw9~`D0s=zu!dKEp(2q5ttEu4k6|ozo>b7 zomDEe?tG01U2f=nn=>~BPJm&`gb*?FT1>`5(U)yMqNJjI(-0S|Q_?d*L863}@n^5xFkf!FcfL)g|GPV_HAn(u;7<$m2NlDt z5enxvj2gVP_1UB1_4sq&Srd1APn|Iu)(W{5yXF~VL+n{2#iSl z&u{D79l+uLA@1-Dw$wToKU?vfO8IDtzYivPK2FE)PdTNso+W@|q3NAT(@e0+OlkdNR*!o_t%f8sB6%~y+IVo}XJa+A zBa!u?@Jtip#V}2>WIT*GnIx&xicS#$*AZM#h<*aym=pEiF03TE&CE>kHH(lo`y2S` z5UUbz1S>x3!P}xztnex!tO=|*^mUlmI*iFG;mEGVYBQQ3;uCoxvxOp}3$Y_Ph7ouW zHs=&J=e56KOsO^*MKw!gkF0jo&@r*~1ft4-WQI?BO@-A^>2WVUD2(BUY1?sW43}#G zMtaeUE8zNs7_&&?`?!@2;5AhiX`S*b_8p@*BaS3ODxhp0%vpw4}V@mpM}VEY&kVA5a+kW{B6;2`w#R zHo5-!Hb-GtpOueZvMqY>F%Ih>9X_(O6*YRuiWyCv*n0LCh~cqn&|4=GJm2_eFYNhl z^BW#y5bA3zwGq$EK@eLk)bF{*Rycp+-|OFG$@&}rID^McEE+rQLw3{h;HH*6(VN^M^PZ4Gs?)J-Xuo6#>9MHB^t&qeea3D=QhRdU$Q6dUg&cRCl6-u zs^p|{+FWF^k_yc^w7kH$miy-KC^Zb22d+bEHLzfFj{||G0}%UzA&*vG0(m4exzB!K zBUmu;L}Ts|4Uzu}=rHj)LwV=Ff)=gqi``p|L3k2oo z`no44C&C5%|NNx9ZP4fT5{biI^E2XIIW#q5$z(MYBH~>wEDlkQ!EZw3^G9M9WRQ`l z=4y$NyiHBTGDZKr|G{D2e>?WBlL{>`>Kw(c6{CZUt(3*lu1!7spZX~OKlPYS>Vaos ze#{RxiB-gX)3Mu=;w2$QszK?h*49TDS1sRsm7F7S=O`YpRR&D)%z%qijZHu<5e{I* zzwe1&i@~tj)%m8gjlGPXa}QiWtf!&FTGk2rUDEoi?z?ab&g9e>cpTB}ao(?in~`%U z`P`j18GUz;Flp1GDML{z*r0roMJ{3aPyP&&v_s`X_`zsG zgfhm}9hpP*L%3h}k30tMR?US8v^g?v2`c^0qL#u^p%m%o$uhoxO;t4-6hA6pZV#An zTzv54@|+y(EFtl$v9L0}-w5e|QzGf@4hgNLC$HsuKO|DMJQ=$sC+C8XZ?_W-v)Jj{ zYc%=4`1qAzF(|j9tcV(>BAmrhdM`((;EivgsjgaFIu;G+jDf%Nj~HmeigOC zygI9e#9}VN8~iO}m(dHJuRBZ6Dcw-F;BMF6>e0WKhauO`6w-FuFLw8d5kuYP5_Jeb7E{FoN{v);c|Dmq1fpQfBVsyN@+Pk8@m) z^HLC1TVad2nH|BAKQ-1wEl(~tI(dBzERqCAU<4D(>h>R9ULR2m%NY>~f{Z_o+N*4| z=X=fAk+rra#Q1STzrx`(5?k9t%Q~Q z9kHV-N4#j!eD&-~loKoec+8vjB)==&Fm!>O9VC)v367YKFF5_E9fJbIS`ub{>aNey1i#Q`HC=UA4QWV zL`G(M>HsTk0X6VGx?8Y)WPrKUbMs(Q(l@YNOz%m`^gZJSwsb^&HExGU1f)gIjABVZ`BlKg9B3$)SME)Tb z3j{UO#9ozXOJ)|4L1WshC^AUnr7jgP+C*G=o6N0}&wcwXNtxG(kpe-+?tUA%G-a=< zV46Z_4q}U2`k_p+1A1qbTU3IKp~S_NwuvaO4PTnVKBqZ3A*k|nhgEEprto1&b26B= zHb@DI&Ml-NjB~qf_u9TU+6pQQbpAz3EIP3B&(q(j6O_H~T1E-$xuf~L?l&V@3&YE| zfL6GcN1X8MSGDKk+XC1~SfBDof;0)MpC3(3ktvH)4%cJy{rcn6S}^%LAF2^Kq=U}N zG?%Hvab=@2bp4QM*vGL99}X@GuFSMT8zn=_?@=_i&y%K`QM3yXRemRkv*=N;EpRgC z&hjvbYU^xii&wE5wx8H=1!H#FJ}JG4=?-`k_b)Ow)*msuOoFAH7E;-z?aEq7ll9Z5?b6_w3g|p`0+J-s1f#Bm|r6MJK?Of`h^F zt<^|^0M*iBzu(BPQRp9|bPiEpMxCe?mJl-=P+fvq$^QEsR?H7eNvKAzBe;=0T)vyW z`EOJ$qiE*t$u1RQ_DsvH*M1q_y>k*T zGKxuNJ{!f138BLH?^p)+9$X|S-s*oEzzTx9XrPBCVe$Qa;e?QOJB@!qW<}H0hIVXB%Y51c7v)cDg z5kKnH5%Bt$DB~r`z?MKjC!}p_;<$%#&qd>tO(fwCS`cyfzq-<;=ZEdD3CTtEtdArT z2_b?A?cX9C9c9>h#PlH!5gc(hjihPr*gmDH;jFh@HH`>P7aVVn-?t;F3lj*LO4n_pcp*kRJ0Eay)~3+vYFU{~BIa zF32``lHQ@$zh7VC?QBnReTOq}UzYKt?jR!eTqUv#h7Ax(>dvS}JJ5^|$zG{^=crZB z9(D+ybdV#iqNuxM&ouMlanzUc z4o|h}NI)QL2W0R${JEu;NHKE5gtR$R{8-4!K}6jI>nPKRzdJ7njagMeoyMyVLNh$N z9&~0kxI4r1x#_Pp7kWI4XT}wocr$gPSjW)Jy`mr(&!2OsK8tn z0H^t}hr;V|{ZV4i!UG|df{^Y&Ncc-j`6_=l$IAA^zG7y3Fajo)Y?f>|L+>d5(rU(Je(R0q!4_-! z67P+pk<7a6P}!Bnc<95|jEk-p19t4cvt#Ap0b>V>SR9F3ME@AY7BxP)>PSBq_{kVF zh;1y)j}pE@W<|=%Z)PD)+Gr}=xQE(!!yh)u48d?7qCZ$h5$Yj3)O1*S?!LM z7PAX+6}IIKtSW9!1;X0e4E?tdC@l{i$|fzdmv4_*{x}|Z z)1YLxDW46mj}VH9y)|aRoC#MEC&8YpU(xJ~p9y!C`DYn3>WJ=p$&FrYo_+}^6X(zT73yQ)rB^0?Cy>RUSVheLQe$oK~(K zjowAxGEDfbLbC{4Auv^d<`@uUz-$#;fEQ&Sbr(&$L|v{Oz%`V`IrL01CB+Cb3oZ~E z(zo5zw9QqqovLVBl3Rcy0dc7OAS&M{7cED~DyMh^89sqN{^!}!x<7%A9zEMI zJ&jA@g0griJz)7HXB39cn}qh7zz@13p4Cm7F3c67SbNZl7z+cmPj=Z|TgZER`|Jw? zao~sgwB1Q%$S3JNiQmjkpGgl@3jnv4g$o__lZ5nY7-s)Tt66>(X=EgV80s1 zro5O3Y?(NwycP*xs$OF36YodrUY<|$@5UyrzueS3NR8RQd0lMLw3Q0?5_tLtxYe{Q ziKdYUyR33S(KwDRB;r#ij#-7LE5u+q98d3+o_ay6s4ibJUrWhiJ04&R#%C7sIrbR3 zvSIlbl0o~=TA@Ia{YLAs>cl!B zNR|B3;%T_i7hbj9!OScN8SicQPx$W}P%0&7{gP-h8egX8p`w$9a=ptBPr5Ua$c&9L z-n-?H@j+-8r!D(UMGmvk!_;l`1>pnoax;*I@@=c?I`7X^Z|I;BSYvX_X^ZD_;goR; zE|7-w&7hP|4l-&?@8I%YQ`7NEQ(|mCka4RhhQ?9O_`6(-!0FT!&+gQ%^jO<^r}@oc z?5gZU0kv(4z?;pR$y?jA&U5`Ux}Oa5nc5xgGn9CaDBB7mcJP=H%P}JUSI*u9R1Fkb z(f0+pAf??vrGW~mLCSI3SiuFc$UqZ#vMW#`7Z^{O%sJL7hhK<|Y6W+(s*xP4Z#csE zFh1)UwvCst0#@qSO(m#JQKiUbsfpO1#0mzMw)LJ01%i zWtk2FneK-tP7+g%^Wxy=L5|X|5AG84kZ(PL`hDyuTBnbrM6tlE=LuqO8aH1aRfgTl zEIy$P+{{r&ii-lk@8Uv#n2UYUR%@8H7%B#!M{ll~ScqYrj>2C#dfI;@G#Cy=_D&u}ndmr|5X z`j@dGL;z?S^}eT+Iu&+{6clzdGxth94@2QwaNj`>N1QWmTfBU&KMX-by@Qg`UJxH` zVOwRzv%yU*410FsdHmR~cYO^I&p`a$-Bc304Ys}FX0X5F{w?9q>KPhscsk}MzlGm6 zoxa-8Y7dR_nV4){GgrqmTrE%dRoj11b;Me>JfFSHEU~my)GF%eZf{rq=6cjS8~FNA zTW$NQ1X#|msdBjJQE5)2uiA6l+|>>#UJ7&?2ur^FQ!c(b_w|Dsi?@%m1=B z;D%tj1HqZSdzz&y(Eaa`q5_yxmsl^foSK)5p8#Ww97c})@ffivY+x2@BiEG&dWUsM!JFGQ$HX~EZT~%xs)%YRju3PjYVy{1og>vdB@7LXu%MWck{+3 z!o9w$x|Ot(7X4lbhs%%Wv<*^}dm{TN>Mc5#%;qF|j}wu0AF-H1{D|p|ImgT9z({v!cUr?YqG^L5Ma(qanVIpi!0^Q7X`Z}E4rxBVUCB0*J)rmg9DYx&$m z>Ad6#(TVN6qwID@Hz#0(9h*zAQr%JEgCe&Y4kp;_?sTz=;8YU0%&|~w^UH83xoY`e z90;C^7LRdQ$uvGuR%%t-Nb`7~XZL>ck9jgwXE11z_?HAulZ7p47W$-u@Kx54N zh6*?+V72AAB;^|1{?;zuzQeuSkmNOnI0pl}eEmrld zG$+}obZhgBrm{GzmOwfKBUV%R@nk1%cLt3WTN74Y%T~W0u6Gb%mZ8qFZ02W&#a?y9 zvBd-s`liKbj7cYrNv*ER9Zi@UQMb%VUxPyn`WClGH{WA79J><<{t#lD6n*fxaf9VK;Eeyxs{Mq`kr-h^@Jd-BAQ;?I{~nx3wJlzA_edMMyuK4Zi!MzqKhvd#Qdq9{}q45Q%K<#U7C`U&UuDCnN#&ISYh zYk(q4Itca;wi~LFS(VS?GFoq~Q;OLyzl;#W9YLOi(56-_XC9L23hE&oi|8wewj9FX z|3%na07nw@45GW)Ff%jLhMAd}Cd|yZVdf1p!-knT*}#UGGht?ChMVudt9$>wx_WOs zrLinoR;#-_w%d}Cezpm#R-C9a_G;P=8%P&=eJ|Njxi}lTupF*_MsP|Le#n0n$X!6D zAJl$F2kzM4o5UYB@?22fRLygz$7;|IDqcfIQi;ncH!j?qZluRTUE?6kS)>ypf-~bp zIig+mYjpi4d(KXm#(BY55ow9X8r^EHjiRClZnNBP1KNm?wvEn-#fu_%%F)mj_6-TW z9N{<$iGD0A;!3B(Cki%9uKwG>WfzuL+XATX7r?likbOpwaS6jprvrQER;=YS&6``E z7u3_kD4r=E_r$w?w>qKc?a3y}I*;<+%XDbu)C!alV#HZ_C5)J+tEfK-iGnG6y#3uu(M&;t8<5&t3zq8Sl%6#n_Ax2>+q@h$KeHIgh-U zVtA(MO9SWaCWgolEzGJ?C(Vq05v=uo{?ngvwP~^3Tu@kQv2H!q31`IhQ?^s9MOkAY zn2QhPOE!kbG{Kn~hS)u6RdC&N)(zEPf1jl7M98XxQ05EWc&0<#HpwO?7EqDWNW2>j z-FJ5W*`Bd3cS(tqB$z0ew6RZc3-LAMnyJue#a;c!%@s^fp|%(}hu|vco8s52qKT(7 z?xl&;37XSv&IlFGmaQ7w)&EL-guM9C!T)OeLH=Rfe@7LraF7d|=jDyvKiV&d+imh3 z*MC)cRFM`$9Dsc5#J#kmu^#hQhm&=K+_J087kqS+X?bQ5g8I`573(HRH&ODp6hej+ zWzwiAZicvdf~P5Ah9YJ>L{SPoCEW;YL+X8eUSj1a`#$BZ%T0UKx;#hHR)XYc;XZRi z(mA<51=OhbzUVCu1ccdw^bfWu%o!PN$+KfIp5UpH#uMjNVO~qBixRAPx|1KS0hjr; z4IY?=Buir*%NyrU=T?ufZ!)6no|$0s&4;>^&XbFozL{9;wKm;LUd;%r7^~jRif4Qs zDjR$oFCCq(I&MVA3a4(b8Lxw{;}5mR>Z0G`wzc>|F1B8lI)z%}s^%yy8U8@s$hK*g z{vyZ-|K{q-U}M4AnAS8t;zXImDb1)e%An-mPdUS=1;w2vAU>A?&>2kKktj2T^9|3< zzp_Cc)%A)twXvs&n?=-+9Gq8NO=FMDtZXLwcF(RhOE*IN!P=RrSN-{2ebswZ_~P%q z_(-NG8zIyvv6kc&_%)cGw1ZGmiP{^xMk2(%`>O1S;wYCF!KhiS<>jFAVd)p~@F+nW zy!R-&JS2adM&i%qa8+^y(IU8<{UfE3z5Q>ILGe&4KTK(x(E1e} z2S9s6|9)@X6-W(Vp%!D^2&P>RE9Cl3`3^t09NFna0DGd>rh7pmJ22lOD~v*MR2Y!H z6Oi=H=8rI{{P2e+__l2NHU8<0)MqRa8V=P6aRWQwk%at@TnRY8AUww(srfjHdueU* zt$J@P=Ezj}3E@|RcOGy@Mqr!b%!$e|I`!Y4tZtRyaFS+24R+J|lky{4;UAHI(A4RA@6^Sg!D0W{FZ?g=&o2*h5*UTHZ4nl}YZ9pT z5%JvREv$U(V(MXRwR}`{>?rH(U?=Q2nDhXe^xszLfjH^G-09&j&?Y391vryjNKrGi zn<`oin^Rgeo4DsfSPYl|f59P)HdLG0;L2;&clfh9#J_DYoO_7tfxJ#sev$Ie?0&&- zH%yRB0g@<#hf&lY^a7B~30{e*mRM>k;l1F{}CwL9|f7g>K-*0>@>xkB7; zay_0BQ2Z=$$dZxUnuxNXZi_k_vxd=}M`6L2zcD_QcbdK`}hg6%lKF zz?KJ<^cDI+Fsdxn5|fF4pb8XK7~EbhFEZ|gSNTQ4kDLmk1Soz`u_V-dqG<)0lZk$S zaYL6G5MyPWVnLJ2et>z7Xb4I|bDtiKNY=b^!-D@5Q{oRK-7lw7ijMaJqXK0L7I7cQK9`Np&&Kk)foV$f!GCKK+kh z2$OjQkW(|mEbETLu+-QZmJyyFCVk-74ZwWh|76H~CG>Y!^hLrls`pCn znJgg$y~iz+oDD5LRdzMW@B*)_)8!9Z465maeSl^dl6ipki-ucpcZ$qUE70@BO4|{6 z(0d2RDi3#R&j;kI@<-Fn784F>yOv+&Lv5V$w!u~|xL>PQX>9c^Y8DdyV{g;s53#v1 zc|i6HlU)$i43}L9ZbM9~fO6vZD9ez04Qpj&8lLaR~R8k!8+3?h8^wzNn4 z@XdZuaq&l+{LGqDv)4B~$Cy`4zgU~aR!1nmlI~F6dIeuR_Z^;xZ#@F4NZW+EwS>Kl zj=^5vdIn-nb)Hl5pLQu9e%cO9p6V=zWG(jW2vh;S!VLB_9vq$#>{Y+QwstulG<-`W0&Qc?ht+tHD8$NJ?00Bb8%3P(J=g#^wFpk1mIpt2?5-j{C_To z$30Xw;sD^FhE$&R)Zr`Mu5mxhF8j}dx&sBv->xZtl{0QT1a*fBRx)mX%T+e$UGvb` zhzfp=>c9%G)7gk}zs?5b>Lq;%uVB5vrz@xTsP&cdhP{Kj<9tsEeF8pC3Gw@*KsoAT zNe?YwM4~;62X6SC+EbV3&iuS>^#^YJVH*&FURyc6!`k&nAl!dj-i=dX+u_vShv68@ zLv|y^f7kAT@fj6rv1gl3_$yTO)Zm(ZQ#o%?T3B&M??DTGFXEM64PQO)1*znVDE5R1 zM(#_>@HHH4UJyO~^H1+R5=03RTDqT{;VX0C$uRi;iTD#Uef$VKA%d3s5;A;!1DoH% zO8@Ncy?=rz>7io!;14{xf|2uAG<@|2n~#H)6QpJODC@m11R>CJf}~6z5xw_hph_6& zpEbSr?x0F^OdoZ<_x)h=NN{q3^h_TFAPLZN{*s2Tyr41|IYDx!kNDnuQ4j(WN6qw+ z-+LeYUx>es;p-Ym*5?=U{1tBBJ?CdB(}x?#a|ce&U(fLM3~b&2q+3VtJrL9kCDR8x zhy>D2-tg51L;`70(R&X7kw7Zef(lU5KYxP?z)N~SYMKR}FoMKr8NQB#NT}(biM{t} zAQDI!aS#bs&R^N^RTe}7nXI7q9t$Lfn#s4P_x=NH{s~TQi;>AUulL>;guuvcf$RbS zssy?!VZ&EAP$lH_Zh+yde&C5WM2R4XHLmxbAH)hRw?+2f5PyBcS3(dEc*zw?dN;@* zE@1O{Xz8EXz4u3;GR*XDkOAxiPdXt=z7D>K<&Zw1=D&hZzJ?87H@;-kKUYDZ`#~`M zv*-If&DT!~J6Qi?Jg+9n8U&69}P!mGozz-zVKdO*@u^{f$;LL`~5d$19 zC{)}ium1@88DAi=piepjAy=*G8MnbazO`7{SMsg-5e$6?Uyib8??Q!T)}t&zCPvJH z)oa2zagwal#tqGdmyckDfP!F;@$Q&yuqZ)rB`02OOq6086OdV$Y>L6LWcG8ZwWw*o z37wI@w9(^%+hGhdiXFQa51^ZsWL!U|o5LVeLVL2Of;torgnKVH=ZJnc~sPnPjTJs3I{&NVEprxwuzxNDe_)^nD<85BIKq!+er>(s)w6^t}we zR9vTQ^_DS7*5%*M>g>gUR}h@#vpW`J73?floySXE>fw~?7{Tly-4XldOkw+uLQQo{}zKB!8hW zS%_zuQBX!V3vX@x@mUn99Bk#?nQVFw5PJ~)xbTAn$k&ySV*|91=@EH$vr`Z)%#4)N zpHba-DimdCX&8w17EazzGPHtHQ&QtbC+O)2RP|M|lhgC(skq0-Wg1x;!43}0iZec4 z^^8SSd??ctH&Xx47#6P>>)$UxYULx9;CsbaDuOyzN1p02sP=huzn4 z_15sDWyf$nPYwiO;_`?Tw!9}BTo~G$+~p^6TUhdIar`>m2eXGpbF=&~1-=SA6UG@n z?kg~Wxs7IhAz7(;Jyx?jRQlR^c{{V2!<+SvbDn7$75xcFi}Y6y>1_w!1wQ6uXcOFn$3(FuYok3)w70w;CV*{`$kMD zrpz1ZVjD!>wBNkmC_9HPe_a;g>_hAJoyIkd2OYKCeblq2zhD5({xY z^*4EvKs-Knt>M0x#RrhlhLyAlg>+e_jrdf?^hsqR< z8x74-atC*PDFwyBapgnOL(lvf`io>HL>Lh_7U;EY)vv19x8fH-OD6$R7E#N^(&ewu z!H2Lr1R%)^-6UXYKrFGs$%a4jlDw65v6z!*lEKBg+`Hh->*#00=0V9q#(7Jdo5o>@ zcFuSP=$VQGY8IM?8ne;Dq-GM5F-vGg8YeW3wt@!G%BP#7GQA`6KSH$U3ic@nVK;Vg zZlZR1cY#ATyM0k}<;*9Z{!V?EUCMDNq=36P1q z0JuEFuu1Y7+s0&Q2gO?ScDhI9XYpIqYm=G)=M5LMwB&jlla_~y=%mM<6V@qnbJUzK z`*s(JsU3x3`=7sPOIpIRS_>Nj*4Xb-zB7w1zKxqmQ~^)QHvzi(0%3MquD8fm6t{n| zey*hBI9|OC9{vbbz5KGq*IF0w7h#!2^d}(R8`bOSJpE+X`_WYoHQ0gh38zJdhRnfAsWs+y&EU|gSkycO;k5yS!>8QO|IuVrj zN>0??)BZAhK{7iZSl0!*-q@{@E3 zG|`BODr+?OCiqq7o)+Gh8LP;Qmdy33X#b|hc}hq9QsawpPdg{K+7F(er#W;kYd-_0 zk}N`7EW?3C1bGv%BMqu!!ed}8=~W~jVHKHWp5y9uOka&(L-^q4a2SqHu#HAZ2U0er z7p)u&4p>ZS%gGJ-=uz~e0}WUH90 zBes^NnpCXSzC8UqP5Jq!`?zAaZhWjmA;=?P9zo7wRpeBXX5UvbGW-NvUAl4a=_+1p zUMAmfcx#VOy6m0{&(>glK;A|aQ$m*wydR^R+XV9mICxZgi?>l2nKZz!_jepPj&+^; zV?|N`zh+)arp3(C@x)Whv}KjlWQ{y)j{9Wvfz1_^aNKLGMCIRa829#pke6+FM<~4Y zs;t3!Niq0#j*uC`F|ifY8TsdowbY8{gp;H|(x-xbal)LXie?h`I+@1RuGNm!U-f@k ztLN9WX0$y1?pPeUExEPWoSy|DSrgJH>{d}Msn}>)OIur6YkjB5CyB|X^mcI;q1YSY z(k`TN#VWz@g7ZB}&Q;|Y<)Lw}dC@)dIa$wk2`<;)EzU)C$SC!xv8nO4cW(1A8vQpH zY6drs65*JG3%@=OASZ+=z(;DbCUk@;MAJV;i}?EdD&d~i z48r#|hZ&8SrYCt7F@cdIk%gH^?J`VF))6^x$zB)A?g0-2(}O{&)T@Y1ST5)}rJaku zJnWzxTsGG?*JEggV4vJEpIaml!^%9vU>gS$Hcob*~UWma<>n z^c$=;sb~oE_kLRtjcYQ{-jRFEXu^Vr0^0F7TvRzx!~p0!9s8@i5pkQ@tswRX*X-@u z&4q6ox|#&5yPYoW7}MJSLcFK(o-LB$RzI41pcHJ>e{?+yhfaWrCmwg<=Zsqy-?>6| z=1k3Namhk)vyTq6OPh07$zs9i)@M%fZSE5b7Z0(X}V=Id@PV>*uX zq%r%Y>jOrdg|jEF%A*>YxSA-V`uDJl&s?EFv%14m8drMm29xqyX=(YQ!;|HJyFoWT zkIYo;$o0;(BgjWUukWug3V3j^cd&4`7!n)jRrL<7h$n|hF&?mme+%@5{=FpG{|It=40B8#@_AuPijQ2p#Xg%}T z%6eB8gN_;cgRJwTEz#IQt9%*Ke2@d=NNJL-?Vk~>6C=*Z^45RKvQdCA^=TUsHxjE^Y{rn1t)s^Ql{LPLUr*(r9^M+ue{T5GLi{0EG*Fuo^* z>IO=EvCr-X#+5LR8ZkV0>c>QGfGE=60fN6gZ82$w2^YEwwUHIIzfm*GUe`_5t=YuQ zS>4b2^!f~4CgU}ICnqP3Ux=U)G0zG1fdlpGd6~<>@(}?yl+^ELo|_SxF_8L^iG67u zMQINxpC)NPUkN`b|4R609Rf4`5cbqh5Ts$&jagqB6s4|9C`7Qq=-2r?96*m;3;9&T zN|zP04Hq&z;?RW2m;M4g`<1mlB4$(z2PdaXqtd!ll@f=O58YK-) zivA|8>PE6kaO>2n?j?pSbEjI}(f(nYm#e*r>bs|OR5qNhpv)4Q`NByIWE6)}!=;y3 zp?yv2P*`jwKHG2~{#tt0QC+Cpz-H!`-6{>Ir;>vS`|tw$@Z-PO zB_Zm3c$I&R&@b%~NecK^B(}Nm+&0!G9%+NdNl7TTKu+nUzl1EWwy~9`R32GDbU%Ez z?PjK;OF^_3O8C!0?5m4}W=+*fk(5B%CdppfSegnI)eF98rfyRR4xtqr5jWjb3I1o8 zg6M$bNIol{sqDhGS9gtOE-JuM)t1!`o@119ZP^L0{w5XstR#8$bnX<_1ap0Hga1Me z(Ck{#UfEu$jca`gdmAIjxo^m{d{IVkSz~j0B}G2DL)0{ zn-5f4#kf2r^wlcsjO`&P2S8yo_RJN8&{oTo&Gl|AkJT&;gT+#OU2O#bS8mSr%gf&bmVo@&Ptk_OLGTy3mc$`2sD#Ph|$;eVc z1q&hJ$(ejM%|>@A+~3@P`+oEZTczE@f1T6P!b3`KY$7g;9$36>%Glko*gX zjMZWIga1v8RUa_%?>EF2ruvpow8e_;pDtxA_W(5Xh4YQ^C*$?#;SJ)g91nNON{fe7 z=}ViHX3PtIToq*Pv%T64J2qRkO}dS+++g}Hf-Ua=FMV7k**R4TdwfZ5J!99Y+Q7;b5_PJvY+MH52C=EO9-5Lz%e zG^PWUPJl2_m+81Xe>4$sDyR{CVP%C#wH;sZ8b=5U-*MaoS=iQR#58E{m<%3e^GqD2 zXAs0%UPL|%&`{CU*FU^YOG!?N)n|2@PRkesRxmKvlP9d_p<;6$Ed*wt-&}rvmR#u3 zWmU+`XH%$lZBF=59GGxCEPU{vc;eu!JVc!YNiPBg+e-x05;P7wnFP8mUn=*g0-Ez^xiLA>ib<*p7p0vMgfg! ztSr98W?Ce>^P2Roxr)cvp}M+~)D@pLct>Fcr&wzq64VvB$gd*G4<>^Q(GtgSC|V}9 z?DhQ-CgCWC;R%z7WN^jjb;gXU(HcCpRrp%53w$wz?4l45K8(}|2)^%5x@HF>eo9&#@Fvl){MNP@*!U zbe7LI%{EOJF^iuNB@01!NBcNfnwL>5;LP2JEQMQ=iy6%c+?KDT_ z9S}3nx&a+PP$EDlRO0sD^vkQ1YSWxnEo_*A78DY{nvtHzU&i<#WT(RA9fqNkle@lG zVz-GiPuoYUza1_mJS;Dru~UVGrGcgRr06sD4pznjzBD`SYH}okQ$2fpD$Lf*{nVU-OU1rvRB`PqhgOk3{bs& zYJ}2iq9lbUEBg%ZHqB3j?nWLKHnCHF@uR`Z2>ZEQHEn2`88Hh8x98f$#E>Pdu4;-o z#o-iYl^#R8X1OK*}wV8AdE zSEq-^7!vUp3X$pNZC^-9!W5*~$V|fI9K0HL298;Edqu6G_UKExrU-S26A@WxJCF~_ z47oG9nei!|vzwiRMGc;|n71Vx7DwGBj#tV6J4fWOteo6BmO44wV>_E_{QSQhLT;|j z?|5I{No}YPU9H192%N~$G|2RJ8ErY;*^@3Emm7cZh|Ze8#@DzHP(dd}5IIq?kBY97 z{+8sE(6aJ;KUrkN^j|tYogO9N<=|}fI#0PQSl#L9@QG8-OU+KSBZFDZ@|1A1zg#xj zqIp>#OOB4-KlLnnIPz>*18qY8=U=X6GP@bgd|t`QC&{G61>Tz9&<5`eEd!qNQzLTf zj?yU;M_mxvCRH-oaKbLx;=p?}qJIY3WCjK+k&#ct57nLIr7)Mwf5Bzn#byWgw(zu$ z6ed*+UthjtGy{JG6JzD1>wI{{$h07uKx20-^oyD(Sguu^Z4%WioQPe!>fgWR9f2f; zbpeFfwiMOAzCK~6rxTrFRD2(a)raUNSs!ndEW+-~U(0lVQ`i)!P;$iVIX~B%HoG=& z;okUT4F)#$HZ-{nS=2k|lCY$s@T0meRKg4U-{7p@sqIE&y`upuxkl5=akD0mB*Ror zVYrepl|wotKsALr_pGI51PC@j;tywYVKa#G`1{B?J{Fx!I-B=7AC|4p-23*Tob&X3 z-__@)6TPG*E4jq3a(#y%1P;Iqf=3UV28zX8^Sjf-I?qh*)jBpHGpGJhOKC}o=aJH6 zgcVkD=Kw!4DYL4E)(f_-JxOXFcI0f4i-i__3P$0lg6`)F$X4F}H167Goi+T_KGNzI z-`~f6JMiZ?x~}(M*ln)SXJ@N&amTSJHlUqGM8&18Z|<|X`zdH^fMWaL&daU4o4t%T z!dj2djh!ifbLk>C8`%WUP+GA`D-sbwgZ?8Hb{<2ONpH<5ojA@0=_oNeJf@a*;MurR zG(I-r+NnqB@~3!MBu;0-fW+&F#p?&iXXW@lfQrW zFLQyDQG-0iI~`iu!u`bWTOU}FS2CR)Sud?xb2F~xZ*YYCv+L{YZmTzg-L%!e?ihyE zHEu)6)B&}P2b}|U<8EOI(4a-{8Wg&;un0En!ZVLehQ9*3YEm7YogJC8>++d{fefyZ zbPUoTLedeeK##^fTIpZ9W);S@_vUMKG{dAW#TihsGT!t|LtwIM1m`YluuM354MubL zkJRR*qx7I_#6&ar`dzI)Uv*B+%Qtg5wR@~W%4hF{kjK5_Hta~GlbNAD)RVS& zwwILsF5p*C8|{CVDK62|L`Qg6&3M?$y0?rT)t4r1pX+OMAxZR8fpNky4zFhX-V~|l zYATN-Td*(BvSn?5e?S*10o1FWdld^z8B0qQH^|)$cQe1QD*8Zhs7gReN4uxSAN0TV zV=yEp>@e~AHZDKMpz$LL4wt!nvj%QPi``ylYy!*uQ0}^6FbQVkbfaLJNWknOLLFAP zwlxb*!;M?5aop-KFT=POKM%VQpnB5^0AFwYP&yNvx?ekZ;^{D-r-|b?u@Q-5Thp0a>E#}-g64-Y? z$s`Mhmw2IahI1xp7@{eYZE_mZ>(S*-igeQ<%n@K#v^c}9IeevWPv&(a&T7luG&jF99pqDO{t$ntl8SO zbegIk)p$0Pdb`@v9`$ySGp7juyiepydoC!p+yI49hg6qyk^SuKFQXi_blhvg_^Vkx zHT
    TQ%I@5Tkozfe8}@sLoH}g7tHIZUS8N`1r%RYS)D&? zC2H7P?Dk^sq-2Vuq>HFC5~+U|(95v>)c#FQ2Qv>=ki^Bv7}Y6Y455C;@I8!u(-3d` zO}V;P104bL4@{jI|+e`s_qJSi$*bwBR6xlwXYT&kI=x;wid+de?bT|*CMk|yt{-$ zXis(aP;e`RG4sEZO?Q#s)NVb&eV%z&4tn>Fsl1C-!xQ>ipD}r=4`9x*Zb_r*p6A>0 ze&+LZ1O~ARmd;Y+TA|0VfpIrQlsTcWN3z+X4mt~BN|NoBk;L-s9Sfs)cjNX-G=ALh zI))e}zxB`~j}4v6n{~FzrIDa&0|*)7QisQ58Y<(!IVynTF}JoW{5sw0Ll)zPO}dN* zG-yuRZ0J+p#l!wsc6L~a-dxEHB2XLz6a;PZ)+x}~rqrv|hvaI?7IRHahls~m&lsjc z;`1vg&U0ENhUHV6C+})I{KMvuh;a(8!b31wB4!Habo`NEPg1{yec>LU{5bxDj0BZp z-IPZr#BAGHQg@D)KInEHyiDGZo#|w|fOL?#ZK3FI3uyS#;YswTIBVeVxCwq?UuY91 z?uUV4@@dHuhhP1IjNx&q17b{hLEcYWeNR%$G;C+Xr-c5h9;T*SF-x65S*eDpKqX9r zZl3$~Bq~7cg}fLF`R-!?St+HyUL=-YHxvuF?vS(r)ih4?OT0Dj2Gsmoz< zHCa@jb5#}PV{vrYBXuzEL55t{t`1G(oY|9%jRE3#Ik72Y$z)J05pl35s*$3^DXaSFrc`vyPB8n7SnWzj%QmMu`~9 zV}z*707bGdBLh8~yra*>ZI9XJx1KQSX+P zmyu#5$6$?EF_o%2rL!j2OJH zX5%zEN~Ps}!4A#V7~P>0KGghiD`CUYnQg@Two5GDnslfYUQcEv&v%(of6@tuuz$v+ zol_^Y>Ir414MiK8Nl8=4l5>QdMDb)W_EaT*0 z8CkRvThb88I`h!#!ak#=jz3{C!GD&`omjHXk+-anFeCYEzUzJewqt%Vs^ z+l_<`8W2e%*G$HerG0Dy@^X!WkL(?|Dr8IXjILiy>>Eeb>NZ+Tf%4{!pk_Jp#%vtY zRacWwD*7CLtOgRYTYGvHVAkQlwc^knLJOf9M`P1ju%<>L#vL4B{MiNUl&|tn`dv@&{Y-1_ zXLd7XsVs>_EJlLywLC#4_)?Xa2jcQvOR%`?zt~NeM`CStG5a59)SCj9)8xjmdIZ{@ zic>m$eO0NJt{5OZ`eY}nba|g{Yr13jCG6im^Kv@!nNXMg4xtpaf*IL3w7jYO&=22H zo~4%`NpIr0;)-t)*m?8H3g6Q&dr34dgc)MY-(0f9r{IhC7BriJBi0jUgrwtlxQx`% zk%@ca(kIQ|(8Ppban&0C4e+lKNq1^35;+aa3(5)?Ujdlfi*e}vso>Np`Wk58Z^H>L zXGzZg6q+DtuF7(wQVX+IjY+&~^WaN&=Fn&fk#!AldBBiRtBV{~72Kguau4iu*hZOh|GqXvcQb#Tt_%%aEshbK|t!hv`d|I^Ko?K5KpBwQgl|arzM&VsVFFY23NJkLKGNJ1~?bk?HEl!gv zzAsODUjkK>T<}B_Nrftl8b_`#RVBrz>?&qANUpzxNbQ2t+fX*_y13z8_q{a|P0!ql zo)7aK zcxC0*>Q-;hLCEBoXh|=-QL31?_Czqw6s`16TIdG~>i1fsqR0?n1Z%-WT zJN;%#k}j4}=t9STZerNMXnhby#ITp(A%XNIhTr6Zlax>(=h91<#v|rQs3u0>QYa^& zqrr6}rFv8^KE*vINi(XVB5WK#lS#1v$pUn}~va zff2%{+mlPy#WN)qN+Ewwgh1GamBNH?p%7}=hOG%O{q7&}ZB#@BBqJ0oR2V&5%n^b- zN~j+@p1uh_5`6TZtt9R!%DEm09FqyNSi+hVaVtklAybQ{8&6c0A&yo8yf^M{c4;Z! zwuYhvhmt(`^4ujFcUo_nTv%UUjceDk2KxtHB_0(^zVHn65IQSFN3Y)UhLUFQ*Rr#7 zgC%+HEx@qAURYlUN45;&gR`2K_ zw%0tkI)d1}{U=I*PFVPBe-0VB@%#v;hU{6&Zj3kxHuMKrZ>=5Ku$3xkiUUK)@{^x- znxQ1Yp~9aWql8j$YSj#s0zmb+bt)Wj-T4JBKS5X!tB`z^^rDNZ);?_rr3)E8XZ4q#iU(BnDHdX%`~`o znb~`*3+pi8VWGEMN}~aRsI0ixJO1$Pe419L(?f1{&IHq6vST@qSg1S(J7+-s#z` zn%V%M%!Z~`7X=JdQT?g2@VG%U?#r}6SXq-gS4J&v3??PYi>#1#(nE=n`TV3DV2gNhqW6UD& zjfGl)C%#mBQ3+JMZpxI@4ra%PpGmp^5T5#Ytt19bn7muJ-DEv&cn0 z*D^dSvYRC)r`go5Qh;M&lA;9(_VJAlXssA&MRx%N`A7sl6mze zrljNV0OQdc1rqIY(K4MC0FXA>&>v;?6SMN{6zpqk?2eu1*uqDx{;}cY@l5Q*&*iQ_ zMD9A;&d1J$KmPtb|7iaXHVdZX+&|<>GDz<0*lpr*{V&2k9iI20Zks+UT@Qy}@*ZVw zIQuoQdNGVe3rYRQ>JsGEmARKD5mCp?9mnJMtC=6jPRd)ZpDCiX)}AL+o)86$hizM3aqxEJdB}7 z!)t#RhxhivKJ<;l4#R$2V6HlB+L;h_9Lr3jphA_6~%Hr7`X%aoJ^uca;4Vp20FBX_w5PZ`r&*|l0LNO={eUl zw_^K|I#V$@dyz(dF}k@stjzBU(8bR=@oyCfYhJmBUKy)L8&5Ae@g)`p&-4iOS0r_{ zL%;~%xNmP@;qsp|8OxiHkK(Q`zKQz+g(vRl$Zx;U-gIMuhaAR4##=7K0!zqy5&oTV z{>yJ9pD8;AGlq%-f}qsnbuRM0=KJ3NiFD(z_K_!edPw%k&s9VGIor8sfDudb$&dKX zdi31r8sRVg-YNSP{v7`Nq6`f!bzyM?>)ZpY_58-=3Clkly}N>JtHL zL;6VqO4YkWpYL0HLxSgD$X^w&g72SjyIY9I?TycX1-zT9o1HxF;j13vPmCLqqUSrL z2!Gx`M+$oedXvKMqM+pM^IUq|U?BOQtR=rY?pNk1*6x=59#Qdsv|;Rd>K$Awc@Ccj z3D1T5Ts(l@-MZW(68I8*r@P#EvEKXg2j3;y-E!&cp~VM@z<&<^Alfz1`YQc@^vkPw z-}FE_VCz8{YM<~*Y#5m->-L7^q#Mv=d`WPn~W&#|Cl>`$))s0k?g;&j-UHI z3ilTEzs%_H*7HT-xYy`?!|tI}rsH?zmxG!;HYWdgwA(^$AZ;N^G)>wQRu6F2JF2D2 z9sObcVAFXh!L>!~GN|B7$J@6U9JDTi|F(5h81<0gP*>W!6p6tbwa~54SnHsiR+2t) z;GpiKc3h9qozl0DAz0NPs2fXv_N46B6=&1KISu2iTQ-70$~irnDsOW?kDG`;`1kJO zZeYo$d4v(JTR$pJ9e-r_S9%-8lM$|c*_=68H+nNCYViT@X>f3LX}5*1 zpxj*0J@*0q`AVa=X2)kJIiyU%j$5eT}ydic)RpC--+NSsCMmG-wAVHIlaj~0Z4AW6hCnWNg6*Y zVpc+KZ09FCg75+T;-A5Kfffz~@@e%8+@%;l% zJbNvS(Cv+JK@m+B^#7YsZpSfKYO5DVFtJlITauXc7JGnR~%|&xL>bLcB!ektAaO@N$-ne^y3v!LCH9G#|ebEjyx%I5| zzW2R3v}M*E7(M3g1g_GKzVw)YsIi4qwqlz+pBqsmUV{M;3PqbQVrZc8kRquND@Rp16`I$HBFNkABbrl)%fCO!c42UrFJLxgz zDQv#0UW@r_w0w~!e!HX_Vin%+ne9>F9smAv;^RXwt+?kvR0`}L4R1_xaeUl2=D}5D zAo(U?ID4}@Jl^WP;m6k#K~=b8YIHc=r^Dh@b2IsJpEin3IVcvUbyoVFixaVn{c`23 z8!v9c_0s%k2oy+QdiQzH7524H21&6Wa4$}F*Gb>qYJvA-|YQvOUW7Rbl{LbFaEF;xQ3QN zdV44fOQ>m*SXbMfsu}X^6t|q6fvYBSBHj%D~EKVV3uWC3|CD_g07oqgwve$li|4s{)zV(Gt853Gi?Q=N0AS(S6mqd81%?X!Yfn#<`M zX$t$oI4};l@{$XlPYWm@^ zqqG%xS7Z<`P`qKj8CN?D!#&B`9K<0pI8TDSwHTOMi?N(R73lf7|Du}THbbu-cuT=Hk?Nom~t5a{&& zGuJnL(7;5B{7X6jqs1BN_3zF5q4o2R%%cBHS`-G4GK*7E=bIuMZzhd-8h`dkH+Go2 zII?$Ti*6hfraw9h|BG9BWmKm>;h z-qlZxZ?wz2?C9gf)DGTUuD&k0-#1IJCSP8BTz^N6V|Ni#yP zN;)yn_e^{EKjyOPYThNY8$WG@U!JO%ZC8DBw)0byG$UP-HaJ%zCO}9}L$qYZqY3WwJD;h|So)$Rh}|)7 z4WH1rnO>IzA3DwG;9tzgt)cVvX2jnAof@+;e$~rgZ2ym?3vOzFH24vmS(U0_YRIJ@ zl%HRsU}6H43gXiD%uCBxFodwN4v#=O=@CJxY49Gop%L&T?r?=@pgtoYj0KsJl3D>2 zFaTNr Date: Mon, 30 Sep 2024 08:15:27 +0000 Subject: [PATCH 16/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 12 +++++++++--- moabb/evaluations/utils.py | 4 ++-- moabb/tests/splits.py | 25 +++++++++++++++---------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index ffcf3186a..f41fb2e5b 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -67,7 +67,7 @@ class OfflineSplit(BaseCrossValidator): """ - def __init__(self, n_folds = None): + def __init__(self, n_folds=None): self.n_folds = n_folds def get_n_splits(self, metadata): @@ -170,7 +170,9 @@ def split(self, X, y, metadata): calib_ix = group[:calib_size].index test_ix = group[calib_size:].index - yield list(test_ix), list(calib_ix) # Take first #calib_size samples as calibration + yield list(test_ix), list( + calib_ix + ) # Take first #calib_size samples as calibration class SamplerSplit(BaseCrossValidator): @@ -245,7 +247,11 @@ def split(self, X, y, metadata, **kwargs): sampler = self.sampler for ix_train, ix_test in cv.split(X, y, metadata, **kwargs): - X_train, y_train, meta_train = X[ix_train], y[ix_train], metadata.iloc[ix_train] + X_train, y_train, meta_train = ( + X[ix_train], + y[ix_train], + metadata.iloc[ix_train], + ) for ix_train_sample in sampler.split(X_train, y_train, meta_train): ix_train_sample = ix_train[ix_train_sample] yield ix_train_sample, ix_test diff --git a/moabb/evaluations/utils.py b/moabb/evaluations/utils.py index 79996c293..6c7530a32 100644 --- a/moabb/evaluations/utils.py +++ b/moabb/evaluations/utils.py @@ -222,7 +222,7 @@ def create_save_path( return str(path_save) else: print("No hdf5_path provided, models will not be saved.") - + def sort_group(groups): runs_sort = [] @@ -233,7 +233,7 @@ def sort_group(groups): runs_sort.append(index) sorted_ix = np.argsort(runs_sort) return groups[sorted_ix] - + def _convert_sklearn_params_to_optuna(param_grid: dict) -> dict: """ diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 2c125bd53..5d7e85b84 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -1,15 +1,16 @@ -import os -import os.path as osp import numpy as np -import pytest -import torch -from sklearn.model_selection import StratifiedKFold, LeaveOneGroupOut +from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold -from moabb.evaluations.splitters import (CrossSessionSplitter, CrossSubjectSplitter, WithinSessionSplitter) from moabb.datasets.fake import FakeDataset +from moabb.evaluations.splitters import ( + CrossSessionSplitter, + CrossSubjectSplitter, + WithinSessionSplitter, +) from moabb.paradigms.motor_imagery import FakeImageryParadigm + dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() @@ -26,6 +27,7 @@ def eval_split_within_session(): for train, test in cv.split(X_, y_): yield X_[train], X_[test] + # Split done for the Cross Session evaluation def eval_split_cross_session(): for subject in dataset.subject_list: @@ -35,6 +37,7 @@ def eval_split_cross_session(): for train, test in cv.split(X, y, groups): yield X[train], X[test] + # Split done for the Cross Subject evaluation def eval_split_cross_subject(): X, y, metadata = paradigm.get_data(dataset=dataset) @@ -50,7 +53,8 @@ def test_within_session(): split = WithinSessionSplitter(n_folds=5) for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_within_session(), split.split(X, y, metadata, random_state=42))): + zip(eval_split_within_session(), split.split(X, y, metadata, random_state=42)) + ): X_train, X_test = X[train], X[test] # Check if the output is the same as the input @@ -64,7 +68,8 @@ def test_cross_session(): split = CrossSessionSplitter() for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_session(), split.split(X, y, metadata))): + zip(eval_split_cross_session(), split.split(X, y, metadata)) + ): X_train, X_test = X[train], X[test] # Check if the output is the same as the input @@ -78,10 +83,10 @@ def test_cross_subject(): split = CrossSubjectSplitter() for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_subject(), split.split(X, y, metadata))): + zip(eval_split_cross_subject(), split.split(X, y, metadata)) + ): X_train, X_test = X[train], X[test] # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) - From 698e539023fa947c7ca070d436791495ec8758e9 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Mon, 30 Sep 2024 10:32:09 +0200 Subject: [PATCH 17/39] Fixing pre-commit --- moabb/evaluations/metasplitters.py | 68 ++++++++++++++++-------------- moabb/evaluations/splitters.py | 4 +- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index ffcf3186a..88efc2dc7 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -31,39 +31,45 @@ class OfflineSplit(BaseCrossValidator): ---------- n_folds: int Not used in this case, just so it can be initialized in the same way as - TimeSeriesSplit. + PseudoOnlineSplit. Examples -------- >>> import numpy as np >>> import pandas as pd >>> from moabb.evaluations.splitters import CrossSubjectSplitter - >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8],[8,9],[5,4],[2,5],[1,7]]) - >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) - >>> subjects = np.array([1, 1, 2, 2, 3, 3, 4, 4]) - >>> sessions = np.array([0, 0, 1, 1, 0, 0, 1, 1]) + >>> X = np.array([[[5, 6]]*12])[0] + >>> y = np.array([[1, 2]*12])[0] + >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3]) + >>> sessions = np.array([[0, 0, 1, 1]*3])[0] >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) >>> csubj = CrossSubjectSplitter() + >>> off = OfflineSplit() >>> csubj.get_n_splits(metadata) - 2 + 3 >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): >>> print(f"Fold {i}:") >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}") - >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[train_index]}") + >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] >>> for j, test_session in enumerate(off.split(X_test, y_test, meta_test)): >>> print(f" By session - Test: index={test_session}, group={subjects[test_session]}, sessions={sessions[test_session]}") Fold 0: - Train: index=[2 3 4 5 6 7], group=[2 2 3 3 4 4] - Test: index=[0 1], group=[1 1] + Train: index=[ 4 5 6 7 8 9 10 11], group=[2 2 2 2 3 3 3 3], sessions=[0 0 1 1 0 0 1 1] + Test: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1] By session - Test: index=[0, 1], group=[1 1], sessions=[0 0] By session - Test: index=[2, 3], group=[1 1], sessions=[1 1] Fold 1: - Train: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1] + Train: index=[ 0 1 2 3 8 9 10 11], group=[1 1 1 1 3 3 3 3], sessions=[0 0 1 1 0 0 1 1] Test: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1] By session - Test: index=[4, 5], group=[2 2], sessions=[0 0] By session - Test: index=[6, 7], group=[2 2], sessions=[1 1] + Fold 2: + Train: index=[0 1 2 3 4 5 6 7], group=[1 1 1 1 2 2 2 2], sessions=[0 0 1 1 0 0 1 1] + Test: index=[ 8 9 10 11], group=[3 3 3 3], sessions=[0 0 1 1] + By session - Test: index=[8, 9], group=[3 3], sessions=[0 0] + By session - Test: index=[10, 11], group=[3 3], sessions=[1 1] """ @@ -79,7 +85,7 @@ def split(self, X, y, metadata): for subject in subjects.unique(): mask = subjects == subject - X_, y_, meta_ = X[mask], y[mask], metadata[mask] + _, _, meta_ = X[mask], y[mask], metadata[mask] sessions = meta_.session.unique() for session in sessions: @@ -88,11 +94,11 @@ def split(self, X, y, metadata): yield list(ix_test) -class TimeSeriesSplit(BaseCrossValidator): +class PseudoOnlineSplit(BaseCrossValidator): """Pseudo-online split for evaluation test data. It takes into account the time sequence for obtaining the test data, and uses first run, - or first #calib_size trial as calibration data, and the rest as evaluation data. + or first #calib_size trials as calibration data, and the rest as evaluation data. Calibration data is important in the context where data alignment or filtering is used on training data. @@ -110,7 +116,7 @@ class TimeSeriesSplit(BaseCrossValidator): >>> import numpy as np >>> import pandas as pd >>> from moabb.evaluations.splitters import CrossSubjectSplitter - >>> from moabb.evaluations.metasplitters import TimeSeriesSplit + >>> from moabb.evaluations.metasplitters import PseudoOnlineSplit >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7]]) >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2]) @@ -118,15 +124,15 @@ class TimeSeriesSplit(BaseCrossValidator): >>> runs = np.array(['0', '1', '0', '1', '0', '1', '0', '1']) >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) >>> csubj = CrossSubjectSplitter() - >>> tssplit = TimeSeriesSplit() - >>> tssplit.get_n_splits(metadata) + >>> posplit = PseudoOnlineSplit() + >>> posplit.get_n_splits(metadata) 4 >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): >>> print(f"Fold {i}:") >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}, runs={runs[train_index]}") >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}, runs={runs[test_index]}") >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] - >>> for j, (test_ix, calib_ix) in enumerate(tssplit.split(X_test, y_test, meta_test)): + >>> for j, (test_ix, calib_ix) in enumerate(posplit.split(X_test, y_test, meta_test)): >>> print(f" Evaluation: index={test_ix}, group={subjects[test_ix]}, sessions={sessions[test_ix]}, runs={runs[test_ix]}") >>> print(f" Calibration: index={calib_ix}, group={subjects[calib_ix]}, sessions={sessions[calib_ix]}, runs={runs[calib_ix]}") @@ -163,7 +169,7 @@ def split(self, X, y, metadata): for run in runs: test_ix = group[group["run"] != run].index calib_ix = group[group["run"] == run].index - yield test_ix, calib_ix + yield list(test_ix), list(calib_ix) break # Take the fist run as calibration else: calib_size = self.calib_size @@ -176,10 +182,12 @@ def split(self, X, y, metadata): class SamplerSplit(BaseCrossValidator): """Return subsets of the training data with different number of samples. - Util for estimating the performance of a model when using different number of - training samples and plotting the learning curve. You must define the data - evaluation type (WithinSubject, CrossSession, CrossSubject) so the training set - can be sampled. + This splitter can be used for estimating a model's performance when using different + numbers of training samples, and for plotting the learning curve. You must define the + data evaluation type (WithinSubject, CrossSession, CrossSubject) so the training set + can be sampled. It is also needed to pass a dictionary indicating the policy used + for sampling the training set and the number of examples (or the percentage) that + each sample must contain. Parameters ---------- @@ -187,21 +195,20 @@ class SamplerSplit(BaseCrossValidator): Evaluation splitter already initialized. It can be WithinSubject, CrossSession, or CrossSubject Splitters. data_size : dict - Contains the policy to pick the datasizes to - evaluate, as well as the actual values. The dict has the - key 'policy' with either 'ratio' or 'per_class', and the key + Contains the policy to pick the datasizes to evaluate, as well as the actual values. + The dict has the key 'policy' with either 'ratio' or 'per_class', and the key 'value' with the actual values as a numpy array. This array should be sorted, such that values in data_size are strictly monotonically increasing. Examples -------- - + >>> import numpy as np >>> import pandas as pd - >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7], [8, 9], [5, 4], [2, 5], [1, 7]]) - >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]) - >>> subjects = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) + >>> X = np.array([[[5, 6]]*12])[0] + >>> y = np.array([[1, 2]*12])[0] + >>> subjects = np.array([1]*12) >>> sessions = np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) - >>> runs = np.array(['0', '0', '1', '2', '0', '0', '1', '2', '0', '0', '1', '2']) + >>> runs = np.array(['0', '0', '1', '1', '2', '2', '0', '0', '1', '1', '2', '2',]) >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) >>> from moabb.evaluations.metasplitters import SamplerSplit >>> from moabb.evaluations.splitters import CrossSessionSplitter @@ -226,7 +233,6 @@ class SamplerSplit(BaseCrossValidator): Train: index=[0 2 4 1 3 5], sessions=[0 0 0 0 0 0] Test: index=[ 6 7 8 9 10 11], sessions=[1 1 1 1 1 1] - """ def __init__(self, data_eval, data_size): diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 008bc7a16..ac4eae093 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -80,7 +80,7 @@ def split(self, X, y, metadata, **kwargs): for session in np.unique(sessions): mask_s = sessions == session - X_s, y_s, meta_s = ( + X_s, y_s, _ = ( X_[mask_s], y_[mask_s], meta_[mask_s], @@ -126,7 +126,7 @@ def split(self, X, y, metadata, **kwargs): for session in np.unique(sessions): mask = sessions == session - X_, y_, meta_ = ( + X_, y_, _ = ( X[mask], y[mask], metadata[mask], From 98d12ac91ae15718a468dfbe64fd94abae0e08fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:32:36 +0000 Subject: [PATCH 18/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/tests/splits.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 5d7e85b84..02b326c9d 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -1,4 +1,3 @@ - import numpy as np from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold From 558d27ba29b925a9096fa093412175449b035783 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Tue, 1 Oct 2024 09:21:39 +0200 Subject: [PATCH 19/39] Adding some tests for metasplitters --- moabb/evaluations/metasplitters.py | 28 ++++-- moabb/evaluations/splitters.py | 6 +- moabb/tests/metasplits.py | 147 +++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 moabb/tests/metasplits.py diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index 00e681e90..a6f8bd1c7 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -73,8 +73,9 @@ class OfflineSplit(BaseCrossValidator): """ - def __init__(self, n_folds=None): + def __init__(self, n_folds=None, run=False): self.n_folds = n_folds + self.run = run def get_n_splits(self, metadata): return metadata.groupby(["subject", "session"]).ngroups @@ -85,14 +86,25 @@ def split(self, X, y, metadata): for subject in subjects.unique(): mask = subjects == subject - _, _, meta_ = X[mask], y[mask], metadata[mask] + X_, y_, meta_ = X[mask], y[mask], metadata[mask] sessions = meta_.session.unique() for session in sessions: - ix_test = meta_[meta_["session"] == session].index + session_mask = meta_["session"] == session + X_session, y_session, meta_session = X_[session_mask], y_[session_mask], meta_[session_mask] - yield list(ix_test) + # If you can (amd want) to split by run also + if self.run and "run" in meta_session.columns: + runs = meta_session["run"].unique() + for run in runs: + run_mask = meta_session["run"] == run + ix_test = meta_session[run_mask].index + yield list(ix_test) + + else: + ix_test = meta_session.index + yield list(ix_test) class PseudoOnlineSplit(BaseCrossValidator): """Pseudo-online split for evaluation test data. @@ -185,10 +197,10 @@ class SamplerSplit(BaseCrossValidator): """Return subsets of the training data with different number of samples. This splitter can be used for estimating a model's performance when using different - numbers of training samples, and for plotting the learning curve. You must define the - data evaluation type (WithinSubject, CrossSession, CrossSubject) so the training set - can be sampled. It is also needed to pass a dictionary indicating the policy used - for sampling the training set and the number of examples (or the percentage) that + numbers of training samples, and for plotting the learning curve per number of training + samples. You must define the data evaluation type (WithinSubject, CrossSession, CrossSubject) + so the training set can be sampled. It is also needed to pass a dictionary indicating the + policy used for sampling the training set and the number of examples (or the percentage) that each sample must contain. Parameters diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index ac4eae093..53ca78e51 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,4 +1,5 @@ import numpy as np +from alembic.testing import assert_raises from sklearn.model_selection import ( BaseCrossValidator, GroupKFold, @@ -56,7 +57,7 @@ class WithinSessionSplitter(BaseCrossValidator): """ - def __init__(self, n_folds: int): + def __init__(self, n_folds=5): self.n_folds = n_folds def get_n_splits(self, metadata): @@ -65,6 +66,8 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata, **kwargs): + assert isinstance(self.n_folds, int) + subjects = metadata.subject.values cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) @@ -120,6 +123,7 @@ def get_n_splits(self, metadata): def split(self, X, y, metadata, **kwargs): assert len(np.unique(metadata.subject)) == 1 + assert isinstance(self.n_folds, int) sessions = metadata.subject.values cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py new file mode 100644 index 000000000..af9021814 --- /dev/null +++ b/moabb/tests/metasplits.py @@ -0,0 +1,147 @@ +import os +import os.path as osp + +import numpy as np +import pytest +import torch +from sklearn.model_selection import StratifiedKFold, LeaveOneGroupOut + +from moabb.evaluations.metasplitters import (OfflineSplit,PseudoOnlineSplit, SamplerSplit) +from moabb.evaluations.splitters import (CrossSessionSplitter, CrossSubjectSplitter, WithinSessionSplitter) +from moabb.datasets.fake import FakeDataset +from moabb.paradigms.motor_imagery import FakeImageryParadigm + +dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) +paradigm = FakeImageryParadigm() + +# Split done for the Within Session evaluation +def eval_sampler_split(): + for subject in dataset.subject_list: + X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) + sessions = metadata.session + for session in np.unique(sessions): + ix = sessions == session + cv = StratifiedKFold(5, shuffle=True, random_state=42) + X_, y_, meta_ = X[ix], y[ix], metadata.loc[ix] + for train, test in cv.split(X_, y_): + X_test, y_test, meta_test = X_[test], y_[test], meta_.loc[test] + + yield X_[train], X_[test] + +# Split done for the Cross Session evaluation +def eval_split_cross_session(): + for subject in dataset.subject_list: + X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) + groups = metadata.session.values + cv = LeaveOneGroupOut() + for _, test in cv.split(X, y, groups): + metadata_test = metadata.loc[test] + runs = metadata_test.run.values + for r in np.unique(runs): + ix = runs == r + yield X[test[ix]] + +def pseudo_split_cross_session(): + for subject in dataset.subject_list: + X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) + groups = metadata.session.values + cv = LeaveOneGroupOut() + for _, test in cv.split(X, y, groups): + metadata_test = metadata.loc[test] + runs = metadata_test.run.values + ix = runs == runs[0] + yield X[test[ix]] + + +# Split done for the Cross Subject evaluation +def eval_split_cross_subject(): + X, y, metadata = paradigm.get_data(dataset=dataset) + groups = metadata.subject.values + cv = LeaveOneGroupOut() + for _, test in cv.split(X, y, groups): + metadata_test = metadata.loc[test] + sessions = metadata_test.session.values + for sess in np.unique(sessions): + ix = sessions == sess + yield X[test[ix]] + +# Split done for the Cross Subject evaluation +def pseudo_split_cross_subject(): + X, y, metadata = paradigm.get_data(dataset=dataset) + groups = metadata.subject.values + cv = LeaveOneGroupOut() + for _, test in cv.split(X, y, groups): + metadata_test = metadata.loc[test] + sessions = metadata_test.session.values + runs = metadata_test.run.values + for sess in np.unique(sessions): + ix = sessions == sess + X_sess, metadata_sess = X[test[ix]], metadata_test.loc[test[ix]].reset_index(drop=True) + + runs_in_session = metadata_sess.run.values + # yield just calibration part + yield X_sess[runs_in_session == runs_in_session[0]] + + +@pytest.mark.parametrize("split", [CrossSubjectSplitter, CrossSessionSplitter]) +def test_offline(split): + X, y, metadata = paradigm.get_data(dataset=dataset) + + run = True if isinstance(split, CrossSessionSplitter) else False + + if isinstance(split, CrossSessionSplitter): + eval_split = eval_split_cross_session + else: + eval_split = eval_split_cross_subject + + split = split() + metasplit = OfflineSplit(run=run) + + Tests = [] + for (_,test) in split.split(X, y, metadata): + X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] + for i, (test_index) in enumerate(metasplit.split(X_test, y_test, metadata_test)): + Tests.append(X[test_index]) + + for ix, X_test_t in enumerate(eval_split()): + # Check if the output is the same as the input + assert np.array_equal(Tests[ix], X_test_t) + + +@pytest.mark.parametrize("split", [CrossSubjectSplitter, CrossSessionSplitter]) +def test_pseudoonline(split): + X, y, metadata = paradigm.get_data(dataset=dataset) + + if isinstance(split, CrossSessionSplitter): + eval_split = pseudo_split_cross_session + else: + eval_split = pseudo_split_cross_subject + + split = split() + metasplit = PseudoOnlineSplit() + + Tests = [] + for (_,test) in split.split(X, y, metadata): + X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] + for i, (_,calib_index) in enumerate(metasplit.split(X_test, y_test, metadata_test)): + Tests.append(X[calib_index]) + + for ix, X_calib_t in enumerate(eval_split()): + # Check if the output is the same as the input + assert np.array_equal(Tests[ix], X_calib_t) + +@pytest.mark.skip(reason="Still working on that") +def test_sampler(): + X, y, metadata = paradigm.get_data(dataset=dataset) + data_size = dict(policy="per_class", value=np.array([5, 10, 30, 50])) + + split = SamplerSplit() + + for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( + zip(eval_split_cross_subject(), split.split(X, y, metadata))): + X_train, X_test = X[train], X[test] + + # Check if the output is the same as the input + assert np.array_equal(X_train, X_train_t) + assert np.array_equal(X_test, X_test_t) + From b435bf815d5664f35637ccc23336a00dd7dc3399 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 07:22:05 +0000 Subject: [PATCH 20/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 7 ++++- moabb/evaluations/splitters.py | 1 - moabb/tests/metasplits.py | 42 ++++++++++++++++++------------ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index a6f8bd1c7..2846e395d 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -91,7 +91,11 @@ def split(self, X, y, metadata): for session in sessions: session_mask = meta_["session"] == session - X_session, y_session, meta_session = X_[session_mask], y_[session_mask], meta_[session_mask] + X_session, y_session, meta_session = ( + X_[session_mask], + y_[session_mask], + meta_[session_mask], + ) # If you can (amd want) to split by run also if self.run and "run" in meta_session.columns: @@ -106,6 +110,7 @@ def split(self, X, y, metadata): ix_test = meta_session.index yield list(ix_test) + class PseudoOnlineSplit(BaseCrossValidator): """Pseudo-online split for evaluation test data. diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 53ca78e51..30012fb5e 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,5 +1,4 @@ import numpy as np -from alembic.testing import assert_raises from sklearn.model_selection import ( BaseCrossValidator, GroupKFold, diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py index af9021814..1d4a39e48 100644 --- a/moabb/tests/metasplits.py +++ b/moabb/tests/metasplits.py @@ -1,19 +1,21 @@ -import os -import os.path as osp import numpy as np import pytest -import torch -from sklearn.model_selection import StratifiedKFold, LeaveOneGroupOut +from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold -from moabb.evaluations.metasplitters import (OfflineSplit,PseudoOnlineSplit, SamplerSplit) -from moabb.evaluations.splitters import (CrossSessionSplitter, CrossSubjectSplitter, WithinSessionSplitter) from moabb.datasets.fake import FakeDataset +from moabb.evaluations.metasplitters import OfflineSplit, PseudoOnlineSplit, SamplerSplit +from moabb.evaluations.splitters import ( + CrossSessionSplitter, + CrossSubjectSplitter, +) from moabb.paradigms.motor_imagery import FakeImageryParadigm + dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() + # Split done for the Within Session evaluation def eval_sampler_split(): for subject in dataset.subject_list: @@ -28,6 +30,7 @@ def eval_sampler_split(): yield X_[train], X_[test] + # Split done for the Cross Session evaluation def eval_split_cross_session(): for subject in dataset.subject_list: @@ -41,6 +44,7 @@ def eval_split_cross_session(): ix = runs == r yield X[test[ix]] + def pseudo_split_cross_session(): for subject in dataset.subject_list: X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) @@ -65,6 +69,7 @@ def eval_split_cross_subject(): ix = sessions == sess yield X[test[ix]] + # Split done for the Cross Subject evaluation def pseudo_split_cross_subject(): X, y, metadata = paradigm.get_data(dataset=dataset) @@ -76,7 +81,9 @@ def pseudo_split_cross_subject(): runs = metadata_test.run.values for sess in np.unique(sessions): ix = sessions == sess - X_sess, metadata_sess = X[test[ix]], metadata_test.loc[test[ix]].reset_index(drop=True) + X_sess, metadata_sess = X[test[ix]], metadata_test.loc[test[ix]].reset_index( + drop=True + ) runs_in_session = metadata_sess.run.values # yield just calibration part @@ -98,14 +105,14 @@ def test_offline(split): metasplit = OfflineSplit(run=run) Tests = [] - for (_,test) in split.split(X, y, metadata): + for _, test in split.split(X, y, metadata): X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] for i, (test_index) in enumerate(metasplit.split(X_test, y_test, metadata_test)): Tests.append(X[test_index]) for ix, X_test_t in enumerate(eval_split()): - # Check if the output is the same as the input - assert np.array_equal(Tests[ix], X_test_t) + # Check if the output is the same as the input + assert np.array_equal(Tests[ix], X_test_t) @pytest.mark.parametrize("split", [CrossSubjectSplitter, CrossSessionSplitter]) @@ -121,14 +128,17 @@ def test_pseudoonline(split): metasplit = PseudoOnlineSplit() Tests = [] - for (_,test) in split.split(X, y, metadata): + for _, test in split.split(X, y, metadata): X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] - for i, (_,calib_index) in enumerate(metasplit.split(X_test, y_test, metadata_test)): + for i, (_, calib_index) in enumerate( + metasplit.split(X_test, y_test, metadata_test) + ): Tests.append(X[calib_index]) for ix, X_calib_t in enumerate(eval_split()): - # Check if the output is the same as the input - assert np.array_equal(Tests[ix], X_calib_t) + # Check if the output is the same as the input + assert np.array_equal(Tests[ix], X_calib_t) + @pytest.mark.skip(reason="Still working on that") def test_sampler(): @@ -138,10 +148,10 @@ def test_sampler(): split = SamplerSplit() for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_subject(), split.split(X, y, metadata))): + zip(eval_split_cross_subject(), split.split(X, y, metadata)) + ): X_train, X_test = X[train], X[test] # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) - From d8f26a398bf67c529820c14005febf9a42c2147a Mon Sep 17 00:00:00 2001 From: brunalopes Date: Tue, 1 Oct 2024 09:26:26 +0200 Subject: [PATCH 21/39] Fixing pre-commit --- moabb/evaluations/metasplitters.py | 2 +- moabb/tests/metasplits.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index a6f8bd1c7..f0f6774fd 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -91,7 +91,7 @@ def split(self, X, y, metadata): for session in sessions: session_mask = meta_["session"] == session - X_session, y_session, meta_session = X_[session_mask], y_[session_mask], meta_[session_mask] + _, _, meta_session = X_[session_mask], y_[session_mask], meta_[session_mask] # If you can (amd want) to split by run also if self.run and "run" in meta_session.columns: diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py index af9021814..fa579c6e6 100644 --- a/moabb/tests/metasplits.py +++ b/moabb/tests/metasplits.py @@ -14,7 +14,7 @@ dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() -# Split done for the Within Session evaluation +# Still working on this def eval_sampler_split(): for subject in dataset.subject_list: X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) @@ -24,8 +24,6 @@ def eval_sampler_split(): cv = StratifiedKFold(5, shuffle=True, random_state=42) X_, y_, meta_ = X[ix], y[ix], metadata.loc[ix] for train, test in cv.split(X_, y_): - X_test, y_test, meta_test = X_[test], y_[test], meta_.loc[test] - yield X_[train], X_[test] # Split done for the Cross Session evaluation @@ -73,7 +71,7 @@ def pseudo_split_cross_subject(): for _, test in cv.split(X, y, groups): metadata_test = metadata.loc[test] sessions = metadata_test.session.values - runs = metadata_test.run.values + for sess in np.unique(sessions): ix = sessions == sess X_sess, metadata_sess = X[test[ix]], metadata_test.loc[test[ix]].reset_index(drop=True) @@ -131,11 +129,11 @@ def test_pseudoonline(split): assert np.array_equal(Tests[ix], X_calib_t) @pytest.mark.skip(reason="Still working on that") -def test_sampler(): +def test_sampler(data_eval): X, y, metadata = paradigm.get_data(dataset=dataset) data_size = dict(policy="per_class", value=np.array([5, 10, 30, 50])) - split = SamplerSplit() + split = SamplerSplit(data_eval=data_eval, data_size=data_size) for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( zip(eval_split_cross_subject(), split.split(X, y, metadata))): From e5159f2da6d1dd169639d6b0388193378e336646 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 07:29:23 +0000 Subject: [PATCH 22/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/metasplitters.py | 6 +++++- moabb/tests/metasplits.py | 7 ++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index b2b72061e..fe428ba90 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -91,7 +91,11 @@ def split(self, X, y, metadata): for session in sessions: session_mask = meta_["session"] == session - _, _, meta_session = X_[session_mask], y_[session_mask], meta_[session_mask] + _, _, meta_session = ( + X_[session_mask], + y_[session_mask], + meta_[session_mask], + ) # If you can (amd want) to split by run also if self.run and "run" in meta_session.columns: diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py index 1f80be889..84a8ec5a8 100644 --- a/moabb/tests/metasplits.py +++ b/moabb/tests/metasplits.py @@ -4,16 +4,14 @@ from moabb.datasets.fake import FakeDataset from moabb.evaluations.metasplitters import OfflineSplit, PseudoOnlineSplit, SamplerSplit -from moabb.evaluations.splitters import ( - CrossSessionSplitter, - CrossSubjectSplitter, -) +from moabb.evaluations.splitters import CrossSessionSplitter, CrossSubjectSplitter from moabb.paradigms.motor_imagery import FakeImageryParadigm dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() + # Still working on this def eval_sampler_split(): for subject in dataset.subject_list: @@ -151,4 +149,3 @@ def test_sampler(data_eval): # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) - From 516a5e837f7a176354b4da17a64a626ff7e689f1 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Tue, 1 Oct 2024 09:31:27 +0200 Subject: [PATCH 23/39] Fixing pre-commit --- moabb/tests/metasplits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py index 1f80be889..85b7b1fde 100644 --- a/moabb/tests/metasplits.py +++ b/moabb/tests/metasplits.py @@ -22,7 +22,7 @@ def eval_sampler_split(): for session in np.unique(sessions): ix = sessions == session cv = StratifiedKFold(5, shuffle=True, random_state=42) - X_, y_, meta_ = X[ix], y[ix], metadata.loc[ix] + X_, y_, _ = X[ix], y[ix], metadata.loc[ix] for train, test in cv.split(X_, y_): yield X_[train], X_[test] From 37cff03a3eefd6e2df7874117b2e701e79d85b32 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Thu, 17 Oct 2024 16:13:20 +0200 Subject: [PATCH 24/39] Fix example SamplerSplit Add shuffle and random_state parameters to WithinSession --- moabb/evaluations/metasplitters.py | 229 +++++------------------------ moabb/evaluations/splitters.py | 219 ++++----------------------- moabb/tests/metasplits.py | 54 +------ moabb/tests/splits.py | 37 +---- 4 files changed, 72 insertions(+), 467 deletions(-) diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py index fe428ba90..9061dbfa8 100644 --- a/moabb/evaluations/metasplitters.py +++ b/moabb/evaluations/metasplitters.py @@ -16,187 +16,6 @@ import numpy as np from sklearn.model_selection import BaseCrossValidator -from moabb.evaluations.utils import sort_group - - -class OfflineSplit(BaseCrossValidator): - """Offline split for evaluation test data. - - It can be used for further splitting of the test data based on sessions or runs as needed. - - Assumes that, per session, all test trials are available for inference. It can be used when - no filtering or data alignment is needed. - - Parameters - ---------- - n_folds: int - Not used in this case, just so it can be initialized in the same way as - PseudoOnlineSplit. - - Examples - -------- - >>> import numpy as np - >>> import pandas as pd - >>> from moabb.evaluations.splitters import CrossSubjectSplitter - >>> X = np.array([[[5, 6]]*12])[0] - >>> y = np.array([[1, 2]*12])[0] - >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3]) - >>> sessions = np.array([[0, 0, 1, 1]*3])[0] - >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) - >>> csubj = CrossSubjectSplitter() - >>> off = OfflineSplit() - >>> csubj.get_n_splits(metadata) - 3 - >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): - >>> print(f"Fold {i}:") - >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}") - >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") - >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] - >>> for j, test_session in enumerate(off.split(X_test, y_test, meta_test)): - >>> print(f" By session - Test: index={test_session}, group={subjects[test_session]}, sessions={sessions[test_session]}") - - Fold 0: - Train: index=[ 4 5 6 7 8 9 10 11], group=[2 2 2 2 3 3 3 3], sessions=[0 0 1 1 0 0 1 1] - Test: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1] - By session - Test: index=[0, 1], group=[1 1], sessions=[0 0] - By session - Test: index=[2, 3], group=[1 1], sessions=[1 1] - Fold 1: - Train: index=[ 0 1 2 3 8 9 10 11], group=[1 1 1 1 3 3 3 3], sessions=[0 0 1 1 0 0 1 1] - Test: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1] - By session - Test: index=[4, 5], group=[2 2], sessions=[0 0] - By session - Test: index=[6, 7], group=[2 2], sessions=[1 1] - Fold 2: - Train: index=[0 1 2 3 4 5 6 7], group=[1 1 1 1 2 2 2 2], sessions=[0 0 1 1 0 0 1 1] - Test: index=[ 8 9 10 11], group=[3 3 3 3], sessions=[0 0 1 1] - By session - Test: index=[8, 9], group=[3 3], sessions=[0 0] - By session - Test: index=[10, 11], group=[3 3], sessions=[1 1] - - """ - - def __init__(self, n_folds=None, run=False): - self.n_folds = n_folds - self.run = run - - def get_n_splits(self, metadata): - return metadata.groupby(["subject", "session"]).ngroups - - def split(self, X, y, metadata): - - subjects = metadata["subject"] - - for subject in subjects.unique(): - mask = subjects == subject - X_, y_, meta_ = X[mask], y[mask], metadata[mask] - sessions = meta_.session.unique() - - for session in sessions: - session_mask = meta_["session"] == session - _, _, meta_session = ( - X_[session_mask], - y_[session_mask], - meta_[session_mask], - ) - - # If you can (amd want) to split by run also - if self.run and "run" in meta_session.columns: - runs = meta_session["run"].unique() - - for run in runs: - run_mask = meta_session["run"] == run - ix_test = meta_session[run_mask].index - yield list(ix_test) - - else: - ix_test = meta_session.index - yield list(ix_test) - - -class PseudoOnlineSplit(BaseCrossValidator): - """Pseudo-online split for evaluation test data. - - It takes into account the time sequence for obtaining the test data, and uses first run, - or first #calib_size trials as calibration data, and the rest as evaluation data. - Calibration data is important in the context where data alignment or filtering is used on - training data. - - OBS: Be careful! Since this inference split is based on time disposition of obtained data, - if your data is not organized by time, but by other parameter, such as class, you may want to - be extra careful when using this split. - - Parameters - ---------- - calib_size: int - Size of calibration set, used if there is just one run. - - Examples - -------- - >>> import numpy as np - >>> import pandas as pd - >>> from moabb.evaluations.splitters import CrossSubjectSplitter - >>> from moabb.evaluations.metasplitters import PseudoOnlineSplit - >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7]]) - >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) - >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2]) - >>> sessions = np.array([0, 0, 1, 1, 0, 0, 1, 1]) - >>> runs = np.array(['0', '1', '0', '1', '0', '1', '0', '1']) - >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) - >>> csubj = CrossSubjectSplitter() - >>> posplit = PseudoOnlineSplit() - >>> posplit.get_n_splits(metadata) - 4 - >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): - >>> print(f"Fold {i}:") - >>> print(f" Train: index={train_index}, group={subjects[train_index]}, sessions={sessions[train_index]}, runs={runs[train_index]}") - >>> print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}, runs={runs[test_index]}") - >>> X_test, y_test, meta_test = X[test_index], y[test_index], metadata.loc[test_index] - >>> for j, (test_ix, calib_ix) in enumerate(posplit.split(X_test, y_test, meta_test)): - >>> print(f" Evaluation: index={test_ix}, group={subjects[test_ix]}, sessions={sessions[test_ix]}, runs={runs[test_ix]}") - >>> print(f" Calibration: index={calib_ix}, group={subjects[calib_ix]}, sessions={sessions[calib_ix]}, runs={runs[calib_ix]}") - - Fold 0: - Train: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] - Test: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] - Evaluation: index=[1], group=[1], sessions=[0], runs=['1'] - Calibration: index=[0], group=[1], sessions=[0], runs=['0'] - Evaluation: index=[3], group=[1], sessions=[1], runs=['1'] - Calibration: index=[2], group=[1], sessions=[1], runs=['0'] - Fold 1: - Train: index=[0 1 2 3], group=[1 1 1 1], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] - Test: index=[4 5 6 7], group=[2 2 2 2], sessions=[0 0 1 1], runs=['0' '1' '0' '1'] - Evaluation: index=[5], group=[2], sessions=[0], runs=['1'] - Calibration: index=[4], group=[2], sessions=[0], runs=['0'] - Evaluation: index=[7], group=[2], sessions=[1], runs=['1'] - Calibration: index=[6], group=[2], sessions=[1], runs=['0'] - - """ - - def __init__(self, calib_size: int = None): - self.calib_size = calib_size - - def get_n_splits(self, metadata): - return len(metadata.groupby(["subject", "session"])) - - def split(self, X, y, metadata): - - for _, group in metadata.groupby(["subject", "session"]): - runs = group.run.unique() - if len(runs) > 1: - # To guarantee that the runs are on the right order - runs = sort_group(runs) - for run in runs: - test_ix = group[group["run"] != run].index - calib_ix = group[group["run"] == run].index - yield list(test_ix), list(calib_ix) - break # Take the fist run as calibration - else: - calib_size = self.calib_size - calib_ix = group[:calib_size].index - test_ix = group[calib_size:].index - - yield list(test_ix), list( - calib_ix - ) # Take first #calib_size samples as calibration - class SamplerSplit(BaseCrossValidator): """Return subsets of the training data with different number of samples. @@ -224,15 +43,15 @@ class SamplerSplit(BaseCrossValidator): >>> import numpy as np >>> import pandas as pd >>> X = np.array([[[5, 6]]*12])[0] - >>> y = np.array([[1, 2]*12])[0] + >>> y = np.array([[1, 2]*6])[0] >>> subjects = np.array([1]*12) >>> sessions = np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) >>> runs = np.array(['0', '0', '1', '1', '2', '2', '0', '0', '1', '1', '2', '2',]) >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) >>> from moabb.evaluations.metasplitters import SamplerSplit - >>> from moabb.evaluations.splitters import CrossSessionSplitter - >>> data_size = dict(policy="per_class", value=np.array([2,3])) - >>> data_eval = CrossSessionSplitter() + >>> from moabb.evaluations.splitters import WithinSessionSplitter + >>> data_size = dict(policy="per_class", value=np.array([1,3])) + >>> data_eval = WithinSessionSplitter(n_folds=3) >>> sampler = SamplerSplit(data_eval, data_size) >>> for i, (train_index, test_index) in enumerate(sampler.split(X, y, metadata)): >>> print(f"Fold {i}:") @@ -240,17 +59,41 @@ class SamplerSplit(BaseCrossValidator): >>> print(f" Test: index={test_index}, sessions={sessions[test_index]}") Fold 0: - Train: index=[6 8 7 9], sessions=[1 1 1 1] - Test: index=[0 1 2 3 4 5], sessions=[0 0 0 0 0 0] + Train: index=[2 1], sessions=[0 0] + Test: index=[0 5], sessions=[0 0] Fold 1: - Train: index=[ 6 8 10 7 9 11], sessions=[1 1 1 1 1 1] - Test: index=[0 1 2 3 4 5], sessions=[0 0 0 0 0 0] + Train: index=[2 4 1 3], sessions=[0 0 0 0] + Test: index=[0 5], sessions=[0 0] Fold 2: - Train: index=[0 2 1 3], sessions=[0 0 0 0] - Test: index=[ 6 7 8 9 10 11], sessions=[1 1 1 1 1 1] + Train: index=[0 3], sessions=[0 0] + Test: index=[1 2], sessions=[0 0] Fold 3: - Train: index=[0 2 4 1 3 5], sessions=[0 0 0 0 0 0] - Test: index=[ 6 7 8 9 10 11], sessions=[1 1 1 1 1 1] + Train: index=[0 4 3 5], sessions=[0 0 0 0] + Test: index=[1 2], sessions=[0 0] + Fold 4: + Train: index=[0 1], sessions=[0 0] + Test: index=[3 4], sessions=[0 0] + Fold 5: + Train: index=[0 2 1 5], sessions=[0 0 0 0] + Test: index=[3 4], sessions=[0 0] + Fold 6: + Train: index=[8 7], sessions=[1 1] + Test: index=[ 6 11], sessions=[1 1] + Fold 7: + Train: index=[ 8 10 7 9], sessions=[1 1 1 1] + Test: index=[ 6 11], sessions=[1 1] + Fold 8: + Train: index=[6 9], sessions=[1 1] + Test: index=[7 8], sessions=[1 1] + Fold 9: + Train: index=[ 6 10 9 11], sessions=[1 1 1 1] + Test: index=[7 8], sessions=[1 1] + Fold 10: + Train: index=[6 7], sessions=[1 1] + Test: index=[ 9 10], sessions=[1 1] + Fold 11: + Train: index=[ 6 8 7 11], sessions=[1 1 1 1] + Test: index=[ 9 10], sessions=[1 1] """ diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 30012fb5e..eb0af2b2d 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,8 +1,6 @@ import numpy as np from sklearn.model_selection import ( BaseCrossValidator, - GroupKFold, - LeaveOneGroupOut, StratifiedKFold, ) @@ -22,6 +20,12 @@ class WithinSessionSplitter(BaseCrossValidator): ---------- n_folds : int Number of folds. Must be at least 2. + random_state: int, RandomState instance or None, default=None + Important when `shuffle` is True. Controls the randomness of splits. + Pass an int for reproducible output across multiple function calls. + shuffle : bool, default=True + Whether to shuffle each class's samples before splitting into batches. + Note that the samples within each split will not be shuffled. Examples ----------- @@ -34,7 +38,7 @@ class WithinSessionSplitter(BaseCrossValidator): >>> subjects = np.array([1, 1, 1, 1, 1, 1, 1, 1]) >>> sessions = np.array(['T', 'T', 'E', 'E', 'T', 'T', 'E', 'E']) >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) - >>> csess = WithinSessionSplitter(2) + >>> csess = WithinSessionSplitter(n_folds=2) >>> csess.get_n_splits(metadata) >>> for i, (train_index, test_index) in enumerate(csess.split(X, y, metadata)): ... print(f"Fold {i}:") @@ -56,8 +60,15 @@ class WithinSessionSplitter(BaseCrossValidator): """ - def __init__(self, n_folds=5): + def __init__(self, n_folds=5, random_state=42, shuffle=True): + + # Check type + assert isinstance(n_folds, int) + self.n_folds = n_folds + # Setting random state + self.random_state = random_state + self.shuffle = shuffle def get_n_splits(self, metadata): sessions_subjects = metadata.groupby(["subject", "session"]).ngroups @@ -68,7 +79,8 @@ def split(self, X, y, metadata, **kwargs): assert isinstance(self.n_folds, int) subjects = metadata.subject.values - cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) + cv = StratifiedKFold(n_splits=self.n_folds, shuffle=self.shuffle, + random_state=self.random_state) for subject in np.unique(subjects): mask = subjects == subject @@ -110,11 +122,23 @@ class IndividualWithinSessionSplitter(BaseCrossValidator): ---------- n_folds : int Number of folds. Must be at least 2. + random_state: int, RandomState instance or None, default=None + Important when `shuffle` is True. Controls the randomness of splits. + Pass an int for reproducible output across multiple function calls. + shuffle : bool, default=True + Whether to shuffle each class's samples before splitting into batches. + Note that the samples within each split will not be shuffled. """ - def __init__(self, n_folds: int): + def __init__(self, n_folds=5, random_state=42, shuffle=True): + # Check type + assert isinstance(n_folds, int) + self.n_folds = n_folds + # Setting random state + self.random_state = random_state + self.shuffle = shuffle def get_n_splits(self, metadata): return self.n_folds @@ -125,7 +149,8 @@ def split(self, X, y, metadata, **kwargs): assert isinstance(self.n_folds, int) sessions = metadata.subject.values - cv = StratifiedKFold(n_splits=self.n_folds, shuffle=True, **kwargs) + cv = StratifiedKFold(n_splits=self.n_folds, shuffle=self.shuffle, + random_state=self.random_state) for session in np.unique(sessions): mask = sessions == session @@ -137,183 +162,3 @@ def split(self, X, y, metadata, **kwargs): for ix_train, ix_test in cv.split(X_, y_): yield ix_train, ix_test - - -class CrossSessionSplitter(BaseCrossValidator): - """Data splitter for cross session evaluation. - - Cross-session evaluation uses a Leave-One-Group-Out cross-validation to - evaluate performance across sessions, but for a single subject. This splitter - assumes that all data from all subjects is already known and loaded. - - . image:: images/crosssess.pdf - :alt: The schematic diagram of the CrossSession split - :align: center - - Parameters - ---------- - n_folds : - Not used. For compatibility with other cross-validation splitters. - Default:None - - Examples - ---------- - >>> import numpy as np - >>> import pandas as pd - >>> from moabb.evaluations.splitters import CrossSessionSplitter - >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [8, 9], [5, 4], [2, 5], [1, 7]]) - >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) - >>> subjects = np.array([1, 1, 1, 1, 2, 2, 2, 2]) - >>> sessions = np.array(['T', 'T', 'E', 'E', 'T', 'T', 'E', 'E']) - >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) - >>> csess = CrossSessionSplitter() - >>> csess.get_n_splits(metadata) - 4 - >>> for i, (train_index, test_index) in enumerate(csess.split(X, y, metadata)): - ... print(f"Fold {i}:") - ... print(f" Train: index={train_index}, group={subjects[train_index]}, session={sessions[train_index]}") - ... print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") - Fold 0: - Train: index=[0 1], group=[1 1], session=['T' 'T'] - Test: index=[2 3], group=[1 1], sessions=['E' 'E'] - Fold 1: - Train: index=[2 3], group=[1 1], session=['E' 'E'] - Test: index=[0 1], group=[1 1], sessions=['T' 'T'] - Fold 2: - Train: index=[4 5], group=[2 2], session=['T' 'T'] - Test: index=[6 7], group=[2 2], sessions=['E' 'E'] - Fold 3: - Train: index=[6 7], group=[2 2], session=['E' 'E'] - Test: index=[4 5], group=[2 2], sessions=['T' 'T'] - - """ - - def __init__(self, n_folds=None): - self.n_folds = n_folds - - def get_n_splits(self, metadata): - sessions_subjects = len(metadata.groupby(["subject", "session"]).first()) - return sessions_subjects - - def split(self, X, y, metadata): - - subjects = metadata.subject.values - split = IndividualCrossSessionSplitter() - - for subject in np.unique(subjects): - mask = subjects == subject - X_, y_, meta_ = ( - X[mask], - y[mask], - metadata[mask], - ) - - for ix_train, ix_test in split.split(X_, y_, meta_): - ix_train = np.where(mask)[0][ix_train] - ix_test = np.where(mask)[0][ix_test] - yield ix_train, ix_test - - -class IndividualCrossSessionSplitter(BaseCrossValidator): - """Data splitter for cross session evaluation. - - Cross-session evaluation uses a Leave-One-Group-Out cross-validation to - evaluate performance across sessions, but for a single subject. This splitter does - not assumethat all data and metadata from all subjects is already loaded. If X, y - and metadata are from a single subject, it returns data split for this subject only. - - It can be used as basis for CrossSessionSplitter or to avoid downloading all data at - once when it is not needed, - - Parameters - ---------- - n_folds : - Not used. For compatibility with other cross-validation splitters. - Default:None - - """ - - def __init__(self, n_folds=None): - self.n_folds = n_folds - - def get_n_splits(self, metadata): - sessions = metadata.session.values - return np.unique(sessions) - - def split(self, X, y, metadata): - assert len(np.unique(metadata.subject)) == 1 - - cv = LeaveOneGroupOut() - sessions = metadata.session.values - - for ix_train, ix_test in cv.split(X, y, groups=sessions): - yield ix_train, ix_test - - -class CrossSubjectSplitter(BaseCrossValidator): - """Data splitter for cross session evaluation. - - Cross-session evaluation uses a Leave-One-Group-Out cross-validation to - evaluate performance across sessions, but for a single subject. This splitter - assumes that all data from all subjects is already known and loaded. - - . image:: images/crosssubj.pdf - :alt: The schematic diagram of the CrossSubj split - :align: center - - Parameters - ---------- - n_groups : int or None - If None, Leave-One-Subject-Out is performed. - If int, Leave-k-Subjects-Out is performed. - - Examples - -------- - >>> import numpy as np - >>> import pandas as pd - >>> from moabb.evaluations.splitters import CrossSubjectSplitter - >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8],[8,9],[5,4],[2,5],[1,7]]) - >>> y = np.array([1, 2, 1, 2, 1, 2, 1, 2]) - >>> subjects = np.array([1, 1, 2, 2, 3, 3, 4, 4]) - >>> metadata = pd.DataFrame(data={'subject': subjects}) - >>> csubj = CrossSubjectSplitter() - >>> csubj.get_n_splits(metadata) - 4 - >>> for i, (train_index, test_index) in enumerate(csubj.split(X, y, metadata)): - ... print(f"Fold {i}:") - ... print(f" Train: index={train_index}, group={subjects[train_index]}") - ... print(f" Test: index={test_index}, group={subjects[test_index]}") - Fold 0: - Train: index=[2 3 4 5 6 7], group=[2 2 3 3 4 4] - Test: index=[0 1], group=[1 1] - Fold 1: - Train: index=[0 1 4 5 6 7], group=[1 1 3 3 4 4] - Test: index=[2 3], group=[2 2] - Fold 2: - Train: index=[0 1 2 3 6 7], group=[1 1 2 2 4 4] - Test: index=[4 5], group=[3 3] - Fold 3: - Train: index=[0 1 2 3 4 5], group=[1 1 2 2 3 3] - Test: index=[6 7], group=[4 4] - - - """ - - def __init__(self, n_groups=None): - self.n_groups = n_groups - - def get_n_splits(self, metadata): - return len(metadata.subject.unique()) - - def split(self, X, y, metadata): - - groups = metadata.subject.values - - # Define split - if self.n_groups is None: - cv = LeaveOneGroupOut() - else: - cv = GroupKFold(n_splits=self.n_groups) - - for ix_train, ix_test in cv.split(metadata, groups=groups): - yield ix_train, ix_test diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py index 9651fe71d..04b9c7ef6 100644 --- a/moabb/tests/metasplits.py +++ b/moabb/tests/metasplits.py @@ -3,8 +3,7 @@ from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold from moabb.datasets.fake import FakeDataset -from moabb.evaluations.metasplitters import OfflineSplit, PseudoOnlineSplit, SamplerSplit -from moabb.evaluations.splitters import CrossSessionSplitter, CrossSubjectSplitter +from moabb.evaluations.metasplitters import SamplerSplit from moabb.paradigms.motor_imagery import FakeImageryParadigm @@ -84,57 +83,8 @@ def pseudo_split_cross_subject(): yield X_sess[runs_in_session == runs_in_session[0]] -@pytest.mark.parametrize("split", [CrossSubjectSplitter, CrossSessionSplitter]) -def test_offline(split): - X, y, metadata = paradigm.get_data(dataset=dataset) - - run = True if isinstance(split, CrossSessionSplitter) else False - - if isinstance(split, CrossSessionSplitter): - eval_split = eval_split_cross_session - else: - eval_split = eval_split_cross_subject - - split = split() - metasplit = OfflineSplit(run=run) - - Tests = [] - for _, test in split.split(X, y, metadata): - X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] - for i, (test_index) in enumerate(metasplit.split(X_test, y_test, metadata_test)): - Tests.append(X[test_index]) - - for ix, X_test_t in enumerate(eval_split()): - # Check if the output is the same as the input - assert np.array_equal(Tests[ix], X_test_t) - - -@pytest.mark.parametrize("split", [CrossSubjectSplitter, CrossSessionSplitter]) -def test_pseudoonline(split): - X, y, metadata = paradigm.get_data(dataset=dataset) - - if isinstance(split, CrossSessionSplitter): - eval_split = pseudo_split_cross_session - else: - eval_split = pseudo_split_cross_subject - - split = split() - metasplit = PseudoOnlineSplit() - - Tests = [] - for _, test in split.split(X, y, metadata): - X_test, y_test, metadata_test = X[test], y[test], metadata.loc[test] - for i, (_, calib_index) in enumerate( - metasplit.split(X_test, y_test, metadata_test) - ): - Tests.append(X[calib_index]) - - for ix, X_calib_t in enumerate(eval_split()): - # Check if the output is the same as the input - assert np.array_equal(Tests[ix], X_calib_t) - - @pytest.mark.skip(reason="Still working on that") +# TODO: Test policy and data eval, test correct output def test_sampler(data_eval): X, y, metadata = paradigm.get_data(dataset=dataset) data_size = dict(policy="per_class", value=np.array([5, 10, 30, 50])) diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 02b326c9d..7f634ef97 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -2,11 +2,7 @@ from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold from moabb.datasets.fake import FakeDataset -from moabb.evaluations.splitters import ( - CrossSessionSplitter, - CrossSubjectSplitter, - WithinSessionSplitter, -) +from moabb.evaluations.splitters import WithinSessionSplitter from moabb.paradigms.motor_imagery import FakeImageryParadigm @@ -45,7 +41,7 @@ def eval_split_cross_subject(): for train, test in cv.split(X, y, groups): yield X[train], X[test] - +# TODO: test shuffle and random_state def test_within_session(): X, y, metadata = paradigm.get_data(dataset=dataset) @@ -60,32 +56,3 @@ def test_within_session(): assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) - -def test_cross_session(): - X, y, metadata = paradigm.get_data(dataset=dataset) - - split = CrossSessionSplitter() - - for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_session(), split.split(X, y, metadata)) - ): - X_train, X_test = X[train], X[test] - - # Check if the output is the same as the input - assert np.array_equal(X_train, X_train_t) - assert np.array_equal(X_test, X_test_t) - - -def test_cross_subject(): - X, y, metadata = paradigm.get_data(dataset=dataset) - - split = CrossSubjectSplitter() - - for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_subject(), split.split(X, y, metadata)) - ): - X_train, X_test = X[train], X[test] - - # Check if the output is the same as the input - assert np.array_equal(X_train, X_train_t) - assert np.array_equal(X_test, X_test_t) From 88ee9106654dd59d334ea34d2bb5bf3051194d4b Mon Sep 17 00:00:00 2001 From: brunalopes Date: Fri, 18 Oct 2024 12:26:44 +0200 Subject: [PATCH 25/39] Add shuffle and random_state parameters to WithinSession --- moabb/evaluations/metasplitters.py | 195 ----------------------------- moabb/evaluations/splitters.py | 92 +++----------- moabb/tests/metasplits.py | 101 --------------- 3 files changed, 16 insertions(+), 372 deletions(-) delete mode 100644 moabb/evaluations/metasplitters.py delete mode 100644 moabb/tests/metasplits.py diff --git a/moabb/evaluations/metasplitters.py b/moabb/evaluations/metasplitters.py deleted file mode 100644 index 9061dbfa8..000000000 --- a/moabb/evaluations/metasplitters.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -The data splitters defined in this file are not directly related to a evaluation method -the way that WithinSession, CrossSession and CrossSubject splitters are. - -OfflineSplit and TimeSeriesSplit split the test data, indicating weather model inference -will be computed using a Offline or a Pseudo-Online validation. Pseudo-online evaluation -is important when training data is pre-processed with some data-dependent transformation. -One part of the test data is separated as a calibration to compute the transformation. - -SamplerSplit is an optional subsplit done on the training set to generate subsets with -different numbers of samples. It can be used to estimate the performance of the model -on different training sizes and plot learning curves. - -""" - -import numpy as np -from sklearn.model_selection import BaseCrossValidator - - -class SamplerSplit(BaseCrossValidator): - """Return subsets of the training data with different number of samples. - - This splitter can be used for estimating a model's performance when using different - numbers of training samples, and for plotting the learning curve per number of training - samples. You must define the data evaluation type (WithinSubject, CrossSession, CrossSubject) - so the training set can be sampled. It is also needed to pass a dictionary indicating the - policy used for sampling the training set and the number of examples (or the percentage) that - each sample must contain. - - Parameters - ---------- - data_eval: BaseCrossValidator object - Evaluation splitter already initialized. It can be WithinSubject, CrossSession, - or CrossSubject Splitters. - data_size : dict - Contains the policy to pick the datasizes to evaluate, as well as the actual values. - The dict has the key 'policy' with either 'ratio' or 'per_class', and the key - 'value' with the actual values as a numpy array. This array should be - sorted, such that values in data_size are strictly monotonically increasing. - - Examples - -------- - >>> import numpy as np - >>> import pandas as pd - >>> X = np.array([[[5, 6]]*12])[0] - >>> y = np.array([[1, 2]*6])[0] - >>> subjects = np.array([1]*12) - >>> sessions = np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) - >>> runs = np.array(['0', '0', '1', '1', '2', '2', '0', '0', '1', '1', '2', '2',]) - >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions, 'run':runs}) - >>> from moabb.evaluations.metasplitters import SamplerSplit - >>> from moabb.evaluations.splitters import WithinSessionSplitter - >>> data_size = dict(policy="per_class", value=np.array([1,3])) - >>> data_eval = WithinSessionSplitter(n_folds=3) - >>> sampler = SamplerSplit(data_eval, data_size) - >>> for i, (train_index, test_index) in enumerate(sampler.split(X, y, metadata)): - >>> print(f"Fold {i}:") - >>> print(f" Train: index={train_index}, sessions={sessions[train_index]}") - >>> print(f" Test: index={test_index}, sessions={sessions[test_index]}") - - Fold 0: - Train: index=[2 1], sessions=[0 0] - Test: index=[0 5], sessions=[0 0] - Fold 1: - Train: index=[2 4 1 3], sessions=[0 0 0 0] - Test: index=[0 5], sessions=[0 0] - Fold 2: - Train: index=[0 3], sessions=[0 0] - Test: index=[1 2], sessions=[0 0] - Fold 3: - Train: index=[0 4 3 5], sessions=[0 0 0 0] - Test: index=[1 2], sessions=[0 0] - Fold 4: - Train: index=[0 1], sessions=[0 0] - Test: index=[3 4], sessions=[0 0] - Fold 5: - Train: index=[0 2 1 5], sessions=[0 0 0 0] - Test: index=[3 4], sessions=[0 0] - Fold 6: - Train: index=[8 7], sessions=[1 1] - Test: index=[ 6 11], sessions=[1 1] - Fold 7: - Train: index=[ 8 10 7 9], sessions=[1 1 1 1] - Test: index=[ 6 11], sessions=[1 1] - Fold 8: - Train: index=[6 9], sessions=[1 1] - Test: index=[7 8], sessions=[1 1] - Fold 9: - Train: index=[ 6 10 9 11], sessions=[1 1 1 1] - Test: index=[7 8], sessions=[1 1] - Fold 10: - Train: index=[6 7], sessions=[1 1] - Test: index=[ 9 10], sessions=[1 1] - Fold 11: - Train: index=[ 6 8 7 11], sessions=[1 1 1 1] - Test: index=[ 9 10], sessions=[1 1] - - """ - - def __init__(self, data_eval, data_size): - self.data_eval = data_eval - self.data_size = data_size - - self.sampler = IndividualSamplerSplit(self.data_size) - - def get_n_splits(self, y, metadata): - return self.data_eval.get_n_splits(metadata) * len( - self.sampler.get_data_size_subsets(y) - ) - - def split(self, X, y, metadata, **kwargs): - cv = self.data_eval - sampler = self.sampler - - for ix_train, ix_test in cv.split(X, y, metadata, **kwargs): - X_train, y_train, meta_train = ( - X[ix_train], - y[ix_train], - metadata.iloc[ix_train], - ) - for ix_train_sample in sampler.split(X_train, y_train, meta_train): - ix_train_sample = ix_train[ix_train_sample] - yield ix_train_sample, ix_test - - -class IndividualSamplerSplit(BaseCrossValidator): - """Return subsets of the training data with different number of samples. - - Util for estimating the performance of a model when using different number of - training samples and plotting the learning curve. It must be used after already splitting - data using one of the other evaluation data splitters (WithinSubject, CrossSession, CrossSubject) - since it corresponds to a subsampling of the training data. - - This 'Individual' Sampler Split assumes that data and metadata being passed is training, and was - already split by WithinSubject, CrossSession, or CrossSubject splitters. - - Parameters - ---------- - data_size : dict - Contains the policy to pick the datasizes to - evaluate, as well as the actual values. The dict has the - key 'policy' with either 'ratio' or 'per_class', and the key - 'value' with the actual values as a numpy array. This array should be - sorted, such that values in data_size are strictly monotonically increasing. - - """ - - def __init__(self, data_size): - self.data_size = data_size - - def get_n_splits(self, y=None): - return len(self.get_data_size_subsets(y)) - - def get_data_size_subsets(self, y): - if self.data_size is None: - raise ValueError( - "Cannot create data subsets without valid policy for data_size." - ) - if self.data_size["policy"] == "ratio": - vals = np.array(self.data_size["value"]) - if np.any(vals < 0) or np.any(vals > 1): - raise ValueError("Data subset ratios must be in range [0, 1]") - upto = np.ceil(vals * len(y)).astype(int) - indices = [np.array(range(i)) for i in upto] - elif self.data_size["policy"] == "per_class": - classwise_indices = dict() - n_smallest_class = np.inf - for cl in np.unique(y): - cl_i = np.where(cl == y)[0] - classwise_indices[cl] = cl_i - n_smallest_class = ( - len(cl_i) if len(cl_i) < n_smallest_class else n_smallest_class - ) - indices = [] - for ds in self.data_size["value"]: - if ds > n_smallest_class: - raise ValueError( - f"Smallest class has {n_smallest_class} samples. " - f"Desired samples per class {ds} is too large." - ) - indices.append( - np.concatenate( - [classwise_indices[cl][:ds] for cl in classwise_indices] - ) - ) - else: - raise ValueError(f"Unknown policy {self.data_size['policy']}") - return indices - - def split(self, X, y, metadata): - - data_size_steps = self.get_data_size_subsets(y) - for subset_indices in data_size_steps: - ix_train = subset_indices - yield ix_train diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index eb0af2b2d..1f643b0be 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -4,6 +4,8 @@ StratifiedKFold, ) +from sklearn.utils import check_random_state + class WithinSessionSplitter(BaseCrossValidator): """Data splitter for within session evaluation. @@ -56,109 +58,47 @@ class WithinSessionSplitter(BaseCrossValidator): Fold 3: Train: index=[0 1], group=[1 1], session=['T' 'T'] Test: index=[4 5], group=[1 1], sessions=['T' 'T'] - - """ - def __init__(self, n_folds=5, random_state=42, shuffle=True): - + def __init__(self, n_folds:int=5, random_state:int=42, shuffle:bool=True): # Check type assert isinstance(n_folds, int) self.n_folds = n_folds # Setting random state - self.random_state = random_state + self.random_state = check_random_state(random_state) + self.shuffle = shuffle def get_n_splits(self, metadata): sessions_subjects = metadata.groupby(["subject", "session"]).ngroups return self.n_folds * sessions_subjects - def split(self, X, y, metadata, **kwargs): + def split(self, y, metadata, **kwargs): assert isinstance(self.n_folds, int) + all_index = metadata.index.values subjects = metadata.subject.values cv = StratifiedKFold(n_splits=self.n_folds, shuffle=self.shuffle, random_state=self.random_state) for subject in np.unique(subjects): mask = subjects == subject - X_, y_, meta_ = ( - X[mask], + index_subject, y_subject, meta_subject = ( + all_index[mask], y[mask], metadata[mask], ) - sessions = meta_.session.values + sessions = meta_subject.session.values for session in np.unique(sessions): - mask_s = sessions == session - X_s, y_s, _ = ( - X_[mask_s], - y_[mask_s], - meta_[mask_s], + mask_session = sessions == session + index_session, y_session = ( + index_subject[mask_session], + y_subject[mask_session] ) - for ix_train, ix_test in cv.split(X_s, y_s): - - ix_train_global = np.where(mask)[0][np.where(mask_s)[0][ix_train]] - ix_test_global = np.where(mask)[0][np.where(mask_s)[0][ix_test]] - yield ix_train_global, ix_test_global - - -class IndividualWithinSessionSplitter(BaseCrossValidator): - """Data splitter for within session evaluation. - - Within-session evaluation uses k-fold cross_validation to determine train - and test sets on separate session for each subject. This splitter does not assume - that all data and metadata from all subjects is already loaded. If X, y and metadata - are from a single subject, it returns data split for this subject only. - - It can be used as basis for WithinSessionSplitter or to avoid downloading all data at - once when it is not needed, - - Parameters - ---------- - n_folds : int - Number of folds. Must be at least 2. - random_state: int, RandomState instance or None, default=None - Important when `shuffle` is True. Controls the randomness of splits. - Pass an int for reproducible output across multiple function calls. - shuffle : bool, default=True - Whether to shuffle each class's samples before splitting into batches. - Note that the samples within each split will not be shuffled. - - """ - - def __init__(self, n_folds=5, random_state=42, shuffle=True): - # Check type - assert isinstance(n_folds, int) - - self.n_folds = n_folds - # Setting random state - self.random_state = random_state - self.shuffle = shuffle - - def get_n_splits(self, metadata): - return self.n_folds - - def split(self, X, y, metadata, **kwargs): - - assert len(np.unique(metadata.subject)) == 1 - assert isinstance(self.n_folds, int) - - sessions = metadata.subject.values - cv = StratifiedKFold(n_splits=self.n_folds, shuffle=self.shuffle, - random_state=self.random_state) - - for session in np.unique(sessions): - mask = sessions == session - X_, y_, _ = ( - X[mask], - y[mask], - metadata[mask], - ) - - for ix_train, ix_test in cv.split(X_, y_): - yield ix_train, ix_test + for ix_train, ix_test in cv.split(index_session, y_session): + yield index_session[ix_train], index_session[ix_test] diff --git a/moabb/tests/metasplits.py b/moabb/tests/metasplits.py deleted file mode 100644 index 04b9c7ef6..000000000 --- a/moabb/tests/metasplits.py +++ /dev/null @@ -1,101 +0,0 @@ -import numpy as np -import pytest -from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold - -from moabb.datasets.fake import FakeDataset -from moabb.evaluations.metasplitters import SamplerSplit -from moabb.paradigms.motor_imagery import FakeImageryParadigm - - -dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) -paradigm = FakeImageryParadigm() - - -# Still working on this -def eval_sampler_split(): - for subject in dataset.subject_list: - X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) - sessions = metadata.session - for session in np.unique(sessions): - ix = sessions == session - cv = StratifiedKFold(5, shuffle=True, random_state=42) - X_, y_, _ = X[ix], y[ix], metadata.loc[ix] - for train, test in cv.split(X_, y_): - yield X_[train], X_[test] - - -# Split done for the Cross Session evaluation -def eval_split_cross_session(): - for subject in dataset.subject_list: - X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) - groups = metadata.session.values - cv = LeaveOneGroupOut() - for _, test in cv.split(X, y, groups): - metadata_test = metadata.loc[test] - runs = metadata_test.run.values - for r in np.unique(runs): - ix = runs == r - yield X[test[ix]] - - -def pseudo_split_cross_session(): - for subject in dataset.subject_list: - X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) - groups = metadata.session.values - cv = LeaveOneGroupOut() - for _, test in cv.split(X, y, groups): - metadata_test = metadata.loc[test] - runs = metadata_test.run.values - ix = runs == runs[0] - yield X[test[ix]] - - -# Split done for the Cross Subject evaluation -def eval_split_cross_subject(): - X, y, metadata = paradigm.get_data(dataset=dataset) - groups = metadata.subject.values - cv = LeaveOneGroupOut() - for _, test in cv.split(X, y, groups): - metadata_test = metadata.loc[test] - sessions = metadata_test.session.values - for sess in np.unique(sessions): - ix = sessions == sess - yield X[test[ix]] - - -# Split done for the Cross Subject evaluation -def pseudo_split_cross_subject(): - X, y, metadata = paradigm.get_data(dataset=dataset) - groups = metadata.subject.values - cv = LeaveOneGroupOut() - for _, test in cv.split(X, y, groups): - metadata_test = metadata.loc[test] - sessions = metadata_test.session.values - - for sess in np.unique(sessions): - ix = sessions == sess - X_sess, metadata_sess = X[test[ix]], metadata_test.loc[test[ix]].reset_index( - drop=True - ) - - runs_in_session = metadata_sess.run.values - # yield just calibration part - yield X_sess[runs_in_session == runs_in_session[0]] - - -@pytest.mark.skip(reason="Still working on that") -# TODO: Test policy and data eval, test correct output -def test_sampler(data_eval): - X, y, metadata = paradigm.get_data(dataset=dataset) - data_size = dict(policy="per_class", value=np.array([5, 10, 30, 50])) - - split = SamplerSplit(data_eval=data_eval, data_size=data_size) - - for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_cross_subject(), split.split(X, y, metadata)) - ): - X_train, X_test = X[train], X[test] - - # Check if the output is the same as the input - assert np.array_equal(X_train, X_train_t) - assert np.array_equal(X_test, X_test_t) From ea9cc59f8089b63932a3ea260bbeef046f595ff6 Mon Sep 17 00:00:00 2001 From: brunalopes Date: Fri, 18 Oct 2024 12:43:04 +0200 Subject: [PATCH 26/39] Change nomenclature of variables --- moabb/evaluations/splitters.py | 35 +++++++++++++++++----------------- moabb/tests/splits.py | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 1f643b0be..bb0c6e7d5 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,4 +1,5 @@ import numpy as np + from sklearn.model_selection import ( BaseCrossValidator, StratifiedKFold, @@ -42,7 +43,8 @@ class WithinSessionSplitter(BaseCrossValidator): >>> metadata = pd.DataFrame(data={'subject': subjects, 'session': sessions}) >>> csess = WithinSessionSplitter(n_folds=2) >>> csess.get_n_splits(metadata) - >>> for i, (train_index, test_index) in enumerate(csess.split(X, y, metadata)): + 4 + >>> for i, (train_index, test_index) in enumerate(csess.split(y, metadata)): ... print(f"Fold {i}:") ... print(f" Train: index={train_index}, group={subjects[train_index]}, session={sessions[train_index]}") ... print(f" Test: index={test_index}, group={subjects[test_index]}, sessions={sessions[test_index]}") @@ -66,13 +68,12 @@ def __init__(self, n_folds:int=5, random_state:int=42, shuffle:bool=True): self.n_folds = n_folds # Setting random state - self.random_state = check_random_state(random_state) - + self.random_state = check_random_state(random_state) if shuffle else None self.shuffle = shuffle def get_n_splits(self, metadata): - sessions_subjects = metadata.groupby(["subject", "session"]).ngroups - return self.n_folds * sessions_subjects + num_sessions_subjects = metadata.groupby(["subject", "session"]).ngroups + return self.n_folds * num_sessions_subjects def split(self, y, metadata, **kwargs): @@ -84,21 +85,21 @@ def split(self, y, metadata, **kwargs): random_state=self.random_state) for subject in np.unique(subjects): - mask = subjects == subject - index_subject, y_subject, meta_subject = ( - all_index[mask], - y[mask], - metadata[mask], + subject_mask = subjects == subject + subject_indices, subject_y, subject_metadata = ( + all_index[subject_mask], + y[subject_mask], + metadata[subject_mask], ) - sessions = meta_subject.session.values + sessions = subject_metadata.session.values for session in np.unique(sessions): - mask_session = sessions == session - index_session, y_session = ( - index_subject[mask_session], - y_subject[mask_session] + session_mask = sessions == session + session_indices, session_y = ( + subject_indices[session_mask], + subject_y[session_mask] ) - for ix_train, ix_test in cv.split(index_session, y_session): - yield index_session[ix_train], index_session[ix_test] + for ix_train, ix_test in cv.split(session_indices, session_y): + yield session_indices[ix_train], session_indices[ix_test] diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 7f634ef97..d2a0fc7a2 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -48,7 +48,7 @@ def test_within_session(): split = WithinSessionSplitter(n_folds=5) for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_within_session(), split.split(X, y, metadata, random_state=42)) + zip(eval_split_within_session(), split.split(y, metadata, random_state=42)) ): X_train, X_test = X[train], X[test] From 819c4ff6025f482ed1e55c44b0a584f8cdc39fb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:05:47 +0000 Subject: [PATCH 27/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/evaluations/splitters.py | 16 ++++++---------- moabb/tests/splits.py | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index bb0c6e7d5..73b56bfaa 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,10 +1,5 @@ import numpy as np - -from sklearn.model_selection import ( - BaseCrossValidator, - StratifiedKFold, -) - +from sklearn.model_selection import BaseCrossValidator, StratifiedKFold from sklearn.utils import check_random_state @@ -62,7 +57,7 @@ class WithinSessionSplitter(BaseCrossValidator): Test: index=[4 5], group=[1 1], sessions=['T' 'T'] """ - def __init__(self, n_folds:int=5, random_state:int=42, shuffle:bool=True): + def __init__(self, n_folds: int = 5, random_state: int = 42, shuffle: bool = True): # Check type assert isinstance(n_folds, int) @@ -81,8 +76,9 @@ def split(self, y, metadata, **kwargs): all_index = metadata.index.values subjects = metadata.subject.values - cv = StratifiedKFold(n_splits=self.n_folds, shuffle=self.shuffle, - random_state=self.random_state) + cv = StratifiedKFold( + n_splits=self.n_folds, shuffle=self.shuffle, random_state=self.random_state + ) for subject in np.unique(subjects): subject_mask = subjects == subject @@ -98,7 +94,7 @@ def split(self, y, metadata, **kwargs): session_mask = sessions == session session_indices, session_y = ( subject_indices[session_mask], - subject_y[session_mask] + subject_y[session_mask], ) for ix_train, ix_test in cv.split(session_indices, session_y): diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index d2a0fc7a2..179c61259 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -41,6 +41,7 @@ def eval_split_cross_subject(): for train, test in cv.split(X, y, groups): yield X[train], X[test] + # TODO: test shuffle and random_state def test_within_session(): X, y, metadata = paradigm.get_data(dataset=dataset) @@ -55,4 +56,3 @@ def test_within_session(): # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) - From f1ad58789da95a0d0c3c49d9cbae130e1431bffd Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:06:04 +0200 Subject: [PATCH 28/39] FIX: fixing the whats_new.rst file --- docs/source/whats_new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index b99628474..a7577a3c9 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -17,6 +17,7 @@ Develop branch Enhancements ~~~~~~~~~~~~ +- Adding :class:`moabb.evaluations.splitters.WithinSessionSplitter` (:gh:`664` by `Bruna Lopes_`) Bugs ~~~~ @@ -76,7 +77,6 @@ Enhancements - Add new dataset :class:`moabb.datasets.Rodrigues2017` dataset (:gh:`602` by `Gregoire Cattan`_ and `Pedro L. C. Rodrigues`_) - Change unittest to pytest (:gh:`618` by `Bruno Aristimunha`_) - Remove tensorflow import warning (:gh:`622` by `Bruno Aristimunha`_) -- Add data splitter classes (:gh:`612` by `Bruna Lopes_`) Bugs ~~~~ From 485e7a5b23032f0961a237b196602145d5ba2abc Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:19:31 +0200 Subject: [PATCH 29/39] EHN: playing a little --- moabb/evaluations/splitters.py | 4 ++-- moabb/tests/splits.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 73b56bfaa..56cd52e12 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -11,8 +11,8 @@ class WithinSessionSplitter(BaseCrossValidator): all data from all subjects is already known and loaded. . image:: images/withinsess.pdf - :alt: The schematic diagram of the WithinSession split - :align: center + :alt: The schematic diagram of the WithinSession split + :align: center Parameters ---------- diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 179c61259..22269102e 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -46,10 +46,10 @@ def eval_split_cross_subject(): def test_within_session(): X, y, metadata = paradigm.get_data(dataset=dataset) - split = WithinSessionSplitter(n_folds=5) + split = WithinSessionSplitter(n_folds=5, random_state=42, shuffle=True) for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_within_session(), split.split(y, metadata, random_state=42)) + zip(eval_split_within_session(), split.split(y, metadata)) ): X_train, X_test = X[train], X[test] From 3f3742fad0510add6a3f02f23bbe2ca4b3926596 Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:38:55 +0200 Subject: [PATCH 30/39] FIX: fixing the import and docs/docstring --- docs/source/evaluations.rst | 7 +++++++ moabb/evaluations/__init__.py | 1 + moabb/evaluations/splitters.py | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/evaluations.rst b/docs/source/evaluations.rst index 3a502297e..564d2928a 100644 --- a/docs/source/evaluations.rst +++ b/docs/source/evaluations.rst @@ -19,6 +19,13 @@ Evaluations CrossSubjectEvaluation +.. autosummary:: + :toctree: generated/ + :template: class.rst + + WithinSessionSplitter + + ------------ Base & Utils ------------ diff --git a/moabb/evaluations/__init__.py b/moabb/evaluations/__init__.py index 023e62c45..453be2ed5 100644 --- a/moabb/evaluations/__init__.py +++ b/moabb/evaluations/__init__.py @@ -9,4 +9,5 @@ CrossSubjectEvaluation, WithinSessionEvaluation, ) +from .splitters import WithinSessionSplitter from .utils import create_save_path, save_model_cv, save_model_list diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 56cd52e12..a1448345c 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -10,10 +10,11 @@ class WithinSessionSplitter(BaseCrossValidator): and test sets on separate session for each subject. This splitter assumes that all data from all subjects is already known and loaded. - . image:: images/withinsess.pdf + .. image:: images/withinsess.pdf :alt: The schematic diagram of the WithinSession split :align: center + Parameters ---------- n_folds : int From c181c5954dc68c94f096ba0b8b203bd30ed5c2a0 Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:40:38 +0200 Subject: [PATCH 31/39] FIX: fixing the import and docs/docstring --- moabb/evaluations/splitters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index a1448345c..04d9f530d 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -10,7 +10,7 @@ class WithinSessionSplitter(BaseCrossValidator): and test sets on separate session for each subject. This splitter assumes that all data from all subjects is already known and loaded. - .. image:: images/withinsess.pdf + .. image:: images/withinsess.png :alt: The schematic diagram of the WithinSession split :align: center From 8f034c871099a394c16b5a82cc7f62f6cdc3ebf9 Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:40:53 +0200 Subject: [PATCH 32/39] FIX: fixing the import and docs/docstring --- docs/source/images/withinsess.png | Bin 0 -> 10769 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/source/images/withinsess.png diff --git a/docs/source/images/withinsess.png b/docs/source/images/withinsess.png new file mode 100644 index 0000000000000000000000000000000000000000..45a9d903787bbfbb46ad2a8a452b3dfa4a04a795 GIT binary patch literal 10769 zcmb_>1yGz#wB_IgPw?P`5ZpbuySoN=cXtR9+&#Di2oQpMU~tLc?(XiqkAL^C-F>fi zx3+4Fn(2@JrtS9azUQ19p(rnbiu4`{1OlN-Ns20iK+t5svpFI>@I4)oumS?Tp|lhc zQM5ER27x3al9SC96SZ)`Cl|bXp@Q#(!xcs1X+8!dVQ5qRttb`T%~lk%`Pv9g8~K$@ zEKUpxS2vm5S_}(0DhL|KC72F+u-Oq?Nl43K1wZ?SNTBs4-FwXb*?D`}dF;*siu7x+ zY1fdqk#ckA!axi@9@a4I3RQG&O842N3(U$?%Ue+8AMfUsvAZh7xAB&bTaAsaPIuHn zE}*kgRWJ24(uDNif8P+N4qakVNc2*Q`l+10BtH@rNtl&Xi`qF2 zQVQ9)$nsqHqp`CvvB4UcpY_>Y;38MJMBfo7dSqz~<0Ysny&V`#zq2@M6? zrs^ag5YO=`D{T6US7|aTDq;Zit{^-Wv|8R9m%z77Fe#llTp-+4C?6q=SV*V*BFALHCOCHi-FX&Nl+w7S5l-TF-)~D* zC)HJ%%aL56rvmrImPEc4h?b67uxJxk2LBek7!TaFaKUK?>BZQIHWXBU51&Zfb;m;q z3p6wwW*SW+l_fM|Eyi=kgM>9jhR+shTFRb zVu?T0YtardjbC)%F`#S2Soi7uL0v0$K=yEKovn?<7R1*a`NTZ6gd=~ zsJ4<=lRhVDIkq;2acS_%`c!z--*TTRopHa(2>BioE}Ab|BU<-IF2a%~rCi80Ph9b$ z3|y*FhRGIEEWPpf_E+{V_<8nu3L+8n3C~L|ueVR;_aHqAsbsd4%hchNA1SS*)Fz}T zsPM7zom7TdX3cQ z$)ggcQmzt468Oly<=oU`7|f}`skBL->2D;7Q)W_bQda2vW!nw>_kA5dNpiQk_CcBsL7rB@su}AicI+J%gxv8hA`4-9+&lc1c9Ck`% zu5^xc<{+k85RDGIbiHi7gj!*7;dWI@#dQ9)BvJLKdaRN_;c^Lob=P0S@!86WGUky4 zvolTZA2`!#lv)sRR<%0)i*_9Gs7mZ=SIvVejvu!9LiFXO zoz11?wdaoYj?VhStOA6wry88QPO`qTDM0dtMDaZ-$|U;YJCDzj98?^ow0H!EB#U^w z4WoujU9O2_lw@#Z9}ct*>Sk6JNn6MadqTe@eKRB&3&|IO5zZ9B?cwd-@7eD5Tw`A& z=TG(j`ohx9kxyqt`In}a&534^rku@+XP)L>@KQEfPS$hShK}<$&9!Tmv##x;Ppib^ z*=FOl-nLXk{UK^MxO=L_yncmp{UH)YbQH?PkRmfDzyvy`4hS{FE}OIw=+2l5JDV zHNqr%jR4A_R$GbF*iQG?ZXGdru`RKv-p6$vf%ngvXB&T)OWR;R7JrnuHoX3LL@;Mx z_aa&*YAL_>2kHif$aJ{2yS4#=leksOPFLUis{_?W+|TaH{0iktCxa~`HQ6bLdOcU2 zZ$2YA8FxKz!^2}{i7hMDO6kj=JlL*xgLz7tW4ylIpBtwcr-O4iZ#gfY{ydkJDgCY~ zJ%gMTZggWVfAY#4&dA8v%9PPwO^HrPPbB+v@@(_stWjZStw&H$xr@b>?6}{{_`M@K z%Z+czIn1~C>c{avyd)OK*=d1H@WO+-w1 zM&QZxo`uE-fA zNZ{-|_Ll(8r;FUg+*Ac>Id(Y$x!hdGtfmK}rKo{aYDQc$Jfk-B#M1@p6=8u-VJ+Q9Y z1Ww&K=1e#THXvvdYdRoa7 z&lBsI58RcNgwkJ`=N8!gz@gzGjSTvcSK2A6xS6n);BA)BX5i;v_?glqP}j^&^0qYx z3}*WL(01reXf$ zoETXBIa|K6!d**=HIHKOy{%5svUk%jTsX22SqJ`Hbet$ShE!yRRO|i&^$PZ*hrdUk z<9gP{o2^ob*rBCUOqKNXzL^fC1nziNR&W_c|E>%`ZomaQmR3~s2f4YVKhJsGhB7x z8lRkGGa*Sat+AS-0lBnnDHn(f4lqSOa*kb0ZX@Aa>Og&XfFg|~=Oso5+Elr!fun&f zWWWp}6-=N21@syPQbk(ckfrhiW{KILLR9K zOh}`q!QL~b0J{PymYeKh7yn>On>qxSv%@&AA4Hn>`!^0?f0Uk1v$_EoH83sG>p{TC zf747P^7iut^4j&JQ5kJY(W>hs5;yG8C^|U{7;9a1eU+YP<|54HTy{;&OEF52IbeCN% zH#Jz@m&fk5GiXS6e~6ogP?^$TO8Pk`6&@+ zS(5}ZuLSfptcX2rbLP=fg~U@Uvvs)GHm|km-Oi*Mh}kJIpCieQ?aA&{Zl(MDF(d666OjBg+5)Vdh*yz$;6WRoUN)8>U{CVBvSQG9?gapbncjHOB3|{r{cuDXpiBgt2-LcG5>5k zdbRC*q|*fjR(fQ7c6OzvU%Ulu_R`Tfet?<+f4K*#aD%b9;Z?4F7)E3Ob#)%dB(3(! z3ris*dZP{M$1`L|8&?9;;2iw(WJ!UkhehQK1Ph7|^O4PUf`jHWbdWn)7nS(r2kbT8 zAPp9}V8C}ZdyKBv>Mtv$TdzXfe>jB;7lu0tV$b}!E1 ziQKbompq>|)0n7$hg4|QZ)qcN7&K7xrVN^$| zJ9UFN1wNNr5#!b#^ryzrAWm=ya~? z{>#$p`xPv4;)_(AhbSK0>`KR&9-?qf<-e$e!6J;H4VxB~{F3b6G`{D8C`$gih{}=$ z!(o*d9N{W{+=SVv50$q4LmD@KV11T~CFbYjlLI$|OCVjUngPHOSLLgALHmU$NwrAe z$R98I?ud`vMUm4%ZUC}lI2MkeWTMR3DLwFa@8`)Msj0sLexf4(3R z`TF92KK2qZymXPqFEem+;~4&Emc`>IKIdgpc!~KH55-asR+xb%g2GGWQcq1JcCZ1R zf%ChjZ2hl#i;6lvKKwD(#2_fsF=6@|8zpb3iG$X#E1yivrd=>7P;FXi` z0h5G=CfzFLBI-!;>Y8B)qdW9Cz#;E9fbv0Ky=PB=^V(AKM9@IY?GG-}~)I7~b5>!%dVs z7Jh&4#hp9((5lY}`~tF97zK}Cm(FT#wD-4Y)sy&U(HR}?1284=as)EF1_I((1o;xo zd@7^fNNmtOOhifw4i_QX*L=iSFieW6VZ*|fE$FXZ9T30=a!s`)6GGG~)W0`1;nNL4 z@1pxD`&Vn#!RuFR9E@e?g20abgM-bRFAJ@ntS&r6`4dzgIbc2aC3Y%ON?=4EU_`eT zqD&6k?#VKh8jA_?riSjF3n2_xZY~Q;%fKlsFiYcV1Ug(^i`sW`5Z~%m&NF0!P?jU(S4VR>NH-2zM#lX-9Q4shB zV3kS%e!z&Sszhmn7ra0(05VuO3L7zl8LD++T65rnhr`OJ>b9!yKDZLbfb}FCXj*`O z2Md)zF>qJtC4;D>t)YPnm@zUcs>ba|`-gs;_5t}@;r_;AD~0~fGg3kS>JW;nG3PNR z6kg)k$M-|E7FJdX-tx)~I)7~m`i*M(Jx_TBEQSXpHR*8J1&~;v90Ju3;}{gnY09YJ)zuXSEEL^P%-xwGi({p3lkx#M z;K3pJRv1QgbE6BaT}&wN2?!8u>$EFHu4Qns8g;9L79dNB4GTx-K4|I{S3$9B%}aIw z(gP;_iVU)X&163U^ZiN*V6}fr78c5oxSI2arw_y~bdlr$Q3LQ84x@fFI{z&Q02tNP z=n8ySNz-cNA#p>*S0QmSIe@Q1_6W}&08{ZY><)a&4Oz02pm^6cU8y6Xh6Xsb3y&qd zzA&N^BEZVo>iz(V^%4d<4REb?EJ$OkgKACS~<((25@SBalA-S)wZ zgmp9?nTqpzU~!I#606zGtW%ZYhuE?MMzL3y7v{oH*4@~+ptY^0#H$3P8N8$!QrmOE*3<3~B8A{TO1=xif8^2pnUSG#D8yHlZ0 z)6)-YKW0$7i07GSc6a(%mE#IWQ=GIWyFjgWad3DX@SPOB&U~zMy<+EkLxJAZV~`Pg zq?41wO}$1%3iz=MN0tMdoa(w%J5^Ojj*-4A__r0_Mol!4`wT-`xFTJ^GQX?;Dx$eU z{(i~~iA$)PM7dO2Aqr~Tp%CViqR5;N643l(&g0nL#a9C<==2$%cevay8$?vNDxYgTu z^AL4*PqG55q8QTc1Nph~neY|!@ExVuX9mMXm4iSEbRD<~kDjc;_qv-~JG2?V{-0#A z>exYM{LhW)eCJ`PF%)?_M0o=(azpT=-w1Tv*oi@i%gozvK?P&QQ-r>3l~9JlFQZP=|gT_OnhBEHlg$5Np% zMdoU(M{qS4F{IJr(P?>x=Uc)Esi@+>r;9jBl>Jf2F4fo)_n)MGNlW;SRJu!Dc+?(V|2*dp+A-DBs!VsDNS z7eDQ{!9tHaG1;^Cb3(9Kgr8EX$GQ?KcGhxF8nL4%sO-_~ybIBvc;NKz{!+Py5$1Glj{uFv0;LcD| zSBI~dimYzGN9wqrs%COO*0ot~)mnZ&FnDfVp+;To@kB*8`n@0uaUE(9qU-Qflf4LbRTPE2Ev!Y#<1r0(qP89?cdvdJg;L z=pva^G%s|3O|E@kU^eN-fDKd^0j_b!kbeab;)7RZcI=B2o=z9U?InoWUSS4oh2%>6Z{Zg4Brdc0si z#rlH0bzmt`F)-QkE|s+R$aV6CF2Ic@0|@c~Jo|AHaY;YmK|@ zbVEUgcytEsgaB%zOtkD@_bA(2Te555jS=0;iF@UJHyu82BkK;6wvxFSWo9ygkkD!Cd z14-=~^ijdKZoLrV2wQ~fjb!u{3E5P*Z27sC`7;ZY;ziyc&(|z>Mi*VzzO#UA%(Raj zmtA>?)VyYa*PQj7?4^R6xg_CXGkb>fZjNVjH~FbDBl=TZbl<7{P)FvqQTeaeUO&Eg zOPU;D*;hzXMJGH8KMo~$BTV%^Fob94k_g9;X`J1Q29XWd%Z5YRRna!}ui69ASMKm+ z1J9wn>;;9lE|GZeAqbG?PJBSZ5dY3dD7+L!Fph~v_Gu-6F0^A7Q;8*BkPbsw?E9Mz zuif0fBy|8=0f*(#?Lo|&P8q7N)^zBnC4`a*h1JA#(d7YUu`1~cX^6kUlTI4)&?|)- z+#y7Pb6CG*eq1$Q=~0`eJ?(hDngykHqT|dM1kFAnK z+*Wqo2l({3<8N%*a-7xAGc%aw-xl2Qy!~nSTWHO8$4fv^g3;7IE>alA?y>lj0Phd0 zvFYOvynYQE^jOKMG(#f$w^9Ui5 zIlV`=H#%9!{Sszcybfg*>99=^Nr}AHwQVM1Me6Y1p}!RE>1SlO)=P5yIAif= zE{5Up_^9%>tNWv|v`hj88#_BHl5lxN$FV6lrw4G)Bv1fY6acwj*T;(nAbflUg*OfN zhW!H?^dKlu0U62Gw$$wOFHG?IQXIKql4@x~qrSX6nyeYC{@IDJof{fYFqGN_1mNqb zscYX91pd+?diHi8ixm}>>*Yb^jhhH-6vDK`p5C6x}R+-ynA;^^!PX8dFTCZ_{{lb8Sg&{Dr5_{w*Z z<#iNrQSfhdFHqo5OXSyg*EXnFuq$#5;>++J!B575NUHnX1iAtr(6X(R*kh-UhkvQb z>oc@l2}EHYRZ8wBbsRY}WXmJ$)~&7hgTzJnov!xbqNzjgC<$k0e)#CAGTy;D|GGjs z|90nsY?kE0J-3SrFY)9DAnh>2&I?{EE33|0I?WWhmTK@uASvP(Eu4et%3wgz1{jKp z3g+|=Ef~>iF> zPPNYCWD&*+7FaB%7l3@jMeq9I zU8I%X6Z3cbdBm$waet_f#tlZjNOuD`6;obaLS+DfwQJCMb_ei(oUblJDz6|m==pI+ zV94Kq$`+t5#u5TMKuiLp&3`4%f6FOwNFika3o4#1-zo8<#n=3=4<{W4IXi~HnnPy} zLVJybL_~u3Sxw&o-4y2T^O4aFbsxF)Y&mtUIZ(0S!t^jj!jk?ctz6|{gLGrjd5kZ- zs*0g9>Q!$K+86ptA$9(Ug3^PMk=u20GsO2cpX$Hq?*L?SAVdIJHaHS;Qhfo$TOeCN z^6Nd#*ae{L0$g3bM8)q!kRwy>_|Bn8w_HJ8tgk~P9L?~2bKrI&)=#wSHEzZO0|2ZR z*d)ka2vy|yMlvDzdr=V_S~v_4r8mw3OtkDqU3mb z9SPyc-g_T6?VSFVVYJkXj-sKa?%IQtyhRH?JFFW6K*c(`|1FGqm2R2q1R|MgMGs(lv${Z2E$%cx@D7GXQo`Em6#bX9zaPyY#of5L`_iz0kQ{1~vt$PzPG_xW z=o8~R?Suh=+Vo?Cf}nrswWL4;S`R*)S}=xGLZ@REA(v*Qx?}=uUI%B-##ukBc5#Zf zLS;-dfc6`88D6isISxfDIgHdH*JXAghL8>5_tQdrl$RmV9A=|go&yME1e|uFfK2dT zG=k)K%LSGHE3h%Lk+nI+>zdDG|Yr=kGuGbr>uQOd%)U zy92KRp1+7EP}t1>k~-iZOZ`do*==>NB)51q_*F~z&-7kq1}5H4KHhoeL9&7J>ylmg zl3gl9j&s>~m4_k(1q=Y*dIo49w}i)LO!La>*@OUp{Xdueh}d}1+PHzyDq?1spLfqEEy6ek5}Z&)ufA*c7B6| zjLdS;CiTYOFK)6?SF>>+2*F1IYoyD zFPd?n(3uN0BL4Su?Bv~eQ3_UvcNt&b)o?jEU%m>Dtd*>!aN_(0CzAE9IlWod=4&Zh z&TsdTOLKB3A;EJgmW@}L7Z`8CR67I_?(dn(8u0+!Rz5?Yt))_&e`24=P)uHF^Tr(Pj6l^U>>jB!w5^|U-3zy+xjzy1LW%)iUf#}3&_#U3NQtTRQ{oTN zMb$(zUWVq+QlxDp`0n(?ui%aH&57x`$mO`N_5wcVXS8!xG>*^*=_;YUjvsB5}>lXR|~ys#5-3&){zR5APAn-5ddcx4XaFc{oT7 zq}an=`)eb6YnxqJzKI(PCd=ZE2qK5Up)2n`-_MN892Sn1MrJME*RUD}Cs>!O4tyvA zhAMARldvY()-Jw5AO*F1ZyfDf*@{r%DSsT3qzJ0|?M7Fj9q`!7>$CH{h4fdBDm_Mo z=m#za!d}z{e`J6nN0#KiW?jpZe4r1I!xeAV;a$Q-n9`8qv%Ni^Ye^zJC?ew~qth>f15HC$`8uBqKtZ)zw3U^(N@>Qz}8TG{B@quHssP;2XL!@CbRooOrny71kG zKOTmLxaKCpppDz}@2=orZl|#|)+QEk%?mkcF>SXRNBAus01T$@;g9f6lKhiS6St7@ z8J^Ju-t$`Dg2T?^!RFC}J4vU3z6bJz1&Vlq6IhS#y$IL4pre=r#}kn#?59IzASF=L z?tth&e}#tjE330ChU1@v{;jg&j5)E8=te*8JvF^8}cXYk?P4m zd0EuWCg%|0%-LWxsX6_LNydGbpg;_HB7p)6KNU08!7gwU^cs`=FWLnd*0vwQi8_|) zcC>i{Af%9Fg4E>PRQE1WD_{e?qov3#^I=Yud!YjOe*@%u%vb5}zvciR{o@HAoJ9&< z7=SGQM;9?o7ZXDlQyya{Q=kB`FtM=FGc(aMbEz_O^RO}Tu&~fFvG6c4VUa9)|1S+} z?M*DrJpcC%OodIrKm)AT5gbg+c+^duoGtC`K#HQ`^vn#Lq?}fkEx>z_l$g9|mGGCK F{{XDRW7q%y literal 0 HcmV?d00001 From fbef7262bbd4724f998842090c4f197df1c2efbb Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:43:03 +0200 Subject: [PATCH 33/39] FIX: removing cross-session and cross-subject --- docs/source/images/crosssess.pdf | Bin 24887 -> 0 bytes docs/source/images/crosssubj.pdf | Bin 33857 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/source/images/crosssess.pdf delete mode 100644 docs/source/images/crosssubj.pdf diff --git a/docs/source/images/crosssess.pdf b/docs/source/images/crosssess.pdf deleted file mode 100644 index 7a39c96002972a508148490b2f9698f6c2ab7175..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24887 zcmaHRV|1iJ({`}2oouq<#s5;{JLw$YHdWgEc{dp+h4`mWlojNtuJ7bUaRclqwoV0W zmD@Z%8+>*E0b5U)U=QhVPsH9&@AqT5eZ9@oXrmHKr`P4fQd+3bms-!9Q7!u6c|p=ZZV4KvhQ9= zmQ@KJp~;ltT)8pHqnMPekGcV!1ugqM+%gue)Hx`b{BoVql>LR`<(pgA(5@TPY%ckQ ze>9iaTss#^TFxbMVHa6+QV`axHXB@?H4z?w4MU=C);G9KD7X>rViT-;QaWSMPb z;k&6nQ1=%@aViQ;EC!U8PBRRjJ3`>M`VBJmA4rkEM9`>y>Vd|bR7BOVo8W7)oBS=Y zTGGYG$QlZ`Qa4QI`wQk>^0!^~r@`Fx9|U&;p|mMOydoA7Lg)vUUG^qCmktRJSE)2r?Uwtr{1Ht;!OQO?W2` zPyGit(#7B%kpoM#JjhF+d=)Phsxr zLqgnuUYBF1;X*|(;Tsn7pbsYBS7F~w*_(xMGkYyh-Sy8fCp7f8F+d`lfE=JwNL5XM zc8l)2zc+bvAPRFREFn~lR%|EK54%s@H%{%J^W2PMTh;)EO+WNUnP#(E#I36(cAlGp zjbydQWVI-wzLW4`t@r_N{|r1IQ}6y=XXfsFzjek9bh_^Py!d!N3d6Z0ArZ9l#1uH@ z{lvvmZC@qj@!Hra8A?!GSE^w8qQ6zDNK%}pB$cI1b=}|3-F)Hi-tF;iQ>XgQDK1;t z?Ns*mq4e;ftl_PYvdUfDZ6zM-7hCdvrt`<} z>a$9xqPvmzuO;rS3pax}FOO!!^M!>GY9tKbs4}e!{3B!X_&d$kc;kwzmnV`Yi|%r{CI}P6OVhB+R`By z7SqHDSwO+QM-Lu!4DZ_x>T7gk=_|W832|3#i+OYmd1S!q*R|#MvB58Z&+~U|5@Bq1 zj#0=5vOR~0-lPLQPEP`NoRhHDLTtv*F=BS(qYfe#&q2AN2nCG>b1Tm+@!w8RXXPJc46rqEN+(|^vRvxzEWsap- zJ$-sr7gHH^vreRU^Ni9)@!C?!8(rR&2saAayP=_`;O%U?;?tY90$`$y*E_aGaT`mm z@rHFmH#Qc|7^x7>0*OP?RxUcyNiHYSO~NL^TjmA%z4%tmu=mNdesCv05+Z$8N31pl zalofYO(rv1*ISC7y{Si&zRBm@tN3}@rTTf!CyihDpJ@)kjwq~<}} za3u7{d^;21P}L<-Pck0}Rv_MxxV0Y$fTxZQQh;61c*HECU@H#-?I<3#l?QTo%XZvb ztU=djx9rrXYuktKi4fYo<=5>>kLY>jgn4xgd6f!)S_=dY`e&rNmy~KJ4)u^yKlVikp?yK9n zZ==C4)?9C%_fEdkS*!`TT(PjV+V?(o?CBnrv-*2k&y%=;!QBC^SB65~?|c#ObK&r} zNU_hzs^9c)Nryg8G^;C4I?2l0~-s&|J?&IvU6}U{jVnIsvE{jd3l}xi}!RZ zojY;;#yCCQIAJPVnrzcp&o3yxod`xtz-3IH7{(x(LIR{xmDE-x5KiTCL9`CDnT|dY zgez}(se+a`Kx&&RAgr@|sbCp)EB91dAkTZZXY#h!i_dO0=rrpw>y*dLbLNHSWd;OH zfXMHMbhd`-y2*A9XdH)M3um;F<_scDCs-LbEQj4SM{RfH_ACv^Ge}#}VZgHG3NhfP zyfaarc5{){x%;VTH#6w5CLv`QhzPHc#?m-j2 z$FdvL1lXziekM4^%(+!qq2S@HdChjW(%A>~G8AWu4AAdPdqlj~>OOmHvRB6H)MgoF zAoh7Ff*Y|K^TIRW_!yRfi8JgsMym@YhWjg$FpEP@{HF7?jBhMqDVxhPEZhei9*QSl zk;hsPcGQ4hsSXc~_7^ro=c4jisn1Y>%80ni(wVgD^XQw(h`FFiZlA6AcGJrj9veH0P}Ez#%XnhQ); z4^CDOSf=0+zTjVZqh>^!RPc|NXM~%q!@k&EmKFt9Er8MqXfb-W*OJUHz|THI=SnrA zOfhJtun3`uZ+!=&86hUJ5Keey2aT*p<{kW-P9@1f&?uY2igVotOmEDY1>*DrNO8g3Zsdg-U@A65Ot z^O4~$_dV)#8w+o@D`w$mNEk--cyRNSZPx|tvUWy7vlII%6$MGGXM&;Tx8d>cJ=pX! z8i9}X;!imd%jBcR#HQgp8gviY^Z?hUv0im?nh|$(3J=ftybJqi*|Ca%UdgpdN9Sz3 z;oFV=MYTd5!dt4O2#cZBi#N>1x2V5<+=chK(?)^kP1yPj;zB5B>7?Ou8{Z|ihv^;B zrI#m+T8az4QmtMYwj_kOa%U+f0tbwt%TxlVQCScCi#ANTWV4Ofvhy~39m&3Bo5oy; zP9@B_<(t2At6DA8TNh2=p*rMd-*SPTWFyKf<6~O!2lmh_zACN59+p!-emiWVelwHb z=d*!WL_8olYtekOZkrO0(?l||5|0mrZa}*!D*m{1Hfzoqf8&OEw;%;ojkHi_pS3<9 zbzfW%am~KD`p3*^#ue(QDdD$mDf!enW-ZGEs$u&ATy%n0iEXCF(|eum4lFCLPTkB{ zdDb}RB57%sDwaS!dAJXIB$)<{XD2hYnmQnA5r;vxSOk{1^69xqYco^hP_O(h={-tDbcEY~g<8qM?H>}v z&@sXd__p1@u2To2PuWoNioh>f$7}Y2v3XgwpLL4;X@&Z`C)YO+*XRPF`pO4_~KC#$~ zEb*MY)Cg%Bdi21kGyt;>sY*F32zm4;FxxLR^&iF+w)$T|FadwXW)^%)IxGGysT0X- zo}qo_5HYe>naIvbwD`UR9shI9|Tu7n#u5LMMiN_^K{g>ERCVCE~_MC9GeuA4(AsP1blSq+Y*k}0vZ~b zx^d3&Td7$|*ULGs*eXfKtrMal!xaJ!O8VN^SW{Lp6Cc3P$dc~d@^qq)n{x-uBg?NX zt+&pZ7@K9n%1fmzfz~)qf~JrnH>CtRYm~i2{RXQINPx_Bq~T&oD+{Z~=a;#JLllei z56lr{Y+R?HAn2bWjEqLY(wM_#lkQe^ve)$ZSDkz{`UYBK9TatC-ab-M&41i+X5^=S znMbwL^Buj>X)5AL*_yYXEGg4pT0PLHxi{6`{$y=&@R%$w_mM4BkOO9b9BqD%ja85}&{29SKU;XJaYTswc&06aDe3TZz38G=M#|_E zf{nLi_Uf&g+gBP<*^i|jdSntC|9m;Q(l(nyTo;w7&%(4n#a2 zrvx6K+-WV9^fv8x*=D(DQ^X1WGTCs5pu#ZzjFRY2W8`3*JJR4Y(uFgd)1N}d!!p<0 zVI#UYb)rpKm>cL~$$(a%N2@R#M|hO8Jc-AoUL}Qy9{9oA z`+GQal(jlmvX1OM>a7f8E{N3U2-HfOB~uuof^ zvBb#>VrzFYoWy`gK^y`vaD6l z1gz3M!%~U}F*Jm3mw&FbX}q;2lh`6*YIEi+Tq)c!sldRp*)U+buQbT&8z@SfDPEEM zUa+kYm?`K`WY3F@!UN<3x0&eXn&^dsU$x!}RV4;uk$O7l$BLk~zr2FJb%-1zkVkp* zyX0`ZiN!*U7%`*j_4TaJ&*x3Z3lCp=Thh>JWTfNLlE|$W3Ak5rFQYMZnd^p(0sAz2r^psle`hNROny1fIyyctoWCHfsM z5?okK$0Bha$sA|nur6UCHn1LT#!!3UG_ur*mU~2Q?QM2S{P;I|IK%=A=!4xz@WL)aCH=M5Sb%7 z_NNh1d_5SR1_QK?_gwMrTbYuO*BIG<#j zLCabz8Er&i_NJby8ab~pD3U$M9cgrvL(;@Ycl-%2r%YK)IQSxLciAXc^6{Z5b{~1^ zVnjXOya)WAGR&P1jfWt3N5d7u=3`^gpL!TxFV#|yr&r7!LClZ#4nnx%y)wHM3>ZKZ zG`J-i*b&C!{*nDFK#;)r*4o0dV(H*2n>Jr@9|;{@)%Fwy949@w*YtKSG!7}3Z|omZ zTNm;1wST=)C^(52DVSD-B-P6*d420WjwfzX_w>I)!gwnwL5t&k58O?p0lL6Ip08Ak`bSslwSvB-T;Q1YjC%E3iE;T{ z*gHjEaYGLIOY%IXfZXsX8%^uT;&iR2|}7sG&O({zfDKG{~uXn?oK^d-%rae%(P0Ve51Z zXY01t^M&g~V0WSiUMXIgqzWxSt&@@)(@Bga)kz^6rRvA&1ax1aic^?X=GCiwC?IP= z1Xl-4sQT}NXax@j5WK4z##ImK_V-?Bl|S!F+8ioI+^<7x(b|M z|5u*4$*(b!UuCAh%#DA=sgDZSrrSYr|Rl-B@3VFs61@ANuM$PJ|r zmsba|HaXJ|+5!SeF$J2;DL;d^Hi^g4hxaiO$0;(=WXEYU1Q;q@T!RmXdO0{w3f%wg;b?cyYg@!3XGi$q4l(-+ z8td&M+nCakGWA*@?jb`OpUwd+Jxum>>{aBX#NRNdG9C+=90A-PRxj)5QIk&=ER8JU z5#fgjNh2~c6%z__@;_nAapth??N(Rr3+Y%MU{CF+4i>v})s|4OC|1>5dhY>qTi-WR;ZT#!NUe3E>&~lF^q? zO~}5?q|0UYvHWtxaU$yZSqnpv)JbU+WJD%gbsme*35 z!5tD!H>FYVlFMhK(e(gWvz$4QfFM9gQ^wj04$quznwWMVpBN~gGl?A(j zHS@ubpx$83galm8jjpdV_0$Dcv8niZpIMWgU#MfBbwLG;R{GXcnAp?_k{41J?g>BQ zjZd*!1|@PP#ooKQ$36!oB!M^%b|f;7%p|VdSG8ud|DdsmOt88)0D+tuQcZG@T(I$z~|1FK92=A0V?Z(KoR-5s;i&RPJBiUk;(fE)g&A z8=ErBI$;9Lq`x7cK_-EI4-Nq9m083(Ad0KWay)*Q!=Qtsi`zrlMbyP+io)oogmrsCAD!7Rs&tJ=_fMKR>N3@YIYdusCZ6^ zPDwMFYgjc5!-0zSjE0H|jfPxSw_S=_jItS_Gpcpyb|{?h+G*1*u}Ptmstu;ZTvoK5 zr&*+d%$2S)oz76PTB+LiZZeHlQ!+LxoK09dGisVQV`Bxly@(212_euvbDrswes~M1 z2B513fKUr!Q1?r#f`R+I+nkxG2Doa#d<{qv?oZeq+rFvZ2G7n9FWK@qD+JR_>Y1pKN(ovoMb8mGEv ztQ^`^Cxx#{P=P(RLUm&3fUOkz-DHX&ls{)Fs~W7Dh%7`jU}N@NArJjQkXuu!>NJnc z>5Um!r5UJFIoH&MQmGoRQVHRTG20UIPQF#KEF=XTf(trm{0g5KPbvF%ZWr`P*zarq zUIL&n2>T-@w~|2r17~Y+@cUJ`cC!w&okN=H4jqya=A{6C(Vlw52#IYlCUT5U96Ar> zg+$yd4;zY|aZk3(|Lv^blfw^;<j68hlOE%xZ*GtDpe(3>MZw)qt$!GSJkG4`QVr5^buUTW;m zT_M6{k!87@p8xz>2!EWZH$=%E@!;i&&Dl__J(5GBpm*;VXigk7rV(vTUvnE8ICfES zRezmA*QGP&j2JiOA(UHSQ5 zDC8rHM;!k*eF8s@E|#5IP5#<&7km_ER{IKUnO=5-;}WRvtgOFAW-Xm-393a zS?J*t2qvj4%J1LTK7HG~7E}ya5zN*@CVgDTAs~>kF2`$Mutp5RrGxSHqER z7D3rvvbCYM9-ZehGUpa0nL!k<3G4AT&aChDxV1aJ1a)+}=xRxcU^Slep zA(057&zbR2JW-;|C5Bw4hA1ohy9|CD2W}X8$m9wV5ICYUWXxlVj%e-CL*uwd2$n@Y z8#4T+j4hGfq|pr6Ph7tThvmMZ_32Xb(Ngy{x1wa)2xj%kB}m+hf|3%|fD|dN47LR^ z52S73T~oEj%%Z{&0(^e^kHo{>Gw-5;bRh#8X*43~|F(+r_z zMPeLWx6(cBp=!tQZM{|}Z%0MDAv^GdXTO5~+gHLPBJ@#|O-XbbcRxaPWNnY%62JM4 z6u%p1HYWRk_?F;njyRnU;rt_d4w5T|u2^lc^1Rxa-aFZ6j7bAZYtlo5zB7_{U?=W% zQl}$-LEa)wU)`*2woQsIj(iMGW|rU=m#Q<3$JO;AM~=gX=cpa`SLPwXnbT9G0I~>- zqytP%f2=+E%?O$G0Qg$epF5hIQLBQQTr^4eAsL?E0ukbj<%B3qInuw;#?hIwrO6w| zt(j7%NFK$xHN;cmO^j*JN!8-0$5oCXAB^tX1GgmD;`QSHjFB8EGz4A|`Vzj6zaJSr za6&+sEXf42xp5`Wl)}uDO06P6xFf((WKI$ypGpR(oOAQe1(?@F$w8}m&55Eu{^XsY#-r{%8VxX z89O27p3x=WBjO{vA|uEXwYv{$&3(=2mhOP0VIrPPSM+|*?53W&|C0$<5c`~FW!%=7 z9w#MkP@2>(fv+X2dhXWX@J!8`G+j9NH|LaDnURz|uk!<;cPM^7fw+Y+=4TkXI6rah zLUNUVL~AZQy_Cg5atIPA;OE6zv2h&VV2!cbMXy-)K3&7EJ(f30g|lCYK0nUy)wxa3 zE`ij=zvl532)p{6>3@7o+TyeYN9L>~P&`FD;BM_IHkB_fk;lTnK^wuN4Cn&3km8kj&^!czPo$2k-=s4?3RM zR(r&eeH~?DaDxx%NBdUgD37wVFxwA!er`U{j90RLIw1$j3kJwj!weSC;5He4tX3mV zM17{NweVxWwfUTHG!7scz(q56yK-toX?pQe2#s|Hj_q#+ufR2moKSJk=8{m#2S#}nX^4ur&+`FuYXF8TvV7lY z(aRU}cb|CBM|$;k(fc>d8#!W9WAbIufBEhj8WVfqtXwz}1| z+~OMFf3jn}TmSuhBW1(Uwe9;u;oJ?x3Zdsi4;%XGg-9wJ{SY%mw^Rt7hd!i<>Q|2% zvFc35kA%xase9gVw5ph$W1U*_X}vwMAWEf$=^&-K3N)rW*+3kRDd3E2%kR}=C+St1mM?2}_k zd?A)ZLLQC=z^HDp9NSM;_KWQk8?WyCPkvuML9o)CF$F-ONMOEKkSYtnE-u{gg`h3I z-0P294%poj%=lt28m#_CzzOi!pYIhTV|uEMCpBm50<7KVtTIK`;+p8;X6{} z8w{fSNqr=={2f8~0nm#N#rv!s!)gL)2c%Mn$bX}XjxF|zK)(SkG5;?JN%=Bge?|0j zy*%kT$Dk*YGJ$%ZAITK+v3ZhnmPzNnf(R4Oz4}3UBntW85QXyPzA>80=edQ5tbkK& zT61iYr&$L~u`}Nh?5L&hC}^>B-;fZl&)Kp+**@4tjGFvWkU`Ib=0h3)1D1W7Pv$6L zk#G3J{PSFcfIKvX+HaUd2Gwu0jQKpEHedD)qcm59v~L}?7|MWg_)H{?=FCNQmTL$} zcAjfklGiv_qmDU-1P~kXGUw;GS0F8+8lAMZo+NJ0Fn_80CO! ze2T3DX?%{I3*iL(F(1|bB%vA7{uFl36a7YxWRL@jd)`lBn*Nt3q&oCR;NL9Zs{v5z zbZ71AdA+e1=8LcPbTqAE;h zgp~78d(f22&^zy{x!vL50No(jPaW;3$0yf&@T^*bA<_Wd2)a)~I^k9yWUKk=-ay@_ zv)et#s)gNt40WJMT`$J(S84C>1EO~L4gmc?#=3yAN1$7JIn>lHx*h%d>fb})9Z)k! z3fOGJPra}^gw!tSjmCxOhUo_IRgE44_&%W3&fe`C^IG>Ly$NiGRRCJAj5RB}12(Vu zUzoO|tUx08>c#u(oAhpI?jDynfltI=APau=(r*7WU?&*&skIK2{ldp8VvY3{U6#uujZF6;|6XS5I?wHDp>g{s@< z@+o@Nj|dbPPog{}PF;&!@r-L zUNP)Xr*~Lg<+np99}2tS^gljP?asOH)w-&`@%0RPzdXHY&vxwvzIl84xdFHK%5z3; zYTv(N4&%PK&)npE|LLWD_2Mn^qojR>_dUfzl=#vxd>Z+m6TnG-i|YgafGzT)rhUZ& zp&uo_zZpJBfDnxIx4b?e6bPZGedYH(Aps#mhEH%H1TF2Wrw@1sTinG;`|9a?dVwhE zC1?2L^*_Ibk@l6-2N3z6hyExbKuz-jM(+grVWfT4^gX#kln|h&`2fQT`kzAqNhA!P zzknoQyg&K?P~aSpgq-FhpbzlzKYxcPxy4HB0cHXkc5u?ZTKa&^ZvsQWufeph5ns4P zJz(~ZzNb}SAUcN6zy9a7Kqno2fEC!H9$MO01`zsD(o4qhsRD#xrF~WO0hM5j-@r{^ z04IP+MfCyHKwr$XuZF&-RA3ljRY?8M>4BS&&<98XA>h^v`kt_W5U?tuKnNH{K_3ta zw)g@oy-UsDUEBB64K$#ocS#t$WBZZZ1!vDMzxDm{>9^mpH!4_R% zq<2Xfyi@z0bpG2o1^S?-^+@Rh=zxSDCH%m8a{8ao0+%AI50Ldg=LR~Ur}Zf518%>u z3_Q+&CrU5K;QiRk@c9iPUHajg1fPBY#cJ$4QiIt1r1XI84JJ1B zG!BYg#Nlm7Eu1g~z6re`BQ}`$1oP#oU4(4f5Re5fC;rCdMdQ`t_?zcg9XtN6zRr=w zi$+`F<~kHTRc~_vhIsUN6~GO{W0Kz>?=HG@`@|jSQ<`fm$x+eJUzh>9E07HNmiRUZ z!u&I(KNwV(oL1$r$(_vu(e1Yz3sN@-|BvR?(zxRyeA9;b?~yp@%rUVaAP*RL#+=$k z_=LGuE(iAV%%qTw8>n zDJSNLf?LT^UC35Z&1ieDxg>j_gQ+IwB|EgbwZaA2$cNYa{bd_5a>UD+6m?N#U{pDk zbm&|&dQIW09aK+Gk0Bh728|Q=NqN3GY;cby69RGi>lbcszD_c76xLa=i!?B>7F%s; z6&t+?KcqIV1P7Un6A2<$^oTrZnr-DyG?wvcAFD2$W%1VO)bjCC$R~OeqB8)z7CGmh zH|{wYCCf9*uTRx?aM3ch*vO}_Oc0G6Od-pl>d}zI|Wu-_o6g&DeQ*I<(&D@O} zHxKHKRZ3&!dS|>7sXP}b|C$wPEk?7Y$RdcvZYHNtm5R4;Xq2Q+YgVu>SdpD@R4!CS z5FM5{HxPL{amnoVSM-e`XhAodj*cT_AdCbmDPa~>No!;h7Y~J372nY#nF=f~=_WUq zPShiaOx5nsUhV`X6(eb&oEUv#A{!YZN z5Sc*7-_6yp+}IzfyY|NWGT+W;P8~@+;xmg{xw1HqrC-%x7^P@yNhJr3tw=4}iI)oZ7`YH zXt;|eG58|LD`jJiF;4iSkW^5))8m*>y;J{*!XC%hwtoV#AivN#rQe9dlHPq>lzE>Z z-+q6_Zm4s*X@fW9_>zf>B6a<^c<@|xw)6d${4ACFWw?1iSeO)nYWv;|;7;hT11mhs zHhfx4^wpNe8xo|2M=S>k4w+MCO3ujs>%b(t$!)k427i)_>4sTEKc>FN`qLVcI|A~u zXgOuMW%(Z*p3b^XPfbJ3!=*HTleSUkh)?oi#@y$^r;21l^@OTT<#@$J#Y_byiAj@) zVxx+lfq}#_HdBaL(Zkcjm7`w!chyCsCD|Go`gFro{j|yn`V5T8TUe||r5EcCN+VC> zf|-d3^{Lu)ZH^wZ?@(8~i)lT2iF#KsXrv~4M_pG9c{}-W#}@Yy_rwiNZ?ART9=qsl z2;?Q=YH^Ou&*Ndqi*4zD$dW6lLN0_e8LV2pYJ4&A#6+;x8o4TCql&QS4qJ(L2{_qT@ zto^*WNoQJe3d+=*teBoP+D;bQtNR4sCj z&&8yyuwx(3M%=0`OF7nvo>?h>Hdaa8PU)^9LdUK~R6e8$5!uXK$=Au{sKndo7GfM+MF-S?5du!mO}N5sGM~}vcps8Ow4NV19RH)p2M=( z@sEv4dtpj18uqg#B`Yh&`8h_Cs8cN;N}6g+s-ChNJB{4(<4qRvII)YUgrsqw!oA7G!_2c8P(i_c)UZhAtMb?R*8tJ4%; z?I1xMFX(I80mIv&v%C~zA~H)WZ^D%y(qbJB>OqshBtv>MC8cxA`h~r{W#jz%30~_Y z*2GC|--bJ)G4(oJ1G}lzI=4BfHlQ>^RN>584HC;`_ z9l`p_c_G^{r7HPwl6tmK@_4g}iWi6yn7daW*o<;(Wd+r=o{|cuqewJ!90_i;BcW)= zX1tliJ#F@L5xbq`di`$F?D*g9#=Ja_gsf6*G6G*qRom?`yBO^YYFNn1W&CkVx$`W! z5kJEMzM&eUOoi~qwT>YlA+LO`X!EXpl;{_&3^h803>r)>ZVCV$w3UiBf0af;;`tR7 z7FK!0rJddJFI9}ySaXZ6+{z$pm1-KV_M)VhW=q+UMybj;S`5roH7(Z0;F%Un4UH_W z#B*yaZC}yJ6Cu^grWIkHfJ+oqe9|=9&`8oG1ksQj#9+-ttI5}(4IP)LTHM+om&ArS z+kRLC#2dKa997+Q9R++?_vYl3sUE5f9;Y7?s@~1r@$b{Izw3|WABx+WE<)AKG}Mfy zBwdy%<9EW0XR@r*tgRI@9m!TL;x8MdC0sN_XG9jVSmv4POWcixw}%=}G*dKd*z}T} zlZP26{}x9Ah~bEvrj>&)>@5r=o{fg65!VA_)2yl?QZ%XUJjHaP4YN&)GyY~YPfsZC z>*yy}8OUk->QFZ=$v)HERXx=)&1xb&Yo91V+c~(a!!6p^FR3hO2}xPPMou1DXPqR` z6bF(P714%KWiD&#=m1M+tAc)bRWbUi7LI&Gx?zR#1mPKY8JK{{gICloh^pKsf5*51 zf@1;vMZT7eRyM_QeWzgqR-CU^q%@(rXBwMi^AYuu~67Nd-- z$JE)hFOJD^DNJF>oQ*WSY(BT}MBXU6ST&fLniOrE5;2!H@#u#bt|c?$gmQI4hNYBa zNrJ=`rzBZ9a|ko12pNc~fDMJRva<8racUy|LbUC|cHl}#ioZCMMRdXnvd7PpR_p4y ztL9>d&ptomu0L1JBr2ZOKUewj>Kt$VOzW3Cl%=Y*nwLx+t!C?%$S~9`|6tnlm%A7X zD%vRex4RhoYY04*(?zV5xn_=%m}pfVK7?*^Rj)i)lhXONH8X3`(MM%kYjID~DstE| zAvsH?7_6Oeryz&kJVcn!QQ(4ey0b2m0Pj-D7PFkU>m>`$Vh1Ztk!@e<=nOWDL zCWVZ&%PJ}oqq}k32UQUKV{}{3=UeIV`uMbya4ExV+Ft+8rjCBM3fGqeccv&W#cxKI zvd!SIzEwPCovUSS%Fny)XKh`k937#rCabEZPHE_!;o^1xUT3KPluoqVm2LjCBdXqi zI%wH|fqO-2XS)df9k?&{Dlgw+>8^HVNd+@u)&c^Y(dreG+< z??S>Rz+sFH`VaUOf^<(a@?$p$+=x0#QWj`t7-=+hF+riNq>VIvqZuBaI%wFi9$BXg zpG8g;l{xOA;=~(Sr#hyr&{Wl=FcvwCYK$IfP~giuw-RR(EO1i3v zzS`7LxY>?j^0uFRyG6JlIS+coP?Y-uC8j0DwtA9!pyKqv&q2d zkVzpQ6QRXTXW_GuUHG-&Es8sN)ZZ{jR8b-S$^8UgcLvo|Qpcsj`$s2Xsa&_pwh!_d z)V=e=I-YbkDZ#Eb+04|86_5v@RdmQWW7T7%r2EGhiG#z(Zt5Gv>6kpmQ6-sVJ47Hl@z(ul zALqU^WJjX)*XuoT#{H4ksoN&D`Z(_g<4^vRJ^_g2-UXCsCjsm!~l z2RustmSnjXYSOm7h^U%iOc%6G^gNDk_`OT>DfBZAn_iNwUGXhy`p`5t<6=@aFb%ry z;k;k#l3}K$lVj5M#f02+ut(mUfsyy$jhJ!S^ZCk(d~unTl@)a^rl!XOn4Fad?uMeC zNr*FR<4U#ZN=MU_CPA%=;$V?wVr~Bl*8sV6eV%5W8D_GoXY~0=N|KW{+}0z{?qnPc zGpQ}so=^GIi) zm+?i)i@NCoJJgG|H;+A9^ma)`4GKNVlA`U@(Q2%u?QHIX8yigs7VnEiO)R&QsPPAY z9secGaC%nCEl-`yIPJ-Ix1Q8Hl=xiscG#|FL@bSS&C8DQVeLPPN}e)BTTMG6lQQh- z|1#4I7o=@U=#-|M`;1w5_2m-nf3;GpR*|M|vc9u$2kO*Uy11;#UUiY3gh(v^3%Lq8 z8G^sLIZt=2;;zE8y0*PWq!tEV2R_oLXq@P<&m6$ZH=~^?x=Bp5{-Yn*U4hbJ#|NRkp@$4ZX*`E5JmIoFWHrsX+*A(qZV61<4%3}w(w?T%o3*F$T!Odp z#?cz07u=+$7!>B(qQYFg?vBHOrOO?nqiDIXu1o)plr4pg;}0HY41Fxznm=JHxdE{K zsP*_M=3W}qDDRebVH7STjO%dW;iD1_yfwa6q>te?a+q;Mn1Y@0^uDGp@={F4kd51Z zg>xw6JO0LnFedT&f+h?Np5i!?G#-KPLV**D6OA8_EW58c>i=2>i5>`LyEIkv8QC`x zl!#E!(y-sANRz<*%0@0TM5he^eHyb(yPQC7|7MXPl5|%vXjn8)NJOW<9~qCLWlxSv;P`a(Lk~U>nYV@r z{=L4hZ0o`yPRIU0Yh$xKP+KYW8ZSODTaDavGV!EV+lt1=in-Q#75J~)rI5GV4T_n` z{dIuQU`i=uU;?GS5rF`P`nDDS0_C*T{#9L$!*NPuQ*)#i1D9?gA@Hrdrb}N_cTOcP z{9|cLnyIVez8Ic&yL!H?bvGL~m4A6EdwpG&eNXw|fjrh3lPelBQ3LN)S|S`=Po<1$ zfLkMLy_fqUdhbtnMn%R2is2>s%&oio+m>pyg@ua6(azLfTKece49qAQ0&Mg3mF}+M z8m>(p*74Ix>SZQ=lf>BV%}k}NOs%=LxS34ko||yiXwt$4tB=#-B4rxyzgIuxzak%% z*|wnJ9M2|ht6aGqp9N_nGmRT$)+(myT}*wD*yVjALemccxh)Zof3^ARRPbuvEJeREO`$eR|AGCT>> zfmYL>Vl2s{d9b8d>gZHiU*+mLl*;;Wp_b#vg{36&(QtEIHB(A#NBE!0zB;ImXX};( zcL>2DxVs$OPH-o<93Z&c!5xCTTd)w^gS)#2hv4pdaQ(>r>fO4x>R0d8tLZ;__ujj^ zX7yA}t?B92kFYx)^<3QO@P|iO@0h%|rM9fcq`oej=d~({T<;CiwYuwDrU?`-W)S`g z3p3310x$H!Y%=jPD(Gc3zU-)Hb@Iksa>mebvkF+bgnSMY+_X&;zd4CMRv?OY1BmJ| zd3!TkjGPSXX1B22+E)O_GLvNi3gD)Db#--QUsc_17COZ{{~zadQ&*9eiUuyjc|jH|)8%wWQN5bT+nPIx+j-Zfh7~46`h=E^g}med>!4Xn*@hjDTBUNy z3>C>4waVLAi%(O#2bX2H)>lL~cmmM={+@TwAw@SB*#wfGb8>oXLs71I5<<9rd*nCu zLu^iLsRSpBT=^pl!X*Kuk}d^#NCArYTz69a7!B9M2I&>WpW?_}>@8jh46b_`F7AQK znw%a^U%t3qnH*qW4-gW_w3+hqQ`B@SD5G*;2HJm_+~0aJI?VVpy1X2DZawv~We8Y_ zTY~L=g@-+4T@K~E{9T-UGS?xe=~6N)wQWF|+*pp8J1b*`Wd(W#EtCN&d|Q)h%4j0z zhvhSt-pr1+sJ$uzpE0!VUbN2=Nc=@w%E|x^jV!n{boZi5N+p`I4U6d6!VLLKl|b); za3e_>`d3VlJ6De%^S+Ywol zNMp>qyu!r^;DEk>6P3D4{oV6>%h<|}-r4HD&~b8k(Bfqtk#efdD0<%R(xFPV3gNw? zH(`PZ*k;ZE5Ec>jeRB4w-1vJ^tyW9fQ8{6GNIBe*-_geaL=stn)cbKUZRrLP83Qt% z>94W!-R#?W35uqJ2L<-sAADl<1Yg`JmA@O@y0ar?^jb7tq(AWHR`>qmI4Rt^A14lc zjpcY`Sc;^l&a5VAvEpnWPL6=K2>RiY(1X2xWJebRTMb5*@5Yh)-%ZGVFv=|sSh3|`(rZntMh9GTa{{-C?@g_|jC zk5-I&jA<{`yyExWyT+eWH!%DNLBwKc{Gp_ZL(bdk!}xuwy949Z>?Y>>(~3#nWIhwh z{MI&PRfgyxO(wL842sC$LuMd<;b`AUxNDSyOAtZ?k}BmDKUlt-;BPVX{R$2XsFM8! zq8Bx4&=~c~ zDAh+QD>Q1{(Wb{CZ~*J1Px9RXq%65AHo$h~Uu4aMFCb z)TkIy0rNE6^rr;l@M(Rk)$WqAWpe$E+V1-9iqFLj_TC0%u{la~c|nFFnTI9J)}`&P z<&E67C#M^P2S$h_m{l#IDB|7YfBtM(I2bB!NSjH}rH(JB^?tp=W53J#rf+p23BQt8 zM_8Um;~-O!NG#1sVOwpvt3ba;%au$RSreecPb}T!MK1pNba(^8zJcD}TOP;HR(c1g zIa(?b?L`l1DOMN%e%9X9q~g5Dgl!ZPqcB+#pZ7qw*3oHLX)Ywh=L`z!Nr_84BP}N5 zyHbT@oKp?3d_R(%7#$m%aFnB3>!^JU-_k0tV6?M6y%RT{D?r32iUv;Rkr*y5n)1Yo z%POR$rxmN`cB)q0*M-;AsBPd*%%Ox$aAgfbYa^M-e3k>l%i~ZX7`!5S5w9Y7xk7P4 zoS0L(I6RxMO{I#vB~;48JC($tiYQO(+?C?1%jbH5b6Pq$%8k7AEqyD|3zu9}w*_&J zpqR!iGX60|vo-@RYTsBf>eO_Q<8%e}e0rs0>qe5dwn-Ev+Al({?Zw(md{szXSOHd@ zbaG=_U_4V?@rOyu>Xc~<<@6b<@)X1PFAp;%R669}odc4z6|+|Eh#7^Wg1#HUzWo+B zT`+OfV5)e0D~)9~YIP6jaNZT}9q|2S_r@b8gOdm1y6DNslGJ?iytW0Y`0`N1LkN3G z9OBBhe+b6FaX{j~^$QeIsEOkYDTR$>lyp;=6;bvT6!ngLOv=+*rk_H!RmvVGn%j3i2bzBSc{qscw_sh-TwGHXhE9#G!>M>eU zyDX0c70N6xJa1dGcBo6Qm>3?|RDSVgI?|N`{wlaTg&z$umG2a6i`rGuZiU|6wm15y z)*t|Xvc@nxv$QvS%=a=NhtMhS|@K(ecmxnUalDBtTEBZv&ur59Ki1@T?XQyPYg;L7Kaj2<Aum8Eu{E~}5}6n@{ZSP2=szN=for5? z1+Moz<$P%Fh*vL;AS&Fr2^zYb;LpysUv0JDcEh@d+Z?BEY3AylT0RRyNzFjB|zGmnwprY)-xpdxU!Pc7VMa(&?kj|jTOa?3-%RUmIlhJSrE0* zViwYpD}3qQ7-QYh;T2vxTOT0L+FjRF8=5A@uBKbtNQoDP1IPJ05rmRyN({apY(9u? z6>Pe-5AB?KNPh=`oAkg^`n6JVC57R%1 z(LQb#fD8{l>lXo{sLu>Hux}7#YjlHtt7GYL>T_Gv$nHW_p0j8S;8TSs`3Np6v?E#L z!!K1_RkCeHsat>Do#>Ib4qk6Ff*ztYttKKE>Fmu&G?-Vixc8^mWB zA6n{G(j4_yWevsqt>9r6M4=O4?mZQda2N!H4H#2oOAw+8esqOw5&`rtfv&khfyHAq@@*>c(hW}9 z1GshfJnfCr$u;=?$e~<_Wr)H``QV(GpXUXKc9z=jt=y~J@a-7Azre+^Y^@IxA&fD}?d)oYYL1Qc-DYJ7a7ZzE7hzN11 zv&GEI1!#!wFnNaTeuvUS{HX^OL^PQY$VCAB}A3)LPW+>l@1 ziq>qZtKOZZ)3WJ?R;`ji3X2OxR4yN@Xx0ZUCXv$c9o5^G;3}(ZN zu9<1akv$%Fd5qlOYdB|;r!g8a#ab{Et-plx0q-zX!4!jJ<)|0&bQ()c%lBjHj?!Y^ zf~%Pfx!z=-)<8n#q%=9TyYV`XVeIcUBx)Hb2I4P#ZJU3FW(0=fNsCZvbYd+#nY_D* z_al-OQ6QK3^K7``E!kc-)1y6wZQh)BbfK&-74cD1xYo(C3?p7%xR#Tf+VNIPP@5XP zTkk`{0D*KM9l7;I=XZp9H{+8iICKyC6>syi8?*B94Rn^#j*BZz?GubCi>(^(rJGj! z+lH*HGzw@%)t~XB95km{!Y`NOSs~#2spv_TXoYl4Vwcw27Zvj*hzfeHAiVGa#uC=u zcJFTX)z@{Rq&!DL{NITT3|Aa<-2q?$e_(3~RB3F^XPLZB?pLyW)h+;biZbjODbO4V z*QM8u(XUt$>6@yv!S&nriTo;cVqZZ9Z~gY$@&ad!k61Kq%?24)=_4@S`$y1YcFZj> zPBD(1VbV(Iz3Ufvu}e*GFg?+ORQ^<~!BnxqRM$yr+N8`<>N51DZ-Y@OfxamB!%2x= z#T*`*Oz<->Nuk*H%3qi!%Lj#>M%{qPGfA7^hQMMD_0hCt4gep$m-?skW{op z_d)B^7b+C&e}u(A__er@fHZ^sa5fPhu$v?>0^R&QEQKkqQblT~U2x>me1{1OY$rs684QlUooDrrr-N zg?|t0my%EqEtrp59QXGhgHAWtn=MTOJ;W7d7TYnQVhb%iX)8(!*`J;&E@^aB2*yvG z5SF9FOtIu|!GAhheIp@lBu=!*k(BW^7p(U&79kS3sMKX4rPiL((!R)s?#G+FV5d4T`oSZgNU1t&)VB{hKrO z?)VWg`OS+qT1)ru&TW%;{t0iu{|j$bcY7037DYo#6(<`c76nqkzl40ij!vYU96bM) zFy!C_0RC$}-X2rOrABO@+saOzghh#NVq@+Ee4T183qwWfo73^yC3tbuR!i$iiXr(a z{Wa+Kq`+|Qp~JWJTQ1R17$G8}j1l-yWrXff@&Z-^1Rv}q7zgiYm=E@*fuzL~^q-N& zGgLnIXo=xrqQFY?e&ds)`-w5BgxNY{P9IK54uxaz9ySM=ZRTBLC~f!;p$OU#2F#DC zetdpJ4h0yn+fq;jRLvM&*do!!jF@jGUE|L(t%VUu}C!H`m_|TVG;Bo zTarC-tWhihn@Dt%AA=te@-qwRz=xi<+@xfc{r7{ToZ!5x{T~>POuU93ve1l4B^7{tE%e&l%blxPP)LbnQ&B^$LNk>8Xi@6u( zAed5_&Ex*k+>KX%jdFj{I;}#z)VPiS#8jxid$4|bI1S0&Fi(@4KlmWXFZ;^h(UgYP z2bH^)^~|Ra%GX!lfBIl*=jSVC54s$k{wH_Hc&wj`qdFbrr#LR*^}2P)Ri2s+eQ3+b zP@u+Rh^&I?lMZ7e?s12aM1c>Qw~vuf3Kn(n>;&|&nhx088KM=-5qb-kGFU&V@QCUb zA4SUReoC8n?`W=62>`Zihv{nxj!*ZwHN%<2yh^a`@D#9x@#Jj6N4~!IW&K@qRnk5z zyuRD_Jm5=HQ>XGC)*l~XI1N)(T*9x+h{^Alvj{@)7-c}MG>MK@Oxr~yN^t3bL780Y3!r^11rMLX@9rBu zo8+p7IP)p>(F~sdT&}Nvhv|dgOK)!meuH-}Hlw1eEm|O9P4aynYx5ybt+m|Z1?#lp zN-AeCmsD3&R_zlnPU-$~)QR!*r^bYc#m9tTHntyQEqtcS!JmHM>4Dbb!k(cF7s*fZ7}WKqv-w3NI_ucFKHey-hILx2B?shS zdas{0KH7Wr-^j`ojoIq@+N&Kl(Li6;O@#(r64+|8!vrO6oI#Ywnf)j64Jj$clH*o= zH5WsD!ug*_(;54lk6KG)m{;H>ObMB(V{SrAk@y|NN-??VpK@7zBt|os=ra=9)tBlo z+Gbz-T~=F+Z7;A)Ag0tTEY@ZWW!`-Q(T!<)H`3z1uM_Q${b2Rpqo~vsY!aQ_fjxp4 z(^f$*j&v)3e%?Ek>C7L&Yf>YE*L-AK^H@EPfQh*#jxA23p0sNKF@IBp;OM1eIDb=0 zym;%_rG@OMO$6KUWmH9D<{k_GEiZb>eqOV>vWxY-;o@$D08Y)Ak6MuLy&JQ{NENpv zs`p-m?Ytr=vSEg~VgcsbY0EHO7usO}T>>YYYd>$zvyL(Jnyk|wuf_tS}ArM_gr z=&|+deG-65I$nz0i2gnPMWG~rw3Uh%sBw72@y8XVv?8(*UwhEnKQ)pG_G&~xw$!7GVb7XIDG2gRN??nN<0iECo!`hF(^;7F<-6OgBRb~B12jldA043)yJyw?4`;8C?*Scq8SE;1 zq`QfpY}(jDcSXC2b?4l%9LxANJ0Q_nhgNR|6`-5KrARngvlw zG=wK|%H-8Ag2qOg%B#U7%YqCA!}A<%e1tTOZnlMz*;Kk!Ss9Jcj-rZN-{9>8M88yR zG2Vy=>nCqAt{BKo^|ql3)_m@$o8Wh^c3Ko0@=ZU^1x8DxpYWqI_ao<)M!7HSK9k=* zecQ@NGV93DZK-N{rXVpg(jThuB;qg@W&4{!A-XSV*;KftD^%VPn!{uNf@uO+CBMs5Gn zM#S&PsvgJtuTTnjD;=}IiOryipGHJN5ie#VzGWdoEtmQOt{ZR7#1giMyyyPBv2{yd z2K`rf?OELA{s?TL?JVvKU1~-`Ch}UIk*OK(HX!?C@KeK>;ID)Jie&Kh_tJy@AsuJ} zeoy$|b^jd^GrKbJYy)DvzkXhQ6khvVBE0q##_dkY6zG-zlf|`Dt?RgsHeoE-yqCuM z0ueTwyTbphH&}f^q~-?4AP_LS9E|m+gEfp9U2e6#_YOb@t;?8w!A+p zdBbouA2K9*)BOGnsr9J%yaM`eJmjwnsg>bx%*%GG?0U$F>td7L7MH)Px1=W@=c*co z;!Bwopy5O`y8c3Av#>rtGDN=Bt*KP+@WHt$V4?Nka=I}$CYdcUYYSVRr!rcvjqhLY zuL|zYen3n1HK{c*Io=!Ja4}blFP|3AL=RNMrBC*?Bki|w9>;q59`iykH?qZUr|0I zk$=70QAIWUFn~#(ie`w}6LabYbCS^K5kwB{Q>C{$F?Uuta!ei^XDt4a-+E=aLIVtVJ~s=p-~-eDzkvZ`szvX#){*wuI9wF(hLC(JTxf#h+mR(xRp;4IEqtmqX8v+Vw+1%7)xWEeHgr&<5>5QN#9seiCL zQ_nmLr~x<0#iLswU3Q2Errxtp=bmK*gtf`HTNG`QZ}!s$+x737w9^B2FS{o&0Iu%MFZL!M{YT(GOmRj7M4u@q!@eV?XaqU-I=+X#JMfiiScI!tV zd_KXCdV&H*s0bW03Y&=e!SU`z1dmRr=0*ITmP5{pl&lM%@K7Y$@VEGn8$S%u!#PFz zbWjpD+xe*K_;!pXGpFF@w|=IO7wtJEa*H_Ab_4H~0d6r1{Z6!Jr7GH`DnQp+i+~iw zMMcak-^rp+GA?f+bm%2rp4(gnqPgvSCUtyC#2b}g3d_1iRE7CXvV5z8H7g$$=^!6) zt@b?5&>>nGZPAFd-`Q$l6}v|iSE4N05`?fsLbkBUYVIg&ecT) z!dAx9l(M&wRv}5u(%d{m7T^4y1s^Pczz|BHcrNDs8{F2*GpX=F(SNSE2wKG3py?{;4hC;`(xwe;RO^8SSd}eO%_~Z}~oV#)V zd_u3o1KwM7kvrRMfLJG~6g*+))aLD}Xaa52IeR-kLsl$*%C)U@{}|0CZedX|7-$?R zn~U3He|Gvq?Xd7N=RmcZghmZT?ir7&dAX>X0Y=J5T zu+iR$9IX@|c2tj1@-h@xkcu-1bm5rIcvIv?4hFSAyoHTa8oYvyjH=OB%sGjo)f^jO zkh4`IIMgYwk>~`Z#1ulchqbe?i4(NLy7BYN;+F2$VmIKa@tpfHQnuvINdL3M^z{{W zn-KfsdzTIqH!fWtpPIYGhm9^hcJ2g*nvI+Wui+M$d+>9L`z4iDyFPXHhjaS|QIk%t zQ)1vRos|{2j;QIy9WU>MkLUR(eo^w+ZJ5;=Yvt4Vi-}V&YxSAzO=)_Sq#7tO*KorG z;%xYS%of`>s&ng#;7)r=L&4hZ8(hjEC=MV@ErI-*9u4GZZC*--kMt|a?{m~5dm(;3 zP{6`MsDojBryc@6YO0_1K5P$|l-CWakqtM1HkxNYybqe}SGd#h6e@)NV zB_T2V?VpGq*Z(ikWBV)E`j_Vc{>}2(xmf>;pH8 zJf?cGi1i?q(k>;Pes#K+N11Oz?8sfqo~FAptfaq|$Uv0|@1D#hLpIaXcWh2+%*-%m zIH?qnKpTC z8wIXo9!KfBvo(sG0zLMPF<7Hzwxu$4vpAaV`m}cMn%NSL%T0&id!fUe+#+PJAB$!I z>5|MPr0;Ov(Iaq@HgX5XO4HMBk*aLQ$vN6dpD>MU{cMV#LJnU=h^z|4$e;cyt%U!3 z4O-jn3jHgetg^K7J-WpYAdj#=DccVB47C2pKOxQiA4scMc$km^-gvUIo!uLftZy!5 zBo=8~Q#;bXLb-4CfBeYtzoc7J*~IkU*t2oG&4UK1&RZV)+ok)r7h@B*w+7a?0p9HY zJV1bILK>$_1PX4WLnja~AT=bl5KrTn*bmDMZ5Ia(PnRX#PAxq?u|S<05k+&*AKv{u zs+*Bc1oJ&D5T9Nth+u{+5RNkk7PB@7T6Z5v0>>JN8ENpIP0j$$QM2=d`mbPAYXf*s zsdXqxo-TZ5>htRI(pg>hiIA;p$JUPM7?$UEDzexD|A397lL6St4g7Xn?3@4|b|fk) IaYc#$0bUAf^Z)<= diff --git a/docs/source/images/crosssubj.pdf b/docs/source/images/crosssubj.pdf deleted file mode 100644 index e5c65135d492d3fdbeda81338beb2967f9f58a1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33857 zcmaI7V{~NE7Ovf~ZFFqgwr$%+$LNY}bjLO-X21t(-$f{)RV(Vo`#>>gZg(xC|=;`HQZSIWdn|onwIP-PPJy5jIq5#U5`w2-5zR`MO zJnTCk)sAf#@GEpd{J4AQ>uX_vO_zfYn&=O&O0|9sKu3kwBhUOMcSX024I1#QV9MF$ z--rrnP<8iH?^93j$NSBYcVOOUWgqUC`ST zTHl1|2RO%>`1N5XsUaAvUGyW%1PE??F*#re@qfL%=^jH%TDy1Je*D?K6Z@X$SS}dR zyR_v2?~fLU=&y$~a-^=T~~#c123gC+we6 zBK){g6S$S<$>Ic9%nT&fbE0=9f8Ny1f;$RAbq@GI=Q^IFT?-@KK4JQFO1TGcZziMk2qQKFq|f5tfRuh?mdbR9t>`Kc=I7Dz}UFAQ-n_#k9;vVe2an_TsfNZ+;-TWSJJ63%e#LKdVA}jEe{C}8m zc2MMP-LM&rDa?F@0XSx&)|B(>}$97$FL(*t)B)-Y@qD*?k`w zD@&TBflfC|h9pP|#pL}xFxkV1X$E<&erd{uo?#Q2gOi?JrekU|mUKtmHR3Y8^U#e6 zD@oO${`Xj|gkLWkFL{SychhM0PuYPgS;2$Lb5nlF``c|H>)_i-YBai604mLWCA{Fh zMzrd}7vkTQKqgR;ItF3Y3xBMJF8Pi|1qDYMgc~}orFai*GbDR53=3-29pt=0g7bLo zfWiT!lGy=Ep@0XI+urQ}R#0jmz<#1ad?K!ETlD?yref?;<6)g7Lj8%V1eyAsvvRV) z^{DU1hs}G=%m1K&`>rqOd}8F+MOjzV1@&<4WC%o3-;1QtB&+P?>+O#{Rbo|4KD80s z+y+q09ZXuJ+)m*4(h1ab)vK(I#3tdUSylP^;_?4X8FJx{`5(M;41TkjW~KZE zce%z<*-y&fN9XuFnrjN%i++e#R%DQrYntyC>^Jyg-B>8<5v)FFHOd;MyP6b>>Bi3< zOkkAvcZtTP~z`Vl8rk#Q&2>1roa z-1Rt1(iM{6nXKHRUEe1H>A{o>-5!L=M@F`&0X8b%bSU+|fk15|3wN}^ z@!T!}nT?cN?7@4c{rjq~IUv{KzL<9!@c-ag?A}$IaVLCZ>^r7Eh6Ebsh{k%entwyL zP+D-HUyrdw7-*2i)y(t{m>sStYM=k?gZjmd?@Ahu_^$rLRyVLFH45#N;MPRJgE=n0 z=$t!RJGz+MY2TFKqSg|gG5F!~X0gx7bt~{;-Sz!tIcNRXJ6N~x{pe=V$NSDX;He^t z)Bz1)vD@k(V0A0#!*|~(U2Qv8akcZ_@i>(jNhi;9mcQWF*4EqD{`z+C-Sgw-3D5hO z3zowhNr^Az&$O0=iv-~Pv|bHLqZ`*zUr2J7bX`m(7jsc+;_ z;>=F*=2?QLNXYZ_%@8ZDC!Tw3drkSJ5$bf=Rjy))Y)KiKsUR9eJhHY)=i_dtDRThNCb``K(9ah;j*8chLJ{#nJz5`CsNKI0TcXki)^ z^r9L03R&8mjhZ}g8W?HD*g$7Tk0_sAZ5XR;S9^MAFXR|2{q^8^j=7A!(#=xM%Ssd` zhspq~lvch%yN%e!Q4LUWSn{&ERPW=bGt$TfGO|^gV3lG9A1)ktX|W;tUcn=c}z;icB%VFFd1cgepp9Suqbs9_yq>k`#4E zg+;HAw?-9~^VMR}A?Of8QeU*_JOGf8ve0F+QT)n;DD2nmxp;QKpEyW~$P==ifaeY< zi!SB~!6wMlI-BwIVc?Qv9lkWZBS?Xd&w3 zwsoYkj3P*<7V~Qc*?svI9bKo_RczC0$C=Z=xkekN*{d z_p_4;m*L`rQ}>!eYy*i&S$T1fx!}VT3&q1Zec`m+3Ka*Kn{}8`Wb z4^HextJdAWw%^!&rbKjudVWM=D{Z)G{m~iK_^Xl27wpB^Nw3FRO_i}=<*e<}J9qGN zMawXyxek{?U#%5q05(oaR9U)Eo_DsUDn5-nuV8G#g2Cf<{ND_8yvJ-jU0#Guo1tV+Bg~*4u=Y!L51GKL z7xUOH@zN+2hFr)(-jhq#Rbvi+YL+p}X%@{{^Onh711pomQE6eNOm9cJA)X>!q}AJm zKCQbdN3k_q6;}O}DO-66BcI6|cPTSw>UQGb(LTOJf#;Ohs(EkI07Sl@DKJ$k-~knb zZ$L%mL$92}*<^{pW1nfH_FGEf`k-3+^5^~4(@ozkcCh`irvDpV~KVwF`2bn=cTW)GJ#U%IQe?}hn1(|GFn&ig?L&No-l=%p@WL{7^8^D^SK0$UQgR-KX|gn7bMhe}R7yfv^*BNN{J4Gm zzwzE^>IsYl)d}qcA3V{JJ=e#aEm!}(lehAx|G}T~Csi!JK)22Kl5ny<3RUzP-S1KK ztdWDoSKMuX12;AvFSpxxB>^cRQUG-m494)#o2y+pCCDtC-a$k8LiX!;U zD+ok3S$AEZWItogHRZ6+nA7%zjU4EB#6+^#w`uyHSm@gnRsZ-FqWyp3F@s02G9+L{ z64VO>w;LW|Va}3F-pMJgG3Ox8+PBv-9#jQUPoti0ldzs{yC9f?ECQNBB_Y@H8&0=H zZf$-*cM6MYemth|{E@s`mmPEps@8lXlFUEd#=n(*V_dG}qv`_{~Di0oYDRQDaGd-4M3TBu#O@or-aA1q|?(opE3Lh;gI=5Yb} z)*zPE*~{J)_p002{Hn|5_s>811JOT!{d4@7{tWDHx=6*!f!k11qG{Em*|^7IHG^okE}F*_b^trWJcb0BeSjr|p8#aC~?$j~LI zU2~nSQ`h99-^$=`U9GLYuLq1JoQ>6P?pz1vn{5IHq5RQc5brs&ILt8U{QEuBh`%Se z6O6=TJY?E8(*5qQ?F@v16DxvC1OSr#uyqr6{43c9q{|SwbKCFfP5w0Nlrf3OYzuTu z1$?FS>`wG}D$}7Z?h=9h_h<(=)T~ozbgR;i@oLC(&<2!!rfeafX?Ej((*L++FV!VUEUY;aIbrWP>>b!j+^^Y$}|q z6`o%H25lM>^d=qJP^kNxo=s82K6WxOORry|rAgQtT}sxaXV3mK5Q?G*dWw=SO2Yyh zq{1!CF3$-MgM{@slEy1pftV>eNxCvuH@`ta_8DRs!k@!c@b`lHLBgg4y~mU#PZsek zMOg&K72v~3a>_sE;rFEalT0|vidAlE>0w6wt|ki`A=ifyH1-uW9HzS&My*1De(rL&z=PA*?onM5|ud+!H8Z{ zS=gP)otbDVR`uDB+ivTXj&vwKiCtUd#W26(DgllZ@7J3cRp;~=f0t^=E^inqC5qkA z)b1UiNBO0utM}U z34V{>^$n}t&3z;&-9nPvB+i%HgoH{Fh(M?CM&gk=#Z41C#qNq;U>?V)GGwFDsA!Zg zb%Dr~+uUPI5s-GG@Q%BXIvs_OIvvK9I*rqbo=*sjo{uSsUPzewjECq2@ucAsnUdj? zP3OXW)j8n}r{Lz7sUzN;?f-%)eon~J3He*=33)I)JfM#Kg&qP^zbaTdD`zxZ*hwHPCTJxOR`ZJ=C6N;Da6WIgR`irR+gtL}?0}TCQ)k#j zYa2J((bBDqTee}`TQFM&CJ~hR9Q74>_{onImvI|+3quN}r-QHYZ0y)MUsqq%*D(?9 zl>F|fl;lQ46nR{6{;`__7+p8Rs~T4h5j1UpU(=N;zjhCtEEsH}B$h9&y=!o_F7Xr> zSGBg#6MyEHI$W*aE)Zlj9qm>5liBn9bR*sdeob>$Ng}}ir-$5?d>U4rv>j$iE3vYV z6w-W;+!^9eQF~(Yoq9Rm7iTh!Mf6KS&NP~Et27I{KT z=xdJ1xoR@oLQ3pgkG!~ymvS{}=eu57Jnwh|(WVpaTj3G2b^h!^%|wfZ`G^Y*8hRc` z{L_n;lRBlCO;a(yjC_7R~?!qVBl8Q>hQ130cd%{;PnSvvrevMkL_(Cft6h0{9U#wu51#uGX&SyeqR*}rA3#%Fi%nf4e351rFj$DP$#UuP=g9nD_~ z{g(H;Odh@TQsG(674deR{Zc3$ahamRmxj9JYx0Cy&mvF<43t28Ol#pRM-O>s$a;u~ zGGW{$W+p9N*yreHY6B)fGaTfcTDgeD>lrj*NVYNS@mKN+?ZE)P<~HmZ8iMQWr{Q?fC_g%3}Pe=|7(@(or@2CD_rJTLJ%sev0(X{nhctw&|D8U4OXC5YdaN)!qo2!r)q zDeacTP^=TpjG+Xug)skyMFuZ{;Ta~pRIACz6Vphv3u{QU|2C6oH-3g46WuOwQ2uZ2 z0z7lNMu`R;mnOjb&_G9*el;Ju3h))C3SjgZ`7l)qud?FR63mNXg-ZRMwJq{9*n%hO zj{5#u{Dl5mfoQ4ie!XEN-doS5vaPh?YpARi{#rfkXXC;jcInZM7RW4#={3Am?|2IJ4=0onvn;4CWvK z-{phK-_*_NkK35U8rmUmNh$Y~4q>L20l%r>dR3BNR!4vs=aRI-^5C!_Y zIe+w zko_}94b$xW^~mr4sCA}(@z}Msj=lxkpHc=zWX(t^?M!^T|FmKMdp%Okq~7c&?2$oT z9pjlmop^4>zochTI`F3wILrmdN(UScA3#!IVs99T%h20P!&D@upABRJ4r|6xUMg$Mslnqm?>?kk-$4^H+56y*)=rvX4uhf2P8PBa}(o*9>%uwU%gIWJ$uq^JZbrkR<4*p@1d%LhC zl|Iu}F144lmdkOlB$zJWZc>^n_`qv@{HJvs5~(;)R4D-w>d?UQ`-MhTkx&h=|7)eM zxr?MVY6sq;Gn55CZGl$IM8k?ubGMm}ML0=w5Y(7=*FyurI3 zI84i(Ng%zPZS$xS&sF}T*G~NhIA@PnKT_JiiOdo@d#yAvuolRaNIuWa07$^HP2t4v zPMQDgg}bNdA!VuKu>#_~zN2+nbH{pBzUCd+h7S2Jgq%Qs_;KPMKE#h0k+!@^62xr_ zzvCGmg?AxZyIB3NB>74IDP$t@as6K<&Hq*}*~vILd3ibh|0|pvZ0y{8|4)hYyaypr zOS+5aU2E|~Wb=eOuP#kxu44Wq`M}CFG(0mmIh^_2kuZiTT)V^y!q!V`ud)Od!)Vl6 z24+nB>ugcl7aq`J{C;I9JWB~ES_f__r_LYkFtPq zi){~J9mn#1vwQQ>2XW92F+cPNmzNb>t1SqT%<^R@i`j64n0IvnHTmTWN0xEV?O56} zVdfVlsPW<@9OkLWx##UK_cUJ&SzX@UbIv)Pq2w689Ht({;puMA8X?N{!LD4l;p8zs zC^13GC0De2Gr62Gc0S;eS$}ccfQ5v`vOq@)s~>{c;wA4w5${1Jz*ez)CNO)}17pLg zvB6jT>zEGNfhYT?vSeP5wO#*ugmVldvHydGijSkw`kWMx*$ujni#kVYuwhFzAo5>> zX+Yu0dIqhFhl$xw)c!yND9b}DVPrOfU~@VcIkMB{1U(7kVX>+d0xBqL%q-#Nas4z$ zY&*5aS-1HKV5!z)3N079@iN4>Zgb2l0;(^ClKlGvQdKv}f2jg#@u}$OMwpleBJ4&G zYX@cOArNEPv8MDAb&LS0(~H}p$Md&g!F-3OA87@fR(L@#Ay<@jY}juBq<*m{6-RhX z-fbt3%&+@Oad|j`f4CR9xkb3m$v>#J(w$U(?5tYUqPNiyzxfb8n7U)SX+hpnnH_yV zcz?Y4dlf7Cla_qcysQ`~B^QSLS+b~wUTA8~mLeTk&^&2@6gMbpM~;}0Y?;zQs1#(r zgLh&=iL-;*E3<3=ojq2k#sS6kRTU)yVGig{r(`}>CutrDtA!d-#%YD@MrldsBn4Z5 ztd|%E#LvqHj67r5n8C2faIy?=O%7X44iYoTzzG0}8Ql=DGe(sw?065>2Gmiz{PzvF<05t%+~m${S=q-U=v*y)AvBlr+7 z&5g)$MX!Z6hz^aAU+u732pFT*G`_$r+DX^^?IczNA~Q)x6)%Us49eE{2Te_uX`8Mp%+rD8X}B-g(YP& z?dfd1PK!AKc03;Qk${2QZ1TBOZ9m6s`#umcPKs4#edM1-r=8+ipJ4%Yu_$)@fCrOQ z%??l87GHOQx-es;L3Ol-P7xGyac_mprL08{pDhl{0L5$oqE0?Z8bUU;zisEZPYh^V zo#MlS9Egu+Sp>R}1~5o+3sBd<2Uj#^9oh*B$8NWWRWuMwY^h*JR}C+pyx=y!BI}hunL*&_)*Jq^nu!6Fxu9&Fj>vkSezOT# zPDVa*$N{I{Q$;2Jom3$0h1gG@C%0(5-&^pb$R%yUi{*^iR>ZJfo@b(G3v=0-aPF&P z^BdIWQFHSNgnz5l`n9<8$u9ZJR+)9z zR8-M#tEv#eSTl{ZYxAZHvC*;i9C)9<(EF1+$;)@_a+y%pL=tfEqL zkzf0Fa#fByYdv!rFKfH_E>r^I8%dh&Qb#Pl^=GBtqIr#tj=pnlBG`@++nydhH0ezN z;Y-VIj4%0FSXeHNmiSpjPA0SG554&7B)j&FdeSWRlF#lw53(aXJls?NxT7?^`4sa? z!EN)SG#)&KxB_v@pFqx`AtLcDm--v|7J@R}*(YontJ z_4y8sL=ZoU80iFSa5}$lgu=;rP$KYmidCnkhrR5N{igu{00=4WZrPjEQxR7YPsEeE zA%hWW;N>j)5?8JS5p3p0O-=nF^)!Ar9eRZaCEh~TxGUerT~vvTc6!y`1-bKE_JVX5b!F*%}X1gr$aVhu&dc~ zVqcY?Zsnt45tz%kbkew9n~C2n&qGKhgK&HLYMGPnrlYC%`!G8 z)YS45_$IcOBvWOuFbd_pGla2e41-OK`9@3)@l+m{ZwbTo3+w&Veb??Iy9|;#mr}R*!GWZHIMC&+mD^%%M4r2p`+|GvHB{>CKfK;nR-PtcwIiXO8!Wx@n5d&4cqSp+wkYsh)^tq_X8yc;zYZ=k-op{h3*3sQ1AfEU%ppLAA9o+HCsGCjW|- ziYGgNW(z*6m~m7)?aGM+w*@ch1MR@$^Y)VDn9A_;VtR<-3EvUk1)gpR?g;t9U4mXRG0STbvZF`yoqMYm|k}hWO7EJ4Jb(sF=7o9z`%` zRMwtu^!T5uO)N;(k7YNgz`0U+SK;COxw3iBR(ob`>eM!(jLAUM(Zt7RQs3V4%`&H} zdkmZb+*ID2!svA@GIa-c`!vEX{oT}A_HuZp1W$A4U$5jynA^^CX#6NRKBnK?jPvh@ zY$hE1SrHZ8AN`ZN=vs^W0F71aO(bM!Pc7I)mAdwqzB2cVpD?zi|LZ~Ii7;WO`B6{g zR~4~Na~(-m>XveQrApeg4YpqMC`dbiB7YXApVThKLK>I=rFVhqKvvnjFT}~z_D2_meSyB(z zVxtp=%PCQ_AY)%*kMnSkK7IPNNc)nipAPVAwlvn)zsqh4c_1;lK41N(ZPZ^bOSvYb z68K~9yd!D4junDfi6T2m*X*|-B%2vW5K#id9tIaSV#03XzV%dXB&RJP6i);RI$D?E0!UU;`K7-y-V7haOYzxZt#$slQ8giF?!am=h+VB>lI=O zXE@&_!r|t$dWD4*9j?uHHYBg8to*OW-vk6QKX@a`#KgS-;blzbIlrzW^4%Qegqi1v zFAXZa7*#^3+Yb~{{D?nX5s4u82aTDh0uza#unbG*hp9Gu4IAs7WkY*gV$a@|GI~$ zurnboGpMH&R+4deY?OEM$r(4w!8v$$&f}QBko1X-$ulnPuH2mS1^QSQdia!}idoB; zs3m~OX_+p+ChVOwh9bqB+G`A}KrSmWGPGpYt&{c|Zz3aA@!G>`yaBHwTL@UbOk512 ztPFOjZUc?vWbaLbcs0Ta9yE7 zvGe?n2_Uh4rZ@7G3>@!vX0YXPho{fJV)^RKR6)eql*m(#EMWh!|2$(uH`p#F=|x3} zbw&3BRkOpNzx4>fHm@Atvw!}SexHQJ2+u-!B%JTTc+@BT^Prp{Uzu^#v z_s1oilMSAe(Q!R~?(x%^h^$zRBXiN(dL z@_U%G1o?V6juX}rP&t_`P8bpp$7hY!jFbYw+8wi*2<^K{_$}u;WXeSnvFBX({9EbJ zpG(_zgpeV%*R1nzDerOQ@0;h}s0ROB65d`}wGTkCc86`UycVF1uv*th{mPJhmswiP zx%AelXkIUDTvr=*feIURk;Ul#O4yyZP1Uq6zQSFroxU(+pnjXjZAAR=5d6)$VTN@S z^!r!oL2AMx`?*5+Y6i$w*Fv-#Gj(gGo{!p7T8pN@_&ZxSF7wUYwM{p!u`CG3 zKzy#R%+(coC|6^?R9;m z;6Vd1j2VOE2?MoJ1GPT}WWda=Nr@{J9vHlw!qwiI6Nfef1Lx#ehnO>a9e_PeKN8*_ z*}vfzTQK4rdtC{O%fN_0)iK<`3Ebif!4P3Xcm&y1NOZb$NIEBo*cHgno2tP96D(g= zAo*4y&u#I{LYYG$RE|OiiRmm*&uvkW+2H!vtx!`qE(BGuJPV!a7JUT{wMbY((ks~D zA|WOdDQw_IzBKRCp|K0n1lzu~UDa;G{&vETe7om^&%Niv_9vbPxw8!r6^^|;&+b>u z$41WOfZXw=!Rq9m49fd{C+^IydLgI?$H@u&Rzaw<0PRI;0~u}am_LVIE}NyyY5Dmr zuBk0%bFuO@v_K4u-o6H1W(JI4nb3dh{%M*DGJMstv&>|AV(n&7n*d2KB>WyPB*YV3 ztyyM0@piFd!(WO|b7N|9q>?y6$eJgXdSawu5K$n@!0EX$DgQ3)% zlHQCm%^KQySzxG0EXut)8}^*6x-==y-0WZ60knBpcCu&9r~_WaUr&NDnts{iLqN+d z^oJ(DN{hS3&?!lx95|>)U?<=3k;j`8OOF%F+KQ2rxhFZ$&xs`G&(Yi%$nsttO!sHr zX#~N>g?F+?9j?wv8HoTDt|p7g9Xj)p&)|e6L1lhWb`5+a52Yk#V}7S zF$DoL>BD>c1EADz<2#|w0in1{jYc5{d7Wzd1lw?Qi-r#O0@HJ#JI?0*@{e8lU|oy` z{=HZ7|85R{N?`gmk~6qNc3=~uiU7;uLgKzY!%)kD;a;o<0NQo}2;^Z@G%!+MT# zYsp<$)LvJUui$o+#$YtNShqV77?eRk5j*H1{7nUze!!T)V!3!Y%q@rCNf@+aN& z`Waq=QkZF_T!g9CT3G1y#V|R8t7PQ0a_@UhFNVbiA>a9%;WfgRTcA8~?~~tlNU9Q< z=gvrgmBf|J)gM7$p9lY^Luom)?kDOGtSwP1neki`*ASCoDgR#8V`;k8ql*clbCfGG zEV|Xp&D8dVf7zy`F1NCDvrJnk1A$kpZ+AUDa1`nH{teRssk%^jSa3f|^m~eWX1C-A zrnZ3J;mLbVnDNOWm`4pyuC8^Uc>`o+#>0P2ZS76MHz=r+N$|Qp9odjP+PqL&P3yS6 zoUc-ToAVgA%E+W`NXPr_d2n(|*Qqs~g~P9`lCiJMk~VL}tFO%d8bFd}RZNq;M+pus zzpc>NOHjGIaUFrD!uY14Vt-$Rqo9P1g^_6a&=P!vkNIt}BzP(N8opJ8;*4{UauM8U zzxIM69KJz}n4~py9|=ED8wo$le8=T!#`N7u3-Q^MI6bt;_j_ zfyo(Uva&&$pvwy^ICp4#%p=+Ar-d1AS`4bq_5QeK566`wwdNG#nlM5L!+yU2^=1 zV^y8NL4aCPxwR-695TmDa%YN86Ex+@>ae@)r~KYE@)_mVKNYh&U({7k^zR@43Y+T8 z4`y0#xS9|LulIHE^o*O<+;g?Rh8^JMZ=5$=h%0W^M&{!>yF0r!FBu)X)}uTKcXXA% z*EYEb8S6})h7>w+>*@^s@o#)5rCPo_fb-LT=&AKLSqfVVE3O3-SjxS%M3P{QhC3X| zzKJ-*xG!>X`2`lS$4s})S|rBbb~fCer;Cw9N~1*7y|bI=gv87JZkTR3lE;)v%`;s= zrS5DXJiI%nLPMkGf3q9}zO$?C)*G4xfE${dJ*{bJYjmpe@)&jNJ&=wRT1C!t^hrtn zTx9`v&5ZRNt5GmSNIn~&tX;#r!+eKXMdFcXd}V}WY|?c~EeBV?mfNUqQfa@`Kx(gQ z=afo>m>05jyQLjG1VC*$UD4QQxE1|`oqF+dGfl3CuScy%a)Eb2AS`xTo+tRn0RqJ1 z5Y%ukSv!rdq?#Wf+!3`o4xkVe*fwgDYg2z7k&(VGV|90ReOq&VTbitH{T|AR7E#M2 zjb~2hpUGUC5hJUm$@$4h94BtWvFt3ICI+4iX0yoXt!+l#Oqu&&=UQNsSFd#mwbM!a zc{~2#Xz+gUe&|i*Y$1MJ{7?6vz=;)`bB=mm7p|YnM9Y6#CgyBTn%F9&f!O+7f4JfX z7_n*%TB|)5tF?QQbn0&HR95xQT89_>3F;nxDz}uH!%*+c;aY+`CvO82lm!14sPrk{%TI$t6j#vh^hk#$g73!)<@W~pneX|*tx2O zhSeEV2*1^y<;Opjs!&+at|_cORZtOF%#ezx#&i*66x7CfEl^#a&M?xVwws^> z4-lh$VF6AcQurNICo!7DR12Z7V5Q-(%2BS3ME8{_=Ei-FCOFL3%8?;>bXK}iiWq!1 z-mT$4uN`>U#>iwU}!t5JLl@k1x{7#;q3Xf~a9UM#K#_K<;ilmodUgEoSBBUhF ziMvnhn>L(cQGPOX&ko+q21H%sRnHw~$Lm#Qhr;N^{R6@@?6du%!qrf@q|TYJx#f6i z-VOB~^uS?}#k(8{^My~cfTO1w;fkU8etU(A4+*X+ZcVGf|V#GhaB_YwBM#om5#p ziVejz51rs4Fvfc&d=GxiNp=)Y2e#;7!`~#^Yo=dh+Zvp`v6pG@W*wiKocnNCtjv*5 z*>UFCvEmwG`=nVZqIbLlm;Vv>0$au(!*TJnMi2c6IY3<+J> zKpAf2DFyGlV@6DpjP-*k9XL7HZ117QRNt*q0yU&eXkAkqKy(I*jweuFq(M=}V@vX5 zD58S=ZnyJk+ozk!>T&LN{R~2Yq37JsGVMV!l02Fzjq z?H^s$p6fUb+elq=ecMgqi=J_d4}F%y&{tuh!b@Wu_dc-k;Oxv1QV5oP7yFjtGokw` z$!P~WRX${oQ&(hpqT&CwH*rhlX@2=cmMNpPaj`be8~AVK+hh4K>E#>>btLAH?-1dT z2i>klA09laZHQ_1pALJ*3e0KkM>(SLwqy!&=c6?C|lB|SVF2AC{h}WOB zIgq}{Gy51$f@hA`68j=^_R(3vN@%+z9r`)D{v1RQ_$N$Dzg=t^@H3N3l*nCET-tJt zOGW!f=NeWKIif_AIkK14CNg(uKvtYVStKQ-3J6xXZ`oFg?a5;&h&qxZ^Qw`$WzZ*t zL$Y6jpfc_i=2SoW>&=%zACU?}K?ljZB47?|`_R!{^b^|Uc|nLCsjPW!yX4=<+ZOyk zWP>fAKX=a&D7%t%o^zjw5Ss^sbv@EIzW)JcDW!9qz_b9IMn!VTPVlKN>?lQ}Wb(=2 z#oc-T2rT9^{BIFhDtqeg`?ZQUYIeq%VO9_~$YxrxOJ?l*K#v zC$eqBnl7oQ*T_Gwv1$hhnItIl2ZTEoJEpD~%2TPtG*q^t@NRfm`t^E}FiXjz4cXA1 z6#XhA0SIlf31r#wA4={Iu-8hGterb1-QedRqP$pR z1~__gG;gSQCSaXp@a9l*(EABLZ=wy}UOD8;A7MYc8$^>4B44Bs zu#4*>DysPxC|^?2Xdfbi-RXY(qVRs_=gGOG24w*ywi8-&nfPE#c7TD|tZF}EO)w;n z_(kIh?M|q&C@L*6*nJrB6rd@Z$jA5Kijk!EW=Pdv77`~le6~%2a0A}nl>xl_P*KW% z2x>q-^w))iX_bPu$&~+k`QE$kNLRn3V3(en4n=XjIlh1Ge&>EwyKhKAkp3Woygmt$ z#O1i}GtVwu9bf2^K!pVBj!S30<7gqz_eqd6J+P)E5x*&7lj(k7MgVZd7n6$c{lM($ zWTv8#8sdM{DnD=z>E=FScL9lFz<1qvshD>>twOC2BnI{R510~Y6pM~`yxgc=Fdlva z1{kk$Q0@)Mu5zJIjGN)11V2n2tiC{2Nh-}eJ)cyJdVEq~$y9wm!;bp%oqrTb0Adkb zz>Z;gFIqyv{YeO|pF-dbCM!3q6^X9GuRyc-Svb#b@`23Brv51~c@h3^oRgzeZ<><> zR-b~?0-^b!iZRIb$wOXmSh=jAz{$U&;*>V01v`+FbE2b=lS`s9SVthz5(3YMBp~{y z`~|?iQ8En*eah$ySXK{k_kae({*NFhfTQ$SKoNR{A)#@9OGEm%rRPO=t}ahVcQBb2 zlCIlV?D9i6 zQEgvR!4Bq|un{>Z!)|_iNZW3Cd&r^X?5Y^)E4aIZFb~}QOs{i#wMSSx2j|AjvIyt) zHMkUQ$MzX05_)!H`G)S&Zw)5jh%Nv{{bdFb8Wr`0<+zo-qb}_+ynWqZ0ttAXzyV2g z&*0pVf~!e~>i}Dkj`v|5Uzbl^_m4YNLWq2JY{BatSj%TuJ1iT8qzA{GZ)CyWi5|k9 zsh2=-?r1J^+oQdJt*BP;`bucdLpuL2GTyXuxW$`22bYJI<*=f)j zZ0L*Le&)JMzERW{5#=8BP7@sVxgX{R{&H@6*k+@yFG3b32)S=zd)U=I>7Awxo^Z7H z^DmbJL12h`i?HA?K`WJ{{W=>Br2Rav3L#MIH=*yaBzRl^(=%5yIFbd^F%|KGeo63| zFDBYq!~XU)(OZ(Q!ruy@{s$%art)3(6*cLkVV6KWBx(D!VOKx=Vfh*^{6Y7I*#BPl zMhN^VkZq?87FmVeK;;mafJWmG7}h=aH=J{6sZTPGdgpjFulN)sFh~Ah{w?9EtFani zh3%we=r_vzrKf%G`?G)7u)!5qLlENI!&eVMzDP5tth=g!UkRch)?KpvdDB5bUz6=P zNni2pIEeGPr#<8IrKdgN^I6m3e*luvIB*;vAnP9Q=6fNjMZqsPQuDq~*|!ct6T!IG z?5VyQwe?lrP7r@p-%b!k0`r?J`|Q(g{q@;LBK{OsU(M}=zCGq3yu@8qFiRjkNY~eJ zJECv*>l;j9f+QH}S>|t_rC12W_MY*ZA2HrN^wx60m^|@PK~y3o*z;rX{$n@$y{i9S z6{b9dg6$3K`$+-3(ub58cnor06gT@F*ngk&X)&?AS$;nuMN<0EGXu+hTA11IDW4W} zc?ccbo8+g3sPv&`28R2z(6isa_ur#^T4Zc*(f#+7kn>6KN}`{mH}>BTLC&KgDT#iL z7xew)3P~x%)C~L&a(){t`#rz^{t3FgkCyGt_|rmE3ehzKcSFwqe}ugSaOFVHAUL7P zgqfL{ITL1PW@cv2gqfL{nVA`0GGXQyrU`R$`EG0X{@wcPw%#jSa(7GWwk=y~w*(1Z zittN@05P5Kn_PVVz3?{$C>egz{}*xpZa~8Lk@FRm?ze(i+V`?Ecn|hn;hW!g?LYnh z1%V4Vwf`mKFKqCN^Ud#D$j<)z6^PP4dd3gefKz|e^v}%i1K;$>89&$pPV2q}P&Ih1 z0$tEYOaF}QzgPb*`tAc-z$wGG*qR2fQ{P2M>7R}L_i5i`zI$XEaQX&OdX16Z``szO zZv?#b8YR6~-r$w*yAhO(zofyd^mn80u}kc~?*UzSf|l8#Vf+vcIQ9Onij>|fZt#l# zUy-QqI=^IgNEm(N`tNx`7u;cFcIX&=+xqXfz7Z&y9WqAWwElbDZ=%lx!KY_^j34^x zpZq@;u0OvAg5#5a@Sg5J=a_&o=XOVlx%f7$m#B!%CujYy@C3>|6(-R_nYUp)@qI-N zEE6+=4T%Ih7M%P-*|WW40&e<*C=j$=^;mrXWq@kNcwmAeupBz~7k3<_Ja|+T4{tp2 zEsk#up_7PTQCUjAZaI3T=m?R1H3R!@)I)*_)EL^bzYfc;Hl-5(_i}m4{^(Fs2@!_6 zB=owDtWBIPFgqwafGoqvtia`xqX}0C2 z)Ui~!smWXOaq$TeeI18S$#qs{CNcP8@KK4`ov+IWJdy=VLV<-yhBEt|+^S-4!Fb4e zz<0MO_%Uj}DlS@8NIR5gX`HPqj#J2-_a)?IrW1m0!CyEU6_sbwjxuKCGmm@6o0;88 z&he*u9=nVQT6nB>OjlFx1K$9}99v+M9P=EKP*9|@QtTeci6}`)^P7!a6Z@^jDP(cd zy-b!l#9c{LvQm<*jmP8a2B5l4LOxpU9dGC;D18MjNq>*v>T@6@#b|eM}!;CZLc?7us5homxfGFVF|< zC(c~+GlUC-uI$QO*(sk_xSyOX>snXR<}?G;h=B|rE^hX|jm=l3<88yJ;4xukU=RiN zMvyASx_9(DpnGg3V`AaR*DP}MG2V-FIYotNbXrwWFh?<=MxDG)w$uW4EokL8oI0Pi ze`|YqvpZ{H0>3`Q(p@{?NJhR)&UrYwv1Q3Je*#Ha%AN*ISi7W?8rql!Ix12)_+>KS zwvpC9RPhmkKu>GXdxBDVO`X~2`B$swzzxW}O!6DOW-}2@eD`%)7CgGF&%dlW6T|b( z2C89)+l?Ps!dI_=C4^e5otsVT^ECR8rEgDd= zmf;b?3PcUs(otu6L+UH6WnKml7(G2VnK}8G0+;+jCnknf!Cv>MulG;lD0Vsrq6-@S z1p(qy<)7r}7|1!evRv@yVp`6nNHG4@hQ3H7akUK&3nz-CQLxQwYH3A?sU?!q!g2GV z3BO<}5x&`LuUST^3zbU`X2UOiWJd=~Z|Z>ZyxlE&EfB+_EigR zUfHVZ5P9Q179+3{s9+aU6GoljgnPSypA)rt$4HQl3NS&!lVY)g zIZ~xc&NimO(M#Xg4;N2}XEDST!~~2Ix<~ZStby3gPbte1(-PZ~4<&JJudKG3`kE85 zg<0B_joLQ-^L9O74X^ga)5etbNt;8<5=&Fd!;s?mBWbD5eZa&71D>qd0(Avt2W8K= zHt=`dMfdGZ>rF=!t+S1Dje(qA$AOj3LAp}6c?<{-Nfs6TvMY4z7Cu1)K*l5K_FG$X|PJ_+RyQ&;*+ z(*fXKv9QEE8EGFttckS0k{!|@KQ=GiAQG{K>`j?EB6ji+=S?81o`bs*u777oNhpuu zqnD3S4{EE^p;lkFdj}c{T9RL$-zPxrkr*usfhD~uT)J7_>MnzS!8y8Dejp+)TEHD% zIZ!!}?;Ib5xCiv|4zTK*f2qcqnT>@*KHhB*M6x2Cc^ohn9?jJx9Wyo8yqzu?cQi>! zoNljHBF4y&MoO}$3mM5QgLghkMGoVo;Ml!Q8&t8NB^zAN8K8{7{#a5^5_4f)X0hlD zk!nWIE(T<1ZAEYL@E7^)Ze!cV%^_4(ZBV3*=8Xw&_yj$hWh5V_AR{kX2=*xA7pC|6q$%iR4O0IInzT4p8ey-#}oxbDw@wAWAgk7ZsBQedsr{r|q zlT7+{-oNj*RI{yDEBAs9sKJdLKlnbOo=ATrcgVvWHzS>u=7-b~u9xhl*je#c{V?mS zYs#VQ*l;KaKO?dEUZig9bc)fN7d}tl?QO3_E^Gp$;6p1rqx^5;g(^K{cX}vYJYvC` zR^)_DbW~9jKrJjtIV)y{-JF!nYsn>c-7OdFbmku76lbC2K8F=2;?+EJO&wsVO&9oHRQ< zS&e&h*+A3N5g?E;Po#P2SaUaFp&;`YaxKONM#=$S8)ZvB65~CEI)fj@FHvtBe zJSt=cKD-yEUkiIK&I&y(YKC4GNL+&SkyFz>Bt`PdL>KVnToF%K6Cc8GjPb8mZJM6D zDmOrmKpO z)i%bMcGiikp85%EO=~?FV_7>aWv;58Vj`Dgl5SG1e8xrnm!uL+0-@9A67PCz7*ZgG(}6m6(pqe1n|+$3WulZl-)$&>+R z&atJ53^ex?3Fq*5*eJN>D&TDVUfhLZJBjw|wSp}?Ta6fsnHhMj9Ee4UxwNu@NK;%I zJo&6m(}cnn4r-w9eW*e#=dSfpsoX*2FarKdShQdtNiUyED=<+o?kEuww9(R`)^UWd zqf0S)7RRDiVwzVejlX4%qFKt2y1<%IQL=3IH5x!^{h=&&jR-{;RR@ zSNo7ShG@RLu*l4BXxRv$r-(QQ>6J6MVONJ)uu790{;61(;tiHLqO_+*izXRaLY$)Q z{k$iHdljav@lf5XYAg)r7BX=Ws-b{ zTo1IBj-;8$@l>`xj?Q+u9BG)~@2M|se;jjWw%18aNKn^QP)g-D_lfM^@q9X9XF&RXQl zfF=x@hC^ir86*}}jo$v?R;}tc6f)bG=&HAtO}AO=+}aJ)Kk#XE{qRYn!&>fgu{1lT z1%%3vnPcdso_9u?X_+Y}iU37t9Uo8+{9KMLgK_A7+m^NHrmf@+{<@hJZO={YxvzIl zbIlsf01Xf-NY$3Sut)fmwloZN9pD_jvsr>ZP8&|!M4ktkuv$!wV|!vtq%QSGv@Qeh zIC~b*k-36(1ffZFtYmDZGpRbBC&J-%{4^8Q8}1ygdE&3>s*ypcd`-lk-wRf*>!oY! zHs`iVX|_FdHpcZ_wvjdC=&cVMj~gc1BG>*r8g|yuJjx8E#Cd)L6Ei42XE_!Z;a8U2 zl}1U_!&n+JwHGEKnn2o+5*kvn(VR$=wvkLLDfMvmTPo7;I2EFqb43E(Q^;F||1P1{ zUuZ2!@e1P07(f;=n8<8xOB&Zu5+J>`0c95{b<$<<>Vuz?!;)Sa-_{H}Qzr<9o3xnj zQ&dacyA|NbGV6#F)!nK#XZSEpPsj1I=ZT)0VYXEN@_GNznE$%6+GOJjD{ue4EoYqX z#!d1RMn3EdfPxU%@M_4OU>gu;=0J1KCJ9`m7FCW`v*QqJXv7(jGM0hmoOVj81Bm;+ ze>mDX_cn)rt}nNHaQ~(&yxyK!scAL}9TZgcRkN-3a#O%u*f#G#>v+el{_T7>wR=pG zZIWf3U*XtPoDrHas_7I|Mx?4|)xP*QgnmbDmLU0K@<%M@G|4z8^BC`M*MPo>tz@r? zE|QiFSLe;w)rTv{iMmsCe}Wf0Pdz`r7vqJ!MfqgoVcTL9U1weEZtGefItCi{^K9lD~n3vBxN2 zW?XcI1Sc_dySeQ1stIb-w7nQTYRpTWGg37na;SpBDC_n3hWExl%x3aEidhabQMu_t zY$S-Nkc^ge#ACq4%9vi&(g!q>6q|y!k+)Rbo8|FsE-d)xwmsT)({n4;;NV-MAolC6 zEwD$YWvnI7WsKWOko=D7DCx1eoJg@G*Ixp9!SE_44Kd~1LnsOfw(!*r9k^+J?Z|O) zeLg;M7S&5FDug$C7He@9=P*Ua;0h1r@N!v22`Pw0nSqnsG72 zMd%`q4GsobE&8aoeC5AZH|eK4IVN~PBbvAG?}LmCV$No+r;0Tu0bDCMJj2;lbDDcL^s8Z4uS?Gc`s4wRDO7?rhxq&7#;0V4XQ1U ztPH}9bI(_}zQ*CGt&W#3XR;F1RNh0IM+>#9sB(}t^e?DJb3;=?B|joBZ-Q%*ZCdZ-J{A=mGWZGiyL9Y>M7C~VUnF5wxC8KfCvW&|*rFAQ& z0<}x@C3}z$L?=pzJ@|~9+SkU$ycbXi$w8}qy#6h7M>kzYWSh2mFU)T9lzTtjeFz^k z#l%=elv^7cljNamHChLBF?B%6-qf)OHzsN&6c?+JT2-Lw=~bm?MwpT_=zDYzqBc!| zdz;w_W{joV*oAiyIttG7JiSmc8=4i_uxr9Wp+s}cXGO1Qcjc^<@CtBrwT<+M?YMLw z@`!y+?(4`pB~{2?CEBxo$@vX9s<2CnwCYM{WXw_&@#V|n+Tlx&Wt4fD&uJcoGzV>C zx*OkaIrg!Cag!UZo57Fq$nzJ2EgLi4lzuP=bnBO1ZXWx#>1|VnnHKq!pvDyrgX=AE zs)tw$4xR_b&_>h>nttqE;4*`GaNFPF_(hJ2F6{Jb(Jq7YN{VV@=h^72}L0>nO1AL>5AgnC3KOSk2Ix0Cu$!4&>^jG!1C%WUWF1qnZ@ah-1*2 zt;6Jm!~|>0nehe@4zv{OwD}SB@jnbSUP$-o zn$0J#o-2pmC~i9~rwa_NU;7;>D@kE+Gr-h6mb!u>J)Mrm!?7EVUg`0yJY>&Vk`%ZM zl9s9$ycv~0x_0eQ)HQryXrM1fywfYsjj0S*$UM}J>oLP7vqa?h6!hLAUn0&o5w`A7 zGexbLWDU z+q9F=JwQrX_Oa%)RGVdd*@bIRCUYgPm+0fvB@;gjOM>`}mBJDI@TlD=hMa5klqmOC z##|eH9#H&5y@ymi2C>X+YKDkjy!!6j_B3%Kxwf{VM)5N>U978&e9agcTtQa$H9fx! zmXCpWWcTj#?c1`DpV^p*mcj2NjlsSH4&>jZa{?gaAaV#;@vjiw)dqGWVhxreRwGA+ zK9{rrrkOO1w0ul7@$qO9({dyA&(kW*&6PJLuy~g--|tTpR4l~)r5V}_3&u%*G|WXl zUA(R3s^<)tqonDTW$qKJROl-9b7-%q`ueu9XX@G}rKhIkwr=lINS_|rZLIaT6*NGg~cve#q(lRo1yW(eX2)b`7R9jw1 z7N&eL#zxQ>e0`@q6u?2KaqL1Nuv|u-mb<_*y!ui{W#>&%TCMIPax!KCaElnnW=G+l z=8lQekV8f-$e0f#+l`3G5?~3TOp8D66^K+pO&Gg~HX9A1o%$StOOs zw2xk=V=}~9;G~Y4WhxyLCjf25#WK$%G(~=Xii=N>!F6b)Wyu}R^9ST+t4{N}C?CCs z6H8K^b<3GHG1~T|qtJvy)MFh+%UKZ}fsQ^kRL8kT<<#aFnl-H^+NIjcG@JW&b{?i{ zUK|-mPeQ_nqP3Nkojv^Tg4G( zejBi&@z;W2IoarRGd{Ny1=UGm0bBW8`1YQ-9E~8;c4;uw`0r!x>wip4;IUD2aB!Yg z0{737*w`TOo3e8gGi@nuLbE)D7&+V*C3m1qKoTf*?IxD|;l% zJQFL;bWHA@j9ZnBN#q7N0w@-%W+ARGiT`#t-!2VOa$il3_NDetb6*W z>Pg+Pw~JiVc)*DB#>dR1o(`@*zWTo6R8UGp;GV2>ZfnJ6ja6z1a*&pBP+G&Ic9aUt zwfeiMhmk?}awBdgFlE3CnSLPFbXIFo{s}pfx$iFgmD)C>qUZdh*@9@jz@By%iYNCH zXWh%bwhvARtg%3;3(mM>a9D(hpJ~qP?!q3wvbeeYS%F$d$&Tm}bvuVa#^x;V3=*ZTDYQCWwSa3JuEM1`rzLPC5i?K|*!FSmP(y!cy8$g|Lxi3Odoy+c%p6 ziGu=UR+E`{?S%A6vP=wM;@7?A0^U_ggm!gyKZiNj}9I5<&`>pjCyF>Cl!nAM>P7!^Do(*x&s_Mu@E+=`Uo{Zgo@MyM_MGH_Vxm~AF`_xY&*XY%7R^5ZH*>Ep zzO8)v_A9(3s4fm&s`e{sXWJ#OhujS+VNDNeA4WORERiX@ze%sdhf-4*9}!s8i0k?B z+K^A-u^gmMBQx@FWmVip1v<5c@>V6vJo$E&(Sqz)93ldESY~ON(=a1X5izpMRikiT zirH)dT6gG6jqA&aqM*|CzhEdl1Wl(i@!qzfbTUL`x&s!vimsOa7_E(Z84EREZkx`< z84MxKOPdlGV+Y~euND>jLZ8?v=j{8D1tU@9!j?e|c>7f|OPCAhF3L_A#gpkI2)-(OO5ha4u3 zkiM7w)yCc<-Qz3uXGab(P>Nq21%c*9?T$`eSfpi>Z>b=gl-51Qjj-k1Bo=qjmIA4C zD%|X2Y)O9GU!1?k!1qXzzVK#~ynKs+gx3{t)xG~-4ipZ&+-RHK^gX%$Bc=ud5oq!t zyf<{xzu{`=Tly3b9|1mEB10%`Vx*}*nuYd~55KRlP{543D44$NLiq-PCnhNvMx?eR zI9znbi{*1pP-~4(hl)<9M2dTZ={WtCd|lD6JdZNj@UH=LA=|Q(FR>to3`ODj<@n(|6-pRVc3Lz|B1NNg=sAhEYbH-u8>P$IKT`stT6p5Dd)-@ zKI*qn04z;HAK8ikrpW{wg+ss3G>>7TSuDIWDI9jy;piK{)$m~d%h(}V@LqY*k4Hu;qMnOZ3+mzAa$%983Cfw`UxO!8IZQk>Nmd!1LwpvI zq(GSJjv<3f_g`(PBSZXfG3Ja#LuvbS@V^kgl6~d+%e=4E4?Q(fz1qAyUZw-G=D`m| z=dy~fbQuPOHC&xP6uA4UgGtgK2YYc>41s-K$?YywKVKyid01-}6}~B# z1wmYUW#r=>S|-yADj6EjKS_k_I|2DpSfc5LgaarKQV0$2!o+%$r_?s{ZOHL zQ)_5s0x^*!Sy)RH*D^e$c{^mI`S+sPxVXndpJf}{-sMWq7cI!M8IE_rP1}b(q*`>! zbYJNX+cu3}No{EzHUYY3g`y3*zdvwopUNYWQ?ylxt_IC^L=Tfo^_HZsC=PeOn@Rr$ zu-7&(6Ri@cBoiTCZE zcXI{mW11iN465Gr$WTShRti)_45=RCfeM|;ubyY>RLo06LGZt4hcx|i?e9o;0uJPc z7l}(q!7}y>q09lEO{R&RTvo_OI|6tr1)pasU!X#@Lbx_IgiGyNfdH&7pQkhgI(H(Q z8A9EcNdOdxHXKDT4Vc?d7XlX_Ne=tF&EISDx1%Af72j2!SFW$USHMH=@6KX`moWju zpYo&)*(r+!;DkYcRyKeh?m*5KKW$szbt^l%QcnP)<@|2c08!)?u`$JwFHWtu_dmA$u75}*F| zKu}^d6dW``jfGXVN%Sw*=K`CgV`u62SK-;-f1@+3{Y|hLOt>hdFEF;_XwD}>jFN{{oufzuy+pJp+EEM!bHD0 zcl$kKi?HVU!NsCDCXYxat1N}ay+7s6eZO?)?5QQ?a#A(rjO?A2G2fzSRD-)gx#Z`` z#TzY?>^l@>2(LvM;YRKKKGk*o?LFxGcnmSt&y)^P@ux=fKBVb_=WBi$7$46l8@sNF z2pmp0X5aBSt$Gu0J+7Y5CGmf~I1QE$@|Np;hVWhqy8L|+1mjiSJ^v!9AEh~49 zr%5@Gb|i);ET)V@%1$+u4i!KLlD!oAXnwk{(aete{S#vF&Rcbmb&>VC3wA*9JY}!V zm(tLus0La{OhKR>M)^4^PUA;0{y z&V8{M^NivlfT|+^)oye6w%sJ(Dn7EX-C=a>?T=d50mtGTLMV z&hW!aUx)Sy4+*ScQ*fP*C^0&2Y0aU3ynHWu1uroKcKMG_?ayFUBHf|L_|PF9Kv)xY zgkdB1X!(^Z9I_@gloNd%jdb{9S_G*b2n}k3j7~payKx|x!g(5c(WmYz0wl4#DTf@<5w|jR&#f~#H$uqEs}(UeIhL~V`XLB@y|NR7^?!l9m$g^? zcY*OZ{!3uI|G!*c%v@~$TO2%H#VM;HM#S!On#Xk9K%jWy&uAh_w&%Z)u6H1@&2A7a zv?hnUeGGaXRQH5*ESYP%mrk1cR@R{Z-1#eig1L-Z#VvR|LGOk3G|);4;-IOCv|($F zwO2C$cpz}CHe#wNe>_7znnC-80yN_|Hkm(<*u0FrS9MIAOuIXNKgEEp{|HPWv$()a zW2o`Ag#?#t)vcNPiE^~1%I+UIyGB$ToU|H)%_di?dZN8SLKBQw7+hFcS!rVN(D%gJ zX_xbA#aq#Y3{9k@NKaY!3y;)OeQ9syjxqsADh-d?7~H)U8H@U;DTA5vLUj}~sUvoa zJb{w%M*2E0!6)uSFhK!5=3-03-uP~}HfZc}cE!6cP*vzi7sQNCu;Egqtuyrk(j1xe z{ND}u|24Uqil>7qA%nb;m9mR1G=m%=8{>as6goM(5VEpx{;%9?ES#MG`w6+!wRPTL zNB+Fh53o;@sY4g|0qzMf8BMq-P2)_8*gBooy$-T!;?>2YA(>dY_xqsx^(UAz&|b-A zbej~`5?Y7W4dabRBaxXTS%g%M^Eh}&k6JQ#yNoKKK91yB|9T;ua+L-v3H4Eu`d5)d z{B1C!*nYB;7-qN>md>Dn8n&Vo=C2@fCPP9Bv<70>;Swd0;DBz{&@W9;Na{d#LN3{I zVx%QVNfBZ?t{8}jd~Yc<3ZyWwQhsCy#1RLvK@dnbfxtr0&RA4jiFQ({UBXfa@+C-W zqU^zdL`IFDkc>QoEkKoitoUgG);%GDVuzokP2oaW8U&=@5DR?)oP?PW4jd^9FD^); zChQqhDZ+ekzvKw&32n0pFKz)@S$@-x2_y(2P*QPN7j#0dKz@-w!0@0M;w8xZCh&=b zA30|T~i)<+sYLGGH{E`<2v0>d{>0Fmn6^l=FZva_H;q^QTcHyYs8GWgFDpl z>-_?7K6TgTPsS0o!*c8E_V9*`-8*x%!m;MX`|)J;!jx4nfv5bDrZerIQ-}U&q;T+I z<>~95QZpXaw9v%jKv=?xUO$p7#L^sH!KXPq($iPZvyujOUC#`eq+4&&p6JtACSsp90kgvdUQ z)EDTNLH_a7`78O?9|VSe-nFGsEF#blE=uZlMHMo!fNtZOKh3rRP;vmmJ3$AQlYOtf zmLNK_WrIcRA-MDm5Sk70>!c0#vQmE4gqEeg9=*1O%Pbc}ma6jzj@*iby>%!|nzT}X zx}9rjIax%BRZbS|IdxHny=+&jK!J_9FP*RHho2u4EKSFQj|plfo~_P&@pjW%$Lgkk zRmTck4<=*7vi#0Qi%n1dBoD4ip6ieAqh)n9^sL7~?%nS{-Zk~7x91Qla#8tcc6?5^ zK7A-|y&3PZ;~U}FC(Jda$x{2NG{g*y51aB`u^g-@WFKbL-|F(y{*CM1j@fu!zxF>~ z1pq={nhvWLZhZ`QuyaaM>P(%_?G24L$=E2QDM)4p*{3$xr-d&mKpKG&FFbJ_ncTc?3%J6vzE=H3d7RnYh|p zXQLw{#ks(Hw$WC=b=--FtQ+sh?8;hFo9j8LN!Q4cnAscPojmfM-Ntg~+#4?g_j7VQd+HdG z&9(*MzLVsb$5A>qd@4K3*h+ZYc9Lb%DYkV}*C74De;|a~t)ez98BjA4xMn1l5cgu& zS2;M~<4G?+#xg4(&>I!fc1}Sz*5#_J6%jSw6fcahces%ieWqvr1@5rs^bjm)O!15*Hz=#g_(cq;(Zdv#+p!Q{;#uru7<0dB9-JPA2Y0rV-RLI<})f|Jzdxq$|Va3)Ptc@k8b9{*F#yei*iV(l%~~5ku{G+uK^EwQ#$UOw`-CsCG?L z7lCdP%4QRnt1Fzo9UmV~lIqL+z7tZFPZz>oE{^iXaZ5}jrg^hZXqj-HcI!SKmt)j4 z-fw^ue)k=ZC|T^CtEG`oPl|aFh;gLs_C^Cs*$7y4!;<-V2BdsiYx-(tU%~UQ6Qt<1 zT0aLaJ#6jE=Ym;Zk|)}rgY1;!ejS~zh7RmR2II0;lNDaMR*Q3EwWD~Ss#xAa$T}K$ zU2Rq0t*S?iRqm0W%h#`aO9f}{OP{X}ox(ePQkkZPTXq`LgcwC{&JCrmvmhPmx6YrI zC3(U(>806vH=Yfnk(1`Lcb(BIwXfZtE3<96EOmRfu0B<}MexdHDY$5nn%A9p)jmJ2 zxJA;a>7|@^u7HBX&fKLb6 zbYQ(tRf8MdrW1#IG0*|$#MsFXDLwc(>Ta~r@Jx6;JW9^jv5|~X#1{(ZOTk_;;@uG2 zb<<8vuoL0S^E~!@wF&NP-3hE0uWT*KP^vM+bRBG-hj|< z>)~!`V`&(Mj+J%c`HKy~>?%ooCwibxBgwJVl>T@}ytArn;(vBT$3ioQFC=MB#ESdSdr=ayspL7R1Z&CgrLv2SQMm&U7q-`EP1Gj?DqXAI5Vcl4gShgPEe zr<>Q#fA=q-`|tMTDjL8T5pEHHU7z;AywelSuCP5Iz0()ixnVE-rp)O(NK`lRt++<7 zoY|?ps~l9U=M%oJ;R|}d5m@C;y_S*Gt{q#S2)bhe#<=iruRhY^ZPyLHsj_i%UAXk2 z6d+IxdaKEYq3rQiQH*w6Jbwq@39DQqx)$1IQ`=rbJ@A-4^SjbKHGb0}{@h;wLp$7e z^*q+&r{zb0)ms*<5w2lj!oH7bM8ww@pz++?@Gri_V_0Z-kja<`;SJ~f2KD?#BW}#A z`#{RgMk7uf9vx`#*^M>W;~{$N+m5k(kl-GZ;vVvha!U`?9q)>F{}1ECjRWaBdLMXD z&y|7$@91vn(2tlWKW>8?VSjlVw#%(+{pr58mcR$^EsjF7t9IPfYvFFx8++Vm3X8X< zL)quDyGW}J{Ogx!P;Er8(@#tf`RgmA%YrYD`bP&syO0*hYF@&p*W1mhSbNsZszXip z_u6Gc$Dr=0tJ$u^`o4z8zp*3Fn^gNdC;#ks)`dlGJe7z0%Yp)AuQ3fTL97upy!+RK z4kW%iXDr5SM7w)zq4ij=Qi-Z$E~J1-u=kC;rGVQT0OE;*7JFL=D6Qhl#06k z=hPi%3}^KIe9X9yxbu4+h|!I}3LPy-`+aR2YSRzB3FN%(+#av!8C$j^0A1E9hzveJ z&JhAS!k_+JVR_>R`hSOxNPosyCW`ltN%8)k`CBIHjpYU}d4{Lm-=&9dH307gi~82! z7VCKL-h}m^|4z;ko^h#1u!A=N>Z-9}Y~d8+J&(RWL3aW~w_Q)3@ky=+-32wOH9|)6|u|`d|iAvuMg7Pba`gAJ}okY)jqey zO1!&0Ih-oOV2CA8^Y@z^2Y$S_U0B=_X)|1_sjp?8$ZyxBs$;B$zN7Hv<|Is*K6(n1 z^ZNvs)?N5nJ@Lo;vN4ohjlrY?QeUzwrhs( zxT0Kb>Qm<8;S zlYca=AvXp47y~(?tbILA>b9+{yPY~F8xf|+z5^~Wn!THD7V&#R4m^;*DMwMuCkAJ_ z@lQ|3&D9c%J)g=xBEbriHyti*jmc=AUgC10d8i1&pDmMACS8oD&A(%(PslF8;DeI{6@L zLTLo6Y_TPB7I}kZp$rNcSSE!p>-Pr&AOVWvRnYP(xQI(esuWUQg(?Cnb%vuQ{kM^K z6|s9e;Sbv{s9yPmE^RDwVe70lE4fj2p%PMxNO0j+Ou;BAUWGE?q5+n}3;0Db7U~O) zS_{yRqTEquzpbGT^9e{4?WyPJ@=5en3^FIsg*g?sS-)3p%82>ePQ`%?V^s`321I)>{5t%nJOOwNmc0g$hc0C2fe&hS~*i14G$e%>9PK<*-ZM-t&*P1Lm?$e`5hH71Hx<6CD9SuH%~N_gzL~S37#BXaIl+V%>va4lGu4celhA&gjslR>?hc3w5yK)| zF#tj-Wa)%H#v1QTS~Au_Q2`noP&`VUmdMIY#_&(Ya$Gu5Zm(Rz%Mhwff&koe7Xlaz zZVIh%Edg~f?yaOm6!gb9@-8_UJ=P(LnYx6~5ghZDRmihAoa6*TTI^I{Ac5(J_Z@l0 zCJ%#irjEVZ&J{-=|B?IzFq3>?zt6-;Qms%xeF6x!<&^V}e~T=si3$&ZfzYuVw`iE$ z?VHsNU4pG(hM@B5Tpl&j9c=^h0Po;F_Ch6;#!{K<<`+pJ0kZgPDTe=#bQ2v*_W&hdy*}G5TDXjLh)m_aB{m)>n=)kr1ea8C&KVM@124Z!XizFs)8vw zTJk!WDMJ1!aJR5>yRcIo+uC38$%d7`x`b{AvW#O=N-W2DA*-@FE=@40)zJgYkY%s+}C8%~i zQ5Y7R;p)J2ALIqDW>TY|#azcY6+K0`6EO|>eBCOEK##> zP)h8?DG8a4Mb{2_p=7%;hSI&|)G29m>AT2DJu4i?y(OmIQGMh81ise8ah~eFi~c9T zWrpM!WLWF#x$(A)-gT*X-mt=+nG$v3F}j}FdZ~ZOyun_RTVz|Oid7M7Te+pV0kR}k zb*XYTu-S@}l6Jxwww~WCD{#rSZuFgTrFCD;Y5hCCXwe3@#+0pjYoCNT85BVzWbU0{ z>rhE~wprGNBU2E*8Oylf;l)hTQlI%O*(_q)Rivc{i>TPE*Z`YZcLfN^ioS31DlC3!aPtM`iX>utn`1+el| zWz?Zrop{Mrg)J-Bnzv&IEwL&3`C)aF3}C36wXZ*7%BqV~L3htFo8rGSdYt{aRXVCvjLnf-5IpiR;S<^{nlakt9z{_8 z(%@=XO{X^3fNpmId&K?4X*@;*_s$Y8H@0j<*+hI7a~(CNqe$5YDQc4hZ~j?frhVbQ z#(gZ!U*z|3IaTU&Y_6;~RZqRwlr=(NzHnzFnbZWSlE2DhKeRGy{m=h2sLDsJnxo2` zo7+>vXF_}eShpnHOa1(?_!RTZG(|Rzsjg(Bzk6JxXy|-0CP*`oB`ClGv2}CNIa2Oq zJWX^Gmjo)^&g|YTYR}a6z=;`G<2&|NkD^fXXW47|mHb8N#d>85MX_vS6clfssT6B- z%x(g&FYwen`qXy)3+&Iztj>R<+_@P4i*i@C^fD#nVuEH+vbT33+JYz*T0=3k4|M z?>yrBUU5Kq-iIFXJ+gR?VNYs From 837c061ccb483b11a7f49e1853c9b096dddc177d Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Fri, 18 Oct 2024 19:49:49 +0200 Subject: [PATCH 34/39] FIX: focus only in the within-session --- moabb/tests/splits.py | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index 22269102e..ce1a3b63f 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -1,12 +1,12 @@ import numpy as np -from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold +from sklearn.model_selection import StratifiedKFold from moabb.datasets.fake import FakeDataset from moabb.evaluations.splitters import WithinSessionSplitter from moabb.paradigms.motor_imagery import FakeImageryParadigm -dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) +dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=1, seed=12) paradigm = FakeImageryParadigm() @@ -20,26 +20,7 @@ def eval_split_within_session(): cv = StratifiedKFold(5, shuffle=True, random_state=42) X_, y_ = X[ix], y[ix] for train, test in cv.split(X_, y_): - yield X_[train], X_[test] - - -# Split done for the Cross Session evaluation -def eval_split_cross_session(): - for subject in dataset.subject_list: - X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) - groups = metadata.session.values - cv = LeaveOneGroupOut() - for train, test in cv.split(X, y, groups): - yield X[train], X[test] - - -# Split done for the Cross Subject evaluation -def eval_split_cross_subject(): - X, y, metadata = paradigm.get_data(dataset=dataset) - groups = metadata.subject.values - cv = LeaveOneGroupOut() - for train, test in cv.split(X, y, groups): - yield X[train], X[test] + yield train, test # TODO: test shuffle and random_state @@ -48,11 +29,9 @@ def test_within_session(): split = WithinSessionSplitter(n_folds=5, random_state=42, shuffle=True) - for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( + for ix, ((idx_train_t, idx_test_t), (train, test)) in enumerate( zip(eval_split_within_session(), split.split(y, metadata)) ): - X_train, X_test = X[train], X[test] - # Check if the output is the same as the input - assert np.array_equal(X_train, X_train_t) - assert np.array_equal(X_test, X_test_t) + assert np.array_equal(train, idx_train_t) + assert np.array_equal(test, idx_test_t) From 39e92e5990d4923858ffc3211b6e5e8a05e79a1c Mon Sep 17 00:00:00 2001 From: brunalopes Date: Sat, 19 Oct 2024 21:51:17 +0200 Subject: [PATCH 35/39] Fix test --- moabb/evaluations/splitters.py | 2 +- moabb/tests/splits.py | 55 ++++++++++++++++------------------ 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index bb0c6e7d5..327cc1695 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -68,8 +68,8 @@ def __init__(self, n_folds:int=5, random_state:int=42, shuffle:bool=True): self.n_folds = n_folds # Setting random state - self.random_state = check_random_state(random_state) if shuffle else None self.shuffle = shuffle + self.random_state = check_random_state(random_state) if self.shuffle else None def get_n_splits(self, metadata): num_sessions_subjects = metadata.groupby(["subject", "session"]).ngroups diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index d2a0fc7a2..e8d4f742e 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -1,58 +1,55 @@ +import pytest + import numpy as np + from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold +from sklearn.utils import check_random_state from moabb.datasets.fake import FakeDataset from moabb.evaluations.splitters import WithinSessionSplitter from moabb.paradigms.motor_imagery import FakeImageryParadigm - dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() # Split done for the Within Session evaluation -def eval_split_within_session(): +def eval_split_within_session(shuffle,random_state): + random_state = check_random_state(random_state) if shuffle else None for subject in dataset.subject_list: X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) sessions = metadata.session for session in np.unique(sessions): ix = sessions == session - cv = StratifiedKFold(5, shuffle=True, random_state=42) - X_, y_ = X[ix], y[ix] - for train, test in cv.split(X_, y_): + cv = StratifiedKFold(n_splits=5, shuffle=shuffle, random_state=random_state) + X_, metadata_, y_ = X[ix], y[ix], metadata[ix] + for train, test in cv.split(y_, metadata_): yield X_[train], X_[test] -# Split done for the Cross Session evaluation -def eval_split_cross_session(): - for subject in dataset.subject_list: - X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) - groups = metadata.session.values - cv = LeaveOneGroupOut() - for train, test in cv.split(X, y, groups): - yield X[train], X[test] - - -# Split done for the Cross Subject evaluation -def eval_split_cross_subject(): +@pytest.mark.parametrize("shuffle", [True, False]) +@pytest.mark.parametrize("random_state", [0, 42]) +def test_within_session(shuffle, random_state): X, y, metadata = paradigm.get_data(dataset=dataset) - groups = metadata.subject.values - cv = LeaveOneGroupOut() - for train, test in cv.split(X, y, groups): - yield X[train], X[test] -# TODO: test shuffle and random_state -def test_within_session(): - X, y, metadata = paradigm.get_data(dataset=dataset) - - split = WithinSessionSplitter(n_folds=5) + split = WithinSessionSplitter(n_folds=5,shuffle=shuffle, random_state=random_state) - for ix, ((X_train_t, X_test_t), (train, test)) in enumerate( - zip(eval_split_within_session(), split.split(y, metadata, random_state=42)) - ): + for (X_train_t, X_test_t), (train, test) in zip(eval_split_within_session( + shuffle=shuffle, random_state=random_state),split.split(y, metadata)): X_train, X_test = X[train], X[test] # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) +def test_is_shuffling(): + X, y, metadata = paradigm.get_data(dataset=dataset) + + split = WithinSessionSplitter(n_folds=5, shuffle=False) + split_shuffle = WithinSessionSplitter(n_folds=5,shuffle=True, random_state=3) + + for (train, test), (train_shuffle, test_shuffle) in zip(split.split(y, metadata), + split_shuffle.split(y, metadata)): + # Check if the output is the same as the input + assert np.array_equal(train, train_shuffle)==False + assert np.array_equal(test, test_shuffle)==False From 590edb144e1ad2f4a483b3715f53ecdd25ace862 Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Wed, 23 Oct 2024 20:15:04 +0200 Subject: [PATCH 36/39] [FIX] I think it is fixed. --- moabb/evaluations/splitters.py | 51 ++++++++++++++++------------------ 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 04d9f530d..88d6649b9 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -1,4 +1,3 @@ -import numpy as np from sklearn.model_selection import BaseCrossValidator, StratifiedKFold from sklearn.utils import check_random_state @@ -64,7 +63,7 @@ def __init__(self, n_folds: int = 5, random_state: int = 42, shuffle: bool = Tru self.n_folds = n_folds # Setting random state - self.random_state = check_random_state(random_state) if shuffle else None + self.random_state = check_random_state(random_state) self.shuffle = shuffle def get_n_splits(self, metadata): @@ -72,31 +71,29 @@ def get_n_splits(self, metadata): return self.n_folds * num_sessions_subjects def split(self, y, metadata, **kwargs): - - assert isinstance(self.n_folds, int) - all_index = metadata.index.values subjects = metadata.subject.values - cv = StratifiedKFold( - n_splits=self.n_folds, shuffle=self.shuffle, random_state=self.random_state - ) - - for subject in np.unique(subjects): - subject_mask = subjects == subject - subject_indices, subject_y, subject_metadata = ( - all_index[subject_mask], - y[subject_mask], - metadata[subject_mask], + sessions = metadata.session.values + + # Get the unique combinations of subject and session + group_keys = metadata.groupby(["subject", "session"]).groups.keys() + group_keys = list(group_keys) + + # Shuffle the order of groups if shuffle is True + if self.shuffle: + self.random_state.shuffle(group_keys) + + for subject, session in group_keys: + # Get the indices for the current group + group_mask = (subjects == subject) & (sessions == session) + group_indices = all_index[group_mask] + group_y = y[group_mask] + + # Use StratifiedKFold with the group-specific random state + cv = StratifiedKFold( + n_splits=self.n_folds, + shuffle=self.shuffle, + random_state=self.random_state, ) - - sessions = subject_metadata.session.values - - for session in np.unique(sessions): - session_mask = sessions == session - session_indices, session_y = ( - subject_indices[session_mask], - subject_y[session_mask], - ) - - for ix_train, ix_test in cv.split(session_indices, session_y): - yield session_indices[ix_train], session_indices[ix_test] + for ix_train, ix_test in cv.split(group_indices, group_y): + yield group_indices[ix_train], group_indices[ix_test] From b151d61f3826a115c48fb70db52a25ce26c04cea Mon Sep 17 00:00:00 2001 From: bruAristimunha Date: Wed, 23 Oct 2024 20:42:25 +0200 Subject: [PATCH 37/39] [FIX] shuffle everything --- moabb/evaluations/splitters.py | 74 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index 88d6649b9..200abf81c 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -21,9 +21,12 @@ class WithinSessionSplitter(BaseCrossValidator): random_state: int, RandomState instance or None, default=None Important when `shuffle` is True. Controls the randomness of splits. Pass an int for reproducible output across multiple function calls. - shuffle : bool, default=True + shuffle_session : bool, default=True Whether to shuffle each class's samples before splitting into batches. Note that the samples within each split will not be shuffled. + shuffle_subjects : bool, default=False + Apply shuffle in mixing subjects and sessions, this parameter allows + sample iterations of the sppliter. Examples ----------- @@ -57,14 +60,17 @@ class WithinSessionSplitter(BaseCrossValidator): Test: index=[4 5], group=[1 1], sessions=['T' 'T'] """ - def __init__(self, n_folds: int = 5, random_state: int = 42, shuffle: bool = True): - # Check type - assert isinstance(n_folds, int) - + def __init__( + self, + n_folds: int = 5, + random_state: int = 42, + shuffle_subjects: bool = False, + shuffle_session: bool = True, + ): self.n_folds = n_folds - # Setting random state + self.shuffle_subjects = shuffle_subjects + self.shuffle_session = shuffle_session self.random_state = check_random_state(random_state) - self.shuffle = shuffle def get_n_splits(self, metadata): num_sessions_subjects = metadata.groupby(["subject", "session"]).ngroups @@ -72,28 +78,32 @@ def get_n_splits(self, metadata): def split(self, y, metadata, **kwargs): all_index = metadata.index.values - subjects = metadata.subject.values - sessions = metadata.session.values - - # Get the unique combinations of subject and session - group_keys = metadata.groupby(["subject", "session"]).groups.keys() - group_keys = list(group_keys) - - # Shuffle the order of groups if shuffle is True - if self.shuffle: - self.random_state.shuffle(group_keys) - - for subject, session in group_keys: - # Get the indices for the current group - group_mask = (subjects == subject) & (sessions == session) - group_indices = all_index[group_mask] - group_y = y[group_mask] - - # Use StratifiedKFold with the group-specific random state - cv = StratifiedKFold( - n_splits=self.n_folds, - shuffle=self.shuffle, - random_state=self.random_state, - ) - for ix_train, ix_test in cv.split(group_indices, group_y): - yield group_indices[ix_train], group_indices[ix_test] + subjects = metadata.subject.unique() + + # Shuffle subjects if required + if self.shuffle_subjects: + self.random_state.shuffle(subjects) + + for subject in subjects: + subject_mask = metadata.subject == subject + subject_indices = all_index[subject_mask] + subject_metadata = metadata[subject_mask] + sessions = subject_metadata.session.unique() + + # Shuffle sessions if required + if self.shuffle_session: + self.random_state.shuffle(sessions) + + for session in sessions: + session_mask = subject_metadata.session == session + indices = subject_indices[session_mask] + group_y = y[indices] + + # Use StratifiedKFold with the group-specific random state + cv = StratifiedKFold( + n_splits=self.n_folds, + shuffle=self.shuffle_session, + random_state=self.random_state, + ) + for ix_train, ix_test in cv.split(indices, group_y): + yield indices[ix_train], indices[ix_test] From 74cf24612f822bb5cbc99eec9729c156a58941af Mon Sep 17 00:00:00 2001 From: brunalopes Date: Fri, 25 Oct 2024 19:30:44 +0200 Subject: [PATCH 38/39] Changing WithinSession image --- images/withinsess.png | Bin 0 -> 18638 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 images/withinsess.png diff --git a/images/withinsess.png b/images/withinsess.png new file mode 100644 index 0000000000000000000000000000000000000000..625f6931982be73f6ad471d151b7210deb3cb034 GIT binary patch literal 18638 zcmeHv2Rzkn|Gz|`WMouiri|>YvdP|CRtLvC&fz$+vyzgMo$Qc|h>}eiS&52_vNMx; z?DfA6N4dvyKhOXFfBo*~zMtn;=jD7|-|Ksg&wPL0*Z1YNhMGJv(J3Mv92{ar1z9Z| z99%TGHX=9#e$o<>;eoFM&RX))IJxg=Cvb4i8lWy4pd8$+Y++Cw7JixCD;7Q;8-z28 zgbR3B=tYzxMb6W|x(;upj`qO72!s=~r21Fm7V_E7L8 z54EsIU>=dNK_VPLiySW>7$5T=G-^YvAV}MNLtxf)4T^NOMZkBL&3A!Ih>I5tj=AG( z0f9kx+j)0e%n?XSC~~(649LgAf0>0x3UtMM_+_vg2EV`|VDcS7Xt2oevq=B3ysj%?bg-?hf0>5$OCEqhlJhe zU_bx&&;F1G@`nff{|qN=fwWL_;#Y<^Ig7YxImtNk^BU}#*EI;t1uODhNjan3_v{#n zaDiJwF&e@+o2!j23aaf0vA}e91@;BnY)}p`>}>?NV~fJrl?bqIX=fA?VGor-z!1A0 zz|SKiC?F^e?tnM|+uwd!m`7L`w4<{#}r>eiU`mmwuPg9@ZWdQ*pso#2}h@cR^ zlptvTGt|e=i{19$UVXgSc<`s`@1I80KSh6n*pU49*B>?y{CDY3KnM#7|Ni>Jg1|pb ze}9SiDZmfV1_p5a{q^_X310wO{?E`KW)~Pk{L}pJFOmNQ_bTYWz5nrHarB?2zrRHM z6yg{7Pw_u&BLCC$_t)@0f&ZBJ`Fr_K5C*6xX)6RELiu>CcDWZ8K0zlJ3?(J4W9tCe zI6fXVs4MuXg>Zntf8GQwKe?Q}Com&%I`8m$7$%Ze*b&empwQd953Z~+BWpvQcX&?F z1vKp}0)7qF>oq@T;KT4Y;K_ZI5CmpxjiE{{0E>koLZvaZjV)mFq_8&}Y%MJ@ zWYh0*OF!8s44;IVdLL`Vzw-~Y?=Hg;@;^xt?Xf?95aNZgR`}Ou82Ep$ME!S}n6r(G zl@-uFM!JF!2MnFK`}2Pdj&DyOQn+c|Fd--rf6CjW(B zu{L~I7Vht7V82&&{F@C7Ea|^JWeV{Aik07sXn#Kg`@OR0-)vxDN&oEzCirXK@^5Qk z|3cXYg^P8IWj}`gTCV$w-k9QdOcz;D=?UBm+{FQglboV>#UeMh0nE#1)7Q8jy$;Au~qMr%YS0{ z)LhJoqOA>+i)Z9n7?Fx&tY6j6-m{Oqugo59FANV`=PZ&Dkxvi!LjLHhRtWQ{Q%%qZ zuTQ*Lk?3FaS=y+7o7g?mx9+{11NG@#K#M-CfX`TOep|!-@t2&PQ`tr zDS+h(?pqre`Hz?9+k!VVG-TNn$u|EjuS%&SOBXuu%1^N9)g*S21KWfeZ68lzetTtw zTXT~*!mcB$@eX$-YfTVI4B_#!ZZOE=WJiJ1u$+j~(2c3Siq8&P{MPRpZW7a2Hiol& zNLQww=q+=7yx3RjqD;kk<;-Ccx=6$4R_;>=`SH#jsPsKFR{Ras@j_}_KES`v=Vp!{ zMmSmo#%0%3PYKwaaUOjeHrbJ_tfCSUdEW2?dCKa-*paPfg|;$w_sPtRG?TJxu|#VJ z4}T!3t7V?bz7^%wsAD4(OK8&FJbrZlPW4Hg;SflNnI&|b9O(s z9!thQ;Ft6AbzbF~5*3#*>+RdO$*HK$79skh-`u59uD^GhML*YAE(8a^f~=-iEgXmN zfiwdZuszmEz`k6M>V8x`}bw15IlT)vD{ZJ zQPhxK=LrL+(Mz2#B&RQ)2@4DR@Z4H`qOZcUE#&eu(+VoP zdtTVujhq8ARk9p*hCjDz(aLQ}NUlOngnO+F1|e5wA%*^UF?huvQIqXpP=f5XvWU_O z-%^{qCwQd=9eU0HhfDu_x9{v%*BMmd>!SYaM&1+XuecLJIC1RB$wh=`vKcY9PwKer zh`hZFEU5V~y;m|G(G8`l>zkHSUHMI8bZk$PN^Bm`N)W8_E^TPJQ2 zt|$)`x9OB#+e}ZP^6IPB2fQ+q?U@l;_x$@#IJZ@%f{oywFtyBQB=eL!L_(J%CUdP# z0=KmLDiT>R(03ax6hC_PqOh7d+CVPUxVL>g^}g9!1_>@J=M>gs&v>ZsYn;VWQJCWHh6$QG4I< z$LmSHa6vZ^7z-KK6I}Z=kMIy|$1%$v`+nHp-*lbP{4g*R6{f3WHH2|CFMn&+b z*;&Wjwm3_In`qq(YK`Kg&F=`Gh?fqZl@>-@h#4E;Ysz5RD&Cw1nLxJLA#`F_9tt{4 z&Ok4kCi?d|)xtoCVR%4bxwLwsa$Yw}NAdLm`n4N3@LV2pXaVuun5tnXGe5Jv+UoN- zndGbtQCs2UJT4LMc8`5t=4@1}*s`&2Zeq*0q$D+sN;7U)e=fsX8EkBm3G3z@r+YEYWWTwSy4*? z(o%Vwr1=3mE-UG^QsO+8B|LX4M5DsP#ifUZT+fqKoi1f%{%d3EJ*ps9SMR2}Lz1*t zkyBl~EHK9gMOoP!!L|}kL$Zsm#l2V4=_*%lLATew<&{xN#&DZa(SC9stj6!I@~L`t z?uFjdE0c8t9r|V!V>tBMT?I++NZY_V1!K9*#Eag`l75Jyu}C=SX=2y$`q1ELV1*is zQly3fCg6{^rPv+7U$*MXn7-|Ax=d!ijg9xO4&auS`4BKM;A3rM?pXnM6Sf^bG}h)Elqf{O43) z+T?>^w$pNc-V6t52pB#Ua~!&uRZ|uA=+UG0s_m6w68F`Sm~@bJ1Yf*i|1mX+QBo?e zS>YMIl>AkY*$TW?k%}rRk?Ing?K9uZir@(s9SMR+2VXJLnxn?vown^RNWaK)(8;9; z*3AS&6WJ=huF@34&HXV$eL35tT=u+Sfh;DzK_C$1;-s$oQnz)7W_Nc1bRio~^#eyd zlLLwS)VWcn1M&nRr!VO_DkYxwuXG-JH`%U1t9$zRab_0f=txB6s$8GjFfqtep_5hL zW-2_FqiIEuE1ct5xy=nVb$Y*x z7imhsr!g8l97^fB(kwAodz{Y$ZL>O8PY-?n=yHMcm>QerqZ<@#TC$U$o^lAmzi3F) z#6dXpb8~tEJXP||>uP(6#BQvHoaD#5KF4|_=4A1lW%L011*^9wP72sXif(T#6kL6c z`&!=*WIm1<`m;YQbPV94ORWz$@p%2fa0+0T_#7huKSh90$fi$m3y8ITa&={Hc;xkA z`pEHdo5H4JP9YtLv;zkZvHE=vz7kO7GTE*OkXI_6n3fy|8YI&2j%oAcF?`jO_Cjw6!5TXeVL zgb+>1QUPC=`+fAyP7+absbv3v(2l;q{0?e-)GPVkvgYoFI^93CEqZf&d{ zIF5F^?K`VhQL@+~)URW9bRb}ci&6A!wJsBkMH;d+*`W-qftgt@L?eYS>{-(ck;Me1 zFiC9cv7|^bqhk%3$KQuLGS51*tM_TJ8Kt~vw>8ja*W-+VAf9_n8x@*pui0i0<7aVc zcOlgbpA%#|H(hafH^x`DC0QL=IeAf^b^LUqdVpST7;(_Oh8AN+@^GOf%huSMK`O(F zUcB&#X@#M?-^e{KXZJPoNm2Osbz3C~wBDGXMz{Deh z3k}SnX~r~P0tc8PdOGSCnQyFPFxrS*05$2E;!gC$M|rh);S|AIb7$DMPft6P{S&s( z->Be`sfX_=8z>at7&{UZFXHSjKzGejO{tXAsdM=274vwVOhd&hF7}DJS5A*sQhpXl zPr4Rjl>Ch+kaH~3rTH0OMo#A_Oig==ghFK^`<-*3f8Xm>yn~xl@i$5NHRTCR@Dt&fE2+y^_S!uA6Xo z6-cwlNZH8Bg6M38u04xtzC_G}PF0Q)8l&DZBK(TNl0g?PuZxrUmVuTW^S+%>LrDa# zE4Ru0l7!Dj$SM)frJT|evYAr;1DDHJ6eG@N$|UFHwACz*4AtE!a2%|r;JJnfuFEVS zNVpwgX(En}y(_Jw6G;@>ou$ihQV`Z9W_X;Oyj@95R&4%iMBZt_BoY0VRTU*_4tsf* zm!HTs>)C8SFW`?{PhI$;?ojp^{qd6@kp}efvyu4ueui>)x|RNig{M^VC5-Q~c?&>t zHQJ2g=uO8aFSw?_w!g=JB_vrcJ%40em4T5!Gi~bR-Y2ZJlWshBM<`>nD%+nMXTXAcbi(yo2d+OZph5FAdeopv|?}Zit(IV(t^w zDVeWK9jEtmEZ8Zac|>$ zQNPZ(;#beLi45q(hjjKzF;kvr?v&Y^T`xe{4yQCcJR~kEYTOjXkuh0+C!p*`gW!st zZ-a2LxNL1>7<15kLcNs%?S(*l$#^d`Onh<|mzAtrQ4ho zaK1k-x~%;honq?x1Tl%-bd}gJ0?|EM%tg!3PQ6drz0aIwP;!T^fi* zy`^{=uf1dt18+SDd=RZVkxhv4!Nk@0T&mu??J3R29@@E-`5iv!b{zhM0=;DTsL@$v zdA;fma9^>dthlES&3!;~7rotUe{%c^Z_$}d#bv^gHt1p1!qVPX0S83ug2YnaQhIm_ zTj|}IUiKmK@N|$Y8uMdYl9xgxoz`mmTp!MXK3344Z`TVSxX4#frVhp3D8Hlvf8^z-B5DPF~As1yMOB0f7RDTN?$24f3rfh-6L z51}`u-$L74)iF|92=ebMS{=%doR*uM!+pT04bYvR5GmLYI;E(UOC`~ww0~RmipvFC zwQlRptq#lJ%|UjIBwFc-Jhn$wZ>GBlX$5gzvdI!Kggv;ZaQ2IaDmi^=e=2wqW2OaT z@F)8OF*h!g6d#{6q+4dL$2@&uNR|21V(snYx^HdIeO#ZZ$}=~}1KF$Y#WC4Zmv7PM zue?w=MkmS&((QbSV(+=gR_ZtY!M<>z*{dh0f=N8(V!4Et9N*?p_0DV?rxbM_LV3~w z{R0_wr^ri%9%px=5Iq)$K^1nZAVz;Eb49G8KdN({CYw=r?f0A!W69~vF&tiLKDc&U zTP#B*mdW~EeAxU*b2G?k-qV%S$D78ESV$4Aoi6~X+4Gfl+vge}Klgj+l;ykm(`y0Z z=*qkHR_T0{XxEuO-3+B6ibj1i@5Mw9$0<>@s<#Q3iZ&~n9-YFZMF2WJq$wUh$#d2B zONn!kzxbUu)i zOE{n(I&+SCk2|47@7+iAm6w#SPnRtlpR3LNI4!tD9JDMT<)+c**~jMh`n{j=0gnFVaB>*+k zXHs2L)2kJH`*s-Mj2b{u)Gkv7n54;+gmdb2Q9b~+6u5lTBP1q{%GS?Q0wrbIv5FXwJC$u*b>Ew^+yd!8WXMwKLQxy}D(p`{Jbn4!Ltqv5i+)sDW*8 zo7wSR)^Ji=R>86CX30f&b{lf`v2#G9vQHwUUXeQI(Q@xUpNKYHc%p6-H~zGu+D2=S#-Y zAL-|r+ELOul;DZ2@gct|EN+jx_LjOtAw*3DMOlbE)Nibm<(UryYC&DiwwhStCgF)Ff2g-yD z`_5cCLajDWSqk(?RGD8K4+AjJDCHQg%v@xk_&2!_;KNIg3D|*8!=(1&ZRlnSRUTeN zwqfz0LgD4(`U~*#`-mE&o^^NH7lb~FDN+lRNjP~O!fK_ z#<$M-w3&FrF$VCL{nyu@eg#P`uolC0Rc!*Z$BUa`9skZx{^@HXh+U4o^^GGkBtkR@ z69?l?xj9HHD<-b+KEI+urUUX#m;X+_3H9%zTVG%uAI!sG4WmTQ=@JyDxeSNb*TD}% zS5nl=?51<$0MO7f-vYDQU6xopr0ZIhPKgGA5HV{Q;e!WuzV0gUF6K?J=rmiGehsV8 zyM=c-_`v;F*<=}Xar2m<>Uy|yy1@j(u=bfu_Bs{xs$`}dRQ znVkar#F-(gU^z91P6YKG3tnoM!i1k(OcIJS_qI?0p-sp&efo|tniRv1#*i_~J<;`8 zd2A6#ob)aqS>>^4NBM?)l_7a<=6$NXGGJTQj9{>w;0p)!<<(4W=9Qv1r%BBO0Ti$%X%JohsCuGk zrr@U(^;g-!o@2)dAL~8b_)|^4^QfxH!4mvbwbh zI9k&fy{0#)+oqxdL2@rIJz=FLk4f%w7@7XVFuF39T-nceTCPcRJ`@ysJ{w+a z4%Ng_PUGGu4VR(V1dn9q`I+EjlcQ7~a!{bRTH=l>e-lrkEuBiv)U(d_%TsoAsiEdS zv$1|%0VLTFCjIg6gAJ4*pCAKwPz6bD+qz(GY}7m(EPb=LSi-*DTe>%?{h+I^8leYl z->6n)6nh3BDBE+SW9_Z{Y2Y$uom-8J@)^8M`$lgXW9>~BGJ5?)A zddtxa-+Sy`LdVS|eN*?#=H}*U;D{qyZhqi(fL*_$oPVD#;UlxU`g-lQj1;?vD{Y%} zBVTEsUtrv5up~07JhRT*kQqE_l?~j3^x+5nYDIExZYE+bq1~@)EJX}Y(6L^U-NF7nlDglRx)2v$)!}+b%bd$;`J<1sv zrltU(NtZ5S(m5~vCYoJ0*k8Pb^}$(jRF%AlNPT>zW^DB*JR#iNHc!M;AS?ax3VNel zfLq@pRZG=bo2jcq;Do|#f**+1z~N8hSdtHy#hVaC5hsyUNTW^Q^^KLe0@oQ+!)I6h zD9#&byi1bE^O|y+T)59D!$*O}P-lRM&qk8YDWFqa)sy`D`r-n2L<Vu3GXXs6Ydsv0cTG4kwH%9#(h=>59_vKz+iAW zhxg`U=lp0KBI|fKy*K(j%AYQU---Q#-o#+yu#zFmc>i~PMI}l$PC#xVZBGqrV%AGdX-qnE}<6^cy!D>1=G;H=NqMgidQC|qf3gI!A+i!bs3#8dci zTCJ@n14Fg*S-d%#39y<{((+U{GaxIY*t8!vXse4wskN2$_>qXMg?IR0Z*1>CPN>9F zaB0+%O>gOVtWL79&1)2+=-T8^iR+mTpz>Pd=Q7m&gsHr?jBb5wJ!s}(14@n_K!EHN zUoqY0DQ{m|)Th^nQ-(inI7o%$zoUpZZCc9Ue5ojAEmQ}Y_L)mc!v52FmbUf9Z|5+l zpkN1?OCXpsfOEHsjPJlRvTG8(3{!%>sop&$@PW`{W!gf3?ywFcg_YiswI?;_pdIn8 zX{h{ydydN)AI2rr6h9|&E7WC;$c~jH^;}b($dDKg*M;sDo`J%fqO<8i!1i-ncHk#q zykj}3nvY&7krThleScuo7fv<4Gy%i7|%bPO(g z`Aw}207(;Yj{3EbWD|hKud8E8F}WsXSy3A~RA?KY-s-!^U5=)(rht|vO@}fm^llhY zN0UM}yhVE}Yxs++HQ+t?Nr!D*X?C>k{q^9S=xm`YW^t{6850N0n0EqL+-nUX+KWs= z&VdRFeO~Cyq6XCr*ld_exs!=B;+#^m-S+U42lXx)lxPqQq5w^fp-iDGvxBBbJSY@s zMn(aR7z4OR_Yh=2%QJ4TODo{7(VW1%3m~zHj$kJ#itFk!pd6-4;W~IasLy4vIFnAy z(YRW=nnl#iEqbNB##(&ZBi>j!B!XRlqnCNqWV0PIZIlmqz08kdukFk*OcT^a_Hxc! zwr_I}Ej2UcrByf_@nPe18ML3fO2MnT1?NS>*{L)4CDfm^ z-@A2xdU5jNsMYW!x86{OYTVf%5_&xdOFr8oV0fDjZvnnZsf*;)g@*4Vulwo9oCM)t zSjou9n#|}l9fGMivxeG+g;qW?KO$!8Z_hI`t4nDG?8K{zLvfBp@LqOe8bKY71$FV& zH1b)}g)-L}Jtn$bv93rOQ07eD3ki8~yl|v+cGbBx`XF*r_Qjbm1RL>95kfjmdaRr- zr#|Vx1y82uB(CdWZ2gKL>0Q2u)zQvP58oRkXn(SE7LQ?D$_UUaxcIX49VI|&W5O a18mk!o88&Zz#k~WQM{}sn=5_Q|Nj8VpAZ=U literal 0 HcmV?d00001 From c85928d1cc2e4583a991d42c421066b41554f0c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:31:07 +0000 Subject: [PATCH 39/39] [pre-commit.ci] auto fixes from pre-commit.com hooks --- moabb/tests/splits.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/moabb/tests/splits.py b/moabb/tests/splits.py index e8d4f742e..a89b4f7f7 100644 --- a/moabb/tests/splits.py +++ b/moabb/tests/splits.py @@ -1,20 +1,19 @@ -import pytest - import numpy as np - -from sklearn.model_selection import LeaveOneGroupOut, StratifiedKFold +import pytest +from sklearn.model_selection import StratifiedKFold from sklearn.utils import check_random_state from moabb.datasets.fake import FakeDataset from moabb.evaluations.splitters import WithinSessionSplitter from moabb.paradigms.motor_imagery import FakeImageryParadigm + dataset = FakeDataset(["left_hand", "right_hand"], n_subjects=3, seed=12) paradigm = FakeImageryParadigm() # Split done for the Within Session evaluation -def eval_split_within_session(shuffle,random_state): +def eval_split_within_session(shuffle, random_state): random_state = check_random_state(random_state) if shuffle else None for subject in dataset.subject_list: X, y, metadata = paradigm.get_data(dataset=dataset, subjects=[subject]) @@ -32,24 +31,28 @@ def eval_split_within_session(shuffle,random_state): def test_within_session(shuffle, random_state): X, y, metadata = paradigm.get_data(dataset=dataset) - split = WithinSessionSplitter(n_folds=5,shuffle=shuffle, random_state=random_state) + split = WithinSessionSplitter(n_folds=5, shuffle=shuffle, random_state=random_state) - for (X_train_t, X_test_t), (train, test) in zip(eval_split_within_session( - shuffle=shuffle, random_state=random_state),split.split(y, metadata)): + for (X_train_t, X_test_t), (train, test) in zip( + eval_split_within_session(shuffle=shuffle, random_state=random_state), + split.split(y, metadata), + ): X_train, X_test = X[train], X[test] # Check if the output is the same as the input assert np.array_equal(X_train, X_train_t) assert np.array_equal(X_test, X_test_t) + def test_is_shuffling(): X, y, metadata = paradigm.get_data(dataset=dataset) split = WithinSessionSplitter(n_folds=5, shuffle=False) - split_shuffle = WithinSessionSplitter(n_folds=5,shuffle=True, random_state=3) + split_shuffle = WithinSessionSplitter(n_folds=5, shuffle=True, random_state=3) - for (train, test), (train_shuffle, test_shuffle) in zip(split.split(y, metadata), - split_shuffle.split(y, metadata)): + for (train, test), (train_shuffle, test_shuffle) in zip( + split.split(y, metadata), split_shuffle.split(y, metadata) + ): # Check if the output is the same as the input - assert np.array_equal(train, train_shuffle)==False - assert np.array_equal(test, test_shuffle)==False + assert np.array_equal(train, train_shuffle) == False + assert np.array_equal(test, test_shuffle) == False