From 267d4e8455ab6de8852681a71aeebf614db16edd Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 5 Aug 2022 15:52:15 +0100 Subject: [PATCH 01/27] =?UTF-8?q?=F0=9F=93=9D=20Create=20basic=20for=20ML?= =?UTF-8?q?=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 3 +- docs/source/conf.py | 3 +- docs/source/rst/api/Azure/index.rst | 0 docs/source/rst/api/Common/index.rst | 0 docs/source/rst/api/ML/SSL/index.rst | 0 .../source/rst/api/ML/augmentations/index.rst | 0 docs/source/rst/api/ML/configs.rst | 36 ------------------- docs/source/rst/api/ML/configs/index.rst | 0 docs/source/rst/api/ML/dataset/index.rst | 0 docs/source/rst/api/ML/models/index.rst | 0 docs/source/rst/api/ML/pipelines/index.rst | 0 docs/source/rst/api/ML/utils/index.rst | 0 docs/source/rst/api/ML/visualizers/index.rst | 0 13 files changed, 4 insertions(+), 38 deletions(-) create mode 100644 docs/source/rst/api/Azure/index.rst create mode 100644 docs/source/rst/api/Common/index.rst create mode 100644 docs/source/rst/api/ML/SSL/index.rst create mode 100644 docs/source/rst/api/ML/augmentations/index.rst create mode 100644 docs/source/rst/api/ML/configs/index.rst create mode 100644 docs/source/rst/api/ML/dataset/index.rst create mode 100644 docs/source/rst/api/ML/models/index.rst create mode 100644 docs/source/rst/api/ML/pipelines/index.rst create mode 100644 docs/source/rst/api/ML/utils/index.rst create mode 100644 docs/source/rst/api/ML/visualizers/index.rst diff --git a/.vscode/settings.json b/.vscode/settings.json index 994fb1769..958b52a90 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,5 +12,6 @@ }, "files.trimTrailingWhitespace": true, "files.trimFinalNewlines": true, - "files.insertFinalNewline": true + "files.insertFinalNewline": true, + "esbonio.sphinx.confDir": "" } diff --git a/docs/source/conf.py b/docs/source/conf.py index 7d13372d1..81f768a4b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,9 +44,10 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + "sphinx.ext.autosummary", 'sphinx_rtd_theme', 'recommonmark', - 'sphinx.ext.viewcode' + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/rst/api/Azure/index.rst b/docs/source/rst/api/Azure/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/Common/index.rst b/docs/source/rst/api/Common/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/SSL/index.rst b/docs/source/rst/api/ML/SSL/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/augmentations/index.rst b/docs/source/rst/api/ML/augmentations/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs.rst b/docs/source/rst/api/ML/configs.rst index 8f7fb3cd8..18bedb629 100644 --- a/docs/source/rst/api/ML/configs.rst +++ b/docs/source/rst/api/ML/configs.rst @@ -3,41 +3,5 @@ Segmentation Model Configuration .. autoclass:: InnerEye.ML.config.SegmentationModelBase - .. autoattribute:: activation_map_layers - .. autoattribute:: architecture - .. autoattribute:: loss_type - .. autoattribute:: mixture_loss_components - .. autoattribute:: loss_class_weight_power - .. autoattribute:: focal_loss_gamma - .. autoattribute:: dataset_expected_spacing_xyz - .. autoattribute:: feature_channels - .. autoattribute:: kernel_size - .. autoattribute:: crop_size - .. autoattribute:: image_channels - .. autoattribute:: ground_truth_ids - .. autoattribute:: mask_id - .. autoattribute:: norm_method - .. autoattribute:: window - .. autoattribute:: level - .. autoattribute:: output_range - .. autoattribute:: debug_mode - .. autoattribute:: tail - .. autoattribute:: sharpen - .. autoattribute:: trim_percentiles - .. autoattribute:: padding_mode - .. autoattribute:: inference_batch_size - .. autoattribute:: test_crop_size - .. autoattribute:: class_weights - .. autoattribute:: ensemble_aggregation_type - .. autoattribute:: posterior_smoothing_mm - .. autoattribute:: store_dataset_sample - .. autoattribute:: comparison_blob_storage_paths - .. autoattribute:: slice_exclusion_rules - .. autoattribute:: summed_probability_rules - .. autoattribute:: disable_extra_postprocessing - .. autoattribute:: ground_truth_ids_display_names - .. autoattribute:: col_type_converters - .. autoattribute:: is_plotting_enabled - .. automodule:: InnerEye.ML.config :exclude-members: SegmentationModelBase diff --git a/docs/source/rst/api/ML/configs/index.rst b/docs/source/rst/api/ML/configs/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/dataset/index.rst b/docs/source/rst/api/ML/dataset/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/models/index.rst b/docs/source/rst/api/ML/models/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/pipelines/index.rst b/docs/source/rst/api/ML/pipelines/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/utils/index.rst b/docs/source/rst/api/ML/utils/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/visualizers/index.rst b/docs/source/rst/api/ML/visualizers/index.rst new file mode 100644 index 000000000..e69de29bb From 806f26125a82c2dc5c95a6a6be945710f9b2461c Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 5 Aug 2022 15:59:10 +0100 Subject: [PATCH 02/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20ML/configs=20base=20?= =?UTF-8?q?doc=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/ML/augmentations.rst | 8 -------- docs/source/rst/api/ML/augmentations/index.rst | 8 ++++++++ docs/source/rst/api/ML/configs/classification/index.rst | 0 docs/source/rst/api/ML/configs/other/index.rst | 0 docs/source/rst/api/ML/configs/regression/index.rst | 0 docs/source/rst/api/ML/configs/segmentation/index.rst | 0 docs/source/rst/api/ML/configs/ssl/index.rst | 0 docs/source/rst/api/ML/configs/unit_testing/index.rst | 0 docs/source/rst/api/ML/index.rst | 2 +- docs/source/rst/api/Scripts/index.rst | 0 10 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 docs/source/rst/api/ML/augmentations.rst create mode 100644 docs/source/rst/api/ML/configs/classification/index.rst create mode 100644 docs/source/rst/api/ML/configs/other/index.rst create mode 100644 docs/source/rst/api/ML/configs/regression/index.rst create mode 100644 docs/source/rst/api/ML/configs/segmentation/index.rst create mode 100644 docs/source/rst/api/ML/configs/ssl/index.rst create mode 100644 docs/source/rst/api/ML/configs/unit_testing/index.rst create mode 100644 docs/source/rst/api/Scripts/index.rst diff --git a/docs/source/rst/api/ML/augmentations.rst b/docs/source/rst/api/ML/augmentations.rst deleted file mode 100644 index 3c327aab4..000000000 --- a/docs/source/rst/api/ML/augmentations.rst +++ /dev/null @@ -1,8 +0,0 @@ -Data augmentation -================= - -.. automodule:: InnerEye.ML.augmentations.augmentation_for_segmentation_utils - -.. automodule:: InnerEye.ML.augmentations.image_transforms - -.. automodule:: InnerEye.ML.augmentations.transform_pipeline diff --git a/docs/source/rst/api/ML/augmentations/index.rst b/docs/source/rst/api/ML/augmentations/index.rst index e69de29bb..3c327aab4 100644 --- a/docs/source/rst/api/ML/augmentations/index.rst +++ b/docs/source/rst/api/ML/augmentations/index.rst @@ -0,0 +1,8 @@ +Data augmentation +================= + +.. automodule:: InnerEye.ML.augmentations.augmentation_for_segmentation_utils + +.. automodule:: InnerEye.ML.augmentations.image_transforms + +.. automodule:: InnerEye.ML.augmentations.transform_pipeline diff --git a/docs/source/rst/api/ML/configs/classification/index.rst b/docs/source/rst/api/ML/configs/classification/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs/other/index.rst b/docs/source/rst/api/ML/configs/other/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs/regression/index.rst b/docs/source/rst/api/ML/configs/regression/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs/segmentation/index.rst b/docs/source/rst/api/ML/configs/segmentation/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs/ssl/index.rst b/docs/source/rst/api/ML/configs/ssl/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/configs/unit_testing/index.rst b/docs/source/rst/api/ML/configs/unit_testing/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst index 4d8bfe304..65328a5c0 100644 --- a/docs/source/rst/api/ML/index.rst +++ b/docs/source/rst/api/ML/index.rst @@ -5,6 +5,6 @@ Machine learning configs runner - augmentations + augmentations/index.rst photometric_normalization pipelines diff --git a/docs/source/rst/api/Scripts/index.rst b/docs/source/rst/api/Scripts/index.rst new file mode 100644 index 000000000..e69de29bb From 0cbd7c71cc2e4b70f99b374bbed48dfff2ea8289 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 09:29:36 +0100 Subject: [PATCH 03/27] =?UTF-8?q?=F0=9F=93=9D=20Finish=20ML/configs=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ML/configs/classification/CovidModel.py | 14 ++++-- InnerEye/ML/configs/other/fastmri_varnet.py | 3 +- .../configs/segmentation/HeadAndNeckBase.py | 1 + .../ML/configs/segmentation/HelloWorld.py | 1 + .../ML/configs/segmentation/ProstatePaper.py | 1 + InnerEye/ML/lightning_container.py | 1 + InnerEye/ML/model_config_base.py | 5 +++ InnerEye/ML/scalar_config.py | 5 +-- .../api/ML/configs/classification/index.rst | 10 +++++ docs/source/rst/api/ML/configs/index.rst | 10 +++++ .../source/rst/api/ML/configs/other/index.rst | 7 +++ .../rst/api/ML/configs/regression/index.rst | 7 +++ .../rst/api/ML/configs/segmentation/index.rst | 45 +++++++++++++++++++ docs/source/rst/api/ML/configs/ssl/index.rst | 19 ++++++++ .../rst/api/ML/configs/unit_testing/index.rst | 0 docs/source/rst/api/ML/index.rst | 4 +- 16 files changed, 123 insertions(+), 10 deletions(-) delete mode 100644 docs/source/rst/api/ML/configs/unit_testing/index.rst diff --git a/InnerEye/ML/configs/classification/CovidModel.py b/InnerEye/ML/configs/classification/CovidModel.py index 88a102e2b..22ecd7ba1 100644 --- a/InnerEye/ML/configs/classification/CovidModel.py +++ b/InnerEye/ML/configs/classification/CovidModel.py @@ -47,12 +47,12 @@ class CovidModel(ScalarModelBase): """ Model to train a CovidDataset model from scratch or finetune from SSL-pretrained model. - For AML you need to provide the run_id of your SSL training job as a command line argument - --pretraining_run_recovery_id=id_of_your_ssl_model, this will download the checkpoints of the run to your + For AML you need to provide the run_id of your SSL training job as a command line argument: + ``--pretraining_run_recovery_id=``. This will download the checkpoints of the run to your machine and load the corresponding pretrained model. - To recover from a particular checkpoint from your SSL run e.g. "recovery_epoch=499.ckpt" please use the - --name_of_checkpoint argument. + To recover from a particular checkpoint from your SSL run e.g. ``"recovery_epoch=499.ckpt"`` please use the + ``--name_of_checkpoint`` argument. """ use_pretrained_model = param.Boolean(default=False, doc="If True, start training from a model pretrained with SSL." "If False, start training a DenseNet model from scratch" @@ -242,6 +242,7 @@ def generate_custom_report(self, report_dir: Path, model_proc: ModelProcessing) Generate a custom report for the Covid model. This report will read the file model_output.csv generated for the training, validation or test sets and compute both the multiclass accuracy and the accuracy for each of the hierarchical tasks. + :param report_dir: Directory report is to be written to :param model_proc: Whether this is a single or ensemble model (model_output.csv will be located in different paths for single vs ensemble runs.) @@ -364,6 +365,11 @@ def compute_binary_accuracy(model_outputs: pd.Series, labels: pd.Series) -> floa class DicomPreparation: def __call__(self, item: torch.Tensor) -> PIL.Image: + """Call class as a function. This will act as a transformation function for the dataset. + + :param item: tensor to transform. + :return: transformed data. + """ # Item will be of dimension [C, Z, X, Y] images = item.numpy() assert images.shape[0] == 1 and images.shape[1] == 1 diff --git a/InnerEye/ML/configs/other/fastmri_varnet.py b/InnerEye/ML/configs/other/fastmri_varnet.py index 29618a49c..c84c711df 100644 --- a/InnerEye/ML/configs/other/fastmri_varnet.py +++ b/InnerEye/ML/configs/other/fastmri_varnet.py @@ -47,11 +47,12 @@ def get_fastmri_data_module(azure_dataset_id: str, Creates a LightningDataModule that consumes data from the FastMRI challenge. The type of challenge (single/multicoil) is determined from the name of the dataset in Azure blob storage. The mask type is set to equispaced, with 4x acceleration. + :param azure_dataset_id: The name of the dataset (folder name in blob storage). :param local_dataset: The local folder at which the dataset has been mounted or downloaded. :param sample_rate: Fraction of slices of the training data split to use. Set to a value <1.0 for rapid prototyping. :param test_path: The name of the folder inside the dataset that contains the test data. - :return: A LightningDataModule object. + :return: The FastMRI LightningDataModule object. """ if not azure_dataset_id: raise ValueError("The azure_dataset_id argument must be provided.") diff --git a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py index 1ced1590d..ad3a12c88 100644 --- a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py +++ b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py @@ -47,6 +47,7 @@ def __init__(self, **kwargs: Any) -> None: """ Creates a new instance of the class. + :param ground_truth_ids: List of ground truth ids. :param ground_truth_ids_display_names: Optional list of ground truth id display names. If present then must be of the same length as ground_truth_ids. diff --git a/InnerEye/ML/configs/segmentation/HelloWorld.py b/InnerEye/ML/configs/segmentation/HelloWorld.py index 704a5c1a2..9b45528fa 100644 --- a/InnerEye/ML/configs/segmentation/HelloWorld.py +++ b/InnerEye/ML/configs/segmentation/HelloWorld.py @@ -94,6 +94,7 @@ def get_parameter_search_hyperdrive_config(self, run_config: ScriptRunConfig) -> https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters A reference is provided at https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train .hyperdrive?view=azure-ml-py + :param run_config: The configuration for running an individual experiment. :return: An Azure HyperDrive run configuration (configured PyTorch environment). """ diff --git a/InnerEye/ML/configs/segmentation/ProstatePaper.py b/InnerEye/ML/configs/segmentation/ProstatePaper.py index 039afac97..441200c48 100644 --- a/InnerEye/ML/configs/segmentation/ProstatePaper.py +++ b/InnerEye/ML/configs/segmentation/ProstatePaper.py @@ -20,6 +20,7 @@ class ProstatePaper(ProstateBase): def __init__(self, **kwargs: Any) -> None: """ Creates a new instance of the class. + :param kwargs: Additional arguments that will be passed through to the SegmentationModelBase constructor. """ ground_truth_ids = fg_classes diff --git a/InnerEye/ML/lightning_container.py b/InnerEye/ML/lightning_container.py index 613d4c00e..1f9491952 100644 --- a/InnerEye/ML/lightning_container.py +++ b/InnerEye/ML/lightning_container.py @@ -183,6 +183,7 @@ def get_data_module(self) -> LightningDataModule: Because the method deals with data loaders, not loaded data, we cannot check automatically that cross validation is handled correctly within the base class, i.e. if the cross validation split is not handled in the method then nothing will fail, but each child run will be identical since they will each be given the full dataset. + :return: A LightningDataModule """ return None # type: ignore diff --git a/InnerEye/ML/model_config_base.py b/InnerEye/ML/model_config_base.py index edd849e7f..5a42835e2 100644 --- a/InnerEye/ML/model_config_base.py +++ b/InnerEye/ML/model_config_base.py @@ -55,6 +55,7 @@ def get_parameter_search_hyperdrive_config(self, run_config: ScriptRunConfig) -> Returns a configuration for AzureML Hyperdrive that should be used when running hyperparameter tuning. This is an abstract method that each specific model should override. + :param run_config: The AzureML estimator object that runs model training. :return: A hyperdrive configuration object. """ @@ -66,6 +67,7 @@ def get_model_train_test_dataset_splits(self, dataset_df: pd.DataFrame) -> Datas """ Computes the training, validation and test splits for the model, from a dataframe that contains the full dataset. + :param dataset_df: A dataframe that contains the full dataset that the model is using. :return: An instance of DatasetSplits with dataframes for training, validation and testing. """ @@ -83,6 +85,7 @@ def create_and_set_torch_datasets(self, for_training: bool = True, for_inference are False, the derived method *may* still create the corresponding datasets, but should not assume that the relevant splits (train/test/val) are non-empty. If either or both is True, they *must* create the corresponding datasets, and should be able to make the assumption. + :param for_training: whether to create the datasets required for training. :param for_inference: whether to create the datasets required for inference. """ @@ -103,6 +106,8 @@ def get_torch_dataset_for_inference(self, mode: ModelExecutionMode) -> Any: """ Returns a torch Dataset for running the model in inference mode, on the given split of the full dataset. The torch dataset must return data in the format required for running the model in inference mode. + + :param mode: The mode of the model, either test, train or val. :return: A torch Dataset object. """ if self._datasets_for_inference is None: diff --git a/InnerEye/ML/scalar_config.py b/InnerEye/ML/scalar_config.py index b156b94e4..9828cd888 100644 --- a/InnerEye/ML/scalar_config.py +++ b/InnerEye/ML/scalar_config.py @@ -542,16 +542,15 @@ def compute_and_log_metrics(self, data_split: ModelExecutionMode) -> None: """ Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers. + :param logits: The model output before normalization. :param targets: The expected model outputs. :param subject_ids: The subject IDs for the present minibatch. :param is_training: If True, write the metrics as training metrics, otherwise as validation metrics. - :param metrics: A dictionary mapping from names of prediction targets to a list of metric computers, - as returned by create_metric_computers. + :param metrics: A dictionary mapping from names of prediction targets to a list of metric computers, as returned by create_metric_computers. :param logger: An object of type DataframeLogger which can be be used for logging within this function. :param current_epoch: Current epoch number. :param data_split: ModelExecutionMode object indicating if this is the train or validation split. - :return: """ per_subject_outputs: List[Tuple[str, str, torch.Tensor, torch.Tensor]] = [] for i, (prediction_target, metric_list) in enumerate(metrics.items()): diff --git a/docs/source/rst/api/ML/configs/classification/index.rst b/docs/source/rst/api/ML/configs/classification/index.rst index e69de29bb..b9f283e41 100644 --- a/docs/source/rst/api/ML/configs/classification/index.rst +++ b/docs/source/rst/api/ML/configs/classification/index.rst @@ -0,0 +1,10 @@ +Classification Configs +============================= + +COVID Model +----------- + +.. autoclass:: InnerEye.ML.configs.classification.CovidModel.CovidModel + +.. autoclass:: InnerEye.ML.configs.classification.CovidModel.DicomPreparation + :special-members: __call__ diff --git a/docs/source/rst/api/ML/configs/index.rst b/docs/source/rst/api/ML/configs/index.rst index e69de29bb..fc7c85afe 100644 --- a/docs/source/rst/api/ML/configs/index.rst +++ b/docs/source/rst/api/ML/configs/index.rst @@ -0,0 +1,10 @@ +Configurations +============== + +.. toctree:: + + classification/index + regression/index + segmentation/index + ssl/index + other/index diff --git a/docs/source/rst/api/ML/configs/other/index.rst b/docs/source/rst/api/ML/configs/other/index.rst index e69de29bb..73a05f5ab 100644 --- a/docs/source/rst/api/ML/configs/other/index.rst +++ b/docs/source/rst/api/ML/configs/other/index.rst @@ -0,0 +1,7 @@ +Other configs +========================== + +FastMRT Varnet +---------------- + +.. automodule:: InnerEye.ML.configs.other.fastmri_varnet diff --git a/docs/source/rst/api/ML/configs/regression/index.rst b/docs/source/rst/api/ML/configs/regression/index.rst index e69de29bb..b157fa521 100644 --- a/docs/source/rst/api/ML/configs/regression/index.rst +++ b/docs/source/rst/api/ML/configs/regression/index.rst @@ -0,0 +1,7 @@ +Regression Configs +=================== + +Dummy Regression +----------------- + +.. automodule:: InnerEye.ML.configs.regression.DummyRegression diff --git a/docs/source/rst/api/ML/configs/segmentation/index.rst b/docs/source/rst/api/ML/configs/segmentation/index.rst index e69de29bb..217767d1e 100644 --- a/docs/source/rst/api/ML/configs/segmentation/index.rst +++ b/docs/source/rst/api/ML/configs/segmentation/index.rst @@ -0,0 +1,45 @@ +Segmentation Configs +===================== + +Basic Models +------------ + +.. automodule:: InnerEye.ML.configs.segmentation.BasicModel2Epochs + +.. automodule:: InnerEye.ML.configs.segmentation.BasicModel2Epochs1Channel + +.. automodule:: InnerEye.ML.configs.segmentation.BasicModel2EpochsMoreData + +HelloWorld +------------ + +.. automodule:: InnerEye.ML.configs.segmentation.HelloWorld + +Head and Neck +------------- + +.. automodule:: InnerEye.ML.configs.segmentation.HeadAndNeckBase + +.. automodule:: InnerEye.ML.configs.segmentation.HeadAndNeckPaper + +Prostate +------------- + +.. automodule:: InnerEye.ML.configs.segmentation.ProstateBase + +.. automodule:: InnerEye.ML.configs.segmentation.ProstatePaper + +Hippocampus +------------ + +.. automodule:: InnerEye.ML.configs.segmentation.Hippocampus + +Lung +------------ + +.. automodule:: InnerEye.ML.configs.segmentation.Lung + +Glioblastoma +------------ + +.. automodule:: InnerEye.ML.configs.segmentation.GbmBase diff --git a/docs/source/rst/api/ML/configs/ssl/index.rst b/docs/source/rst/api/ML/configs/ssl/index.rst index e69de29bb..a042733e7 100644 --- a/docs/source/rst/api/ML/configs/ssl/index.rst +++ b/docs/source/rst/api/ML/configs/ssl/index.rst @@ -0,0 +1,19 @@ +Self-Supervised Learning Configs +================================ + +CIFAR +----- + +.. automodule:: InnerEye.ML.configs.ssl.CIFAR_classifier_configs + +.. automodule:: InnerEye.ML.configs.ssl.CIFAR_SSL_configs + +COVID +----- + +.. automodule:: InnerEye.ML.configs.ssl.CovidContainers + +Chest X-Rays +------------ + +.. automodule:: InnerEye.ML.configs.ssl.CXR_SSL_configs diff --git a/docs/source/rst/api/ML/configs/unit_testing/index.rst b/docs/source/rst/api/ML/configs/unit_testing/index.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst index 65328a5c0..6d54375d1 100644 --- a/docs/source/rst/api/ML/index.rst +++ b/docs/source/rst/api/ML/index.rst @@ -3,8 +3,8 @@ Machine learning .. toctree:: - configs + configs/index + augmentations/index runner - augmentations/index.rst photometric_normalization pipelines From f5c82cfda53a9388156a65aa2ae0f5ab56f3860e Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 11:27:26 +0100 Subject: [PATCH 04/27] =?UTF-8?q?=F0=9F=93=9D=20Update=20augmentations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../augmentation_for_segmentation_utils.py | 10 +++++----- InnerEye/ML/configs/classification/CovidModel.py | 1 + docs/source/rst/api/ML/augmentations/index.rst | 11 ++++++++++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/InnerEye/ML/augmentations/augmentation_for_segmentation_utils.py b/InnerEye/ML/augmentations/augmentation_for_segmentation_utils.py index 5d36b3baa..1bcfa7629 100644 --- a/InnerEye/ML/augmentations/augmentation_for_segmentation_utils.py +++ b/InnerEye/ML/augmentations/augmentation_for_segmentation_utils.py @@ -20,9 +20,9 @@ class among the available classes then samples a center point among the pixels o class. :param sample: A set of Image channels, ground truth labels and mask to randomly crop. - :param class_weights: A weighting vector with values [0, 1] to influence the class the center crop - voxel belongs to (must sum to 1), uniform distribution assumed if none provided. - :return numpy int array (3x1) containing patch center spatial coordinates + :param class_weights: A weighting vector with values [0, 1] to influence the class the center crop voxel belongs + to (must sum to 1), uniform distribution assumed if none provided. + :return: numpy int array (3x1) containing patch center spatial coordinates """ num_classes = sample.labels.shape[0] @@ -70,9 +70,9 @@ def slicers_for_random_crop(sample: Sample, :param sample: A set of Image channels, ground truth labels and mask to randomly crop. :param crop_size: The size of the crop expressed as a list of 3 ints, one per spatial dimension. :param class_weights: A weighting vector with values [0, 1] to influence the class the center crop - voxel belongs to (must sum to 1), uniform distribution assumed if none provided. + voxel belongs to (must sum to 1), uniform distribution assumed if none provided. :return: Tuple element 1: The slicers that convert the input image to the chosen crop. Tuple element 2: The - indices of the center point of the crop. + indices of the center point of the crop. :raises ValueError: If there are shape mismatches among the arguments or if the crop size is larger than the image. """ shape = sample.image.shape[1:] diff --git a/InnerEye/ML/configs/classification/CovidModel.py b/InnerEye/ML/configs/classification/CovidModel.py index 22ecd7ba1..d15f94a06 100644 --- a/InnerEye/ML/configs/classification/CovidModel.py +++ b/InnerEye/ML/configs/classification/CovidModel.py @@ -246,6 +246,7 @@ def generate_custom_report(self, report_dir: Path, model_proc: ModelProcessing) :param report_dir: Directory report is to be written to :param model_proc: Whether this is a single or ensemble model (model_output.csv will be located in different paths for single vs ensemble runs.) + """ label_prefix = LoggingColumns.Label.value diff --git a/docs/source/rst/api/ML/augmentations/index.rst b/docs/source/rst/api/ML/augmentations/index.rst index 3c327aab4..dce679565 100644 --- a/docs/source/rst/api/ML/augmentations/index.rst +++ b/docs/source/rst/api/ML/augmentations/index.rst @@ -1,8 +1,17 @@ Data augmentation ================= -.. automodule:: InnerEye.ML.augmentations.augmentation_for_segmentation_utils +Image Transformations +---------------------- .. automodule:: InnerEye.ML.augmentations.image_transforms +Transformation Pipeline +----------------------- + .. automodule:: InnerEye.ML.augmentations.transform_pipeline + +Utils +----- + +.. automodule:: InnerEye.ML.augmentations.augmentation_for_segmentation_utils From ac0171af775997f399d39f49ab87cf8a2a29c282 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 11:54:44 +0100 Subject: [PATCH 05/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20ML/dataset=20API=20d?= =?UTF-8?q?ocs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/dataset/cropping_dataset.py | 2 + InnerEye/ML/dataset/full_image_dataset.py | 6 +- InnerEye/ML/dataset/sample.py | 6 +- InnerEye/ML/dataset/scalar_dataset.py | 67 +++++++++++++---------- InnerEye/ML/dataset/scalar_sample.py | 17 +++--- docs/source/rst/api/ML/dataset/index.rst | 21 +++++++ docs/source/rst/api/ML/index.rst | 1 + 7 files changed, 82 insertions(+), 38 deletions(-) diff --git a/InnerEye/ML/dataset/cropping_dataset.py b/InnerEye/ML/dataset/cropping_dataset.py index 6d9d93f18..4908b517d 100644 --- a/InnerEye/ML/dataset/cropping_dataset.py +++ b/InnerEye/ML/dataset/cropping_dataset.py @@ -57,6 +57,7 @@ def create_possibly_padded_sample_for_cropping(sample: Sample, """ Pad the original sample such the the provided images has the same (or slightly larger in case of uneven difference) shape to the output_size, using the provided padding mode. + :param sample: Sample to pad. :param crop_size: Crop size to match. :param padding_mode: The padding scheme to apply. @@ -89,6 +90,7 @@ def create_random_cropped_sample(sample: Sample, class_weights: Optional[List[float]] = None) -> CroppedSample: """ Creates an instance of a cropped sample extracted from full 3D images. + :param sample: the full size 3D sample to use for extracting a cropped sample. :param crop_size: the size of the crop to extract. :param center_size: the size of the center of the crop (this should be the same as the spatial dimensions diff --git a/InnerEye/ML/dataset/full_image_dataset.py b/InnerEye/ML/dataset/full_image_dataset.py index f2105b0be..cab2136e7 100644 --- a/InnerEye/ML/dataset/full_image_dataset.py +++ b/InnerEye/ML/dataset/full_image_dataset.py @@ -31,6 +31,7 @@ def collate_with_metadata(batch: List[Dict[str, Any]]) -> Dict[str, Any]: The collate function that the dataloader workers should use. It does the same thing for all "normal" fields (all fields are put into tensors with outer dimension batch_size), except for the special "metadata" field. Those metadata objects are collated into a simple list. + :param batch: A list of samples that should be collated. :return: collated result """ @@ -123,6 +124,7 @@ def __init__(self, **kwargs: Any): """ Creates a new data loader. + :param dataset: The dataset that should be loaded. :param batch_size: The number of samples per minibatch. :param shuffle: If true, the dataset will be shuffled randomly. @@ -204,11 +206,12 @@ class FullImageDataset(GeneralDataset): """ Dataset class that loads and creates samples with full 3D images from a given pd.Dataframe. The following are the operations performed to generate a sample from this dataset: - ------------------------------------------------------------------------------------------------- + 1) On initialization parses the provided pd.Dataframe with dataset information, to cache the set of file paths and patient mappings to load as PatientDatasetSource. The sources are then saved in a list: dataset_sources. 2) dataset_sources is iterated in a batched fashion, where for each batch it loads the full 3D images, and applies pre-processing functions (e.g. normalization), returning a sample that can be used for full image operations. + """ def __init__(self, args: SegmentationModelBase, data_frame: pd.DataFrame, @@ -313,6 +316,7 @@ def load_dataset_sources(dataframe: pd.DataFrame, The dataframe contains per-patient per-channel image information, relative to a root directory. This method converts that into a per-patient dictionary, that contains absolute file paths separated for for image channels, ground truth channels, and mask channels. + :param dataframe: A dataframe read directly from a dataset CSV file. :param local_dataset_root_folder: The root folder that contains all images. :param image_channels: The names of the image channels that should be used in the result. diff --git a/InnerEye/ML/dataset/sample.py b/InnerEye/ML/dataset/sample.py index a233e01ec..8ffef1011 100644 --- a/InnerEye/ML/dataset/sample.py +++ b/InnerEye/ML/dataset/sample.py @@ -40,6 +40,7 @@ def from_dataframe(dataframe: pd.DataFrame, patient_id: str) -> PatientMetadata: For each of the columns "seriesId", "instituionId" and "tags", the distinct values for the given patient are computed. If there is exactly 1 distinct value, that is returned as the respective patient metadata. If there is more than 1 distinct value, the metadata column is set to None. + :param dataframe: The dataset to read from. :param patient_id: The ID of the patient for which the metadata should be extracted. :return: An instance of PatientMetadata for the given patient_id @@ -101,8 +102,9 @@ def __init__(self) -> None: def from_dict(cls: Type[T], sample: Dict[str, Any]) -> T: """ Create an instance of the sample class, based on the provided sample dictionary + :param sample: dictionary of arguments - :return: + :return: an instance of the SampleBase class """ return cls(**sample) # type: ignore @@ -110,6 +112,7 @@ def clone_with_overrides(self: T, **overrides: Any) -> T: """ Create a clone of the current sample, with the provided overrides to replace the existing properties if they exist. + :param overrides: :return: """ @@ -118,7 +121,6 @@ def clone_with_overrides(self: T, **overrides: Any) -> T: def get_dict(self) -> Dict[str, Any]: """ Get the current sample as a dictionary of property names and their values. - :return: """ return vars(self) diff --git a/InnerEye/ML/dataset/scalar_dataset.py b/InnerEye/ML/dataset/scalar_dataset.py index b7ac3f1f3..5a11327c4 100644 --- a/InnerEye/ML/dataset/scalar_dataset.py +++ b/InnerEye/ML/dataset/scalar_dataset.py @@ -33,7 +33,7 @@ def extract_label_classification(label_string: str, sample_id: str, num_classes: Converts a string from a dataset.csv file that contains a model's label to a scalar. For classification datasets: - If num_classes is 1 (binary classification tasks): + If num_classes is 1 (binary classification tasks) The function maps ["1", "true", "yes"] to [1], ["0", "false", "no"] to [0]. If the entry in the CSV file was missing (no string given at all) or an empty string, it returns math.nan. If num_classes is greater than 1 (multilabel datasets): @@ -42,17 +42,16 @@ def extract_label_classification(label_string: str, sample_id: str, num_classes: map "1|3|4" to [0, 1, 0, 1, 1, 0]). If the entry in the CSV file was missing (no string given at all) or an empty string, this function returns an all-zero tensor (none of the label classes were positive for this sample). - For regression datasets: - The function casts a string label to float. Raises an exception if the conversion is - not possible. - If the entry in the CSV file was missing (no string given at all) or an empty string, it returns math.nan. + The function casts a string label to float. Raises an exception if the conversion is + not possible. + If the entry in the CSV file was missing (no string given at all) or an empty string, it returns math.nan. :param label_string: The value of the label as read from CSV via a DataFrame. :param sample_id: The sample ID where this label was read from. This is only used for creating error messages. :param num_classes: Number of classes. This should be equal the size of the model output. - For binary classification tasks, num_classes should be one. For multilabel classification tasks, num_classes should - correspond to the number of label classes in the problem. + For binary classification tasks, num_classes should be one. For multilabel classification tasks, num_classes + should correspond to the number of label classes in the problem. :param is_classification_dataset: If the model is a classification model :return: A list of floats with the same size as num_classes """ @@ -120,6 +119,7 @@ def _get_single_channel_row(subject_rows: pd.DataFrame, 'channel' argument. Throws a ValueError if there is no or more than 1 such row. The result is returned as a dictionary, not a DataFrame! If the 'channel' argument is null, the input is expected to be already 1 row, which is returned as a dictionary. + :param subject_rows: A set of rows all belonging to the same subject. :param channel: The value to look for in the `channel_column` column. This can be null. If it is null, the input `subject_rows` is expected to have exactly 1 row. @@ -144,6 +144,7 @@ def _string_to_float(text: Union[str, float], error_message_prefix: str = None) """ Converts a string coming from a dataset.csv file to a floating point number, taking into account all the corner cases that can happen when the dataset file is malformed. + :param text: The element coming from the dataset.csv file. :param error_message_prefix: A prefix string that will go into the error message if the conversion fails. :return: A floating point number, possibly np.nan. @@ -181,27 +182,28 @@ def load_single_data_source(subject_rows: pd.DataFrame, """ Converts a set of dataset rows for a single subject to a ScalarDataSource instance, which contains the labels, the non-image features, and the paths to the image files. + :param num_classes: Number of classes, this is equivalent to model output tensor size :param channel_column: The name of the column that contains the row identifier ("channels") :param metadata_columns: A list of columns that well be added to the item metadata as key/value pairs. :param subject_rows: All dataset rows that belong to the same subject. :param subject_id: The identifier of the subject that is being processed. :param image_channels: The names of all channels (stored in the CSV_CHANNEL_HEADER column of the dataframe) - that are expected to be loaded from disk later because they are large images. + that are expected to be loaded from disk later because they are large images. :param image_file_column: The name of the column that contains the image file names. :param label_channels: The name of the channel where the label scalar or vector is read from. :param label_value_column: The column that contains the value for the label scalar or vector. :param non_image_feature_channels: non_image_feature_channels: A dictonary of the names of all channels where - additional scalar values should be read from. THe keys should map each feature to its channels. + additional scalar values should be read from. THe keys should map each feature to its channels. :param numerical_columns: The names of all columns where additional scalar values should be read from. :param categorical_data_encoder: Encoding scheme for categorical data. :param is_classification_dataset: If True, the dataset will be used in a classification model. If False, - assume that the dataset will be used in a regression model. + assume that the dataset will be used in a regression model. :param transform_labels: a label transformation or a list of label transformation to apply to the labels. - If a list is provided, the transformations are applied in order from left to right. + If a list is provided, the transformations are applied in order from left to right. :param sequence_position_numeric: Numeric position of the data source in a data sequence. Assumed to be - a non-sequential dataset item if None provided (default). - :return: + a non-sequential dataset item if None provided (default). + :return: A ScalarDataSource containing the specified data. """ def _get_row_for_channel(channel: Optional[str]) -> Dict[str, str]: @@ -234,6 +236,7 @@ def get_none_list_from_dict(non_image_channels: Dict[str, List[str]], feature: s """ Return either the list of channels for a given column or if None was passed as numerical channels i.e. there are no channel to be specified return [None]. + :param non_image_channels: Dict mapping features name to their channels :param feature: feature name for which to return the channels :return: List of channels for the given feature. @@ -359,7 +362,7 @@ def __init__(self, metadata.sequence_position. If this column name is not provided, the sequence_position will be 0. :param subject_column: The name of the column that contains the subject identifier :param channel_column: The name of the column that contains the row identifier ("channels") - that are expected to be loaded from disk later because they are large images. + that are expected to be loaded from disk later because they are large images. :param is_classification_dataset: If the current dataset is classification or not. :param categorical_data_encoder: Encoding scheme for categorical data. """ @@ -422,6 +425,7 @@ def load_data_sources_as_per_config(data_frame: pd.DataFrame, """ Loads dataset items from the given dataframe, where all column and channel configurations are taken from their respective model config elements. + :param data_frame: The dataframe to read dataset items from. :param args: The model configuration object. :return: A list of all dataset items that could be read from the dataframe. @@ -452,13 +456,13 @@ def load_data_sources_as_per_config(data_frame: pd.DataFrame, def load_data_sources(self, num_dataset_reader_workers: int = 0) -> List[ScalarDataSource]: """ Extracts information from a dataframe to create a list of ClassificationItem. This will create one entry per - unique - value of subject_id in the dataframe. The file is structured around "channels", indicated by specific values in - the CSV_CHANNEL_HEADER column. The result contains paths to image files, a label vector, and a matrix of - additional values that are specified by rows and columns given in non_image_feature_channels and + unique value of subject_id in the dataframe. The file is structured around "channels", indicated by specific + values in the CSV_CHANNEL_HEADER column. The result contains paths to image files, a label vector, and a matrix + of additional values that are specified by rows and columns given in non_image_feature_channels and numerical_columns. + :param num_dataset_reader_workers: Number of worker processes to use, if 0 then single threaded execution, - otherwise if -1 then multiprocessing with all available cpus will be used. + otherwise if -1 then multiprocessing with all available cpus will be used. :return: A list of ScalarDataSource or SequenceDataSource instances """ subject_ids = self.data_frame[self.subject_column].unique() @@ -512,9 +516,9 @@ def files_by_stem(root_path: Path) -> Dict[str, Path]: """ Lists all files under the given root directory recursively, and returns a mapping from file name stem to full path. The file name stem is computed more restrictively than what Path.stem returns: file.nii.gz will use "file" as the - stem, not "file.nii" as Path.stem would. - Only actual files are returned in the mapping, no directories. - If there are multiple files that map to the same stem, the function raises a ValueError. + stem, not "file.nii" as Path.stem would. Only actual files are returned in the mapping, no directories. If there are + multiple files that map to the same stem, the function raises a ValueError. + :param root_path: The root directory from which the file search should start. :return: A dictionary mapping from file name stem to the full path to where the file is found. """ @@ -546,11 +550,12 @@ def is_valid_item_index(item: ScalarDataSource, min_sequence_position_value: int = 0) -> bool: """ Returns True if the item metadata in metadata.sequence_position is a valid sequence index. + :param item: The item to check. :param min_sequence_position_value: Check if the item has a metadata.sequence_position that is at least - the value given here. Default is 0. + the value given here. Default is 0. :param max_sequence_position_value: If provided then this is the maximum sequence position the sequence can - end with. Longer sequences will be truncated. None is default. + end with. Longer sequences will be truncated. None is default. :return: True if the item has a valid index. """ # If no max_sequence_position_value is given, we don't care about @@ -572,9 +577,9 @@ def filter_valid_classification_data_sources_items(items: Iterable[ScalarDataSou :param items: The list of items to filter. :param min_sequence_position_value: Restrict the data to items with a metadata.sequence_position that is at least - the value given here. Default is 0. + the value given here. Default is 0. :param max_sequence_position_value: If provided then this is the maximum sequence position the sequence can - end with. Longer sequences will be truncated. None is default. + end with. Longer sequences will be truncated. None is default. :param file_to_path_mapping: A mapping from a file name stem (without extension) to its full path. :return: A list of items, all of which are valid now. """ @@ -671,7 +676,8 @@ def __init__(self, args: ScalarModelBase, name: Optional[str] = None, sample_transform: Callable[[ScalarItem], ScalarItem] = ScalarItemAugmentation()): """ - High level class for the scalar dataset designed to be inherited for specific behaviour + High level class for the scalar dataset designed to be inherited for specific behaviour. + :param args: The model configuration object. :param data_frame: The dataframe to read from. :param feature_statistics: If given, the normalization factor for the non-image features is taken @@ -691,7 +697,8 @@ def __init__(self, args: ScalarModelBase, def load_all_data_sources(self) -> List[ScalarDataSource]: """ Uses the dataframe to create data sources to be used by the dataset. - :return: + + :return: List of data sources. """ all_data_sources = DataSourceReader.load_data_sources_as_per_config(self.data_frame, self.args) # type: ignore self.status += f"Loading: {self.create_status_string(all_data_sources)}" @@ -722,6 +729,7 @@ def load_item(self, item: ScalarDataSource) -> ScalarItem: """ Loads the images and/or segmentations as given in the ClassificationDataSource item and applying the optional transformation specified by the class. + :param item: The item to load. :return: A ClassificationItem instances with the loaded images, and the labels and non-image features copied from the argument. @@ -738,6 +746,7 @@ def load_item(self, item: ScalarDataSource) -> ScalarItem: def create_status_string(self, items: List[ScalarDataSource]) -> str: """ Creates a human readable string that contains the number of items, and the distinct number of subjects. + :param items: Use the items provided to create the string :return: A string like "12 items for 5 subjects" """ @@ -757,6 +766,7 @@ def __init__(self, args: ScalarModelBase, sample_transform: Callable[[ScalarItem], ScalarItem] = ScalarItemAugmentation()): """ Creates a new scalar dataset from a dataframe. + :param args: The model configuration object. :param data_frame: The dataframe to read from. :param feature_statistics: If given, the normalization factor for the non-image features is taken @@ -802,6 +812,7 @@ def get_class_counts(self) -> Dict[int, int]: one class index. The value stored will be the number of samples that belong to the positive class. In the multilabel case, this returns a dictionary with class indices and samples per class as the key-value pairs. + :return: Dictionary of {class_index: count} """ all_labels = [torch.flatten(torch.nonzero(item.label).int()).tolist() for item in self.items] # [N, 1] diff --git a/InnerEye/ML/dataset/scalar_sample.py b/InnerEye/ML/dataset/scalar_sample.py index 6df95eb3d..ef2e404a4 100644 --- a/InnerEye/ML/dataset/scalar_sample.py +++ b/InnerEye/ML/dataset/scalar_sample.py @@ -39,7 +39,6 @@ def __post_init__(self) -> None: def id(self) -> str: """ Gets the identifier of the present object from metadata. - :return: """ return self.metadata.id # type: ignore @@ -47,7 +46,6 @@ def id(self) -> str: def props(self) -> Dict[str, Any]: """ Gets the general metadata dictionary for the present object. - :return: """ return self.metadata.props # type: ignore @@ -94,6 +92,7 @@ def to_device(self, device: Any) -> ScalarItem: """ Creates a copy of the present object where all tensors live on the given CUDA device. The metadata field is left unchanged. + :param device: The CUDA or GPU device to move to. :return: A new `ScalarItem` with all tensors on the chosen device. """ @@ -124,14 +123,15 @@ def load_images(self, root_path argument, or it must contain a file name stem only (without extension). In this case, the actual mapping from file name stem to full path is expected in the file_mapping argument. Either of 'root_path' or 'file_mapping' must be provided. + :param root_path: The root path where all channel files for images are expected. This is ignored if - file_mapping is given. + file_mapping is given. :param file_mapping: A mapping from a file name stem (without extension) to its full path. :param load_segmentation: If True it loads segmentation if present on the same file as the image. :param center_crop_size: If supplied, all loaded images will be cropped to the size given here. The crop will - be taken from the center of the image. + be taken from the center of the image. :param image_size: If given, all loaded images will be reshaped to the size given here, prior to the - center crop. + center crop. :return: An instance of ClassificationItem, with the same label and numerical_non_image_features fields, and all images loaded. """ @@ -156,6 +156,7 @@ def is_valid(self) -> bool: """ Checks if all file paths and non-image features are present in the object. All image channel files must be not None, and none of the non imaging features may be NaN or infinity. + :return: True if channel files is a list with not-None entries, and all non imaging features are finite floating point numbers. """ @@ -169,8 +170,9 @@ def get_all_image_filepaths(self, file_mapping: Optional[Dict[str, Path]]) -> List[Path]: """ Get a list of image paths for the object. Either root_path or file_mapping must be specified. + :param root_path: The root path where all channel files for images are expected. This is ignored if - file_mapping is given. + file_mapping is given. :param file_mapping: A mapping from a file name stem (without extension) to its full path. """ full_channel_files: List[Path] = [] @@ -188,9 +190,10 @@ def get_full_image_filepath(file: str, """ Get the full path of an image file given the path relative to the dataset folder and one of root_path or file_mapping. + :param file: Image filepath relative to the dataset folder :param root_path: The root path where all channel files for images are expected. This is ignored if - file_mapping is given. + file_mapping is given. :param file_mapping: A mapping from a file name stem (without extension) to its full path. """ if file is None: diff --git a/docs/source/rst/api/ML/dataset/index.rst b/docs/source/rst/api/ML/dataset/index.rst index e69de29bb..a0be27b37 100644 --- a/docs/source/rst/api/ML/dataset/index.rst +++ b/docs/source/rst/api/ML/dataset/index.rst @@ -0,0 +1,21 @@ +Dataset +======= + +Full Datasets +-------------- + +.. automodule:: InnerEye.ML.dataset.full_image_dataset + +.. automodule:: InnerEye.ML.dataset.sample + +Scalar Datasets +--------------- + +.. automodule:: InnerEye.ML.dataset.scalar_dataset + +.. automodule:: InnerEye.ML.dataset.scalar_sample + +CroppingDataset +--------------- +.. automodule:: InnerEye.ML.dataset.cropping_dataset + :special-members: __getitem__ diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst index 6d54375d1..ed8684efe 100644 --- a/docs/source/rst/api/ML/index.rst +++ b/docs/source/rst/api/ML/index.rst @@ -5,6 +5,7 @@ Machine learning configs/index augmentations/index + dataset/index runner photometric_normalization pipelines From e9fff6a9432c1334de1ab52bd8f1744e44836160 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 12:02:32 +0100 Subject: [PATCH 06/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20rst=20skeleton=20for?= =?UTF-8?q?=20ML/models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/ML/configs/index.rst | 2 ++ docs/source/rst/api/ML/index.rst | 3 ++- docs/source/rst/api/ML/models/architectures/index.rst | 0 docs/source/rst/api/ML/models/blocks/index.rst | 0 docs/source/rst/api/ML/models/index.rst | 11 +++++++++++ docs/source/rst/api/ML/models/layers/index.rst | 0 docs/source/rst/api/ML/models/losses/index.rst | 0 docs/source/rst/api/ML/models/parallel/index.rst | 0 8 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/source/rst/api/ML/models/architectures/index.rst create mode 100644 docs/source/rst/api/ML/models/blocks/index.rst create mode 100644 docs/source/rst/api/ML/models/layers/index.rst create mode 100644 docs/source/rst/api/ML/models/losses/index.rst create mode 100644 docs/source/rst/api/ML/models/parallel/index.rst diff --git a/docs/source/rst/api/ML/configs/index.rst b/docs/source/rst/api/ML/configs/index.rst index fc7c85afe..e592f92a1 100644 --- a/docs/source/rst/api/ML/configs/index.rst +++ b/docs/source/rst/api/ML/configs/index.rst @@ -1,6 +1,8 @@ Configurations ============== +The following is a list of the pre-defined configurations that can be used or inherited and adapted to your own purposes. + .. toctree:: classification/index diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst index ed8684efe..4c5d53a7c 100644 --- a/docs/source/rst/api/ML/index.rst +++ b/docs/source/rst/api/ML/index.rst @@ -4,8 +4,9 @@ Machine learning .. toctree:: configs/index - augmentations/index + models/index dataset/index + augmentations/index runner photometric_normalization pipelines diff --git a/docs/source/rst/api/ML/models/architectures/index.rst b/docs/source/rst/api/ML/models/architectures/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/models/blocks/index.rst b/docs/source/rst/api/ML/models/blocks/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/models/index.rst b/docs/source/rst/api/ML/models/index.rst index e69de29bb..d5432a1d3 100644 --- a/docs/source/rst/api/ML/models/index.rst +++ b/docs/source/rst/api/ML/models/index.rst @@ -0,0 +1,11 @@ +Models +====== + +Below is a list of models and various deep learning components that are available within InnerEye-DeepLearning. + +.. toctree:: + architectures/index + blocks/index + layers/index + losses/index + parallel/index diff --git a/docs/source/rst/api/ML/models/layers/index.rst b/docs/source/rst/api/ML/models/layers/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/models/losses/index.rst b/docs/source/rst/api/ML/models/losses/index.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/rst/api/ML/models/parallel/index.rst b/docs/source/rst/api/ML/models/parallel/index.rst new file mode 100644 index 000000000..e69de29bb From 4fd8de4716af0f66e2d2f534e28ce80874ed96e2 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 15:13:36 +0100 Subject: [PATCH 07/27] =?UTF-8?q?=F0=9F=93=9D=20Fix=20docstring=20missing?= =?UTF-8?q?=20newlines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Azure/azure_config.py | 2 + InnerEye/Azure/azure_runner.py | 9 +++++ InnerEye/Azure/azure_util.py | 13 +++++++ InnerEye/Azure/parser_util.py | 3 ++ InnerEye/Azure/run_pytest.py | 2 + InnerEye/Azure/secrets_handling.py | 7 ++++ InnerEye/Azure/tensorboard_monitor.py | 2 + .../Common/Statistics/mann_whitney_test.py | 3 ++ .../Statistics/report_structure_extremes.py | 1 + .../Statistics/wilcoxon_signed_rank_test.py | 5 +++ InnerEye/Common/common_util.py | 9 +++++ InnerEye/Common/fixed_paths.py | 2 + InnerEye/Common/generic_parsing.py | 8 ++++ InnerEye/Common/output_directories.py | 2 + InnerEye/Common/resource_monitor.py | 7 ++++ InnerEye/Common/spawn_subprocess.py | 1 + .../datamodules_and_datasets/cxr_datasets.py | 1 + .../datamodules_and_datasets/datamodules.py | 3 ++ .../transforms_utils.py | 2 + InnerEye/ML/SSL/encoders.py | 1 + .../SSL/lightning_containers/ssl_container.py | 3 ++ .../SSL/lightning_modules/byol/byol_module.py | 5 +++ .../ML/SSL/lightning_modules/simclr_module.py | 1 + .../lightning_modules/ssl_online_evaluator.py | 2 + InnerEye/ML/SSL/utils.py | 1 + InnerEye/ML/augmentations/image_transforms.py | 1 + .../ML/augmentations/transform_pipeline.py | 3 ++ InnerEye/ML/baselines_util.py | 2 + InnerEye/ML/common.py | 1 + InnerEye/ML/configs/other/HelloContainer.py | 7 ++++ .../configs/segmentation/HeadAndNeckBase.py | 5 +++ .../configs/segmentation/HeadAndNeckPaper.py | 1 + .../ML/configs/segmentation/ProstateBase.py | 6 +++ InnerEye/ML/dataset/cropping_dataset.py | 1 + InnerEye/ML/dataset/scalar_dataset.py | 15 +++++++ InnerEye/ML/dataset/scalar_sample.py | 4 ++ InnerEye/ML/deep_learning_config.py | 9 +++++ InnerEye/ML/lightning_base.py | 6 +++ InnerEye/ML/lightning_container.py | 6 +++ InnerEye/ML/lightning_helpers.py | 3 ++ InnerEye/ML/lightning_loggers.py | 6 +++ InnerEye/ML/lightning_metrics.py | 4 ++ InnerEye/ML/lightning_models.py | 11 ++++++ InnerEye/ML/metrics.py | 8 ++++ InnerEye/ML/metrics_dict.py | 39 +++++++++++++++++++ InnerEye/ML/model_config_base.py | 5 +++ InnerEye/ML/model_testing.py | 14 +++++++ InnerEye/ML/model_training.py | 4 ++ .../ML/models/architectures/base_model.py | 8 ++++ .../classification/image_encoder_with_mlp.py | 15 +++++++ .../classification/segmentation_encoder.py | 3 ++ InnerEye/ML/models/architectures/complex.py | 2 + InnerEye/ML/models/architectures/mlp.py | 1 + InnerEye/ML/models/architectures/unet_2d.py | 3 ++ InnerEye/ML/models/architectures/unet_3d.py | 11 ++++++ InnerEye/ML/models/layers/basic.py | 3 ++ InnerEye/ML/models/losses/cross_entropy.py | 5 +++ InnerEye/ML/models/losses/mixture.py | 2 + InnerEye/ML/models/losses/soft_dice.py | 1 + InnerEye/ML/models/parallel/model_parallel.py | 3 ++ InnerEye/ML/photometric_normalization.py | 3 ++ InnerEye/ML/pipelines/ensemble.py | 3 ++ InnerEye/ML/pipelines/inference.py | 3 ++ InnerEye/ML/pipelines/scalar_inference.py | 2 + InnerEye/ML/plotting.py | 20 ++++++++++ .../classification_multilabel_report.py | 3 ++ InnerEye/ML/reports/classification_report.py | 37 ++++++++++++++++++ InnerEye/ML/reports/notebook_report.py | 5 +++ InnerEye/ML/reports/segmentation_report.py | 4 ++ InnerEye/ML/run_ml.py | 15 +++++++ InnerEye/ML/runner.py | 4 ++ InnerEye/ML/scalar_config.py | 3 ++ InnerEye/ML/surface_distance_heatmaps.py | 1 + InnerEye/ML/utils/checkpoint_handling.py | 9 +++++ InnerEye/ML/utils/config_loader.py | 1 + InnerEye/ML/utils/csv_util.py | 6 +++ InnerEye/ML/utils/device_aware_module.py | 1 + InnerEye/ML/utils/hdf5_util.py | 2 + InnerEye/ML/utils/image_util.py | 10 +++++ InnerEye/ML/utils/io_util.py | 11 ++++++ InnerEye/ML/utils/layer_util.py | 5 +++ InnerEye/ML/utils/metrics_util.py | 3 ++ InnerEye/ML/utils/ml_util.py | 5 +++ InnerEye/ML/utils/model_metadata_util.py | 2 + InnerEye/ML/utils/model_util.py | 4 ++ InnerEye/ML/utils/plotting_util.py | 2 + InnerEye/ML/utils/sequence_utils.py | 3 ++ InnerEye/ML/utils/split_dataset.py | 3 ++ InnerEye/ML/utils/supervised_criterion.py | 2 + InnerEye/ML/utils/surface_distance_utils.py | 11 ++++++ InnerEye/ML/utils/temperature_scaling.py | 1 + InnerEye/ML/utils/transforms.py | 1 + InnerEye/ML/visualizers/activation_maps.py | 4 ++ .../ML/visualizers/metrics_scatterplot.py | 1 + InnerEye/ML/visualizers/model_hooks.py | 2 + InnerEye/ML/visualizers/model_summary.py | 7 ++++ InnerEye/ML/visualizers/patch_sampling.py | 2 + .../ML/visualizers/plot_cross_validation.py | 20 ++++++++++ .../visualizers/regression_visualization.py | 1 + InnerEye/ML/visualizers/reliability_curve.py | 2 + .../Scripts/download_model_and_run_scoring.py | 1 + InnerEye/Scripts/move_model.py | 5 +++ InnerEye/Scripts/prepare_fastmri.py | 6 +++ InnerEye/Scripts/submit_for_inference.py | 5 +++ Tests/AfterTraining/test_after_training.py | 5 +++ Tests/Azure/test_secrets_handling.py | 1 + Tests/Common/test_segmentation_configs.py | 3 ++ .../models/architectures/DummyScalarModel.py | 1 + .../architectures/DummyScalarModel2D.py | 1 + Tests/ML/pipelines/test_scalar_inference.py | 1 + Tests/ML/test_lightning_containers.py | 1 + Tests/ML/test_model_testing.py | 1 + Tests/ML/test_regression_tests.py | 2 + Tests/ML/util.py | 5 +++ Tests/ML/utils/test_checkpoint_handling.py | 1 + Tests/ML/utils/test_io_util.py | 1 + Tests/ML/utils/test_model_util.py | 1 + .../visualizers/test_plot_cross_validation.py | 2 + .../ML/visualizers/test_visualize_patches.py | 2 + Tests/SSL/test_ssl_containers.py | 2 + TestsOutsidePackage/test_register_model.py | 1 + mypy_runner.py | 1 + rewrite_params.py | 20 ++++++++++ score.py | 3 ++ 124 files changed, 609 insertions(+) create mode 100644 rewrite_params.py diff --git a/InnerEye/Azure/azure_config.py b/InnerEye/Azure/azure_config.py index d3902fb95..366536b07 100755 --- a/InnerEye/Azure/azure_config.py +++ b/InnerEye/Azure/azure_config.py @@ -168,6 +168,7 @@ def from_yaml(yaml_file_path: Path, project_root: Optional[Path]) -> AzureConfig """ Creates an AzureConfig object with default values, with the keys/secrets populated from values in the given YAML file. If a `project_root` folder is provided, a private settings file is read from there as well. + :param yaml_file_path: Path to the YAML file that contains values to create the AzureConfig :param project_root: A folder in which to search for a private settings file. :return: AzureConfig with values populated from the yaml files. @@ -231,6 +232,7 @@ def get_service_principal_auth(self) -> Optional[Union[InteractiveLoginAuthentic def fetch_run(self, run_recovery_id: str) -> Run: """ Gets an instantiated Run object for a given run recovery ID (format experiment_name:run_id). + :param run_recovery_id: A run recovery ID (format experiment_name:run_id) """ return fetch_run(workspace=self.get_workspace(), run_recovery_id=run_recovery_id) diff --git a/InnerEye/Azure/azure_runner.py b/InnerEye/Azure/azure_runner.py index 6c7d58d60..3a6623a9e 100644 --- a/InnerEye/Azure/azure_runner.py +++ b/InnerEye/Azure/azure_runner.py @@ -39,6 +39,7 @@ def get_git_tags(azure_config: AzureConfig) -> Dict[str, str]: Creates a dictionary with git-related information, like branch and commit ID. The dictionary key is a string that can be used as a tag on an AzureML run, the dictionary value is the git information. If git information is passed in via commandline arguments, those take precedence over information read out from the repository. + :param azure_config: An AzureConfig object specifying git-related commandline args. :return: A dictionary mapping from tag name to git info. """ @@ -56,6 +57,7 @@ def get_git_tags(azure_config: AzureConfig) -> Dict[str, str]: def additional_run_tags(azure_config: AzureConfig, commandline_args: str) -> Dict[str, str]: """ Gets the set of tags that will be added to the AzureML run as metadata, like git status and user name. + :param azure_config: The configurations for the present AzureML job :param commandline_args: A string that holds all commandline arguments that were used for the present run. """ @@ -77,6 +79,7 @@ def additional_run_tags(azure_config: AzureConfig, commandline_args: str) -> Dic def create_experiment_name(azure_config: AzureConfig) -> str: """ Gets the name of the AzureML experiment. This is taken from the commandline, or from the git branch. + :param azure_config: The object containing all Azure-related settings. :return: The name to use for the AzureML experiment. """ @@ -147,6 +150,7 @@ def create_runner_parser(model_config_class: type = None) -> argparse.ArgumentPa """ Creates a commandline parser, that understands all necessary arguments for running a script in Azure, plus all arguments for the given class. The class must be a subclass of GenericConfig. + :param model_config_class: A class that contains the model-specific parameters. :return: An instance of ArgumentParser. """ @@ -167,6 +171,7 @@ def parse_args_and_add_yaml_variables(parser: ArgumentParser, """ Reads arguments from sys.argv, modifies them with secrets from local YAML files, and parses them using the given argument parser. + :param project_root: The root folder for the whole project. Only used to access a private settings file. :param parser: The parser to use. :param yaml_config_file: The path to the YAML file that contains values to supply into sys.argv. @@ -183,6 +188,7 @@ def parse_args_and_add_yaml_variables(parser: ArgumentParser, def _create_default_namespace(parser: ArgumentParser) -> Namespace: """ Creates an argparse Namespace with all parser-specific default values set. + :param parser: The parser to work with. :return: """ @@ -207,10 +213,12 @@ def parse_arguments(parser: ArgumentParser, Parses a list of commandline arguments with a given parser, and adds additional information read from YAML files. Returns results broken down into a full arguments dictionary, a dictionary of arguments that were set to non-default values, and unknown arguments. + :param parser: The parser to use :param settings_from_yaml: A dictionary of settings read from a YAML config file. :param fail_on_unknown_args: If True, raise an exception if the parser encounters an argument that it does not recognize. If False, unrecognized arguments will be ignored, and added to the "unknown" field of the parser result. + :param args: Arguments to parse. If not given, use those in sys.argv :return: The parsed arguments, and overrides """ @@ -261,6 +269,7 @@ def run_duration_string_to_seconds(s: str) -> Optional[int]: Parse a string that represents a timespan, and returns it converted into seconds. The string is expected to be floating point number with a single character suffix s, m, h, d for seconds, minutes, hours, day. Examples: '3.5h', '2d'. If the argument is an empty string, None is returned. + :param s: The string to parse. :return: The timespan represented in the string converted to seconds. """ diff --git a/InnerEye/Azure/azure_util.py b/InnerEye/Azure/azure_util.py index 72bbba201..18b516a0e 100644 --- a/InnerEye/Azure/azure_util.py +++ b/InnerEye/Azure/azure_util.py @@ -47,6 +47,7 @@ def split_recovery_id(id: str) -> Tuple[str, str]: The argument can be in the format 'experiment_name:run_id', or just a run ID like user_branch_abcde12_123. In the latter case, everything before the last two alphanumeric parts is assumed to be the experiment name. + :param id: :return: experiment name and run name """ @@ -74,6 +75,7 @@ def fetch_run(workspace: Workspace, run_recovery_id: str) -> Run: Finds an existing run in an experiment, based on a recovery ID that contains the experiment ID and the actual RunId. The run can be specified either in the experiment_name:run_id format, or just the run_id. + :param workspace: the configured AzureML workspace to search for the experiment. :param run_recovery_id: The Run to find. Either in the full recovery ID format, experiment_name:run_id or just the run_id @@ -85,6 +87,7 @@ def fetch_run(workspace: Workspace, run_recovery_id: str) -> Run: def fetch_runs(experiment: Experiment, filters: List[str]) -> List[Run]: """ Fetch the runs in an experiment. + :param experiment: the experiment to fetch runs from :param filters: a list of run status to include. Must be subset of [Running, Completed, Failed, Canceled]. :return: the list of runs in the experiment @@ -107,6 +110,7 @@ def fetch_child_runs( """ Fetch child runs for the provided runs that have the provided AML status (or fetch all by default) and have a run_recovery_id tag value set (this is to ignore superfluous AML infrastructure platform runs). + :param run: parent run to fetch child run from :param status: if provided, returns only child runs with this status :param expected_number_cross_validation_splits: when recovering child runs from AML hyperdrive @@ -159,6 +163,7 @@ def to_azure_friendly_string(x: Optional[str]) -> Optional[str]: def to_azure_friendly_container_path(path: Path) -> str: """ Converts a path an Azure friendly container path by replacing "\\", "//" with "/" so it can be in the form: a/b/c. + :param path: Original path :return: Converted path """ @@ -168,6 +173,7 @@ def to_azure_friendly_container_path(path: Path) -> str: def is_offline_run_context(run_context: Run) -> bool: """ Tells if a run_context is offline by checking if it has an experiment associated with it. + :param run_context: Context of the run to check :return: """ @@ -177,6 +183,7 @@ def is_offline_run_context(run_context: Run) -> bool: def get_run_context_or_default(run: Optional[Run] = None) -> Run: """ Returns the context of the run, if run is not None. If run is None, returns the context of the current run. + :param run: Run to retrieve context for. If None, retrieve ocntext of current run. :return: Run context """ @@ -186,6 +193,7 @@ def get_run_context_or_default(run: Optional[Run] = None) -> Run: def get_cross_validation_split_index(run: Run) -> int: """ Gets the cross validation index from the run's tags or returns the default + :param run: Run context from which to get index :return: The cross validation split index """ @@ -204,6 +212,7 @@ def is_cross_validation_child_run(run: Run) -> bool: """ Checks the provided run's tags to determine if it is a cross validation child run (which is the case if the split index >=0) + :param run: Run to check. :return: True if cross validation run. False otherwise. """ @@ -213,6 +222,7 @@ def is_cross_validation_child_run(run: Run) -> bool: def strip_prefix(string: str, prefix: str) -> str: """ Returns the string without the prefix if it has the prefix, otherwise the string unchanged. + :param string: Input string. :param prefix: Prefix to remove from input string. :return: Input string with prefix removed. @@ -226,6 +236,7 @@ def get_all_environment_files(project_root: Path) -> List[Path]: """ Returns a list of all Conda environment files that should be used. This is firstly the InnerEye conda file, and possibly a second environment.yml file that lives at the project root folder. + :param project_root: The root folder of the code that starts the present training run. :return: A list with 1 or 2 entries that are conda environment files. """ @@ -260,6 +271,7 @@ def download_run_output_file(blob_path: Path, destination: Path, run: Run) -> Pa Downloads a single file from the run's default output directory: DEFAULT_AML_UPLOAD_DIR ("outputs"). For example, if blobs_path = "foo/bar.csv", then the run result file "outputs/foo/bar.csv" will be downloaded to /bar.csv (the directory will be stripped off). + :param blob_path: The name of the file to download. :param run: The AzureML run to download the files from :param destination: Local path to save the downloaded blob to. @@ -287,6 +299,7 @@ def download_run_outputs_by_prefix( have a given prefix (folder structure). When saving, the prefix string will be stripped off. For example, if blobs_prefix = "foo", and the run has a file "outputs/foo/bar.csv", it will be downloaded to destination/bar.csv. If there is in addition a file "foo.txt", that file will be skipped. + :param blobs_prefix: The prefix for all files in "outputs" that should be downloaded. :param run: The AzureML run to download the files from. :param destination: Local path to save the downloaded blobs to. diff --git a/InnerEye/Azure/parser_util.py b/InnerEye/Azure/parser_util.py index 77d33f386..af37e2aae 100644 --- a/InnerEye/Azure/parser_util.py +++ b/InnerEye/Azure/parser_util.py @@ -9,6 +9,7 @@ def _is_empty(item: Any) -> bool: """ Returns True if the argument has length 0. + :param item: Object to check. :return: True if the argument has length 0. False otherwise. """ @@ -18,6 +19,7 @@ def _is_empty(item: Any) -> bool: def _is_empty_or_empty_string_list(item: Any) -> bool: """ Returns True if the argument has length 0, or a list with a single element that has length 0. + :param item: Object to check. :return: True if argument has length 0, or a list with a single element that has length 0. False otherwise. """ @@ -32,6 +34,7 @@ def value_to_string(x: object) -> str: """ Returns a string representation of x, with special treatment of Enums (return their value) and lists (return comma-separated list). + :param x: Object to convert to string :return: The string representation of the object. Special cases: For Enums, returns their value, for lists, returns a comma-separated list. diff --git a/InnerEye/Azure/run_pytest.py b/InnerEye/Azure/run_pytest.py index be8ac24f8..271877208 100644 --- a/InnerEye/Azure/run_pytest.py +++ b/InnerEye/Azure/run_pytest.py @@ -19,6 +19,7 @@ def run_pytest(pytest_mark: str, outputs_folder: Path) -> Tuple[bool, Path]: """ Runs pytest on the whole test suite, restricting to the tests that have the given PyTest mark. + :param pytest_mark: The PyTest mark to use for filtering out the tests to run. :param outputs_folder: The folder into which the test result XML file should be written. :return: True if PyTest found tests to execute and completed successfully, False otherwise. @@ -43,6 +44,7 @@ def download_pytest_result(run: Run, destination_folder: Path = Path.cwd()) -> P """ Downloads the pytest result file that is stored in the output folder of the given AzureML run. If there is no pytest result file, throw an Exception. + :param run: The run from which the files should be read. :param destination_folder: The folder into which the PyTest result file is downloaded. :return: The path (folder and filename) of the downloaded file. diff --git a/InnerEye/Azure/secrets_handling.py b/InnerEye/Azure/secrets_handling.py index 0907fdd83..20a2f7e62 100644 --- a/InnerEye/Azure/secrets_handling.py +++ b/InnerEye/Azure/secrets_handling.py @@ -28,6 +28,7 @@ class SecretsHandling: def __init__(self, project_root: Path) -> None: """ Creates a new instance of the class. + :param project_root: The root folder of the project that starts the InnerEye run. """ self.project_root = project_root @@ -36,6 +37,7 @@ def read_secrets_from_file(self, secrets_to_read: List[str]) -> Optional[Dict[st """ Reads the secrets from file in YAML format, and returns the contents as a dictionary. The YAML file is expected in the project root directory. + :param secrets_to_read: The list of secret names to read from the YAML file. These will be converted to uppercase. :return: A dictionary with secrets, or None if the file does not exist. @@ -57,6 +59,7 @@ def get_secrets_from_environment_or_file(self, secrets_to_read: List[str]) -> Di Attempts to read secrets from the project secret file. If there is no secrets file, it returns all secrets in secrets_to_read read from environment variables. When reading from environment, if an expected secret is not found, its value will be None. + :param secrets_to_read: The list of secret names to read from the YAML file. These will be converted to uppercase. """ @@ -69,6 +72,7 @@ def get_secrets_from_environment_or_file(self, secrets_to_read: List[str]) -> Di def get_secret_from_environment(self, name: str, allow_missing: bool = False) -> Optional[str]: """ Gets a password or key from the secrets file or environment variables. + :param name: The name of the environment variable to read. It will be converted to uppercase. :param allow_missing: If true, the function returns None if there is no entry of the given name in any of the places searched. If false, missing entries will raise a ValueError. @@ -99,6 +103,7 @@ def read_all_settings(project_settings_file: Optional[Path] = None, the `project_root` folder. Settings in the private settings file override those in the project settings. Both settings files are expected in YAML format, with an entry called 'variables'. + :param project_settings_file: The first YAML settings file to read. :param project_root: The folder that can contain a 'InnerEyePrivateSettings.yml' file. :return: A dictionary mapping from string to variable value. The dictionary key is the union of variable names @@ -117,6 +122,7 @@ def read_settings_and_merge(project_settings_file: Optional[Path] = None, file is read into a dictionary, then the private settings file is read. Settings in the private settings file override those in the project settings. Both settings files are expected in YAML format, with an entry called 'variables'. + :param project_settings_file: The first YAML settings file to read. :param private_settings_file: The second YAML settings file to read. Settings in this file has higher priority. :return: A dictionary mapping from string to variable value. The dictionary key is the union of variable names @@ -138,6 +144,7 @@ def read_settings_yaml_file(yaml_file: Path) -> Dict[str, Any]: """ Reads a YAML file, that is expected to contain an entry 'variables'. Returns the dictionary for the 'variables' section of the file. + :param yaml_file: The yaml file to read. :return: A dictionary with the variables from the yaml file. """ diff --git a/InnerEye/Azure/tensorboard_monitor.py b/InnerEye/Azure/tensorboard_monitor.py index 62fcf9647..9d14cc0b1 100644 --- a/InnerEye/Azure/tensorboard_monitor.py +++ b/InnerEye/Azure/tensorboard_monitor.py @@ -45,6 +45,7 @@ def validate(self) -> None: def monitor(monitor_config: AMLTensorBoardMonitorConfig, azure_config: AzureConfig) -> None: """ Starts TensorBoard monitoring as per the provided arguments. + :param monitor_config: The config containing information on which runs that need be monitored. :param azure_config: An AzureConfig object with secrets/keys to access the workspace. """ @@ -93,6 +94,7 @@ def main(settings_yaml_file: Optional[Path] = None, """ Parses the commandline arguments, and based on those, starts the Tensorboard monitoring for the AzureML runs supplied on the commandline. + :param settings_yaml_file: The YAML file that contains all information for accessing Azure. :param project_root: The root folder that contains all code for the present run. This is only used to locate a private settings file InnerEyePrivateSettings.yml. diff --git a/InnerEye/Common/Statistics/mann_whitney_test.py b/InnerEye/Common/Statistics/mann_whitney_test.py index 7db0a79e0..3eec955f0 100644 --- a/InnerEye/Common/Statistics/mann_whitney_test.py +++ b/InnerEye/Common/Statistics/mann_whitney_test.py @@ -88,6 +88,7 @@ def compose_distribution_comparisons(file_contents: List[List[List[str]]]) -> List[str]: """ Composes comparisons as detailed above. + :param file_contents: two or more lists of rows, where each "rows" is returned by read_csv_file on (typically) a statistics.csv file :return a list of lines to print @@ -121,6 +122,7 @@ def mann_whitney_on_key(key: str, lists: List[List[float]]) -> List[Tuple[Tuple[ Applies Mann-Whitney test to all sets of values (in lists) for the given key, and return a line of results, paired with some values for ordering purposes. Member lists with fewer than MINIMUM_INSTANCE_COUNT items are discarded. + :param key: statistic name; "Vol" statistics have mm^3 replaced by cm^3 for convenience. :param lists: list of lists of values """ @@ -256,6 +258,7 @@ def read_csv_file(input_file: str) -> List[List[str]]: """ Reads and returns the contents of a csv file. Empty rows (which can result from end-of-line mismatches) are dropped. + :param input_file: path to a file in csv format :return: list of rows from the file """ diff --git a/InnerEye/Common/Statistics/report_structure_extremes.py b/InnerEye/Common/Statistics/report_structure_extremes.py index 10740d04b..61450d3ae 100644 --- a/InnerEye/Common/Statistics/report_structure_extremes.py +++ b/InnerEye/Common/Statistics/report_structure_extremes.py @@ -72,6 +72,7 @@ def report_structure_extremes(dataset_dir: str, azure_config: AzureConfig) -> No Writes structure-extreme lines for the subjects in a directory. If there are any structures with missing slices, a ValueError is raised after writing all the lines. This allows a build failure to be triggered when such structures exist. + :param azure_config: An object with all necessary information for accessing Azure. :param dataset_dir: directory containing subject subdirectories with integer names. """ diff --git a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py index f2ed99b19..a9cb0f074 100644 --- a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py +++ b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py @@ -97,6 +97,7 @@ class WilcoxonTestConfig(GenericConfig): def calculate_statistics(dist1: Dict[str, float], dist2: Dict[str, float], factor: float) -> Dict[str, float]: """ Select common pairs and run the hypothesis test. + :param dist1: mapping from keys to scores :param dist2: mapping from keys (some or all in common with dist1) to scores :param factor: factor to divide the Wilcoxon "z" value by to determine p-value @@ -131,6 +132,7 @@ def difference_counts(values1: List[float], values2: List[float]) -> Tuple[int, """ Returns the number of corresponding pairs from vals1 and vals2 in which val1 > val2 and val2 > val1 respectively. + :param values1: list of values :param values2: list of values, same length as values1 :return: number of pairs in which first value is greater than second, and number of pairs @@ -159,6 +161,7 @@ def evaluate_data_pair(data1: Dict[str, Dict[str, float]], data2: Dict[str, Dict -> Dict[str, Dict[str, float]]: """ Find and compare dice scores for each structure + :param data1: dictionary from structure names, to dictionary from subjects to scores :param data2: another such dictionary, sharing some structure names :param is_raw_p_value: whether to use "raw" Wilcoxon z values when calculating p values (rather than reduce) @@ -240,6 +243,7 @@ def wilcoxon_signed_rank_test(args: WilcoxonTestConfig, """ Reads data from a csv file, and performs all pairwise comparisons, except if --against was specified, compare every other run against the "--against" run. + :param args: parsed command line parameters :param name_shortener: optional function to shorten names to make graphs and tables more legible """ @@ -262,6 +266,7 @@ def run_wilcoxon_test_on_data(data: Dict[str, Dict[str, Dict[str, float]]], """ Performs all pairwise comparisons on the provided data, except if "against" was specified, compare every other run against the "against" run. + :param data: scores such that data[run][structure][subject] = dice score :param against: runs to compare against; or None to compare all against all :param raw: whether to interpret Wilcoxon Z values "raw" or apply a correction diff --git a/InnerEye/Common/common_util.py b/InnerEye/Common/common_util.py index aaf49e72e..383d31ce5 100644 --- a/InnerEye/Common/common_util.py +++ b/InnerEye/Common/common_util.py @@ -74,6 +74,7 @@ def get_best_epoch_results_path(mode: ModelExecutionMode, """ For a given model execution mode, creates the relative results path in the form BEST_EPOCH_FOLDER_NAME/(Train, Test or Val) + :param mode: model execution mode :param model_proc: whether this is for an ensemble or single model. If ensemble, we return a different path to avoid colliding with the results from the single model that may have been created earlier in the same run. @@ -108,6 +109,7 @@ def any_pairwise_larger(items1: Any, items2: Any) -> bool: def check_is_any_of(message: str, actual: Optional[str], valid: Iterable[Optional[str]]) -> None: """ Raises an exception if 'actual' is not any of the given valid values. + :param message: The prefix for the error message. :param actual: The actual value. :param valid: The set of valid strings that 'actual' is allowed to take on. @@ -136,6 +138,7 @@ def logging_to_stdout(log_level: Union[int, str] = logging.INFO) -> None: """ Instructs the Python logging libraries to start writing logs to stdout up to the given logging level. Logging will use a timestamp as the prefix, using UTC. + :param log_level: The logging level. All logging message with a level at or above this level will be written to stdout. log_level can be numeric, or one of the pre-defined logging strings (INFO, DEBUG, ...). """ @@ -186,6 +189,7 @@ def logging_to_file(file_path: Path) -> None: Instructs the Python logging libraries to start writing logs to the given file. Logging will use a timestamp as the prefix, using UTC. The logging level will be the same as defined for logging to stdout. + :param file_path: The path and name of the file to write to. """ # This function can be called multiple times, and should only add a handler during the first call. @@ -219,6 +223,7 @@ def logging_only_to_file(file_path: Path, stdout_log_level: Union[int, str] = lo Redirects logging to the specified file, undoing that on exit. If logging is currently going to stdout, messages at level stdout_log_level or higher (typically ERROR) are also sent to stdout. Usage: with logging_only_to_file(my_log_path): do_stuff() + :param file_path: file to log to :param stdout_log_level: mininum level for messages to also go to stdout """ @@ -253,6 +258,7 @@ def logging_section(gerund: str) -> Generator: to help people locate particular sections. Usage: with logging_section("doing this and that"): do_this_and_that() + :param gerund: string expressing what happens in this section of the log. """ from time import time @@ -354,6 +360,7 @@ def is_gpu_tensor(data: Any) -> bool: def print_exception(ex: Exception, message: str, logger_fn: Callable = logging.error) -> None: """ Prints information about an exception, and the full traceback info. + :param ex: The exception that was caught. :param message: An additional prefix that is printed before the exception itself. :param logger_fn: The logging function to use for logging this exception @@ -366,6 +373,7 @@ def print_exception(ex: Exception, message: str, logger_fn: Callable = logging.e def namespace_to_path(namespace: str, root: PathOrString = repository_root_directory()) -> Path: """ Given a namespace (in form A.B.C) and an optional root directory R, create a path R/A/B/C + :param namespace: Namespace to convert to path :param root: Path to prefix (default is project root) :return: @@ -377,6 +385,7 @@ def path_to_namespace(path: Path, root: PathOrString = repository_root_directory """ Given a path (in form R/A/B/C) and an optional root directory R, create a namespace A.B.C. If root is provided, then path must be a relative child to it. + :param path: Path to convert to namespace :param root: Path prefix to remove from namespace (default is project root) :return: diff --git a/InnerEye/Common/fixed_paths.py b/InnerEye/Common/fixed_paths.py index b9d544d06..60f530263 100755 --- a/InnerEye/Common/fixed_paths.py +++ b/InnerEye/Common/fixed_paths.py @@ -14,6 +14,7 @@ def repository_root_directory(path: Optional[PathOrString] = None) -> Path: """ Gets the full path to the root directory that holds the present repository. + :param path: if provided, a relative path to append to the absolute path to the repository root. :return: The full path to the repository's root directory, with symlinks resolved if any. """ @@ -28,6 +29,7 @@ def repository_root_directory(path: Optional[PathOrString] = None) -> Path: def repository_parent_directory(path: Optional[PathOrString] = None) -> Path: """ Gets the full path to the parent directory that holds the present repository. + :param path: if provided, a relative path to append to the absolute path to the repository root. :return: The full path to the repository's root directory, with symlinks resolved if any. """ diff --git a/InnerEye/Common/generic_parsing.py b/InnerEye/Common/generic_parsing.py index 062c8269a..b4a97f2ed 100644 --- a/InnerEye/Common/generic_parsing.py +++ b/InnerEye/Common/generic_parsing.py @@ -99,9 +99,11 @@ class GenericConfig(param.Parameterized): def __init__(self, should_validate: bool = True, throw_if_unknown_param: bool = False, **params: Any): """ Instantiates the config class, ignoring parameters that are not overridable. + :param should_validate: If True, the validate() method is called directly after init. :param throw_if_unknown_param: If True, raise an error if the provided "params" contains any key that does not correspond to an attribute of the class. + :param params: Parameters to set. """ # check if illegal arguments are passed in @@ -157,6 +159,7 @@ def add_args(cls: Type[GenericConfig], """ Adds all overridable fields of the current class to the given argparser. Fields that are marked as readonly, constant or private are ignored. + :param parser: Parser to add properties to. """ @@ -165,6 +168,7 @@ def parse_bool(x: str) -> bool: Parse a string as a bool. Supported values are case insensitive and one of: 'on', 't', 'true', 'y', 'yes', '1' for True 'off', 'f', 'false', 'n', 'no', '0' for False. + :param x: string to test. :return: Bool value if string valid, otherwise a ValueError is raised. """ @@ -179,6 +183,7 @@ def _get_basic_type(_p: param.Parameter) -> Union[type, Callable]: """ Given a parameter, get its basic Python type, e.g.: param.Boolean -> bool. Throw exception if it is not supported. + :param _p: parameter to get type and nargs for. :return: Type """ @@ -222,6 +227,7 @@ def add_boolean_argument(parser: argparse.ArgumentParser, k: str, p: Parameter) Add a boolean argument. If the parameter default is False then allow --flag (to set it True) and --flag=Bool as usual. If the parameter default is True then allow --no-flag (to set it to False) and --flag=Bool as usual. + :param parser: parser to add a boolean argument to. :param k: argument name. :param p: boolean parameter. @@ -318,6 +324,7 @@ def report_on_overrides(self, values: Dict[str, Any], keys_to_ignore: Set[str]) """ Logs a warning for every parameter whose value is not as given in "values", other than those in keys_to_ignore. + :param values: override dictionary, parameter names to values :param keys_to_ignore: set of dictionary keys not to report on :return: None @@ -347,6 +354,7 @@ def create_from_matching_params(from_object: param.Parameterized, cls_: Type[T]) Creates an object of the given target class, and then copies all attributes from the `from_object` to the newly created object, if there is a matching attribute. The target class must be a subclass of param.Parameterized. + :param from_object: The object to read attributes from. :param cls_: The name of the class for the newly created object. :return: An instance of cls_ diff --git a/InnerEye/Common/output_directories.py b/InnerEye/Common/output_directories.py index e4b05eba5..5c0054376 100644 --- a/InnerEye/Common/output_directories.py +++ b/InnerEye/Common/output_directories.py @@ -33,6 +33,7 @@ def create_file_or_folder_path(self, file_or_folder_name: str) -> Path: """ Creates a full path for the given file or folder name relative to the root directory stored in the present object. + :param file_or_folder_name: Name of file or folder to be created under root_dir """ return self.root_dir / file_or_folder_name @@ -40,6 +41,7 @@ def create_file_or_folder_path(self, file_or_folder_name: str) -> Path: def make_sub_dir(self, dir_name: str) -> Path: """ Makes a sub directory under root_dir + :param dir_name: Name of subdirectory to be created. """ sub_dir_path = self.create_file_or_folder_path(dir_name) diff --git a/InnerEye/Common/resource_monitor.py b/InnerEye/Common/resource_monitor.py index ba8cb6d72..bed251e2b 100644 --- a/InnerEye/Common/resource_monitor.py +++ b/InnerEye/Common/resource_monitor.py @@ -28,6 +28,7 @@ def memory_in_gb(bytes: int) -> float: """ Converts a memory amount in bytes to gigabytes. + :param bytes: :return: """ @@ -63,6 +64,7 @@ def __add__(self, other: GpuUtilization) -> GpuUtilization: def max(self, other: GpuUtilization) -> GpuUtilization: """ Computes the metric-wise maximum of the two GpuUtilization objects. + :param other: :return: """ @@ -103,6 +105,7 @@ def enumerate(self, prefix: str = "") -> List[Tuple[str, float]]: """ Lists all metrics stored in the present object, as (metric_name, value) pairs suitable for logging in Tensorboard. + :param prefix: If provided, this string as used as an additional prefix for the metric name itself. If prefix is "max", the metric would look like "maxLoad_Percent" :return: A list of (name, value) tuples. @@ -118,6 +121,7 @@ def enumerate(self, prefix: str = "") -> List[Tuple[str, float]]: def from_gpu(gpu: GPU) -> GpuUtilization: """ Creates a GpuUtilization object from data coming from the gputil library. + :param gpu: GPU diagnostic data from gputil. :return: """ @@ -145,6 +149,7 @@ def __init__(self, csv_results_folder: Path): """ Creates a process that will monitor CPU and GPU utilization. + :param interval_seconds: The interval in seconds at which usage statistics should be written. :param tensorboard_folder: The path in which to create a tensorboard logfile. :param csv_results_folder: The path in which the CSV file with aggregate metrics will be created. @@ -163,6 +168,7 @@ def __init__(self, def log_to_tensorboard(self, label: str, value: float) -> None: """ Write a scalar metric value to Tensorboard, marked with the present step. + :param label: The name of the metric. :param value: The value. """ @@ -172,6 +178,7 @@ def update_metrics(self, gpus: List[GPU]) -> None: """ Updates the stored GPU utilization metrics with the current status coming from gputil, and logs them to Tensorboard. + :param gpus: The current utilization information, read from gputil, for all available GPUs. """ for gpu in gpus: diff --git a/InnerEye/Common/spawn_subprocess.py b/InnerEye/Common/spawn_subprocess.py index 38aba7d15..8b37aa57f 100644 --- a/InnerEye/Common/spawn_subprocess.py +++ b/InnerEye/Common/spawn_subprocess.py @@ -14,6 +14,7 @@ def spawn_and_monitor_subprocess(process: str, """ Helper function to start a subprocess, passing in a given set of arguments, and monitor it. Returns the subprocess exit code and the list of lines written to stdout. + :param process: The name and path of the executable to spawn. :param args: The args to the process. :param env: The environment variables that the new process will run with. If not provided, copy the diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/cxr_datasets.py b/InnerEye/ML/SSL/datamodules_and_datasets/cxr_datasets.py index 5d6d38d64..4d66dbd9a 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/cxr_datasets.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/cxr_datasets.py @@ -202,6 +202,7 @@ def num_classes(self) -> int: def _split_dataset(self, val_split: float, seed: int) -> Tuple[Subset, Subset]: """ Implements val - train split. + :param val_split: proportion to use for validation :param seed: random seed for splitting :return: dataset_train, dataset_val diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py b/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py index b96d03bf2..45a175045 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py @@ -37,8 +37,10 @@ def __init__(self, :param dataset_cls: class to load the dataset. Expected to inherit from InnerEyeDataClassBaseWithReturnIndex and VisionDataset. See InnerEyeCIFAR10 for an example. BEWARE VisionDataModule expects the first positional argument of your class to be the data directory. + :param return_index: whether the return the index in __get_item__, the dataset_cls is expected to implement this logic. + :param train_transforms: transforms to use at training time :param val_transforms: transforms to use at validation time :param data_dir: data directory where to find the data @@ -151,6 +153,7 @@ def get_combined_loader(self, encoder_loader: Sized, linear_head_loader: Sized) """ Creates a CombinedLoader from the data loaders for the encoder and the linear head. The cycle mode is chosen such that in all cases the encoder dataset is only cycled through once. + :param encoder_loader: The dataloader to use for the SSL encoder. :param linear_head_loader: The dataloader to use for the linear head. """ diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py index 89a197b90..778d4df94 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py @@ -31,9 +31,11 @@ def get_ssl_transforms_from_config(config: CfgNode, :param config: configuration defining which augmentations to apply as well as their intensities. :param return_two_views_per_sample: if True the resulting transforms will return two versions of each sample they are called on. If False, simply return one transformed version of the sample centered and cropped. + :param use_training_augmentations_for_validation: If True, use augmentation at validation time too. This is required for SSL validation loss to be meaningful. If False, only apply basic processing step (no augmentations) + :param expand_channels: if True the expand channel transformation from InnerEye.ML.augmentations.image_transforms will be added to the transformation passed through the config. This is needed for single channel images as CXR. """ diff --git a/InnerEye/ML/SSL/encoders.py b/InnerEye/ML/SSL/encoders.py index 9a83836de..d597acbe8 100644 --- a/InnerEye/ML/SSL/encoders.py +++ b/InnerEye/ML/SSL/encoders.py @@ -63,6 +63,7 @@ def get_encoder_output_dim( ) -> int: """ Calculates the output dimension of ssl encoder by making a single forward pass. + :param pl_module: pl encoder module :param dm: pl datamodule """ diff --git a/InnerEye/ML/SSL/lightning_containers/ssl_container.py b/InnerEye/ML/SSL/lightning_containers/ssl_container.py index b39a07468..5bd07ba23 100644 --- a/InnerEye/ML/SSL/lightning_containers/ssl_container.py +++ b/InnerEye/ML/SSL/lightning_containers/ssl_container.py @@ -237,10 +237,13 @@ def _get_transforms(self, augmentation_config: Optional[CfgNode], is_ssl_encoder_module: bool) -> Tuple[Any, Any]: """ Returns the transformation pipeline for training and validation. + :param augmentation_config: optional yaml config defining strength of augmenentations. Ignored for CIFAR examples. + :param dataset_name: name of the dataset, value has to be in SSLDatasetName, determines which transformation pipeline to return. + :param is_ssl_encoder_module: if True the transformation pipeline will yield two versions of the image it is applied on and it applies the training transformations also at validation time. Note that if your transformation does not contain any randomness, the pipeline will return two identical copies. If False, it diff --git a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py index 565e7c3c8..8201730bb 100644 --- a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py +++ b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py @@ -40,15 +40,18 @@ def __init__(self, **kwargs: Any) -> None: """ Args: + :param num_samples: Number of samples present in training dataset / dataloader. :param learning_rate: Optimizer learning rate. :param batch_size: Sample batch size used in gradient updates. :param encoder_name: Type of CNN encoder used to extract image embeddings. The options are: {'resnet18', 'resnet50', 'resnet101', 'densenet121'}. + :param warmup_epochs: Number of epochs for scheduler warm up (linear increase from 0 to base_lr). :param use_7x7_first_conv_in_resnet: If True, use a 7x7 kernel (default) in the first layer of resnet. If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not shrink them. + :param weight_decay: L2-norm weight decay. """ super().__init__() @@ -78,10 +81,12 @@ def shared_step(self, batch: BatchType, batch_idx: int) -> T: """ Returns the BYOL loss for a given batch of images, used in validation and training step. + :param batch: assumed to be a batch a Tuple(List[tensor, tensor, tensor], tensor) to match lightning-bolts SimCLRTrainDataTransform API; the first tuple element contains a list of three tensor where the two first elements contain two are two strong augmented versions of the original images in the batch and the last is a milder augmentation (ignored here). + :param batch_idx: index of the batch :return: BYOL loss """ diff --git a/InnerEye/ML/SSL/lightning_modules/simclr_module.py b/InnerEye/ML/SSL/lightning_modules/simclr_module.py index 0ab1dd566..0c51dcdfe 100644 --- a/InnerEye/ML/SSL/lightning_modules/simclr_module.py +++ b/InnerEye/ML/SSL/lightning_modules/simclr_module.py @@ -41,6 +41,7 @@ def __init__(self, encoder_name: str, dataset_name: str, use_7x7_first_conv_in_r **kwargs: Any) -> None: """ Returns SimCLR pytorch-lightning module, based on lightning-bolts implementation. + :param encoder_name: Image encoder name (predefined models) :param dataset_name: Dataset name (e.g. cifar10, kaggle, etc.) :param use_7x7_first_conv_in_resnet: If True, use a 7x7 kernel (default) in the first layer of resnet. diff --git a/InnerEye/ML/SSL/lightning_modules/ssl_online_evaluator.py b/InnerEye/ML/SSL/lightning_modules/ssl_online_evaluator.py index e4a9caf11..4cdc54373 100644 --- a/InnerEye/ML/SSL/lightning_modules/ssl_online_evaluator.py +++ b/InnerEye/ML/SSL/lightning_modules/ssl_online_evaluator.py @@ -38,6 +38,7 @@ def __init__(self, :param class_weights: The class weights to use when computing the cross entropy loss. If set to None, no weighting will be done. + :param length_linear_head_loader: The maximum number of batches in the dataloader for the linear head. """ @@ -123,6 +124,7 @@ def on_pretrain_routine_start(self, trainer: pl.Trainer, pl_module: pl.Lightning def to_device(batch: Any, device: Union[str, torch.device]) -> Tuple[T, T]: """ Moves batch to device. + :param device: device to move the batch to. """ _, x, y = batch diff --git a/InnerEye/ML/SSL/utils.py b/InnerEye/ML/SSL/utils.py index 35fee8736..a185365da 100644 --- a/InnerEye/ML/SSL/utils.py +++ b/InnerEye/ML/SSL/utils.py @@ -38,6 +38,7 @@ def load_yaml_augmentation_config(config_path: Path) -> CfgNode: def create_ssl_encoder(encoder_name: str, use_7x7_first_conv_in_resnet: bool = True) -> torch.nn.Module: """ Creates SSL encoder. + :param encoder_name: available choices: resnet18, resnet50, resnet101 and densenet121. :param use_7x7_first_conv_in_resnet: If True, use a 7x7 kernel (default) in the first layer of resnet. If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not shrink them. diff --git a/InnerEye/ML/augmentations/image_transforms.py b/InnerEye/ML/augmentations/image_transforms.py index d9fd28fe7..b0e0dc0d6 100644 --- a/InnerEye/ML/augmentations/image_transforms.py +++ b/InnerEye/ML/augmentations/image_transforms.py @@ -59,6 +59,7 @@ class AddGaussianNoise: def __init__(self, p_apply: float, std: float) -> None: """ Transformation to add Gaussian noise N(0, std) to an image. + :param: p_apply: probability of applying the transformation. :param: std: standard deviation of the gaussian noise to add to the image. """ diff --git a/InnerEye/ML/augmentations/transform_pipeline.py b/InnerEye/ML/augmentations/transform_pipeline.py index 5c772b942..269a93483 100644 --- a/InnerEye/ML/augmentations/transform_pipeline.py +++ b/InnerEye/ML/augmentations/transform_pipeline.py @@ -36,6 +36,7 @@ def __init__(self, :param transforms: List of transformations to apply to images. Supports out of the boxes torchvision transforms as they accept data of arbitrary dimension. You can also define your own transform class but be aware that you function should expect input of shape [C, Z, H, W] and apply the same transformation to each Z slice. + :param use_different_transformation_per_channel: if True, apply a different version of the augmentation pipeline for each channel. If False, applies the same transformation to each channel, separately. """ @@ -93,9 +94,11 @@ def create_transforms_from_config(config: CfgNode, Defines the image transformations pipeline from a config file. It has been designed for Chest X-Ray images but it can be used for other types of images data, type of augmentations to use and strength are expected to be defined in the config. The channel expansion is needed for gray images. + :param config: config yaml file fixing strength and type of augmentation to apply :param apply_augmentations: if True return transformation pipeline with augmentations. Else, disable augmentations i.e. only resize and center crop the image. + :param expand_channels: if True the expand channel transformation from InnerEye.ML.augmentations.image_transforms will be added to the transformation passed through the config. This is needed for single channel images as CXR. """ diff --git a/InnerEye/ML/baselines_util.py b/InnerEye/ML/baselines_util.py index 52a7229b1..e53e38cd7 100755 --- a/InnerEye/ML/baselines_util.py +++ b/InnerEye/ML/baselines_util.py @@ -112,6 +112,7 @@ def download_and_compare_scores(outputs_folder: Path, azure_config: AzureConfig, :param azure_config: Azure configuration to use for downloading data :param comparison_blob_storage_paths: list of paths to directories containing metrics.csv and dataset.csv files, each of the form run_recovery_id/rest_of_path + :param model_dataset_df: dataframe containing contents of dataset.csv for the current model :param model_metrics_df: dataframe containing contents of metrics.csv for the current model :return: a dataframe for all the data (current model and all baselines); whether any comparisons were @@ -196,6 +197,7 @@ def compare_files(expected: Path, actual: Path, csv_relative_tolerance: float = :param expected: A file that contains the expected contents. The type of comparison (text or binary) is chosen based on the extension of this file. + :param actual: A file that contains the actual contents. :param csv_relative_tolerance: When comparing CSV files, use this as the maximum allowed relative discrepancy. If 0.0, do not allow any discrepancy. diff --git a/InnerEye/ML/common.py b/InnerEye/ML/common.py index e6bc814fe..50dcb88e0 100644 --- a/InnerEye/ML/common.py +++ b/InnerEye/ML/common.py @@ -86,6 +86,7 @@ def create_unique_timestamp_id() -> str: def get_best_checkpoint_path(path: Path) -> Path: """ Given a path and checkpoint, formats a path based on the checkpoint file name format. + :param path to checkpoint folder """ return path / LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX diff --git a/InnerEye/ML/configs/other/HelloContainer.py b/InnerEye/ML/configs/other/HelloContainer.py index 0d8e08979..9c872a105 100644 --- a/InnerEye/ML/configs/other/HelloContainer.py +++ b/InnerEye/ML/configs/other/HelloContainer.py @@ -35,6 +35,7 @@ class HelloDataset(Dataset): def __init__(self, raw_data: List[List[float]]) -> None: """ Creates the 1-dim regression dataset. + :param raw_data: The raw data, e.g. from a cross validation split or loaded from file. This must be numeric data which can be converted into a tensor. See the static method from_path_and_indexes for an example call. @@ -55,6 +56,7 @@ def from_path_and_indexes( end_index: int) -> 'HelloDataset': ''' Static method to instantiate a HelloDataset from the root folder with the start and end indexes. + :param root_folder: The folder in which the data file lives ("hellocontainer.csv") :param start_index: The first row to read. :param end_index: The last row to read (exclusive) @@ -137,6 +139,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # type: ignore This method is part of the standard PyTorch Lightning interface. For an introduction, please see https://pytorch-lightning.readthedocs.io/en/stable/starter/converting.html It runs a forward pass of a tensor through the model. + :param x: The input tensor(s) :return: The model output. """ @@ -148,6 +151,7 @@ def training_step(self, batch: Dict[str, torch.Tensor], *args: Any, **kwargs: An https://pytorch-lightning.readthedocs.io/en/stable/starter/converting.html It consumes a minibatch of training data (coming out of the data loader), does forward propagation, and computes the loss. + :param batch: The batch of training data :return: The loss value with a computation graph attached. """ @@ -162,6 +166,7 @@ def validation_step(self, batch: Dict[str, torch.Tensor], *args: Any, # type: i https://pytorch-lightning.readthedocs.io/en/stable/starter/converting.html It consumes a minibatch of validation data (coming out of the data loader), does forward propagation, and computes the loss. + :param batch: The batch of validation data :return: The loss value on the validation data. """ @@ -173,6 +178,7 @@ def shared_step(self, batch: Dict[str, torch.Tensor]) -> torch.Tensor: """ This is a convenience method to reduce code duplication, because training, validation, and test step share large amounts of code. + :param batch: The batch of data to process, with input data and targets. :return: The MSE loss that the model achieved on this batch. """ @@ -207,6 +213,7 @@ def test_step(self, batch: Dict[str, torch.Tensor], batch_idx: int) -> torch.Ten https://pytorch-lightning.readthedocs.io/en/stable/starter/converting.html It evaluates the model in "inference mode" on data coming from the test set. It could, for example, also write each model prediction to disk. + :param batch: The batch of test data. :param batch_idx: The index (0, 1, ...) of the batch when the data loader is enumerated. :return: The loss on the test data. diff --git a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py index ad3a12c88..68b117eb6 100644 --- a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py +++ b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py @@ -51,14 +51,19 @@ def __init__(self, :param ground_truth_ids: List of ground truth ids. :param ground_truth_ids_display_names: Optional list of ground truth id display names. If present then must be of the same length as ground_truth_ids. + :param colours: Optional list of colours. If present then must be of the same length as ground_truth_ids. + :param fill_holes: Optional list of fill hole flags. If present then must be of the same length as ground_truth_ids. + :param roi_interpreted_types: Optional list of roi_interpreted_types. If present then must be of the same length as ground_truth_ids. + :param class_weights: Optional list of class weights. If present then must be of the same length as ground_truth_ids + 1. + :param slice_exclusion_rules: Optional list of SliceExclusionRules. :param summed_probability_rules: Optional list of SummedProbabilityRule. :param num_feature_channels: Optional number of feature channels. diff --git a/InnerEye/ML/configs/segmentation/HeadAndNeckPaper.py b/InnerEye/ML/configs/segmentation/HeadAndNeckPaper.py index 34e2e9b2e..302fb2a48 100644 --- a/InnerEye/ML/configs/segmentation/HeadAndNeckPaper.py +++ b/InnerEye/ML/configs/segmentation/HeadAndNeckPaper.py @@ -29,6 +29,7 @@ class HeadAndNeckPaper(HeadAndNeckBase): def __init__(self, num_structures: Optional[int] = None, **kwargs: Any) -> None: """ Creates a new instance of the class. + :param num_structures: number of structures from STRUCTURE_LIST to predict (default: all structures) :param kwargs: Additional arguments that will be passed through to the SegmentationModelBase constructor. """ diff --git a/InnerEye/ML/configs/segmentation/ProstateBase.py b/InnerEye/ML/configs/segmentation/ProstateBase.py index feb2c08d2..cee8dcd7f 100644 --- a/InnerEye/ML/configs/segmentation/ProstateBase.py +++ b/InnerEye/ML/configs/segmentation/ProstateBase.py @@ -28,17 +28,23 @@ def __init__(self, **kwargs: Any) -> None: """ Creates a new instance of the class. + :param ground_truth_ids: List of ground truth ids. :param ground_truth_ids_display_names: Optional list of ground truth id display names. If present then must be of the same length as ground_truth_ids. + :param colours: Optional list of colours. If present then must be of the same length as ground_truth_ids. + :param fill_holes: Optional list of fill hole flags. If present then must be of the same length as ground_truth_ids. + :param interpreted_types: Optional list of interpreted_types. If present then must be of the same length as ground_truth_ids. + :param class_weights: Optional list of class weights. If present then must be of the same length as ground_truth_ids + 1. + :param kwargs: Additional arguments that will be passed through to the SegmentationModelBase constructor. """ ground_truth_ids_display_names = ground_truth_ids_display_names or [f"zz_{name}" for name in ground_truth_ids] diff --git a/InnerEye/ML/dataset/cropping_dataset.py b/InnerEye/ML/dataset/cropping_dataset.py index 4908b517d..93c46f48c 100644 --- a/InnerEye/ML/dataset/cropping_dataset.py +++ b/InnerEye/ML/dataset/cropping_dataset.py @@ -95,6 +95,7 @@ def create_random_cropped_sample(sample: Sample, :param crop_size: the size of the crop to extract. :param center_size: the size of the center of the crop (this should be the same as the spatial dimensions of the posteriors that the model produces) + :param class_weights: the distribution to use for the crop center class. :return: CroppedSample """ diff --git a/InnerEye/ML/dataset/scalar_dataset.py b/InnerEye/ML/dataset/scalar_dataset.py index 5a11327c4..eff40adee 100644 --- a/InnerEye/ML/dataset/scalar_dataset.py +++ b/InnerEye/ML/dataset/scalar_dataset.py @@ -52,6 +52,7 @@ def extract_label_classification(label_string: str, sample_id: str, num_classes: :param num_classes: Number of classes. This should be equal the size of the model output. For binary classification tasks, num_classes should be one. For multilabel classification tasks, num_classes should correspond to the number of label classes in the problem. + :param is_classification_dataset: If the model is a classification model :return: A list of floats with the same size as num_classes """ @@ -123,6 +124,7 @@ def _get_single_channel_row(subject_rows: pd.DataFrame, :param subject_rows: A set of rows all belonging to the same subject. :param channel: The value to look for in the `channel_column` column. This can be null. If it is null, the input `subject_rows` is expected to have exactly 1 row. + :param subject_id: A string describing the presently processed subject. This is only used for error reporting. :return: A dictionary mapping from column names to values, created from the unique row that was found. """ @@ -190,17 +192,21 @@ def load_single_data_source(subject_rows: pd.DataFrame, :param subject_id: The identifier of the subject that is being processed. :param image_channels: The names of all channels (stored in the CSV_CHANNEL_HEADER column of the dataframe) that are expected to be loaded from disk later because they are large images. + :param image_file_column: The name of the column that contains the image file names. :param label_channels: The name of the channel where the label scalar or vector is read from. :param label_value_column: The column that contains the value for the label scalar or vector. :param non_image_feature_channels: non_image_feature_channels: A dictonary of the names of all channels where additional scalar values should be read from. THe keys should map each feature to its channels. + :param numerical_columns: The names of all columns where additional scalar values should be read from. :param categorical_data_encoder: Encoding scheme for categorical data. :param is_classification_dataset: If True, the dataset will be used in a classification model. If False, assume that the dataset will be used in a regression model. + :param transform_labels: a label transformation or a list of label transformation to apply to the labels. If a list is provided, the transformations are applied in order from left to right. + :param sequence_position_numeric: Numeric position of the data source in a data sequence. Assumed to be a non-sequential dataset item if None provided (default). :return: A ScalarDataSource containing the specified data. @@ -355,14 +361,18 @@ def __init__(self, :param label_channels: The name of the channel where the label scalar or vector is read from. :param transform_labels: a label transformation or a list of label transformation to apply to the labels. If a list is provided, the transformations are applied in order from left to right. + :param non_image_feature_channels: non_image_feature_channels: A dictionary of the names of all channels where additional scalar values should be read from. The keys should map each feature to its channels. + :param numerical_columns: The names of all columns where additional scalar values should be read from. :param sequence_column: The name of the column that contains the sequence index, that will be stored in metadata.sequence_position. If this column name is not provided, the sequence_position will be 0. + :param subject_column: The name of the column that contains the subject identifier :param channel_column: The name of the column that contains the row identifier ("channels") that are expected to be loaded from disk later because they are large images. + :param is_classification_dataset: If the current dataset is classification or not. :param categorical_data_encoder: Encoding scheme for categorical data. """ @@ -554,6 +564,7 @@ def is_valid_item_index(item: ScalarDataSource, :param item: The item to check. :param min_sequence_position_value: Check if the item has a metadata.sequence_position that is at least the value given here. Default is 0. + :param max_sequence_position_value: If provided then this is the maximum sequence position the sequence can end with. Longer sequences will be truncated. None is default. :return: True if the item has a valid index. @@ -578,8 +589,10 @@ def filter_valid_classification_data_sources_items(items: Iterable[ScalarDataSou :param items: The list of items to filter. :param min_sequence_position_value: Restrict the data to items with a metadata.sequence_position that is at least the value given here. Default is 0. + :param max_sequence_position_value: If provided then this is the maximum sequence position the sequence can end with. Longer sequences will be truncated. None is default. + :param file_to_path_mapping: A mapping from a file name stem (without extension) to its full path. :return: A list of items, all of which are valid now. """ @@ -643,6 +656,7 @@ def __init__(self, """ :param image_transform: transformation function to apply to images field. If None, images field is unchanged by call. + :param segmentation_transform: transformation function to apply to segmentations field. If None segmentations field is unchanged by call. """ @@ -771,6 +785,7 @@ def __init__(self, args: ScalarModelBase, :param data_frame: The dataframe to read from. :param feature_statistics: If given, the normalization factor for the non-image features is taken from the values provided. If None, the normalization factor is computed from the data in the present dataset. + :param sample_transform: Sample transforms that should be applied. :param name: Name of the dataset, used for diagnostics logging """ diff --git a/InnerEye/ML/dataset/scalar_sample.py b/InnerEye/ML/dataset/scalar_sample.py index ef2e404a4..fe16c48d8 100644 --- a/InnerEye/ML/dataset/scalar_sample.py +++ b/InnerEye/ML/dataset/scalar_sample.py @@ -126,10 +126,12 @@ def load_images(self, :param root_path: The root path where all channel files for images are expected. This is ignored if file_mapping is given. + :param file_mapping: A mapping from a file name stem (without extension) to its full path. :param load_segmentation: If True it loads segmentation if present on the same file as the image. :param center_crop_size: If supplied, all loaded images will be cropped to the size given here. The crop will be taken from the center of the image. + :param image_size: If given, all loaded images will be reshaped to the size given here, prior to the center crop. :return: An instance of ClassificationItem, with the same label and numerical_non_image_features fields, @@ -173,6 +175,7 @@ def get_all_image_filepaths(self, :param root_path: The root path where all channel files for images are expected. This is ignored if file_mapping is given. + :param file_mapping: A mapping from a file name stem (without extension) to its full path. """ full_channel_files: List[Path] = [] @@ -194,6 +197,7 @@ def get_full_image_filepath(file: str, :param file: Image filepath relative to the dataset folder :param root_path: The root path where all channel files for images are expected. This is ignored if file_mapping is given. + :param file_mapping: A mapping from a file name stem (without extension) to its full path. """ if file is None: diff --git a/InnerEye/ML/deep_learning_config.py b/InnerEye/ML/deep_learning_config.py index d2d460d1d..eef5eab97 100644 --- a/InnerEye/ML/deep_learning_config.py +++ b/InnerEye/ML/deep_learning_config.py @@ -117,12 +117,15 @@ def create(project_root: Path, Creates a new object that holds output folder configurations. When running inside of AzureML, the output folders will be directly under the project root. If not running inside AzureML, a folder with a timestamp will be created for all outputs and logs. + :param project_root: The root folder that contains the code that submitted the present training run. When running inside the InnerEye repository, it is the git repo root. When consuming InnerEye as a package, this should be the root of the source code that calls the package. + :param is_offline_run: If true, this is a run outside AzureML. If False, it is inside AzureML. :param model_name: The name of the model that is trained. This is used to generate a run-specific output folder. + :param output_to: If provided, the output folders will be created as a subfolder of this argument. If not given, the output folders will be created inside of the project root. """ @@ -165,6 +168,7 @@ def add_subfolder(self, subfolder: str) -> DeepLearningFileSystemConfig: """ Creates a new output folder configuration, where both outputs and logs go into the given subfolder inside the present outputs folder. + :param subfolder: The subfolder that should be created. :return: """ @@ -281,6 +285,7 @@ def is_inference_required(self, data_split: ModelExecutionMode) -> bool: """ Returns True if inference is required for this model_proc (single or ensemble) and data_split (Train/Val/Test). + :param model_proc: Whether we are testing an ensemble or single model. :param data_split: Indicates which of the 3 sets (training, test, or validation) is being processed. :return: True if inference required. @@ -442,6 +447,7 @@ def model_name(self) -> str: def set_output_to(self, output_to: PathOrString) -> None: """ Adjusts the file system settings in the present object such that all outputs are written to the given folder. + :param output_to: The absolute path to a folder that should contain the outputs. """ if isinstance(output_to, Path): @@ -453,6 +459,7 @@ def create_filesystem(self, project_root: Path = fixed_paths.repository_root_dir """ Creates new file system settings (outputs folder, logs folder) based on the information stored in the present object. If any of the folders do not yet exist, they are created. + :param project_root: The root folder for the codebase that triggers the training run. """ self.file_system_config = DeepLearningFileSystemConfig.create( @@ -783,6 +790,7 @@ def dataset_data_frame(self) -> Optional[DataFrame]: def dataset_data_frame(self, data_frame: Optional[DataFrame]) -> None: """ Sets the pandas data frame that the model uses. + :param data_frame: The data frame to set. """ self._dataset_data_frame = data_frame @@ -844,6 +852,7 @@ def load_checkpoint_and_modify(self, path_to_checkpoint: Path) -> Dict[str, Any] See https://pytorch.org/tutorials/beginner/saving_loading_models.html#warmstarting-model-using-parameters -from-a-different-model for an explanation on why strict=False is useful when loading parameters from other models. + :param path_to_checkpoint: Path to the checkpoint file. :return: Dictionary with model and optimizer state dicts. The dict should have at least the following keys: 1. Key ModelAndInfo.MODEL_STATE_DICT_KEY and value set to the model state dict. diff --git a/InnerEye/ML/lightning_base.py b/InnerEye/ML/lightning_base.py index 8fe0df094..7e2c6a690 100644 --- a/InnerEye/ML/lightning_base.py +++ b/InnerEye/ML/lightning_base.py @@ -275,6 +275,7 @@ def on_validation_epoch_start(self) -> None: def validation_epoch_end(self, outputs: List[Any]) -> None: """ Resets the random number generator state to what it was before the current validation epoch started. + :param outputs: The list of outputs from the individual validation minibatches. """ # reset the random state for training, so that we get continue from where we were before the validation step. @@ -316,6 +317,7 @@ def log_on_epoch(self, :param sync_dist_override: If not None, use this value for the sync_dist argument to self.log. If None, set it automatically depending on the use of DDP. + :param name: The name of the metric to log :param value: The value of the metric. This can be a tensor, floating point value, or a Metric class. :param is_training: If true, give the metric a "train/" prefix, otherwise a "val/" prefix. @@ -334,6 +336,7 @@ def store_epoch_results(self, metrics: DictStrFloat, epoch: int, is_training: bo """ Stores a set of metrics (key/value pairs) to a file logger. That file logger is either one that only holds training or only holds validation metrics. + :param metrics: A dictionary with all the metrics to write, as key/value pairs. :param epoch: The epoch to which the metrics belong. :param is_training: If true, write the metrics to the logger for training metrics, if False, write to the logger @@ -346,6 +349,7 @@ def on_load_checkpoint(self, checkpoint: Dict[str, Any]) -> None: """ This hook is called when loading a model from a checkpoint. It just prints out diagnostics about which epoch created the present checkpoint. + :param checkpoint: The checkpoint dictionary loaded from disk. """ keys = ['epoch', 'global_step'] @@ -371,6 +375,7 @@ def training_or_validation_step(self, """ This is the shared method that handles the training (when `is_training==True`) and validation steps (when `is_training==False`) + :param sample: The minibatch of data that should be processed. :param batch_index: The index of the current minibatch. :param is_training: If true, this has been called from `training_step`, otherwise it has been called from @@ -382,6 +387,7 @@ def write_loss(self, is_training: bool, loss: torch.Tensor) -> None: """ Writes the given loss value to Lightning, labelled either "val/loss" or "train/loss". If this comes from a training step, then also log the learning rate. + :param loss: The loss value that should be logged. :param is_training: If True, the logged metric will be called "train/Loss". If False, the metric will be called "val/Loss" diff --git a/InnerEye/ML/lightning_container.py b/InnerEye/ML/lightning_container.py index 1f9491952..d55a92a79 100644 --- a/InnerEye/ML/lightning_container.py +++ b/InnerEye/ML/lightning_container.py @@ -55,6 +55,7 @@ def on_inference_epoch_start(self, dataset_split: ModelExecutionMode, is_ensembl Runs initialization for inference, when starting inference on a new dataset split (train/val/test). Depending on the settings, this can be called anywhere between 0 (no inference at all) to 3 times (inference on all of train/val/test split). + :param dataset_split: Indicates whether the item comes from the training, validation or test set. :param is_ensemble_model: If False, the model_outputs come from an individual model. If True, the model outputs come from multiple models. @@ -65,6 +66,7 @@ def inference_step(self, batch: Any, batch_idx: int, model_output: torch.Tensor) """ This hook is called when the model has finished making a prediction. It can write the results to a file, or compute metrics and store them. + :param batch: The batch of data for which the model made a prediction. :param model_output: The model outputs. This would usually be a torch.Tensor, but can be any datatype. """ @@ -91,6 +93,7 @@ def aggregate_ensemble_model_outputs(self, model_outputs: Iterator[torch.Tensor] """ Aggregates the outputs of multiple models when using an ensemble model. In the default implementation, this averages the tensors coming from all the models. + :param model_outputs: An iterator over the model outputs for all ensemble members. :return: The aggregate model outputs. """ @@ -230,6 +233,7 @@ def update_azure_config(self, azure_config: AzureConfig) -> None: This can be avoided by always using unique parameter names. Also note that saving a reference to `azure_config` and updating its attributes at any other point may lead to unexpected behaviour. + :param azure_config: The initialised AzureConfig whose parameters to override in-place. """ pass @@ -298,6 +302,7 @@ def get_cross_validation_hyperdrive_config(self, run_config: ScriptRunConfig) -> Because this adds a val/Loss metric it is important that when subclassing LightningContainer your implementation of LightningModule logs val/Loss. There is an example of this in HelloRegression's validation_step method. + :param run_config: The AzureML run configuration object that training for an individual model. :return: A hyperdrive configuration object. """ @@ -316,6 +321,7 @@ def get_hyperdrive_config(self, run_config: ScriptRunConfig) -> HyperDriveConfig """ Returns the HyperDrive config for either parameter search or cross validation (if number_of_cross_validation_splits > 1). + :param run_config: AzureML estimator :return: HyperDriveConfigs """ diff --git a/InnerEye/ML/lightning_helpers.py b/InnerEye/ML/lightning_helpers.py index 7f40d9dad..2f1d30baf 100644 --- a/InnerEye/ML/lightning_helpers.py +++ b/InnerEye/ML/lightning_helpers.py @@ -17,6 +17,7 @@ def load_from_lightning_checkpoint(config: ModelConfigBase, checkpoint_path: Pat """ Reads a PyTorch model from a checkpoint. First, a PyTorch Lightning model is created matching the InnerEye model configuration, its parameter tensors are then populated from the given checkpoint. + :param config: An InnerEye model configuration object :param checkpoint_path: The location of the checkpoint file. :return: A PyTorch Lightning model object. @@ -37,6 +38,7 @@ def adjust_model_for_inference(config: ModelConfigBase, lightning_model: InnerEy Makes all necessary adjustments to use a given model for inference, possibly on multiple GPUs via model parallelization. The method also computes parameters like output patch size for segmentation model, and stores them in the model configuration. + :param config: The model configuration object. It may be modified in place. :param lightning_model: The trained model that should be adjusted. """ @@ -65,6 +67,7 @@ def load_from_checkpoint_and_adjust_for_inference(config: ModelConfigBase, check """ Reads a PyTorch model from a checkpoint, and makes all necessary adjustments to use the model for inference, possibly on multiple GPUs. + :param config: An InnerEye model configuration object :param checkpoint_path: The location of the checkpoint file. :return: A PyTorch Lightning model object. diff --git a/InnerEye/ML/lightning_loggers.py b/InnerEye/ML/lightning_loggers.py index b4481b60c..24df64095 100644 --- a/InnerEye/ML/lightning_loggers.py +++ b/InnerEye/ML/lightning_loggers.py @@ -80,6 +80,7 @@ def extract_by_prefix(self, epoch: int, prefix_filter: str = "") -> DictStrFloat Reads the set of metrics for a given epoch, filters them to retain only those that have the given prefix, and returns the filtered ones. This is used to break a set of results down into those for training data (prefix "Train/") or validation data (prefix "Val/"). + :param epoch: The epoch for which results should be read. :param prefix_filter: If empty string, return all metrics. If not empty, return only those metrics that have a name starting with `prefix`, and strip off the prefix. @@ -103,6 +104,7 @@ def to_metrics_dicts(self, prefix_filter: str = "") -> Dict[int, DictStrFloat]: Converts the results stored in the present object into a two-level dictionary, mapping from epoch number to metric name to metric value. Only metrics where the name starts with the given prefix are retained, and the prefix is stripped off in the result. + :param prefix_filter: If empty string, return all metrics. If not empty, return only those metrics that have a name starting with `prefix`, and strip off the prefix. :return: A dictionary mapping from epoch number to metric name to metric value. @@ -113,8 +115,10 @@ def get_metric(self, is_training: bool, metric_type: str) -> List[float]: """ Gets a scalar metric out of either the list of training or the list of validation results. This returns the value that a specific metric attains in all of the epochs. + :param is_training: If True, read metrics that have a "train/" prefix, otherwise those that have a "val/" prefix. + :param metric_type: The metric to extract. :return: A list of floating point numbers, with one entry per entry in the the training or validation results. """ @@ -132,6 +136,7 @@ def get_train_metric(self, metric_type: str) -> List[float]: """ Gets a scalar metric from the list of training results. This returns the value that a specific metric attains in all of the epochs. + :param metric_type: The metric to extract. :return: A list of floating point numbers, with one entry per entry in the the training results. """ @@ -141,6 +146,7 @@ def get_val_metric(self, metric_type: str) -> List[float]: """ Gets a scalar metric from the list of validation results. This returns the value that a specific metric attains in all of the epochs. + :param metric_type: The metric to extract. :return: A list of floating point numbers, with one entry per entry in the the validation results. """ diff --git a/InnerEye/ML/lightning_metrics.py b/InnerEye/ML/lightning_metrics.py index b2c9d96ac..934015df0 100644 --- a/InnerEye/ML/lightning_metrics.py +++ b/InnerEye/ML/lightning_metrics.py @@ -20,6 +20,7 @@ def nanmean(values: torch.Tensor) -> torch.Tensor: """ Computes the average of all values in the tensor, skipping those entries that are NaN (not a number). If all values are NaN, the result is also NaN. + :param values: The values to average. :return: A scalar tensor containing the average. """ @@ -287,9 +288,11 @@ def __init__(self, ground_truth_ids: List[str], is_training: bool, use_average_across_structures: bool = True) -> None: """ Creates a new MetricForMultipleStructures object. + :param ground_truth_ids: The list of anatomical structures that should be stored. :param metric_name: The name of the metric that should be stored. This is used in the names of the individual metrics. + :param is_training: If true, use "train/" as the prefix for all metric names, otherwise "val/" :param use_average_across_structures: If True, keep track of the average metric value across structures, while skipping NaNs. If false, only store the per-structure metric values. @@ -307,6 +310,7 @@ def update(self, values_per_structure: torch.Tensor) -> None: """ Stores a vector of per-structure Dice scores in the present object. It updates the per-structure values, and the aggregate value across all structures. + :param values_per_structure: A row tensor that has as many entries as there are ground truth IDs. """ if values_per_structure.dim() != 1 or values_per_structure.numel() != self.count: diff --git a/InnerEye/ML/lightning_models.py b/InnerEye/ML/lightning_models.py index 223b66557..a3f0743b0 100644 --- a/InnerEye/ML/lightning_models.py +++ b/InnerEye/ML/lightning_models.py @@ -51,6 +51,7 @@ def forward(self, patches: torch.Tensor) -> torch.Tensor: # type: ignore """ Runs a set of 3D crops through the segmentation model, and returns the result. This method is used at inference time. + :param patches: A tensor of size [batches, channels, Z, Y, X] """ return self.logits_to_posterior(self.model(patches)) @@ -67,8 +68,10 @@ def training_or_validation_step(self, is_training: bool) -> torch.Tensor: """ Runs training for a single minibatch of training or validation data, and computes all metrics. + :param is_training: If true, the method is called from `training_step`, otherwise it is called from `validation_step`. + :param sample: The batched sample on which the model should be trained. :param batch_index: The index of the present batch (supplied only for diagnostics). """ @@ -103,6 +106,7 @@ def compute_metrics(self, cropped_sample: CroppedSample, segmentation: torch.Ten is_training: bool) -> None: """ Computes and stores all metrics coming out of a single training step. + :param cropped_sample: The batched image crops used for training or validation. :param segmentation: The segmentation that was produced by the model. :param is_training: If true, the method is called from `training_step`, otherwise it is called from @@ -167,6 +171,7 @@ def training_or_validation_epoch_end(self, is_training: bool) -> None: def get_subject_output_file_per_rank(rank: int) -> str: """ Gets the name of a file that will store the per-rank per-subject model outputs. + :param rank: The rank of the current model in distributed training. :return: A string like "rank7_metrics.csv" """ @@ -228,8 +233,10 @@ def training_or_validation_step(self, is_training: bool) -> torch.Tensor: """ Runs training for a single minibatch of training or validation data, and computes all metrics. + :param is_training: If true, the method is called from `training_step`, otherwise it is called from `validation_step`. + :param sample: The batched sample on which the model should be trained. :param batch_index: The index of the present batch (supplied only for diagnostics). Runs a minibatch of training or validation data through the model. @@ -283,6 +290,7 @@ def transfer_batch_to_device(self, batch: Any, device: torch.device, dataloader_ """ For sequence models, transfer the nested lists of items to the given GPU device. For all other models, this relies on the superclass to move the batch of data to the GPU. + :param batch: A batch of data coming from the dataloader. :param device: The target CUDA device. :return: A modified batch of data, where all tensor now live on the given CUDA device. @@ -294,6 +302,7 @@ def transfer_batch_to_device(batch: Any, device: torch.device) -> Any: """ For sequence models, transfer the nested lists of items to the given GPU device. For all other models, this relies on Lightning's default code to move the batch of data to the GPU. + :param batch: A batch of data coming from the dataloader. :param device: The target CUDA device. :return: A modified batch of data, where all tensor now live on the given CUDA device. @@ -314,8 +323,10 @@ def create_lightning_model(config: ModelConfigBase, set_optimizer_and_scheduler: """ Creates a PyTorch Lightning model that matches the provided InnerEye model configuration object. The `optimizer` and `l_rate_scheduler` object of the Lightning model will also be populated. + :param set_optimizer_and_scheduler: If True (default), initialize the optimizer and LR scheduler of the model. If False, skip that step (this is only meant to be used for unit tests.) + :param config: An InnerEye model configuration object :return: A PyTorch Lightning model object. """ diff --git a/InnerEye/ML/metrics.py b/InnerEye/ML/metrics.py index 4780c6579..53384069f 100644 --- a/InnerEye/ML/metrics.py +++ b/InnerEye/ML/metrics.py @@ -65,6 +65,7 @@ def get_metrics_log_key(self) -> str: def log_metrics(self, run_context: Run = None) -> None: """ Log metrics for each epoch to the provided runs logs, or the current run context if None provided + :param run_context: Run for which to log the metrics to, use the current run context if None provided :return: """ @@ -79,6 +80,7 @@ def surface_distance(seg: sitk.Image, reference_segmentation: sitk.Image) -> flo """ Symmetric surface distances taking into account the image spacing https://github.com/InsightSoftwareConsortium/SimpleITK-Notebooks/blob/master/Python/34_Segmentation_Evaluation.ipynb + :param seg: mask 1 :param reference_segmentation: mask 2 :return: mean distance @@ -118,6 +120,7 @@ def surface_distance(seg: sitk.Image, reference_segmentation: sitk.Image) -> flo def _add_zero_distances(num_segmented_surface_pixels: int, seg2ref_distance_map_arr: np.ndarray) -> List[float]: """ # Get all non-zero distances and then add zero distances if required. + :param num_segmented_surface_pixels: :param seg2ref_distance_map_arr: :return: list of distances, augmented with zeros. @@ -137,6 +140,7 @@ def calculate_metrics_per_class(segmentation: np.ndarray, Returns a MetricsDict with metrics for each of the foreground structures. Metrics are NaN if both ground truth and prediction are all zero for a class. If first element of a ground truth image channel is NaN, the image is flagged as NaN and not use. + :param ground_truth_ids: The names of all foreground classes. :param segmentation: predictions multi-value array with dimensions: [Z x Y x X] :param ground_truth: ground truth binary array with dimensions: [C x Z x Y x X]. @@ -217,6 +221,7 @@ def compute_dice_across_patches(segmentation: torch.Tensor, allow_multiple_classes_for_each_pixel: bool = False) -> torch.Tensor: """ Computes the Dice scores for all classes across all patches in the arguments. + :param segmentation: Tensor containing class ids predicted by a model. :param ground_truth: One-hot encoded torch tensor containing ground-truth label ids. :param allow_multiple_classes_for_each_pixel: If set to False, ground-truth tensor has @@ -255,6 +260,7 @@ def store_epoch_metrics(metrics: DictStrFloat, """ Writes all metrics (apart from ones that measure run time) into a CSV file, with an additional columns for epoch number. + :param file_logger: An instance of DataframeLogger, for logging results to csv. :param epoch: The epoch corresponding to the results. :param metrics: The metrics of the specified epoch, averaged along its batches. @@ -291,6 +297,7 @@ def compute_scalar_metrics(metrics_dict: ScalarMetricsDict, of class 1. The label vector is expected to contain class indices 0 and 1 only. Metrics for each model output channel will be isolated, and a non-default hue for each model output channel is expected, and must exist in the provided metrics_dict. The Default hue is used for single model outputs. + :param metrics_dict: An object that holds all metrics. It will be updated in-place. :param subject_ids: Subject ids for the model output and labels. :param model_output: A tensor containing model outputs. @@ -346,6 +353,7 @@ def add_average_foreground_dice(metrics: MetricsDict) -> None: """ If the given metrics dictionary contains an entry for Dice score, and only one value for the Dice score per class, then add an average Dice score for all foreground classes to the metrics dictionary (modified in place). + :param metrics: The object that holds metrics. The average Dice score will be written back into this object. """ all_dice = [] diff --git a/InnerEye/ML/metrics_dict.py b/InnerEye/ML/metrics_dict.py index 405fbc600..a4735068c 100644 --- a/InnerEye/ML/metrics_dict.py +++ b/InnerEye/ML/metrics_dict.py @@ -35,6 +35,7 @@ def average_metric_values(values: List[float], skip_nan_when_averaging: bool) -> """ Returns the average (arithmetic mean) of the values provided. If skip_nan_when_averaging is True, the mean will be computed without any possible NaN values in the list. + :param values: The individual values that should be averaged. :param skip_nan_when_averaging: If True, compute mean with any NaN values. If False, any NaN value present in the argument will make the function return NaN. @@ -61,6 +62,7 @@ def get_column_name_for_logging(metric_name: Union[str, MetricType], """ Computes the column name that should be used when logging a metric to disk. Raises a value error when no column name has yet been defined. + :param metric_name: The name of the metric. :param hue_name: If provided will be used as a prefix hue_name/column_name """ @@ -104,9 +106,11 @@ def add_predictions(self, labels: np.ndarray) -> None: """ Adds predictions and labels for later computing the area under the ROC curve. + :param subject_ids: Subject ids associated with the predictions and labels. :param predictions: A numpy array with model predictions, of size [N x C] for N samples in C classes, or size [N x 1] or size [N] for binary. + :param labels: A numpy array with labels, of size [N x C] for N samples in C classes, or size [N x 1] or size [N] for binary. """ @@ -155,6 +159,7 @@ def get_predictions_and_labels_per_subject(self) -> List[PredictionEntry[float]] def _concat_if_needed(arrays: List[np.ndarray]) -> np.ndarray: """ Joins a list of arrays into a single array, taking empty lists into account correctly. + :param arrays: Array list to be concatenated. """ if arrays: @@ -193,6 +198,7 @@ def __init__(self, hues: Optional[List[str]] = None, is_classification_metrics: """ :param hues: Supported hues for this metrics dict, otherwise all records will belong to the default hue. + :param is_classification_metrics: If this is a classification metrics dict """ @@ -210,6 +216,7 @@ def __init__(self, hues: Optional[List[str]] = None, is_classification_metrics: def subject_ids(self, hue: str = DEFAULT_HUE_KEY) -> List[str]: """ Return the subject ids that have metrics associated with them in this dictionary. + :param hue: If provided then subject ids belonging to this hue only will be returned. Otherwise subject ids for the default hue will be returned. """ @@ -218,6 +225,7 @@ def subject_ids(self, hue: str = DEFAULT_HUE_KEY) -> List[str]: def get_hue_names(self, include_default: bool = True) -> List[str]: """ Returns all of the hues supported by this metrics dict + :param include_default: Include the default hue if True, otherwise exclude the default hue. """ _hue_names = list(self.hues.keys()) @@ -228,6 +236,7 @@ def get_hue_names(self, include_default: bool = True) -> List[str]: def delete_hue(self, hue: str) -> None: """ Removes all data stored for the given hue from the present object. + :param hue: The hue to remove. """ del self.hues[hue] @@ -236,6 +245,7 @@ def get_single_metric(self, metric_name: MetricTypeOrStr, hue: str = DEFAULT_HUE """ Gets the value stored for the given metric. The method assumes that there is a single value stored for the metric, and raises a ValueError if that is not the case. + :param metric_name: The name of the metric to retrieve. :param hue: The hue to retrieve the metric from. :return: @@ -249,6 +259,7 @@ def get_single_metric(self, metric_name: MetricTypeOrStr, hue: str = DEFAULT_HUE def has_prediction_entries(self, hue: str = DEFAULT_HUE_KEY) -> bool: """ Returns True if the present object stores any entries for computing the Area Under Roc Curve metric. + :param hue: will be used to check a particular hue otherwise default hue will be used. :return: True if entries exist. False otherwise. """ @@ -257,6 +268,7 @@ def has_prediction_entries(self, hue: str = DEFAULT_HUE_KEY) -> bool: def values(self, hue: str = DEFAULT_HUE_KEY) -> Dict[str, Any]: """ Returns values held currently in the dict + :param hue: will be used to restrict values for the provided hue otherwise values in the default hue will be returned. :return: Dictionary of values for this object. @@ -267,6 +279,7 @@ def add_diagnostics(self, name: str, value: Any) -> None: """ Adds a diagnostic value to the present object. Multiple diagnostics can be stored per unique value of name, the values get concatenated. + :param name: The name of the diagnostic value to store. :param value: The value to store. """ @@ -292,10 +305,12 @@ def add_metric(self, hue: str = DEFAULT_HUE_KEY) -> None: """ Adds values for a single metric to the present object, when the metric value is a scalar. + :param metric_name: The name of the metric to add. This can be a string or a value in the MetricType enum. :param metric_value: The values of the metric, as a float or integer. :param skip_nan_when_averaging: If True, averaging this metric will skip any NaN (not a number) values. If False, NaN will propagate through the mean computation. + :param hue: The hue for which this record belongs to, default hue will be used if None provided. """ _metric_name = MetricsDict._metric_name(metric_name) @@ -315,6 +330,7 @@ def delete_metric(self, hue: str = DEFAULT_HUE_KEY) -> None: """ Deletes all values that are stored for a given metric from the present object. + :param metric_name: The name of the metric to add. This can be a string or a value in the MetricType enum. :param hue: The hue for which this record belongs to, default hue will be used if None provided. """ @@ -327,11 +343,14 @@ def add_predictions(self, subject_ids: Sequence[str], hue: str = DEFAULT_HUE_KEY) -> None: """ Adds predictions and labels for later computing the area under the ROC curve. + :param subject_ids: Subject ids associated with the predictions and labels. :param predictions: A numpy array with model predictions, of size [N x C] for N samples in C classes, or size [N x 1] or size [N] for binary. + :param labels: A numpy array with labels, of size [N x C] for N samples in C classes, or size [N x 1] or size [N] for binary. + :param hue: The hue this prediction belongs to, default hue will be used if None provided. """ self._get_hue(hue).add_predictions(subject_ids=subject_ids, @@ -341,6 +360,7 @@ def add_predictions(self, subject_ids: Sequence[str], def num_entries(self, hue: str = DEFAULT_HUE_KEY) -> Dict[str, int]: """ Gets the number of values that are stored for each individual metric. + :param hue: The hue to count entries for, otherwise all entries will be counted. :return: A dictionary mapping from metric name to number of values stored. """ @@ -355,6 +375,7 @@ def average(self, object. Computing the average will respect the skip_nan_when_averaging value that has been provided when adding the metric. + :param add_metrics_from_entries: average existing metrics in the dict. :param across_hues: If True then same metric types will be averaged regardless of hues, otherwise separate averages for each metric type for each hue will be computed, Default is True. @@ -434,6 +455,7 @@ def get_metrics_at_optimal_cutoff(self, hue: str = DEFAULT_HUE_KEY) -> Tuple: difference between true positive rate and false positive rate is smallest. Then, computes the false positive rate, false negative rate and accuracy at this threshold (i.e. when the predicted probability is higher than the threshold the predicted label is 1 otherwise 0). + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :returns: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) """ @@ -450,6 +472,7 @@ def get_metrics_at_optimal_cutoff(self, hue: str = DEFAULT_HUE_KEY) -> Tuple: def get_roc_auc(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Computes the Area Under the ROC curve, from the entries that were supplied in the add_roc_entries method. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: The AUC score, or np.nan if no entries are available in the present object. """ @@ -469,6 +492,7 @@ def get_pr_auc(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Computes the Area Under the Precision Recall Curve, from the entries that were supplied in the add_roc_entries method. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: The PR AUC score, or np.nan if no entries are available in the present object. """ @@ -488,6 +512,7 @@ def get_cross_entropy(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Computes the binary cross entropy from the entries that were supplied in the add_roc_entries method. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: The cross entropy score. """ @@ -498,6 +523,7 @@ def get_cross_entropy(self, hue: str = DEFAULT_HUE_KEY) -> float: def get_mean_absolute_error(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Get the mean absolute error. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: Mean absolute error. """ @@ -506,6 +532,7 @@ def get_mean_absolute_error(self, hue: str = DEFAULT_HUE_KEY) -> float: def get_mean_squared_error(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Get the mean squared error. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: Mean squared error """ @@ -514,6 +541,7 @@ def get_mean_squared_error(self, hue: str = DEFAULT_HUE_KEY) -> float: def get_r2_score(self, hue: str = DEFAULT_HUE_KEY) -> float: """ Get the R2 score. + :param hue: The hue to restrict the values used for computation, otherwise all values will be used. :return: R2 score """ @@ -524,6 +552,7 @@ def enumerate_single_values(self, hue: Optional[str] = None) -> Iterable[Tuple[s Returns an iterator that contains all (hue name, metric name, metric values) tuples that are stored in the present object. This method assumes that for each hue/metric combination there is exactly 1 value, and it throws an exception if that is more than 1 value. + :param hue: The hue to restrict the values, otherwise all values will be used if set to None. :return: An iterator with (hue name, metric name, metric values) pairs. """ @@ -536,6 +565,7 @@ def _enumerate_values(self, hue: Optional[str] = None, """ Returns an iterator that contains all (hue name, metric name, metric values) tuples that are stored in the present object. + :param hue: The hue to restrict the values, otherwise all values will be used if set to None. :param ensure_singleton_values_only: Ensure that each of the values return is a singleton. :return: An iterator with (hue name, metric name, metric values) pairs. @@ -566,6 +596,7 @@ def enumerate_single_values_groupwise(self) -> Iterable[Tuple[str, Iterable[Tupl def get_predictions(self, hue: str = DEFAULT_HUE_KEY) -> np.ndarray: """ Return a concatenated copy of the roc predictions stored internally. + :param hue: The hue to restrict the values, otherwise all values will be used. :return: concatenated roc predictions as np array """ @@ -574,6 +605,7 @@ def get_predictions(self, hue: str = DEFAULT_HUE_KEY) -> np.ndarray: def get_labels(self, hue: str = DEFAULT_HUE_KEY) -> np.ndarray: """ Return a concatenated copy of the roc labels stored internally. + :param hue: The hue to restrict the values, otherwise all values will be used. :return: roc labels as np array """ @@ -583,6 +615,7 @@ def get_predictions_and_labels_per_subject(self, hue: str = DEFAULT_HUE_KEY) \ -> List[PredictionEntry[float]]: """ Gets the per-subject labels and predictions that are stored in the present object. + :param hue: The hue to restrict the values, otherwise the default hue will be used. :return: List of per-subject labels and predictions """ @@ -591,6 +624,7 @@ def get_predictions_and_labels_per_subject(self, hue: str = DEFAULT_HUE_KEY) \ def to_string(self, tabulate: bool = True) -> str: """ Creates a multi-line human readable string from the given metrics. + :param tabulate: If True then create a pretty printable table string. :return: Formatted metrics string """ @@ -623,6 +657,7 @@ def _get_hue(self, hue: str = DEFAULT_HUE_KEY) -> Hue: """ Get the hue record for the provided key. Raises a KeyError if the provided hue key does not exist. + :param hue: The hue to retrieve record for """ if hue not in self.hues: @@ -654,6 +689,7 @@ def store_metrics_per_subject(self, cross_validation_split_index: int = DEFAULT_CROSS_VALIDATION_SPLIT_INDEX) -> None: """ Store metrics using the provided df_logger at subject level for classification models. + :param df_logger: A data frame logger to use to write the metrics to disk. :param mode: Model execution mode these metrics belong to. :param cross_validation_split_index: cross validation split index for the epoch if performing cross val @@ -720,6 +756,7 @@ def aggregate_and_save_execution_mode_metrics( Given metrics dicts for execution modes and epochs, compute the aggregate metrics that are computed from the per-subject predictions. The metrics are written to the dataframe logger with the string labels (column names) taken from the `MetricType` enum. + :param metrics: Mapping between epoch and subject level metrics :param data_frame_logger: DataFrame logger to write to and flush :param log_info: If True then log results as an INFO string to the default logger also. @@ -781,6 +818,7 @@ def get_target_index_from_hue_name(hue_name: str) -> int: """ Extracts a sequence target index from a metrics hue name. For example, from metrics hue "Seq_pos 07", it would return 7. + :param hue_name: hue name containing sequence target index """ if hue_name.startswith(SEQUENCE_POSITION_HUE_NAME_PREFIX): @@ -807,6 +845,7 @@ def add_record(self, record: Dict[str, Any]) -> None: def flush(self, log_info: bool = False) -> None: """ Save the internal records to a csv file. + :param log_info: If true, write the final dataframe also to logging.info. """ import pandas as pd diff --git a/InnerEye/ML/model_config_base.py b/InnerEye/ML/model_config_base.py index 5a42835e2..713063389 100644 --- a/InnerEye/ML/model_config_base.py +++ b/InnerEye/ML/model_config_base.py @@ -166,6 +166,7 @@ def get_cross_validation_hyperdrive_sampler(self) -> GridParameterSampling: def get_cross_validation_hyperdrive_config(self, run_config: ScriptRunConfig) -> HyperDriveConfig: """ Returns a configuration for AzureML Hyperdrive that varies the cross validation split index. + :param run_config: The AzureML run configuration object that training for an individual model. :return: A hyperdrive configuration object. """ @@ -181,6 +182,7 @@ def get_cross_validation_dataset_splits(self, dataset_split: DatasetSplits) -> D """ When running cross validation, this method returns the dataset split that should be used for the currently executed cross validation split. + :param dataset_split: The full dataset, split into training, validation and test section. :return: The dataset split with training and validation sections shuffled according to the current cross validation index. @@ -192,6 +194,7 @@ def get_hyperdrive_config(self, run_config: ScriptRunConfig) -> HyperDriveConfig """ Returns the HyperDrive config for either parameter search or cross validation (if number_of_cross_validation_splits > 1). + :param run_config: AzureML estimator :return: HyperDriveConfigs """ @@ -244,6 +247,7 @@ def set_derived_model_properties(self, model: Any) -> None: A hook to adjust the model configuration that is stored in the present object to match the torch model given in the argument. This hook is called after adjusting the model for mixed precision and parallel training. + :param model: The torch model. """ pass @@ -273,6 +277,7 @@ def __init__(self, train: Optional[Callable] = None, :param train: the transformation(s) to apply to the training set. Should be a function that takes a sample as input and outputs sample. + :param val: the transformation(s) to apply to the validation set :param test: the transformation(s) to apply to the test set """ diff --git a/InnerEye/ML/model_testing.py b/InnerEye/ML/model_testing.py index 1423a656b..3a46609f7 100644 --- a/InnerEye/ML/model_testing.py +++ b/InnerEye/ML/model_testing.py @@ -51,6 +51,7 @@ def model_test(config: ModelConfigBase, Runs model inference on segmentation or classification models, using a given dataset (that could be training, test or validation set). The inference results and metrics will be stored and logged in a way that may differ for model categories (classification, segmentation). + :param config: The configuration of the model :param data_split: Indicates which of the 3 sets (training, test, or validation) is being processed. :param checkpoint_paths: Checkpoint paths to initialize model. @@ -82,6 +83,7 @@ def segmentation_model_test(config: SegmentationModelBase, """ The main testing loop for segmentation models. It loads the model and datasets, then proceeds to test the model for all requested checkpoints. + :param config: The arguments object which has a valid random seed attribute. :param execution_mode: Indicates which of the 3 sets (training, test, or validation) is being processed. :param checkpoint_handler: Checkpoint handler object to find checkpoint paths for model initialization. @@ -121,6 +123,7 @@ def segmentation_model_test_epoch(config: SegmentationModelBase, The main testing loop for a given epoch. It loads the model and datasets, then proceeds to test the model. Returns a list with an entry for each image in the dataset. The entry is the average Dice score, where the average is taken across all non-background structures in the image. + :param checkpoint_paths: Checkpoint paths to run inference on. :param config: The arguments which specify all required information. :param execution_mode: Is the model evaluated on train, test, or validation set? @@ -204,6 +207,7 @@ def evaluate_model_predictions(process_id: int, Evaluates model segmentation predictions, dice scores and surface distances are computed. Generated contours are plotted and saved in results folder. The function is intended to be used in parallel for loop to process each image in parallel. + :param process_id: Identifier for the process calling the function :param config: Segmentation model config object :param dataset: Dataset object, it is used to load intensity image, labels, and patient metadata. @@ -235,8 +239,10 @@ def populate_metrics_writer( config: SegmentationModelBase) -> Tuple[MetricsPerPatientWriter, List[FloatOrInt]]: """ Populate a MetricsPerPatientWriter with the metrics for each patient + :param model_prediction_evaluations: The list of PatientMetadata/MetricsDict tuples obtained from evaluate_model_predictions + :param config: The SegmentationModelBase config from which we read the ground_truth_ids :returns: A new MetricsPerPatientWriter and a list of foreground DICE score averages """ @@ -263,6 +269,7 @@ def get_patient_results_folder(results_folder: Path, patient_id: int) -> Path: """ Gets a folder name that will contain all results for a given patient, like root/017 for patient 17. The folder name is constructed such that string sorting gives numeric sorting. + :param results_folder: The root folder in which the per-patient results should sit. :param patient_id: The numeric ID of the patient. :return: A path like "root/017" @@ -276,8 +283,10 @@ def store_inference_results(inference_result: InferencePipeline.Result, image_header: ImageHeader) -> List[Path]: """ Store the segmentation, posteriors, and binary predictions into Nifti files. + :param inference_result: The inference result for a given patient_id and epoch. Posteriors must be in (Classes x Z x Y x X) shape, segmentation in (Z, Y, X) + :param config: The test configurations. :param results_folder: The folder where the prediction should be stored. :param image_header: The image header that was used in the input image. @@ -286,6 +295,7 @@ def store_inference_results(inference_result: InferencePipeline.Result, def create_file_path(_results_folder: Path, _file_name: str) -> Path: """ Create filename with Nifti extension + :param _results_folder: The results folder :param _file_name: The name of the file :return: A full path to the results folder for the file @@ -339,6 +349,7 @@ def store_run_information(results_folder: Path, image_channels: List[str]) -> None: """ Store dataset id and ground truth ids into files in the results folder. + :param image_channels: The names of the image channels that the model consumes. :param results_folder: The folder where the files should be stored. :param dataset_id: The dataset id @@ -359,6 +370,7 @@ def create_inference_pipeline(config: ModelConfigBase, """ If multiple checkpoints are found in run_recovery then create EnsemblePipeline otherwise InferencePipeline. If no checkpoint files exist in the run recovery or current run checkpoint folder, None will be returned. + :param config: Model related configs. :param epoch: The epoch for which to create pipeline for. :param run_recovery: RunRecovery data if applicable @@ -408,9 +420,11 @@ def classification_model_test(config: ScalarModelBase, """ The main testing loop for classification models. It runs a loop over all epochs for which testing should be done. It loads the model and datasets, then proceeds to test the model for all requested checkpoints. + :param config: The model configuration. :param data_split: The name of the folder to store the results inside each epoch folder in the outputs_dir, used mainly in model evaluation using different dataset splits. + :param checkpoint_paths: Checkpoint paths to initialize model :param model_proc: whether we are testing an ensemble or single model :return: InferenceMetricsForClassification object that contains metrics related for all of the checkpoint epochs. diff --git a/InnerEye/ML/model_training.py b/InnerEye/ML/model_training.py index 89aab6ea8..77331ddb8 100644 --- a/InnerEye/ML/model_training.py +++ b/InnerEye/ML/model_training.py @@ -38,6 +38,7 @@ def upload_output_file_as_temp(file_path: Path, outputs_folder: Path) -> None: """ Uploads a file to the AzureML run. It will get a name that is composed of a "temp/" prefix, plus the path of the file relative to the outputs folder that is used for training. + :param file_path: The path of the file to upload. :param outputs_folder: The root folder that contains all training outputs. """ @@ -65,6 +66,7 @@ def create_lightning_trainer(container: LightningContainer, Creates a Pytorch Lightning Trainer object for the given model configuration. It creates checkpoint handlers and loggers. That includes a diagnostic logger for use in unit tests, that is also returned as the second return value. + :param container: The container with model and data. :param resume_from_checkpoint: If provided, training resumes from this checkpoint point. :param num_nodes: The number of nodes to use in distributed training. @@ -204,6 +206,7 @@ def model_train(checkpoint_path: Optional[Path], The main training loop. It creates the Pytorch model based on the configuration options passed in, creates a Pytorch Lightning trainer, and trains the model. If a checkpoint was specified, then it loads the checkpoint before resuming training. + :param checkpoint_path: Checkpoint path for model initialization :param num_nodes: The number of nodes to use in distributed training. :param container: A container object that holds the training data in PyTorch Lightning format @@ -325,6 +328,7 @@ def aggregate_and_create_subject_metrics_file(outputs_folder: Path) -> None: This functions takes all the subject metrics file written by each GPU (one file per GPU) and aggregates them into one single metrics file. Results is saved in config.outputs_folder / mode.value / SUBJECT_METRICS_FILE_NAME. This is done for the metrics files for training and for validation data separately. + :param config: model config """ for mode in [ModelExecutionMode.TRAIN, ModelExecutionMode.VAL]: diff --git a/InnerEye/ML/models/architectures/base_model.py b/InnerEye/ML/models/architectures/base_model.py index 2ac8881bc..b2970e3ef 100644 --- a/InnerEye/ML/models/architectures/base_model.py +++ b/InnerEye/ML/models/architectures/base_model.py @@ -25,6 +25,7 @@ def __init__(self, multiple_of: Optional[IntOrTuple3] = None, :param minimum_size: Training crops must have a size that is a multiple of this value, along each dimension. For example, if set to (1, 16, 16), the crop size has to be a multiple of 16 along X and Y, and a multiple of 1 (i.e., any number) along the Z dimension. + :param num_dimensions: Training crops must have a size that is at least this value. """ self.multiple_of = multiple_of @@ -56,6 +57,7 @@ def validate(self, crop_size: TupleInt3, message_prefix: Optional[str] = None) - """ Checks if the given crop size is a valid crop size for the present model. If it is not valid, throw a ValueError. + :param crop_size: The crop size that should be checked. :param message_prefix: A string prefix for the error message if the crop size is found to be invalid. :return: @@ -83,6 +85,7 @@ def restrict_crop_size_to_image(self, (at test time). The new crop size will be the largest multiple of self.multiple_of that fits into the image_shape. The stride size will attempt to maintain the stride-to-crop ratio before adjustment. + :param image_shape: The shape of the image to process. :param crop_size: The present test crop size. :param stride_size: The present inference stride size. @@ -121,6 +124,7 @@ def __init__(self, ): """ Creates a new instance of the base model class. + :param name: A human readable name of the model. :param input_channels: The number of image input channels. :param crop_size_constraints: The size constraints for the training crop size. If not provided, @@ -144,6 +148,7 @@ def get_output_shape(self, input_shape: Union[TupleInt2, TupleInt3]) -> Tuple[in The argument is expected to be either a 2-tuple or a 3-tuple. A batch dimension (1) and the number of channels are added as the first dimensions. The result tuple has batch and channel dimension stripped off. + :param input_shape: A tuple (2D or 3D) representing incoming tensor shape. """ # Create a sample tensor for inference @@ -166,6 +171,7 @@ def validate_crop_size(self, crop_size: TupleInt3, message_prefix: Optional[str] """ Checks if the given crop size is a valid crop size for the present model. If it is not valid, throw a ValueError. + :param crop_size: The crop size that should be checked. :param message_prefix: A string prefix for the error message if the crop size is found to be invalid. """ @@ -178,8 +184,10 @@ def generate_model_summary(self, crop_size: Optional[TupleInt3] = None, Stores a model summary, containing information about layers, memory consumption and runtime in the model.summary field. When called again with the same crop_size, the summary is not created again. + :param crop_size: The crop size for which the summary should be created. If not provided, the minimum allowed crop size is used. + :param log_summaries_to_files: whether to write the summary to a file """ if crop_size is None: diff --git a/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py b/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py index 79a97f2ac..6f1af363a 100644 --- a/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py +++ b/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py @@ -64,21 +64,27 @@ def __init__(self, """ Creates an image classifier that has UNet encoders sections for each image channel. The encoder output is fed through average pooling and an MLP. + :param encode_channels_jointly: If False, create a UNet encoder structure separately for each channel. If True, encode all channels jointly (convolution will run over all channels). + :param num_encoder_blocks: Number of UNet encoder blocks. :param initial_feature_channels: Number of feature channels in the first UNet encoder. :param num_image_channels: Number of channels of the input. Input is expected to be of size B x num_image_channels x Z x Y x X, where B is the batch dimension. + :param num_non_image_features: Number of non imaging features will be used in the model. :param kernel_size_per_encoding_block: The size of the kernels per encoding block, assumed to be the same if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default performs convolutions only in X and Y. + :param stride_size_per_encoding_block: The stride size for the encoding block, assumed to be the same if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default reduces spatial dimensions only in X and Y. + :param encoder_dimensionality_reduction_factor: how to reduce the dimensionality of the image features in the combined model to balance with non imaging features. + :param scan_size: should be a tuple representing 3D tensor shape and if specified it's usedd in initializing gated pooling or z-adaptive. The first element should be representing the z-direction for classification images """ @@ -168,6 +174,7 @@ def get_last_encoder_layer_names(self) -> List[str]: def _get_aggregation_layer(self, aggregation_type: AggregationType, scan_size: Optional[TupleInt3]) -> Any: """ Returns the aggregation layer as specified by the config + :param aggregation_type: name of the aggregation :param scan_size: [Z, Y, X] size of the scans """ @@ -191,6 +198,7 @@ def _get_aggregation_layer(self, aggregation_type: AggregationType, scan_size: O def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into a torch.Tensor that the forward pass can consume + :param item: ClassificationItem :return: Tensor """ @@ -290,23 +298,29 @@ def __init__(self, Creates an image classifier that has UNet encoders sections for each image channel. The encoder output is fed through average pooling and an MLP. Extension of the ImageEncoder class using an MLP as classification layer. + :param encode_channels_jointly: If False, create a UNet encoder structure separately for each channel. If True, encode all channels jointly (convolution will run over all channels). + :param num_encoder_blocks: Number of UNet encoder blocks. :param initial_feature_channels: Number of feature channels in the first UNet encoder. :param num_image_channels: Number of channels of the input. Input is expected to be of size B x num_image_channels x Z x Y x X, where B is the batch dimension. + :param mlp_dropout: The amount of dropout that should be applied between the two layers of the classifier MLP. :param final_activation: Activation function to normalize the logits default is Identity. :param num_non_image_features: Number of non imaging features will be used in the model. :param kernel_size_per_encoding_block: The size of the kernels per encoding block, assumed to be the same if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default performs convolutions only in X and Y. + :param stride_size_per_encoding_block: The stride size for the encoding block, assumed to be the same if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default reduces spatial dimensions only in X and Y. + :param encoder_dimensionality_reduction_factor: how to reduce the dimensionality of the image features in the combined model to balance with non imaging features. + :param scan_size: should be a tuple representing 3D tensor shape and if specified it's usedd in initializing gated pooling or z-adaptive. The first element should be representing the z-direction for classification images """ @@ -369,6 +383,7 @@ def create_mlp(input_num_feature_channels: int, hidden_layer_num_feature_channels: Optional[int] = None) -> MLP: """ Create an MLP with 1 hidden layer. + :param input_num_feature_channels: The number of input channels to the first MLP layer. :param dropout: The drop out factor that should be applied between the first and second MLP layer. :param final_output_channels: if provided, the final number of output channels. diff --git a/InnerEye/ML/models/architectures/classification/segmentation_encoder.py b/InnerEye/ML/models/architectures/classification/segmentation_encoder.py index 091f545db..40cda17d5 100644 --- a/InnerEye/ML/models/architectures/classification/segmentation_encoder.py +++ b/InnerEye/ML/models/architectures/classification/segmentation_encoder.py @@ -161,8 +161,10 @@ def __init__(self, """ :param encode_channels_jointly: If False, create an encoder structure separately for each channel. If True, encode all channels jointly (convolution will run over all channels). + :param num_image_channels: Number of channels of the input. Input is expected to be of size B x (num_image_channels * 10) x Z x Y x X, where B is the batch dimension. + :param use_mixed_precision: If True, assume that training happens with mixed precision. Segmentations will be converted to float16 tensors right away. If False, segmentations will be converted to float32 tensors. """ @@ -194,6 +196,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # type: ignore def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into a torch.Tensor that the forward pass can consume + :param item: ClassificationItem :return: Tensor """ diff --git a/InnerEye/ML/models/architectures/complex.py b/InnerEye/ML/models/architectures/complex.py index 8da211ef9..5755cb8de 100644 --- a/InnerEye/ML/models/architectures/complex.py +++ b/InnerEye/ML/models/architectures/complex.py @@ -27,9 +27,11 @@ def __init__(self, crop_size_constraints: Optional[CropSizeConstraints] = None): """ Creates a new instance of the class. + :param args: The full model configuration. :param full_channels_list: A vector of channel sizes. First entry is the number of image channels, then all feature channels, then the number of classes. + :param network_definition: :param crop_size_constraints: The size constraints for the training crop size. """ diff --git a/InnerEye/ML/models/architectures/mlp.py b/InnerEye/ML/models/architectures/mlp.py index 83e819675..2a9f76bc9 100644 --- a/InnerEye/ML/models/architectures/mlp.py +++ b/InnerEye/ML/models/architectures/mlp.py @@ -59,6 +59,7 @@ def forward(self, x: Tensor) -> Tensor: # type: ignore def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into a torch.Tensor that the forward pass can consume + :param item: ClassificationItem :return: Tensor """ diff --git a/InnerEye/ML/models/architectures/unet_2d.py b/InnerEye/ML/models/architectures/unet_2d.py index 950b300e6..a3d7ae7ce 100644 --- a/InnerEye/ML/models/architectures/unet_2d.py +++ b/InnerEye/ML/models/architectures/unet_2d.py @@ -26,15 +26,18 @@ def __init__(self, """ Initializes a 2D UNet model, where the input image is expected as a 3 dimensional tensor with a vanishing Z dimension. + :param input_image_channels: The number of image channels that the model should consume. :param initial_feature_channels: The number of feature maps used in the model in the first convolution layer. Subsequent layers will contain number of feature maps that are multiples of `initial_channels` (2^(image_level) * initial_channels) + :param num_classes: Number of output classes :param num_downsampling_paths: Number of image levels used in Unet (in encoding and decoding paths) :param downsampling_dilation: An additional dilation that is used in the second convolution layer in each of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good choice is (1, 2, 2), to increase the receptive field only in X and Y. + :param padding_mode: The type of padding that should be applied. """ super().__init__(input_image_channels=input_image_channels, diff --git a/InnerEye/ML/models/architectures/unet_3d.py b/InnerEye/ML/models/architectures/unet_3d.py index 67b0bb31a..6b103f313 100644 --- a/InnerEye/ML/models/architectures/unet_3d.py +++ b/InnerEye/ML/models/architectures/unet_3d.py @@ -30,6 +30,7 @@ class UNet3D(BaseSegmentationModel): 4) Support for more downsampling operations to capture larger image context and improve the performance. The network has `num_downsampling_paths` downsampling steps on the encoding side and same number upsampling steps on the decoding side. + :param num_downsampling_paths: Number of downsampling paths used in Unet model (default 4 image level are used) :param num_classes: Number of output segmentation classes :param kernel_size: Spatial support of convolution kernels used in Unet model @@ -39,11 +40,14 @@ class UNetDecodeBlock(torch.nn.Module): """ Implements upsampling block for UNet architecture. The operations carried out on the input tensor are 1) Upsampling via strided convolutions 2) Concatenating the skip connection tensor 3) Two convolution layers + :param channels: A tuple containing the number of input and output channels :param upsample_kernel_size: Spatial support of upsampling kernels. If an integer is provided, the same value will be repeated for all three dimensions. For non-cubic kernels please pass a list or tuple with three elements + :param upsampling_stride: Upsamling factor used in deconvolutional layer. Similar to the `upsample_kernel_size` parameter, if an integer is passed, the same upsampling factor will be used for all three dimensions. + :param activation: Linear/Non-linear activation function that is used after linear deconv/conv mappings. :param depth: The depth inside the UNet at which the layer operates. This is only for diagnostic purposes. """ @@ -120,11 +124,14 @@ class UNetEncodeBlock(torch.nn.Module): Implements a EncodeBlock for UNet. A EncodeBlock is two BasicLayers without dilation and with same padding. The first of those BasicLayer can use stride > 1, and hence will downsample. + :param channels: A list containing two elements representing the number of input and output channels :param kernel_size: Spatial support of convolution kernels. If an integer is provided, the same value will be repeated for all three dimensions. For non-cubic kernels please pass a tuple with three elements. + :param downsampling_stride: Downsampling factor used in the first convolutional layer. If an integer is passed, the same downsampling factor will be used for all three dimensions. + :param dilation: Dilation of convolution kernels - If set to > 1, kernels capture content from wider range. :param activation: Linear/Non-linear activation function that is used after linear convolution mappings. :param use_residual: If set to True, block2 learns the residuals while preserving the output of block1 @@ -182,18 +189,22 @@ def __init__(self, crop_size_constraints=crop_size_constraints) """ Modified 3D-Unet Class + :param input_image_channels: Number of image channels (scans) that are fed into the model. :param initial_feature_channels: Number of feature-maps used in the model - Subsequent layers will contain number of featuremaps that is multiples of `initial_feature_channels` (e.g. 2^(image_level) * initial_feature_channels) + :param num_classes: Number of output classes :param kernel_size: Spatial support of conv kernels in each spatial axis. :param num_downsampling_paths: Number of image levels used in Unet (in encoding and decoding paths) :param downsampling_factor: Spatial downsampling factor for each tensor axis (depth, width, height). This will be used as the stride for the first convolution layer in each encoder block. + :param downsampling_dilation: An additional dilation that is used in the second convolution layer in each of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good choice is (1, 2, 2), to increase the receptive field only in X and Y. + :param crop_size: The size of the crop that should be used for training. """ diff --git a/InnerEye/ML/models/layers/basic.py b/InnerEye/ML/models/layers/basic.py index 073b95d18..dbc589309 100644 --- a/InnerEye/ML/models/layers/basic.py +++ b/InnerEye/ML/models/layers/basic.py @@ -14,15 +14,18 @@ class BasicLayer(torch.nn.Module): """ A Basic Layer applies a 3D convolution and BatchNorm with the given channels, kernel_size, and dilation. The output of BatchNorm layer is passed through an activation function and its output is returned. + :param channels: Number of input and output channels. :param kernel_size: Spatial support of convolution kernels :param stride: Kernel stride lenght for convolution op :param padding: Feature map padding after convolution op {"constant/zero", "no_padding"}. When it is set to "no_padding", no padding is applied. For "constant", feature-map tensor size is kept the same at the output by padding with zeros. + :param dilation: Kernel dilation used in convolution layer :param use_bias: If set to True, a bias parameter will be added to the layer. Default is set to False as batch normalisation layer has an affine parameter which are used applied after the bias term is added. + :param activation: Activation layer (e.g. nonlinearity) to be used after the convolution and batch norm operations. """ diff --git a/InnerEye/ML/models/losses/cross_entropy.py b/InnerEye/ML/models/losses/cross_entropy.py index f03f12134..9b43f98a7 100644 --- a/InnerEye/ML/models/losses/cross_entropy.py +++ b/InnerEye/ML/models/losses/cross_entropy.py @@ -21,13 +21,16 @@ def __init__(self, super().__init__(smoothing_eps) """ Multi-class cross entropy loss. + :param class_weight_power: if 1.0, weights the cross-entropy term for each class equally. Class weights are inversely proportional to the number of pixels belonging to each class, raised to class_weight_power + :param focal_loss_gamma: Gamma term used in focal loss to weight negative log-likelihood term: https://arxiv.org/pdf/1708.02002.pdf equation(4-5). When gamma equals to zero, it is equivalent to standard CE with no class balancing. (Gamma >= 0.0) + :param ignore_index: Specifies a target value that is ignored and does not contribute to the input gradient """ @@ -60,6 +63,7 @@ def _get_class_weights(target_labels: torch.Tensor, num_classes: int) -> torch.T def get_focal_loss_pixel_weights(self, logits: torch.Tensor, target: torch.Tensor) -> torch.Tensor: """ Computes weights for each pixel/sample inversely proportional to the posterior likelihood. + :param logits: Logits tensor. :param target: Target label tensor in one-hot encoding. """ @@ -103,6 +107,7 @@ def forward_minibatch(self, output: torch.Tensor, target: torch.Tensor, **kwargs Wrapper for multi-class cross entropy function implemented in PyTorch. The implementation supports tensors with arbitrary spatial dimension. Input logits are normalised internally in `F.cross_entropy` function. + :param output: Class logits (unnormalised), e.g. in 3D : BxCxWxHxD or in 1D BxC :param target: Target labels encoded in one-hot representation, e.g. in 3D BxCxWxHxD or in 1D BxC """ diff --git a/InnerEye/ML/models/losses/mixture.py b/InnerEye/ML/models/losses/mixture.py index 7ca4890f1..a47830376 100644 --- a/InnerEye/ML/models/losses/mixture.py +++ b/InnerEye/ML/models/losses/mixture.py @@ -14,6 +14,7 @@ class MixtureLoss(SupervisedLearningCriterion): def __init__(self, components: List[Tuple[float, SupervisedLearningCriterion]]): """ Loss function defined as a weighted mixture (interpolation) of other loss functions. + :param components: a non-empty list of weights and loss function instances. """ super().__init__() @@ -25,6 +26,7 @@ def forward_minibatch(self, output: torch.Tensor, target: torch.Tensor, **kwargs """ Wrapper for mixture loss function implemented in PyTorch. Arguments should be suitable for the component loss functions, typically: + :param output: Class logits (unnormalised), e.g. in 3D : BxCxWxHxD or in 1D BxC :param target: Target labels encoded in one-hot representation, e.g. in 3D BxCxWxHxD or in 1D BxC """ diff --git a/InnerEye/ML/models/losses/soft_dice.py b/InnerEye/ML/models/losses/soft_dice.py index 4568e381f..305d1d8ee 100644 --- a/InnerEye/ML/models/losses/soft_dice.py +++ b/InnerEye/ML/models/losses/soft_dice.py @@ -25,6 +25,7 @@ def __init__(self, :param eps: A small constant to smooth Sorensen-Dice Loss function. Additionally, it avoids division by zero. :param apply_softmax: If true, the input to the loss function will be first fed through a Softmax operation. If false, the input to the loss function will be used as is. + :param class_weight_power: power to raise 1/C to, where C is the number of voxels in each class. Should be non-negative to help increase accuracy on small structures. """ diff --git a/InnerEye/ML/models/parallel/model_parallel.py b/InnerEye/ML/models/parallel/model_parallel.py index 6db78a857..9d54ae8b7 100644 --- a/InnerEye/ML/models/parallel/model_parallel.py +++ b/InnerEye/ML/models/parallel/model_parallel.py @@ -14,6 +14,7 @@ def move_to_device(input_tensors: List[torch.Tensor], non_blocking: bool = False) -> Iterable[torch.Tensor]: """ Updates the memory location of tensors stored in a list. + :param input_tensors: List of torch tensors :param target_device: Target device (e.g. cuda:0, cuda:1, etc). If the device is None, the tensors are not moved. :param non_blocking: bool @@ -40,6 +41,7 @@ def group_layers_with_balanced_memory(inputs: List[torch.nn.Module], summary: Optional[OrderedDict]) -> Generator: """ Groups layers in the model in a balanced way as such each group has similar size of memory requirement + :param inputs: List of input torch modules. :param num_groups: Number of groups to be produced. :param summary: Model summary of the input layers which is used to retrieve memory requirements. @@ -121,6 +123,7 @@ def partition_layers(layers: List[torch.nn.Module], target_devices: List[torch.device]) -> None: """ Splits the models into multiple chunks and assigns each sub-model to a particular GPU + :param layers: The layers to partition :param summary: Model architecture summary to use for partitioning :param target_devices: The devices to partition layers into diff --git a/InnerEye/ML/photometric_normalization.py b/InnerEye/ML/photometric_normalization.py index 558e93772..c05cebc3b 100644 --- a/InnerEye/ML/photometric_normalization.py +++ b/InnerEye/ML/photometric_normalization.py @@ -242,6 +242,7 @@ def robust_mean_std(data: np.ndarray) -> Tuple[float, float, float, float]: Computes robust estimates of mean and standard deviation in the given array. The median is the robust estimate for the mean, the standard deviation is computed from the inter-quartile ranges. + :param data: The data for which mean and std should be computed. :return: A 4-tuple with values (median, robust_std, minimum data value, maximum data value) """ @@ -272,9 +273,11 @@ def mri_window(image_in: np.ndarray, around the mean of the remaining values and with a range controlled by the standard deviation and the sharpen input parameter. The larger sharpen is, the wider the range. The resulting values are the normalised to the given output_range, with values below and above the range being set the the boundary values. + :param image_in: The image to normalize. :param mask: Consider only pixel values of the input image for which the mask is non-zero. If None the whole image is considered. + :param output_range: The desired value range of the result image. :param sharpen: number of standard deviation either side of mean to include in the window :param tail: Default 1, allow window range to include more of tail of distribution. diff --git a/InnerEye/ML/pipelines/ensemble.py b/InnerEye/ML/pipelines/ensemble.py index 1f119bf31..c1d062947 100644 --- a/InnerEye/ML/pipelines/ensemble.py +++ b/InnerEye/ML/pipelines/ensemble.py @@ -49,9 +49,11 @@ def aggregate_results(results: Iterable[InferencePipeline.Result], aggregation_type: EnsembleAggregationType) -> InferencePipeline.Result: """ Helper method to aggregate results from multiple inference pipelines, based on the aggregation type provided. + :param results: inference pipeline results to aggregate. This may be a Generator to prevent multiple large posterior arrays being held at the same time. The first element of the sequence is modified in place to minimize memory use. + :param aggregation_type: aggregation function to use to combine the results. :return: InferenceResult: contains a Segmentation for each of the classes and their posterior probabilities. @@ -78,6 +80,7 @@ def predict_whole_image(self, image_channels: np.ndarray, """ Performs a single inference pass for each model in the ensemble, and aggregates the results based on the provided aggregation type. + :param image_channels: The input image channels to perform inference on in format: Channels x Z x Y x X. :param voxel_spacing_mm: Voxel spacing to use for each dimension in (Z x Y x X) order :param mask: A binary image used to ignore results outside it in format: Z x Y x X. diff --git a/InnerEye/ML/pipelines/inference.py b/InnerEye/ML/pipelines/inference.py index ae7a91ef8..1d0edda5e 100644 --- a/InnerEye/ML/pipelines/inference.py +++ b/InnerEye/ML/pipelines/inference.py @@ -56,6 +56,7 @@ def post_process(self, results: InferencePipeline.Result) -> InferencePipeline.R """ Perform connected component analysis to update segmentation with largest connected component based on the configurations + :param results: inference results to post-process :return: post-processed version of results """ @@ -199,8 +200,10 @@ def create_from_checkpoint(path_to_checkpoint: Path, Creates an instance of the inference pipeline for a given epoch from a stored checkpoint. After loading, the model parameters are checked for NaN and Infinity values. If there is no checkpoint file for the given epoch, return None. + :param path_to_checkpoint: The path to the checkpoint that we want to load model_config.checkpoint_folder + :param model_config: Model related configurations. :param pipeline_id: Numeric identifier for the pipeline (useful for logging when ensembling) :return InferencePipeline: an instantiated inference pipeline instance, or None if there was no checkpoint diff --git a/InnerEye/ML/pipelines/scalar_inference.py b/InnerEye/ML/pipelines/scalar_inference.py index 3134a21a6..ff5cc70a8 100644 --- a/InnerEye/ML/pipelines/scalar_inference.py +++ b/InnerEye/ML/pipelines/scalar_inference.py @@ -140,6 +140,7 @@ def create_from_checkpoint(paths_to_checkpoint: List[Path], config: ScalarModelBase) -> ScalarEnsemblePipeline: """ Creates an ensemble pipeline from a list of checkpoints. + :param paths_to_checkpoint: List of paths to the checkpoints to be recovered. :param config: Model configuration information. :return: @@ -179,6 +180,7 @@ def predict(self, sample: Dict[str, Any]) -> ScalarInferencePipelineBase.Result: def aggregate_model_outputs(self, model_outputs: torch.Tensor) -> torch.Tensor: """ Aggregates the forward pass results from the individual models in the ensemble. + :param model_outputs: List of model outputs for every model in the ensemble. (Number of ensembles) x (batch_size) x 1 """ diff --git a/InnerEye/ML/plotting.py b/InnerEye/ML/plotting.py index b467eeb2c..37cec242a 100644 --- a/InnerEye/ML/plotting.py +++ b/InnerEye/ML/plotting.py @@ -28,6 +28,7 @@ def is_val_dice(name: str) -> bool: """ Returns true if the given metric name is a Dice score on the validation set, for a class that is not the background class. + :param name: :return: """ @@ -37,6 +38,7 @@ def is_val_dice(name: str) -> bool: def get_val_dice_names(metric_names: Iterable[str]) -> List[str]: """ Returns a list of those metric names from the argument that fulfill the is_val_dice predicate. + :param metric_names: :return: """ @@ -47,6 +49,7 @@ def plot_loss_per_epoch(metrics: Dict[str, Any], metric_name: str, label: Option """ Adds a plot of loss (y-axis) versus epoch (x-axis) to the current plot, if the metric is present in the metrics dictionary. + :param metrics: A dictionary of metrics. :param metric_name: The name of the single metric to plot. :param label: The label for the series that will be plotted. @@ -64,6 +67,7 @@ def plot_loss_per_epoch(metrics: Dict[str, Any], metric_name: str, label: Option def plot_val_dice_per_epoch(metrics: Dict[str, Any]) -> int: """ Creates a plot of all validation Dice scores per epoch, for all classes apart from background. + :param metrics: :return: The number of series that were plotted in the graph. Can return 0 if the metrics dictionary does not contain any validation Dice score. @@ -83,6 +87,7 @@ def plot_val_dice_per_epoch(metrics: Dict[str, Any]) -> int: def add_legend(series_count: int) -> None: """ Adds a legend to the present plot, with the column layout depending on the number of series. + :param series_count: :return: """ @@ -93,6 +98,7 @@ def add_legend(series_count: int) -> None: def resize_and_save(width_inch: int, height_inch: int, filename: PathOrString, dpi: int = 150) -> None: """ Resizes the present figure to the given (width, height) in inches, and saves it to the given filename. + :param width_inch: The width of the figure in inches. :param height_inch: The height of the figure in inches. :param filename: The filename to save to. @@ -113,13 +119,17 @@ def plot_image_and_label_contour(image: np.ndarray, """ Creates a plot that shows the given 2D image in greyscale, and overlays a contour that shows where the 'labels' array has value 1. + :param image: A 2D image :param labels: A binary 2D image, or a list of binary 2D images. A contour will be plotted for each of those binary images. + :param contour_arguments: A dictionary of keyword arguments that will be passed directly into matplotlib's contour function. Can also be a list of dictionaries, with one dict per entry in the 'labels' argument. + :param image_range: If provided, the image will be plotted using the given range for the color limits. If None, the minimum and maximum image values will be mapped to the endpoints of the color map. + :param plot_file_name: The file name that should be used to save the plot. """ if image.ndim != 2: @@ -183,6 +193,7 @@ def plot_before_after_statistics(image_before: np.ndarray, that were obtained before and after a transformation of pixel values. The plot contains histograms, box plots, and visualizations of a single XY slice at z_slice. If a mask argument is provided, only the image pixel values inside of the mask will be plotted. + :param image_before: The first image for which to plot statistics. :param image_after: The second image for which to plot statistics. :param mask: Indicators with 1 for foreground, 0 for background. If None, plot statistics for all image pixels. @@ -237,10 +248,13 @@ def plot_normalization_result(loaded_images: Sample, The first plot contains pixel value histograms before and after photometric normalization. The second plot contains the normalized image, overlayed with contours for the foreground pixels, at the slice where the foreground has most pixels. + :param loaded_images: An instance of Sample with the image and the labels. The first channel of the image will be plotted. + :param image_range: The image value range that will be mapped to the color map. If None, the full image range will be mapped to the colormap. + :param normalizer: The photometric normalization that should be applied. :param result_folder: The folder into which the resulting PNG files should be written. :param result_prefix: The prefix for all output filenames. @@ -283,6 +297,7 @@ def plot_contours_for_all_classes(sample: Sample, """ Creates a plot with the image, the ground truth, and the predicted segmentation overlaid. One plot is created for each class, each plotting the Z slice where the ground truth has most pixels. + :param sample: The image sample, with the photonormalized image and the ground truth labels. :param segmentation: The predicted segmentation: multi-value, size Z x Y x X. :param foreground_class_names: The names of all classes, excluding the background class. @@ -290,6 +305,7 @@ def plot_contours_for_all_classes(sample: Sample, :param result_prefix: A string prefix that will be used for all plots. :param image_range: The minimum and maximum image values that will be mapped to the color map ranges. If None, use the actual min and max values. + :param channel_index: The index of the image channel that should be plotted. :return: The paths to all generated PNG files. """ @@ -337,6 +353,7 @@ def segmentation_and_groundtruth_plot(prediction: np.ndarray, ground_truth: np.n """ Plot predicted and the ground truth segmentations. Always plots the middle slice (to match surface distance plots), which can sometimes lead to an empty plot. + :param prediction: 3D volume (X x Y x Z) of predicted segmentation :param ground_truth: 3D volume (X x Y x Z) of ground truth segmentation :param subject_id: ID of subject for annotating plot @@ -389,6 +406,7 @@ def surface_distance_ground_truth_plot(ct: np.ndarray, ground_truth: np.ndarray, annotator: str = None) -> None: """ Plot surface distances where prediction > 0, with ground truth contour + :param ct: CT scan :param ground_truth: Ground truth segmentation :param sds_full: Surface distances (full= where prediction > 0) @@ -471,10 +489,12 @@ def scan_with_transparent_overlay(scan: np.ndarray, information in the range [0, 1]. High values of the `overlay` are shown as opaque red, low values as transparent red. Plots are created in the current axis. + :param scan: A 3-dimensional image in (Z, Y, X) ordering :param overlay: A 3-dimensional image in (Z, Y, X) ordering, with values between 0 and 1. :param dimension: The array dimension along with the plot should be created. dimension=0 will generate an axial slice. + :param position: The index in the chosen dimension where the plot should be created. :param spacing: The tuple of voxel spacings, in (Z, Y, X) order. """ diff --git a/InnerEye/ML/reports/classification_multilabel_report.py b/InnerEye/ML/reports/classification_multilabel_report.py index d38dd7e56..ad915813c 100644 --- a/InnerEye/ML/reports/classification_multilabel_report.py +++ b/InnerEye/ML/reports/classification_multilabel_report.py @@ -73,6 +73,7 @@ def get_dataframe_with_exact_label_matches(metrics_df: pd.DataFrame, The dataframe must have at least the following columns (defined in the LoggingColumns enum): LoggingColumns.Hue, LoggingColumns.Patient, LoggingColumns.Label, LoggingColumns.ModelOutput. Any other columns will be ignored. + :param prediction_target_set_to_match: The set of prediction targets to which each sample is compared :param all_prediction_targets: The entire set of prediction targets on which the model is trained :param thresholds_per_prediction_target: Thresholds per prediction target to decide if model has predicted True or @@ -140,8 +141,10 @@ def print_metrics_for_thresholded_output_for_all_prediction_targets(csv_to_set_o :param csv_to_set_optimal_threshold: Csv written during inference time for the val set. This is used to determine the optimal threshold for classification. + :param csv_to_compute_metrics: Csv written during inference time for the test set. Metrics are calculated for this csv. + :param config: Model config """ diff --git a/InnerEye/ML/reports/classification_report.py b/InnerEye/ML/reports/classification_report.py index f88ddfc35..7de18a93a 100644 --- a/InnerEye/ML/reports/classification_report.py +++ b/InnerEye/ML/reports/classification_report.py @@ -69,9 +69,11 @@ def read_csv_and_filter_prediction_target(csv: Path, prediction_target: str, :param csv: Path to the metrics CSV file. Must contain at least the following columns (defined in the LoggingColumns enum): LoggingColumns.Patient, LoggingColumns.Hue. + :param prediction_target: Target ("hue") by which to filter. :param crossval_split_index: If specified, filter rows only for the respective run (requires LoggingColumns.CrossValidationSplitIndex). + :param data_split: If specified, filter rows by Train/Val/Test (requires LoggingColumns.DataSplit). :param epoch: If specified, filter rows for given epoch (default: last epoch only; requires LoggingColumns.Epoch). :return: Filtered dataframe. @@ -122,9 +124,11 @@ def get_labels_and_predictions(csv: Path, prediction_target: str, :param csv: Path to the metrics CSV file. Must contain at least the following columns (defined in the LoggingColumns enum): LoggingColumns.Patient, LoggingColumns.Hue. + :param prediction_target: Target ("hue") by which to filter. :param crossval_split_index: If specified, filter rows only for the respective run (requires LoggingColumns.CrossValidationSplitIndex). + :param data_split: If specified, filter rows by Train/Val/Test (requires LoggingColumns.DataSplit). :param epoch: If specified, filter rows for given epoch (default: last epoch only; requires LoggingColumns.Epoch). :return: Filtered labels and model outputs. @@ -150,6 +154,7 @@ def get_labels_and_predictions_from_dataframe(df: pd.DataFrame) -> LabelsAndPred def format_pr_or_roc_axes(plot_type: str, ax: Axes) -> None: """ Format PR or ROC plot with appropriate title, axis labels, limits, and grid. + :param plot_type: Either 'pr' or 'roc'. :param ax: Axes object to format. """ @@ -171,9 +176,11 @@ def plot_pr_and_roc_curves(labels_and_model_outputs: LabelsAndPredictions, axs: plot_kwargs: Optional[Dict[str, Any]] = None) -> None: """ Given labels and model outputs, plot the ROC and PR curves. + :param labels_and_model_outputs: :param axs: Pair of axes objects onto which to plot the ROC and PR curves, respectively. New axes are created by default. + :param plot_kwargs: Plotting options to be passed to both `ax.plot(...)` calls. """ if axs is None: @@ -202,11 +209,14 @@ def plot_scores_and_summary(all_labels_and_model_outputs: Sequence[LabelsAndPred Each plotted curve is interpolated onto a common horizontal grid, and the median and CI are computed vertically at each horizontal location. + :param all_labels_and_model_outputs: Collection of ground-truth labels and model predictions (e.g. for various cross-validation runs). + :param scoring_fn: A scoring function mapping a `LabelsAndPredictions` object to X and Y coordinates for plotting. :param interval_width: A value in [0, 1] representing what fraction of the data should be contained in the shaded area. The edges of the interval are `median +/- interval_width/2`. + :param ax: Axes object onto which to plot (default: use current axes). :return: A tuple of `(line_handles, summary_handle)` to use in setting a legend for the plot: `line_handles` is a list corresponding to the curves for each `LabelsAndPredictions`, and `summary_handle` references the median line @@ -236,8 +246,10 @@ def plot_pr_and_roc_curves_crossval(all_labels_and_model_outputs: Sequence[Label """ Given a list of LabelsAndPredictions objects, plot the corresponding ROC and PR curves, along with median line and shaded 80% confidence interval (computed over TPRs and precisions for each fixed FPR and recall value). + :param all_labels_and_model_outputs: Collection of ground-truth labels and model predictions (e.g. for various cross-validation runs). + :param axs: Pair of axes objects onto which to plot the ROC and PR curves, respectively. New axes are created by default. """ @@ -276,6 +288,7 @@ def plot_pr_and_roc_curves_from_csv(metrics_csv: Path, config: ScalarModelBase, """ Given the CSV written during inference time and the model config, plot the ROC and PR curves for all prediction targets. + :param metrics_csv: Path to the metrics CSV file. :param config: Model config. :param data_split: Whether to filter the CSV file for Train, Val, or Test results (default: no filtering). @@ -300,8 +313,10 @@ def get_metric(predictions_to_set_optimal_threshold: LabelsAndPredictions, optimal_threshold: Optional[float] = None) -> float: """ Given LabelsAndPredictions objects for the validation and test sets, return the specified metric. + :param predictions_to_set_optimal_threshold: This set of ground truth labels and model predictions is used to determine the optimal threshold for classification. + :param predictions_to_compute_metrics: The set of labels and model outputs to calculate metrics for. :param metric: The name of the metric to calculate. :param optimal_threshold: If provided, use this threshold instead of calculating an optimal threshold. @@ -351,6 +366,7 @@ def get_all_metrics(predictions_to_set_optimal_threshold: LabelsAndPredictions, is_thresholded: bool = False) -> Dict[str, float]: """ Given LabelsAndPredictions objects for the validation and test sets, compute some metrics. + :param predictions_to_set_optimal_threshold: This is used to determine the optimal threshold for classification. :param predictions_to_compute_metrics: Metrics are calculated for this set. :param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point) @@ -379,6 +395,7 @@ def print_metrics(predictions_to_set_optimal_threshold: LabelsAndPredictions, is_thresholded: bool = False) -> None: """ Given LabelsAndPredictions objects for the validation and test sets, print out some metrics. + :param predictions_to_set_optimal_threshold: This is used to determine the optimal threshold for classification. :param predictions_to_compute_metrics: Metrics are calculated for this set. :param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point) @@ -402,16 +419,21 @@ def get_metrics_table_for_prediction_target(csv_to_set_optimal_threshold: Path, :param csv_to_set_optimal_threshold: CSV written during inference time for the val set. This is used to determine the optimal threshold for classification. + :param csv_to_compute_metrics: CSV written during inference time for the test set. Metrics are calculated for this CSV. + :param config: Model config :param prediction_target: The prediction target for which to compute metrics. :param data_split_to_set_optimal_threshold: Whether to filter the validation CSV file for Train, Val, or Test results (default: no filtering). + :param data_split_to_compute_metrics: Whether to filter the test CSV file for Train, Val, or Test results (default: no filtering). + :param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point) or are floating point numbers. + :param is_crossval_report: If True, assumes CSVs contain results for multiple cross-validation runs and formats the metrics along with means and standard deviations. Otherwise, collect metrics for a single run. :return: Tuple of rows and header, where each row and the header are lists of strings of same length (2 if @@ -468,15 +490,20 @@ def print_metrics_for_all_prediction_targets(csv_to_set_optimal_threshold: Path, :param csv_to_set_optimal_threshold: CSV written during inference time for the val set. This is used to determine the optimal threshold for classification. + :param csv_to_compute_metrics: CSV written during inference time for the test set. Metrics are calculated for this CSV. + :param config: Model config :param data_split_to_set_optimal_threshold: Whether to filter the validation CSV file for Train, Val, or Test results (default: no filtering). + :param data_split_to_compute_metrics: Whether to filter the test CSV file for Train, Val, or Test results (default: no filtering). + :param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point) or are floating point numbers. + :param is_crossval_report: If True, assumes CSVs contain results for multiple cross-validation runs and prints the metrics along with means and standard deviations. Otherwise, prints metrics for a single run. """ @@ -563,11 +590,14 @@ def print_k_best_and_worst_performing(val_metrics_csv: Path, """ Print the top "k" best predictions (i.e. correct classifications where the model was the most certain) and the top "k" worst predictions (i.e. misclassifications where the model was the most confident). + :param val_metrics_csv: Path to one of the metrics csvs written during inference. This set of metrics will be used to determine the thresholds for predicting labels on the test set. The best and worst performing subjects will not be printed out for this csv. + :param test_metrics_csv: Path to one of the metrics csvs written during inference. This is the csv for which best and worst performing subjects will be printed out. + :param k: Number of subjects of each category to print out. :param prediction_target: The class label to filter on """ @@ -605,6 +635,7 @@ def get_image_filepath_from_subject_id(subject_id: str, config: ScalarModelBase) -> List[Path]: """ Return the filepaths for images associated with a subject. If the subject is not found, raises a ValueError. + :param subject_id: Subject to retrive image for :param dataset: scalar dataset object :param config: model config @@ -623,6 +654,7 @@ def get_image_labels_from_subject_id(subject_id: str, config: ScalarModelBase) -> List[str]: """ Return the ground truth labels associated with a subject. If the subject is not found, raises a ValueError. + :param subject_id: Subject to retrive image for :param dataset: scalar dataset object :param config: model config @@ -657,6 +689,7 @@ def get_image_outputs_from_subject_id(subject_id: str, def plot_image_from_filepath(filepath: Path, im_width: int) -> bool: """ Plots a 2D image given the filepath. Returns false if the image could not be plotted (for example, if it was 3D). + :param filepath: Path to image :param im_width: Display width for image :return: True if image was plotted, False otherwise @@ -693,6 +726,7 @@ def plot_image_for_subject(subject_id: str, metrics_df: Optional[pd.DataFrame] = None) -> None: """ Given a subject ID, plots the corresponding image. + :param subject_id: Subject to plot image for :param dataset: scalar dataset object :param im_width: Display width for image @@ -741,11 +775,14 @@ def plot_k_best_and_worst_performing(val_metrics_csv: Path, test_metrics_csv: Pa """ Plot images for the top "k" best predictions (i.e. correct classifications where the model was the most certain) and the top "k" worst predictions (i.e. misclassifications where the model was the most confident). + :param val_metrics_csv: Path to one of the metrics csvs written during inference. This set of metrics will be used to determine the thresholds for predicting labels on the test set. The best and worst performing subjects will not be printed out for this csv. + :param test_metrics_csv: Path to one of the metrics csvs written during inference. This is the csv for which best and worst performing subjects will be printed out. + :param k: Number of subjects of each category to print out. :param prediction_target: The class label to filter on :param config: scalar model config object diff --git a/InnerEye/ML/reports/notebook_report.py b/InnerEye/ML/reports/notebook_report.py index cbb122c85..f071ad7b8 100644 --- a/InnerEye/ML/reports/notebook_report.py +++ b/InnerEye/ML/reports/notebook_report.py @@ -26,6 +26,7 @@ def get_ipynb_report_name(report_type: str) -> str: """ Constructs the name of the report (as an ipython notebook). + :param report_type: suffix describing the report, added to the filename :return: """ @@ -35,6 +36,7 @@ def get_ipynb_report_name(report_type: str) -> str: def get_html_report_name(report_type: str) -> str: """ Constructs the name of the report (as an html file). + :param report_type: suffix describing the report, added to the filename :return: """ @@ -49,6 +51,7 @@ def print_header(message: str, level: int = 2) -> None: """ Displays a message, and afterwards repeat it as Markdown with the given indentation level (level=1 is the outermost, `# Foo`. + :param message: The header string to display. :param level: The Markdown indentation level. level=1 for top level, level=3 for `### Foo` """ @@ -59,6 +62,7 @@ def print_header(message: str, level: int = 2) -> None: def print_table(rows: Sequence[Sequence[str]], header: Optional[Sequence[str]] = None) -> None: """ Displays the provided content in a formatted HTML table, with optional column headers. + :param rows: List of rows, where each row is a list of string-valued cell contents. :param header: List of column headers. If given, this special first row is rendered with emphasis. """ @@ -74,6 +78,7 @@ def print_table(rows: Sequence[Sequence[str]], header: Optional[Sequence[str]] = def generate_notebook(template_notebook: Path, notebook_params: Dict, result_notebook: Path) -> Path: """ Generates a notebook report as jupyter notebook and html page + :param template_notebook: path to template notebook :param notebook_params: parameters for the notebook :param result_notebook: the path for the executed notebook diff --git a/InnerEye/ML/reports/segmentation_report.py b/InnerEye/ML/reports/segmentation_report.py index 44959571a..37c420570 100644 --- a/InnerEye/ML/reports/segmentation_report.py +++ b/InnerEye/ML/reports/segmentation_report.py @@ -40,6 +40,7 @@ def plot_scores_for_csv(path_csv: str, outlier_range: float, max_row_count: int) def display_without_index(df: pd.DataFrame) -> None: """ Prints the given dataframe as HTML via the `display` function, but without the index column. + :param df: The dataframe to print. """ display(HTML(df.to_html(index=False))) @@ -52,6 +53,7 @@ def display_metric(df: pd.DataFrame, high_values_are_good: bool) -> None: """ Displays a dataframe with a metric per structure, first showing the + :param max_row_count: The number of rows to print when showing the lowest score patients :param df: The dataframe with metrics per structure :param metric_name: The metric to sort by. @@ -80,11 +82,13 @@ def worst_patients_and_outliers(df: pd.DataFrame, Prints a dataframe that contains the worst patients by the given metric, and a column indicating whether the performance is so poor that it is considered an outlier: metric value which is outside of outlier_range * standard deviation from the mean. + :param df: The dataframe with metrics. :param outlier_range: The multiplier for standard deviation when constructing the interval for outliers. :param metric_name: The metric for which the "worst" patients should be computed. :param high_values_are_good: If True, high values for the metric are considered good, and hence low values are marked as outliers. If False, low values are considered good, and high values are marked as outliers. + :param max_row_count: The maximum number of rows to print. :return: """ diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py index 31498f43e..5d2bde49a 100644 --- a/InnerEye/ML/run_ml.py +++ b/InnerEye/ML/run_ml.py @@ -69,6 +69,7 @@ def check_dataset_folder_exists(local_dataset: PathOrString) -> Path: """ Checks if a folder with a local dataset exists. If it does exist, return the argument converted to a Path instance. If it does not exist, raise a FileNotFoundError. + :param local_dataset: The dataset folder to check. :return: The local_dataset argument, converted to a Path. """ @@ -83,6 +84,7 @@ def log_metrics(metrics: Dict[ModelExecutionMode, InferenceMetrics], run_context: Run) -> None: """ Log metrics for each split to the provided run, or the current run context if None provided + :param metrics: Dictionary of inference results for each split. :param run_context: Run for which to log the metrics to, use the current run context if None provided """ @@ -111,18 +113,24 @@ def __init__(self, """ Driver class to run a ML experiment. Note that the project root argument MUST be supplied when using InnerEye as a package! + :param model_config: If None, run the training as per the `container` argument (bring-your-own-model). If not None, this is the model configuration for a built-in InnerEye model. + :param container: The LightningContainer object to use for training. If None, assume that the training is for a built-in InnerEye model. + :param azure_config: Azure related configurations :param project_root: Project root. This should only be omitted if calling run_ml from the test suite. Supplying it is crucial when using InnerEye as a package or submodule! + :param post_cross_validation_hook: A function to call after waiting for completion of cross validation runs. The function is called with the model configuration and the path to the downloaded and merged metrics files. + :param model_deployment_hook: an optional function for deploying a model in an application-specific way. If present, it should take a LightningContainer, an AzureConfig, an AzureML Model and a ModelProcessing object as arguments, and return an object of any type. + :param output_subfolder: If provided, the output folder structure will have an additional subfolder, when running outside AzureML. """ @@ -145,6 +153,7 @@ def setup(self, azure_run_info: Optional[AzureRunInfo] = None) -> None: """ If the present object is using one of the InnerEye built-in models, create a (fake) container for it and call the setup method. It sets the random seeds, and then creates the actual Lightning modules. + :param azure_run_info: When running in AzureML or on a local VM, this contains the paths to the datasets. This can be missing when running in unit tests, where the local dataset paths are already populated. """ @@ -404,6 +413,7 @@ def lightning_data_module_dataloaders(data: LightningDataModule) -> Dict[ModelEx def run_inference_for_lightning_models(self, checkpoint_paths: List[Path]) -> None: """ Run inference on the test set for all models that are specified via a LightningContainer. + :param checkpoint_paths: The path to the checkpoint that should be used for inference. """ if len(checkpoint_paths) != 1: @@ -461,6 +471,7 @@ def run_inference(self, checkpoint_paths: List[Path], model_proc: ModelProcessing) -> None: """ Run inference on InnerEyeContainer models + :param checkpoint_paths: Checkpoint paths to initialize model :param model_proc: whether we are running an ensemble model from within a child run with index 0. If we are, then outputs will be written to OTHER_RUNS/ENSEMBLE under the main outputs directory. @@ -515,6 +526,7 @@ def register_model(self, """ Registers a new model in the workspace's model registry on AzureML to be deployed further. The AzureML run's tags are updated to describe with information about ensemble creation and the parent run ID. + :param checkpoint_paths: Checkpoint paths to register. :param model_proc: whether it's a single or ensemble model. :returns Tuple element 1: AML model object, or None if no model could be registered. @@ -601,9 +613,11 @@ def copy_child_paths_to_folder(self, extra_code_directory, and all checkpoints in a newly created "checkpoints" folder inside the model. In addition, the name of the present AzureML Python environment will be written to a file, for later use in the inference code. + :param model_folder: The folder into which all files should be copied. :param checkpoint_paths: A list with absolute paths to checkpoint files. They are expected to be inside of the model's checkpoint folder. + :param python_environment: The Python environment that is used in the present AzureML run. """ @@ -704,6 +718,7 @@ def model_inference_train_and_test(self, def wait_for_runs_to_finish(self, delay: int = 60) -> None: """ Wait for cross val runs (apart from the current one) to finish and then aggregate results of all. + :param delay: How long to wait between polls to AML to get status of child runs """ with logging_section("Waiting for sibling runs"): diff --git a/InnerEye/ML/runner.py b/InnerEye/ML/runner.py index f41fd409f..81a00f231 100755 --- a/InnerEye/ML/runner.py +++ b/InnerEye/ML/runner.py @@ -110,10 +110,12 @@ class Runner: """ This class contains the high-level logic to start a training run: choose a model configuration by name, submit to AzureML if needed, or otherwise start the actual training and test loop. + :param project_root: The root folder that contains all of the source code that should be executed. :param yaml_config_file: The path to the YAML file that contains values to supply into sys.argv. :param post_cross_validation_hook: A function to call after waiting for completion of cross validation runs. The function is called with the model configuration and the path to the downloaded and merged metrics files. + :param model_deployment_hook: an optional function for deploying a model in an application-specific way. If present, it should take a model config (SegmentationModelBase), an AzureConfig, and an AzureML Model as arguments, and return an optional Path and a further object of any type. @@ -378,6 +380,7 @@ def print_git_tags(self) -> None: def run_in_situ(self, azure_run_info: AzureRunInfo) -> None: """ Actually run the AzureML job; this method will typically run on an Azure VM. + :param azure_run_info: Contains all information about the present run in AzureML, in particular where the datasets are mounted. """ @@ -423,6 +426,7 @@ def create_ml_runner(self) -> MLRunner: def default_post_cross_validation_hook(config: ModelConfigBase, root_folder: Path) -> None: """ A function to run after cross validation results have been aggregated, before they are uploaded to AzureML. + :param config: The configuration of the model that should be trained. :param root_folder: The folder with all aggregated and per-split files. """ diff --git a/InnerEye/ML/scalar_config.py b/InnerEye/ML/scalar_config.py index 9828cd888..8f6abd5a1 100644 --- a/InnerEye/ML/scalar_config.py +++ b/InnerEye/ML/scalar_config.py @@ -89,6 +89,7 @@ def difference(labels: List) -> Any: def get_scaling_transform(max_value: int = 100, min_value: int = 0, last_in_pipeline: bool = True) -> Callable: """ Defines the function to scale labels. + :param max_value: :param min_value: :param last_in_pipeline: if the transformation is the last @@ -340,6 +341,7 @@ def get_non_image_feature_channels_dict(self) -> Dict: def filter_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: """ Filter dataframes based on expected values on columns + :param df: the input dataframe :return: the filtered dataframe """ @@ -592,6 +594,7 @@ def get_non_image_features_dict(default_channels: List[str], :param default_channels: the channels to use for all features except the features specified in specific_channels + :param specific_channels: a dictionary mapping feature names to channels for all features that do not use the default channels """ diff --git a/InnerEye/ML/surface_distance_heatmaps.py b/InnerEye/ML/surface_distance_heatmaps.py index 4db34a7ed..a653d167e 100644 --- a/InnerEye/ML/surface_distance_heatmaps.py +++ b/InnerEye/ML/surface_distance_heatmaps.py @@ -43,6 +43,7 @@ def load_predictions(run_type: SurfaceDistanceRunType, azure_config: AzureConfig ) -> List[Segmentation]: """ For each run type (IOV or outliers), instantiate a list of predicted Segmentations and return + :param run_type: either "iov" or "outliers: :param azure_config: AzureConfig :param model_config: GenericConfig diff --git a/InnerEye/ML/utils/checkpoint_handling.py b/InnerEye/ML/utils/checkpoint_handling.py index f02fa8516..d54c82269 100644 --- a/InnerEye/ML/utils/checkpoint_handling.py +++ b/InnerEye/ML/utils/checkpoint_handling.py @@ -74,6 +74,7 @@ def download_recovery_checkpoints_or_weights(self, only_return_path: bool = Fals Download checkpoints from a run recovery object or from a weights url. Set the checkpoints path based on the run_recovery_object, weights_url or local_weights_path. This is called at the start of training. + :param: only_return_path: if True, return a RunRecovery object with the path to the checkpoint without actually downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple nodes. If False, return the RunRecovery object and download the checkpoint to disk. @@ -246,6 +247,7 @@ def download_folder_from_run_to_temp_folder(folder: str, :param run: If provided, download the files from that run. If omitted, download the files from the current run (taken from RUN_CONTEXT) + :param workspace: The AML workspace where the run is located. If omitted, the hi-ml defaults of finding a workspace are used (current workspace when running in AzureML, otherwise expecting a config.json file) :return: The path to which the files were downloaded. The files are located in that folder, without any further @@ -276,6 +278,7 @@ def find_recovery_checkpoint_on_disk_or_cloud(path: Path) -> Optional[Path]: """ Looks at all the checkpoint files and returns the path to the one that should be used for recovery. If no checkpoint files are found on disk, the function attempts to download from the current AzureML run. + :param path: The folder to start searching in. :return: None if there is no suitable recovery checkpoints, or else a full path to the checkpoint file. """ @@ -294,6 +297,7 @@ def get_recovery_checkpoint_path(path: Path) -> Path: """ Returns the path to the last recovery checkpoint in the given folder or the provided filename. Raises a FileNotFoundError if no recovery checkpoint file is present. + :param path: Path to checkpoint folder """ recovery_checkpoint = find_recovery_checkpoint(path) @@ -308,6 +312,7 @@ def find_recovery_checkpoint(path: Path) -> Optional[Path]: Finds the checkpoint file in the given path that can be used for re-starting the present job. This can be an autosave checkpoint, or the last checkpoint. All existing checkpoints are loaded, and the one for the highest epoch is used for recovery. + :param path: The folder to search in. :return: Returns the checkpoint file to use for re-starting, or None if no such file was found. """ @@ -337,6 +342,7 @@ def find_recovery_checkpoint(path: Path) -> Optional[Path]: def cleanup_checkpoints(path: Path) -> None: """ Remove autosave checkpoints from the given checkpoint folder, and check if a "last.ckpt" checkpoint is present. + :param path: The folder that contains all checkpoint files. """ logging.info(f"Files in checkpoint folder: {' '.join(p.name for p in path.glob('*'))}") @@ -359,6 +365,7 @@ def download_best_checkpoints_from_child_runs(config: OutputParams, run: Run) -> The checkpoints for the sibling runs will go into folder 'OTHER_RUNS/' in the checkpoint folder. There is special treatment for the child run that is equal to the present AzureML run, its checkpoints will be read off the checkpoint folder as-is. + :param config: Model related configs. :param run: The Hyperdrive parent run to download from. :return: run recovery information @@ -392,10 +399,12 @@ def download_all_checkpoints_from_run(config: OutputParams, run: Run, only_return_path: bool = False) -> RunRecovery: """ Downloads all checkpoints of the provided run inside the checkpoints folder. + :param config: Model related configs. :param run: Run whose checkpoints should be recovered :param subfolder: optional subfolder name, if provided the checkpoints will be downloaded to CHECKPOINT_FOLDER / subfolder. If None, the checkpoint are downloaded to CHECKPOINT_FOLDER of the current run. + :param: only_return_path: if True, return a RunRecovery object with the path to the checkpoint without actually downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple nodes. If False, return the RunRecovery object and download the checkpoint to disk. diff --git a/InnerEye/ML/utils/config_loader.py b/InnerEye/ML/utils/config_loader.py index 187623360..e67f4da1c 100644 --- a/InnerEye/ML/utils/config_loader.py +++ b/InnerEye/ML/utils/config_loader.py @@ -60,6 +60,7 @@ def _get_model_config(module_spec: ModuleSpec) -> Optional[DeepLearningConfig]: Given a module specification check to see if it has a class property with the provided, and instantiate that config class with the provided . Otherwise, return None. + :param module_spec: :return: Instantiated model config if it was found. """ diff --git a/InnerEye/ML/utils/csv_util.py b/InnerEye/ML/utils/csv_util.py index d7b8f21fa..e0a686db9 100644 --- a/InnerEye/ML/utils/csv_util.py +++ b/InnerEye/ML/utils/csv_util.py @@ -61,6 +61,7 @@ def drop_rows_missing_important_values(df: pd.DataFrame, important_cols: List[st """ Remove rows from the DataFrame in which the columns that have been specified by the user as "important" contain null values or only whitespace. + :param df: DataFrame :param important_cols: Columns which must not contain null values :return: df: DataFrame without the dropped rows. @@ -86,6 +87,7 @@ def extract_outliers(df: pd.DataFrame, outlier_range: float, outlier_col: str = to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above mean + outlier_range * std (if outlier_type is HIGH) or below mean - outlier_range * std (if outlier_type is LOW). + :param outlier_col: The column from which to calculate outliers, e.g. Dice :param outlier_type: Either LOW (i.e. below accepted range) or HIGH (above accepted range) outliers. :return: DataFrame containing only the outliers @@ -108,11 +110,13 @@ def mark_outliers(df: pd.DataFrame, Rows that are not considered outliers have an empty string in the new column. Outliers are taken from the column `outlier_col`, that have a value that falls outside of mean +- outlier_range * std. + :param df: DataFrame from which to extract the outliers :param outlier_range: The number of standard deviation from the mean which the points have to be apart to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above mean + outlier_range * std (if outlier_type is HIGH) or below mean - outlier_range * std (if outlier_type is LOW). + :param outlier_col: The column from which to calculate outliers, e.g. Dice :param high_values_are_good: If True, high values for the metric are considered good, and hence low values are marked as outliers. If False, low values are considered good, and high values are marked as outliers. @@ -137,10 +141,12 @@ def get_worst_performing_outliers(df: pd.DataFrame, """ Returns a sorted list (worst to best) of all the worst performing outliers in the metrics table according to metric provided by outlier_col_name + :param df: Metrics DataFrame :param outlier_col_name: The column by which to determine outliers :param outlier_range: The standard deviation from the mean which the points have to be below to be considered an outlier. + :param max_n_outliers: the number of (worst performing) outlier IDs to return. :return: a sorted list (worst to best) of all the worst performing outliers """ diff --git a/InnerEye/ML/utils/device_aware_module.py b/InnerEye/ML/utils/device_aware_module.py index 74df1f663..51718166d 100644 --- a/InnerEye/ML/utils/device_aware_module.py +++ b/InnerEye/ML/utils/device_aware_module.py @@ -51,6 +51,7 @@ def get_input_tensors(self, item: T) -> List[E]: """ Extract the input tensors from a data sample as required by the forward pass of the module. + :param item: a data sample :return: the correct input tensors for the forward pass """ diff --git a/InnerEye/ML/utils/hdf5_util.py b/InnerEye/ML/utils/hdf5_util.py index 78f7e4b9d..9f4788b57 100644 --- a/InnerEye/ML/utils/hdf5_util.py +++ b/InnerEye/ML/utils/hdf5_util.py @@ -72,6 +72,7 @@ def __init__(self, def parse_acquisition_date(date: str) -> Optional[datetime]: """ Converts a string representing a date to a datetime object + :param date: string representing a date :return: converted date, None if the string is invalid for date conversion. @@ -90,6 +91,7 @@ def _hdf5_data_path(data_field: HDF5Field) -> str: def _load_image(hdf5_data: h5py.File, data_field: HDF5Field) -> np.ndarray: """ Load the volume from the HDF5 file. + :param hdf5_data: path to the hdf5 file :param data_field: field of the hdf5 file containing the data :return: image as numpy array diff --git a/InnerEye/ML/utils/image_util.py b/InnerEye/ML/utils/image_util.py index e0eb840c5..99dbaddf8 100644 --- a/InnerEye/ML/utils/image_util.py +++ b/InnerEye/ML/utils/image_util.py @@ -48,6 +48,7 @@ def get_unit_image_header(spacing: Optional[TupleFloat3] = None) -> ImageHeader: """ Creates an ImageHeader object with the origin at 0, and unit direction. The spacing is set to the argument, defaulting to (1, 1, 1) if not provided. + :param spacing: The image spacing, as a (Z, Y, X) tuple. """ if not spacing: @@ -186,6 +187,7 @@ def _pad_images(images: np.ndarray, :param padding_vector: padding before and after in each dimension eg: ((2,2), (3,3), (2,0)) will pad 4 pixels in Z (2 on each side), 6 pixels in Y (3 on each side) and 2 in X (2 on the left and 0 on the right). + :param padding_mode: a valid numpy padding mode. :return: padded copy of the original image. """ @@ -241,9 +243,11 @@ def largest_connected_components(img: np.ndarray, Select the largest connected binary components (plural) in an image. If deletion_limit is set in which case a component is only deleted (i.e. its voxels are False in the output) if its voxel count as a proportion of all the True voxels in the input is less than deletion_limit. + :param img: np.ndarray :param deletion_limit: if set, a component is deleted only if its voxel count as a proportion of all the True voxels in the input is less than deletion_limit. + :param class_index: Optional. Can be used to provide a class index for logging purposes if the image contains only pixels from a specific class. """ @@ -281,6 +285,7 @@ def extract_largest_foreground_connected_component( restrictions: Optional[List[Tuple[int, Optional[float]]]] = None) -> np.ndarray: """ Extracts the largest foreground connected component per class from a multi-label array. + :param multi_label_array: An array of class assignments, i.e. value c at (z, y, x) is a class c. :param restrictions: restrict processing to a subset of the classes (if provided). Each element is a pair (class_index, threshold) where threshold may be None. @@ -311,6 +316,7 @@ def merge_masks(masks: np.ndarray) -> np.ndarray: """ Merges a one-hot encoded mask tensor (Classes x Z x Y x X) into a multi-label map with labels corresponding to their index in the original tensor of shape (Z x Y x X). + :param masks: array of shape (Classes x Z x Y x X) containing the mask for each class :return: merged_mask of shape (Z x Y x X). """ @@ -399,6 +405,7 @@ def check_array_range(data: np.ndarray, expected_range: Optional[Range] = None, :param data: The array to check. It can have any size. :param expected_range: The interval that all array elements must fall into. The first entry is the lower bound, the second entry is the upper bound. + :param error_prefix: A string to use as the prefix for the error message. """ if expected_range is None: @@ -514,6 +521,7 @@ def gaussian_smooth_posteriors(posteriors: np.ndarray, kernel_size_mm: TupleFloa :param posteriors: Normalized probability distribution in range [0, 1] for each class, in shape: Class x Z x Y x X + :param kernel_size_mm: The size of the smoothing kernel in mm to be used in each dimension (Z, Y, X) :param voxel_spacing_mm: Voxel spacing to use to map from mm space to pixel space for the Gaussian sigma parameter for each dimension in (Z x Y x X) order. @@ -558,6 +566,7 @@ def segmentation_to_one_hot(segmentation: torch.Tensor, :param segmentation: A segmentation as a multi-label map of shape [B, C, Z, Y, X] :param use_gpu: If true, and the input is not yet on the GPU, move the intermediate tensors to the GPU. The result will be on the same device as the argument `segmentation` + :param result_dtype: The torch data type that the result tensor should have. This would be either float16 or float32 :return: A torch tensor with one-hot encoding of the segmentation of shape [B, C*HDF5_NUM_SEGMENTATION_CLASSES, Z, Y, X] @@ -702,6 +711,7 @@ def apply_summed_probability_rules(model_config: SegmentationModelBase, :param model_config: Model configuration information :param posteriors: Confidences per voxel per class, in format Batch x Classes x Z x Y x X if batched, or Classes x Z x Y x X if not batched. + :param segmentation: Class labels per voxel, in format Batch x Z x Y x X if batched, or Z x Y x X if not batched. :return: Modified segmentation, as Batch x Z x Y x X if batched, or Z x Y x X if not batched. """ diff --git a/InnerEye/ML/utils/io_util.py b/InnerEye/ML/utils/io_util.py index d25024a18..52ab589d6 100644 --- a/InnerEye/ML/utils/io_util.py +++ b/InnerEye/ML/utils/io_util.py @@ -207,6 +207,7 @@ def load_nifti_image(path: PathOrString, image_type: Optional[Type] = float) -> :param path: The path to the image to load. :return: A numpy array of the image and header data if applicable. + :param image_type: The type to load the image in, set to None to not cast, default is float :raises ValueError: If the path is invalid or the image is not 3D. """ @@ -214,6 +215,7 @@ def load_nifti_image(path: PathOrString, image_type: Optional[Type] = float) -> def _is_valid_image_path(_path: Path) -> bool: """ Validates a path for an image. Image must be .nii, or .nii.gz. + :param _path: The path to the file. :return: True if it is valid, False otherwise """ @@ -241,6 +243,7 @@ def _is_valid_image_path(_path: Path) -> bool: def load_numpy_image(path: PathOrString, image_type: Optional[Type] = None) -> np.ndarray: """ Loads an array from a numpy file (npz or npy). The array is converted to image_type or untouched if None + :param path: The path to the numpy file. :param image_type: The dtype to cast the array :return: ndarray @@ -258,6 +261,7 @@ def load_numpy_image(path: PathOrString, image_type: Optional[Type] = None) -> n def load_dicom_image(path: PathOrString) -> np.ndarray: """ Loads an array from a single dicom file. + :param path: The path to the dicom file. """ ds = dicom.dcmread(path) @@ -278,6 +282,7 @@ def load_dicom_image(path: PathOrString) -> np.ndarray: def load_hdf5_dataset_from_file(path_str: Path, dataset_name: str) -> np.ndarray: """ Loads a hdf5 dataset from a file as an ndarray + :param path_str: The path to the HDF5 file :param dataset_name: The dataset name in the HDF5 file that we want to load :return: ndarray @@ -292,6 +297,7 @@ def load_hdf5_dataset_from_file(path_str: Path, dataset_name: str) -> np.ndarray def load_hdf5_file(path_str: Union[str, Path], load_segmentation: bool = False) -> HDF5Object: """ Loads a single HDF5 file. + :param path_str: The path of the HDF5 file that should be loaded. :param load_segmentation: If True, the `segmentation` field of the result object will be populated. If False, the field will be set to None. @@ -301,6 +307,7 @@ def load_hdf5_file(path_str: Union[str, Path], load_segmentation: bool = False) def _is_valid_hdf5_path(_path: Path) -> bool: """ Validates a path for an image + :param _path: :return: """ @@ -332,8 +339,10 @@ def load_images_and_stack(files: Iterable[Path], :param files: The paths of the files to load. :param load_segmentation: If True it loads segmentation if present on the same file as the image. This is only supported for loading from HDF5 files. + :param center_crop_size: If supplied, all loaded images will be cropped to the size given here. The crop will be taken from the center of the image. + :param image_size: If supplied, all loaded images will be resized immediately after loading. :return: A wrapper class that contains the loaded images, and if load_segmentation is True, also the segmentations that were present in the files. @@ -420,6 +429,7 @@ def load_labels_from_dataset_source(dataset_source: PatientDatasetSource, check_ In the future, this function will be used to load global class and non-imaging information as well. :type image_size: Image size, tuple of integers. + :param dataset_source: The dataset source for which channels are to be loaded into memory. :param check_exclusive: Check that the labels are mutually exclusive (defaults to True). :return: A label sample object containing ground-truth information. @@ -467,6 +477,7 @@ def load_image(path: PathOrString, image_type: Optional[Type] = float) -> ImageW For segmentation binary || For segmentation multimap ||| The expected dimensions to be (channel, Z, Y, X) + :param path: The path to the file :param image_type: The type of the image """ diff --git a/InnerEye/ML/utils/layer_util.py b/InnerEye/ML/utils/layer_util.py index e300e838d..5847c7202 100644 --- a/InnerEye/ML/utils/layer_util.py +++ b/InnerEye/ML/utils/layer_util.py @@ -39,10 +39,13 @@ def get_padding_from_kernel_size(padding: PaddingMode, :param padding: Padding type (Enum) {`zero`, `no_padding`}. Option `zero` is intended to preserve the tensor shape. In `no_padding` option, padding is not applied and the function returns only zeros. + :param kernel_size: Spatial support of the convolution kernel. It is used to determine the padding size. This can be a scalar, tuple or array. + :param dilation: Dilation of convolution kernel. It is used to determine the padding size. This can be a scalar, tuple or array. + :param num_dimensions: The number of dimensions that the returned padding tuple should have, if both kernel_size and dilation are scalars. :return padding value required for convolution layers based on input kernel size and dilation. @@ -72,6 +75,7 @@ def get_upsampling_kernel_size(downsampling_factor: IntOrTuple3, num_dimensions: :param downsampling_factor: downsampling factor use for each dimension of the kernel. Can be either a list of len(num_dimension) with one factor per dimension or an int in which case the same factor will be applied for all dimension. + :param num_dimensions: number of dimensions of the kernel :return: upsampling_kernel_size """ @@ -98,6 +102,7 @@ def set_model_to_eval_mode(model: torch.nn.Module) -> Generator: """ Puts the given torch model into eval mode. At the end of the context, resets the state of the training flag to what is was before the call. + :param model: The model to modify. """ old_mode = model.training diff --git a/InnerEye/ML/utils/metrics_util.py b/InnerEye/ML/utils/metrics_util.py index 3b5cdd29d..fd635a4ca 100644 --- a/InnerEye/ML/utils/metrics_util.py +++ b/InnerEye/ML/utils/metrics_util.py @@ -54,6 +54,7 @@ def to_csv(self, file_name: Path) -> None: """ Writes the per-patient per-structure metrics to a CSV file. Sorting is done first by structure name, then by Dice score ascending. + :param file_name: The name of the file to write to. """ df = self.to_data_frame() @@ -130,6 +131,7 @@ def to_data_frame(self) -> DataFrame: def get_number_of_voxels_per_class(labels: torch.Tensor) -> torch.Tensor: """ Computes the number of voxels for each class in a one-hot label map. + :param labels: one-hot label map in shape Batches x Classes x Z x Y x X or Classes x Z x Y x X :return: A tensor of shape [Batches x Classes] containing the number of non-zero voxels along Z, Y, X """ @@ -268,6 +270,7 @@ def is_missing_ground_truth(ground_truth: np.ndarray) -> bool: calculate_metrics_per_class in metrics.py and plot_contours_for_all_classes in plotting.py both check whether there is ground truth missing using this simple check for NaN value at 0, 0, 0. To avoid duplicate code we bring it here as a utility function. + :param ground_truth: ground truth binary array with dimensions: [Z x Y x X]. :param label_id: Integer index of the label to check. :returns: True if the label is missing (signified by NaN), False otherwise. diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index 137ea9e36..4da218dad 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -108,10 +108,13 @@ def check_size_matches(arg1: Union[np.ndarray, torch.Tensor], :param arg2: The second array to check. :param dim1: The expected number of dimensions of arg1. If zero, no check for number of dimensions will be conducted. + :param dim2: The expected number of dimensions of arg2. If zero, no check for number of dimensions will be conducted. + :param matching_dimensions: The dimensions along which the two arguments have to match. For example, if arg1.ndim==4 and arg2.ndim==5, matching_dimensions==[3] checks if arg1.shape[3] == arg2.shape[3]. + :param arg1_name: If provided, all error messages will use that string to instead of "arg1" :param arg2_name: If provided, all error messages will use that string to instead of "arg2" :raise ValueError if shapes don't match @@ -125,6 +128,7 @@ def check_size_matches(arg1: Union[np.ndarray, torch.Tensor], def check_dim(expected: int, actual_shape: Any, name: str) -> None: """ Check if actual_shape is equal to the expected shape + :param expected: expected shape :param actual_shape: :param name: variable name @@ -151,6 +155,7 @@ def check_dim(expected: int, actual_shape: Any, name: str) -> None: def set_random_seed(random_seed: int, caller_name: Optional[str] = None) -> None: """ Set the seed for the random number generators of python, numpy, torch.random, and torch.cuda for all gpus. + :param random_seed: random seed value to set. :param caller_name: name of the caller for logging purposes. """ diff --git a/InnerEye/ML/utils/model_metadata_util.py b/InnerEye/ML/utils/model_metadata_util.py index 9c28e6144..c56782eb7 100644 --- a/InnerEye/ML/utils/model_metadata_util.py +++ b/InnerEye/ML/utils/model_metadata_util.py @@ -11,6 +11,7 @@ def random_colour(rng: random.Random) -> TupleInt3: """ Generates a random colour in RGB given a random number generator + :param rng: Random number generator :return: Tuple with random colour in RGB """ @@ -23,6 +24,7 @@ def random_colour(rng: random.Random) -> TupleInt3: def generate_random_colours_list(rng: random.Random, size: int) -> List[TupleInt3]: """ Generates a list of random colours in RGB given a random number generator and the size of this list + :param rng: random number generator :param size: size of the list :return: list of random colours in RGB diff --git a/InnerEye/ML/utils/model_util.py b/InnerEye/ML/utils/model_util.py index 6d2ed73ff..dd8fc445d 100644 --- a/InnerEye/ML/utils/model_util.py +++ b/InnerEye/ML/utils/model_util.py @@ -203,6 +203,7 @@ def generate_and_print_model_summary(config: ModelConfigBase, model: DeviceAware """ Writes a human readable summary of the present model to logging.info, and logs the number of trainable parameters to AzureML. + :param config: The configuration for the model. :param model: The instantiated Pytorch model. """ @@ -264,6 +265,7 @@ def __post_init__(self) -> None: def move_to_device(self, device: Union[str, torch.device]) -> None: """ Moves the model_inputs and labels field of the present object to the given device. This is done in-place. + :param device: The target device. """ self.model_inputs = [t.to(device=device) for t in self.model_inputs] @@ -274,10 +276,12 @@ def get_scalar_model_inputs_and_labels(model: torch.nn.Module, sample: Dict[str, Any]) -> ScalarModelInputsAndLabels: """ For a model that predicts scalars, gets the model input tensors from a sample returned by the data loader. + :param model: The instantiated PyTorch model. :param target_indices: If this list is non-empty, assume that the model is a sequence model, and build the model inputs and labels for a model that predicts those specific positions in the sequence. If the list is empty, assume that the model is a normal (non-sequence) model. + :param sample: A training sample, as returned by a PyTorch data loader (dictionary mapping from field name to value) :return: An instance of ScalarModelInputsAndLabels, containing the list of model input tensors, label tensor, subject IDs, and the data item reconstructed from the data loader output diff --git a/InnerEye/ML/utils/plotting_util.py b/InnerEye/ML/utils/plotting_util.py index a64407550..78b55b12e 100644 --- a/InnerEye/ML/utils/plotting_util.py +++ b/InnerEye/ML/utils/plotting_util.py @@ -14,6 +14,7 @@ def get_view_dim_and_origin(plane: Plane) -> Tuple[int, str]: """ Get the axis along which to slice, as well as the orientation of the origin, to ensure images are plotted as expected + :param plane: the plane in which to plot (i.e. axial, sagittal or coronal) :return: """ @@ -33,6 +34,7 @@ def get_cropped_axes(image: np.ndarray, boundary_width: int = 5) -> Tuple[slice, """ Return the min and max values on both x and y axes where the image is not empty Method: find the min and max of all non-zero pixels in the image, and add a border + :param image: the image to be cropped :param boundary_width: number of pixels boundary to add around bounding box :return: diff --git a/InnerEye/ML/utils/sequence_utils.py b/InnerEye/ML/utils/sequence_utils.py index 51bc34692..f70793f43 100644 --- a/InnerEye/ML/utils/sequence_utils.py +++ b/InnerEye/ML/utils/sequence_utils.py @@ -50,6 +50,7 @@ def sequences_to_padded_tensor(sequences: List[torch.Tensor], padding_value: float = 0.0) -> torch.Tensor: """ Method to convert possibly unequal length sequences to a padded tensor. + :param sequences: List of Tensors to pad :param padding_value: Padding value to use, default is 0.0 :return: Output tensor with shape B x * where * is the max dimensions from the list of provided tensors. @@ -80,6 +81,7 @@ def get_masked_model_outputs_and_labels(model_output: torch.Tensor, Helper function to get masked model outputs, labels and their associated subject ids. Masking is performed by excluding the NaN model outputs and labels based on a bool mask created using the occurrences of NaN in the labels provided. + :param model_output: The model output tensor to mask. :param labels: The label tensor to use for mask, and use for masking. :param subject_ids: The associated subject ids. @@ -125,6 +127,7 @@ def apply_sequence_model_loss(loss_fn: torch.nn.Module, """ Applies a loss function to a model output and labels, when the labels come from sequences with unequal length. Missing sequence elements are masked out. + :param loss_fn: The loss function to apply to the sequence elements that are present. :param model_output: The model outputs :param labels: The ground truth labels. diff --git a/InnerEye/ML/utils/split_dataset.py b/InnerEye/ML/utils/split_dataset.py index 8f210536b..7b69c694c 100644 --- a/InnerEye/ML/utils/split_dataset.py +++ b/InnerEye/ML/utils/split_dataset.py @@ -255,6 +255,7 @@ def from_proportions(df: pd.DataFrame, :param subject_column: Subject id column name :param group_column: grouping column name; if given, samples from each group will always be in the same subset (train, val, or test) and cross-validation fold. + :param proportion_val: proportion for the validation set. :param shuffle: If True the subjects in the dataframe will be shuffle before performing splits. :param random_seed: Random seed to be used for shuffle 0 is default. @@ -355,8 +356,10 @@ def from_institutions(df: pd.DataFrame, :param random_seed: Random seed to be used for shuffle 0 is default. :param exclude_institutions: If given, all subjects where institutionId has the given value will be excluded from train, test, and validation set. + :param institutions_for_test_only: If given, all subjects where institutionId has the given value will be placed only in the test set. + :param subject_ids_for_test_only: If given, all images with the provided subject Ids will be placed in the test set. :return: Data splits with respected dataset split proportions per institution. diff --git a/InnerEye/ML/utils/supervised_criterion.py b/InnerEye/ML/utils/supervised_criterion.py index 37e3fed53..fb66cb855 100644 --- a/InnerEye/ML/utils/supervised_criterion.py +++ b/InnerEye/ML/utils/supervised_criterion.py @@ -55,10 +55,12 @@ def __init__(self, num_classes: int, """ :param num_classes: The number of classes the model predicts. For binary classification num_classes is one and for multi-label classification tasks num_classes will be greater than one. + :param class_counts: The number of positive samples for each class. class_counts is a dictionary with key-value pairs corresponding to each class and the positive sample count for the class. For binary classification tasks, class_counts should have a single key-value pair for the positive class. + :param num_train_samples: The total number of training samples in the dataset. """ super().__init__(is_binary_classification=True, **kwargs) diff --git a/InnerEye/ML/utils/surface_distance_utils.py b/InnerEye/ML/utils/surface_distance_utils.py index 138110083..d8398131e 100644 --- a/InnerEye/ML/utils/surface_distance_utils.py +++ b/InnerEye/ML/utils/surface_distance_utils.py @@ -60,6 +60,7 @@ class SurfaceDistanceConfig(GenericConfig): def get_first_child_run(azure_config: AzureConfig) -> Run: """ Download first child run in order to download data + :param azure_config: :return: first child run """ @@ -74,6 +75,7 @@ def load_ground_truth_from_run(model_config: SegmentationModelBase, sd_config: S structure: str) -> np.ndarray: """ For outliers, load individual ground truth file for a given dataset, subject ID and structure name + :param model_config: :param sd_config: :param subject_id: ID of the given subject @@ -92,6 +94,7 @@ def get_run_output_dir(azure_config: AzureConfig, model_config: SegmentationMode """ Get the directory where Azure run's output data will be stored. the total filepath will depend on which container we download data from. + :param azure_config: :param model_config: :return output_dir: directory that all artifact paths use as a prefix @@ -108,6 +111,7 @@ def dir_for_subject(azure_config: AzureConfig, model_config: SegmentationModelBa """ Combine the local data dir and the Azure dir we are downloading images from to get the directory the images for a subject will be downlaoded to + :param azure_config: AzureConfig :param model_config: Config :param prefix: @@ -123,6 +127,7 @@ def dir_for_subject(azure_config: AzureConfig, model_config: SegmentationModelBa def get_metrics_path(azure_config: AzureConfig, model_config: SegmentationModelBase) -> Path: """ Get path to metrics.csv file for a downlaoded run, for the purpose of determining outliers + :param azure_config: AzureConfig :param model_config: Config :return: @@ -141,6 +146,7 @@ def get_metrics_path(azure_config: AzureConfig, model_config: SegmentationModelB def get_subject_prefix(model_config: SegmentationModelBase, train_mode: ModelExecutionMode, subject_id: int) -> Path: """ Returns the path to subject dir for a given model and train mode + :param model_config: Config :param train_mode: Model execution mode -i.e. train, test or val :param subject_id: ID of the subject @@ -156,6 +162,7 @@ def initialise_surface_distance_dictionary(annotators: List[str], arr_shape: Tup Given a list of annotators and the image size expected for all surface distance plots, return a dictionary where keys are annotators and values are zeros in shape of entire image, so that surface distances for each structure can be added to one plot. + :param annotators: List of the annotator names as they appear in filepaths :param arr_shape: Shape of the array to be intialized :return: @@ -166,6 +173,7 @@ def initialise_surface_distance_dictionary(annotators: List[str], arr_shape: Tup def get_majority_vote(arr_list: List[np.ndarray]) -> np.ndarray: """ Given a list of label arrays, get the majority vote at each voxel + :param arr_list: :return: """ @@ -179,6 +187,7 @@ def get_annotations_and_majority_vote(model_config: SegmentationModelBase, annot ) -> np.ndarray: """ Load each annotation and calculate the 'gold standard' segmentation (with majority voting) + :param model_config: Config :param annotators: List of the annotator names as they appear in filepaths :param structure_name: Name of the anatomical structure @@ -199,6 +208,7 @@ def get_annotations_and_majority_vote(model_config: SegmentationModelBase, annot def extract_border(img: np.ndarray, connectivity: int = 1) -> np.ndarray: """ Get contour by calculating eroded version of the image and subtracting from the original. + :param img: Array containing structure from which to extract the border :param connectivity: integer determining which pixels are considered neighbours of the central element, ranging from 1 = no diagonal elements and rank = all elements @@ -218,6 +228,7 @@ def calculate_surface_distances(ground_truth: np.ndarray, pred: np.ndarray, voxe ) -> np.ndarray: """ Calculate the Euclidean surface distance between a given prediction and the 'ground truth' + :param ground_truth: 3D binary array (X x Y x Z) of ground truth segmentation :param pred: 3D binary array (X x Y x Z) of predicted segmentation :param voxel_spacing: voxel spacing, taken from the image header (transposed if applicable) diff --git a/InnerEye/ML/utils/temperature_scaling.py b/InnerEye/ML/utils/temperature_scaling.py index df617fe5d..0b6f49424 100644 --- a/InnerEye/ML/utils/temperature_scaling.py +++ b/InnerEye/ML/utils/temperature_scaling.py @@ -58,6 +58,7 @@ def set_temperature(self, use_gpu: bool) -> float: """ Tune the temperature of the model using the provided logits and labels. + :param logits: Logits to use to learn the temperature parameter :param labels: Labels to use to learn the temperature parameter :param criterion_fn: A criterion function s.t: (logits, labels) => (loss, ECE) diff --git a/InnerEye/ML/utils/transforms.py b/InnerEye/ML/utils/transforms.py index 43f9e298d..526190117 100644 --- a/InnerEye/ML/utils/transforms.py +++ b/InnerEye/ML/utils/transforms.py @@ -59,6 +59,7 @@ def __call__(self, sample: T) -> T: def apply(compose: Optional[Compose3D[T]], sample: T) -> T: """ Apply a composition of transfer functions to the provided sample + :param compose: A composition of transfer functions :param sample: The sample to apply the composition on :return: diff --git a/InnerEye/ML/visualizers/activation_maps.py b/InnerEye/ML/visualizers/activation_maps.py index b35aa9ed0..8b04a026f 100644 --- a/InnerEye/ML/visualizers/activation_maps.py +++ b/InnerEye/ML/visualizers/activation_maps.py @@ -19,6 +19,7 @@ def vis_activation_map(activation_map: np.ndarray) -> np.ndarray: """ Normalizes the activation map and maps it to RGB range for visualization + :param activation_map: :return: """ @@ -35,6 +36,7 @@ def vis_activation_map(activation_map: np.ndarray) -> np.ndarray: def visualize_2d_activation_map(activation_map: np.ndarray, args: ModelConfigBase, slice_index: int = 0) -> None: """ Saves all feature channels of a 2D activation map as png files + :param activation_map: :param args: :param slice_index: @@ -55,6 +57,7 @@ def visualize_3d_activation_map(activation_map: np.ndarray, args: ModelConfigBas slices_to_visualize: Optional[List[int]] = None) -> None: """ Saves all feature channels of a 3D activation map as png files + :param activation_map: :param args: :param slices_to_visualize: @@ -72,6 +75,7 @@ def visualize_3d_activation_map(activation_map: np.ndarray, args: ModelConfigBas def extract_activation_maps(args: ModelConfigBase) -> None: """ Extracts and saves activation maps of a specific layer of a trained network + :param args: :return: """ diff --git a/InnerEye/ML/visualizers/metrics_scatterplot.py b/InnerEye/ML/visualizers/metrics_scatterplot.py index f9aab7f02..2ffa4f0a4 100644 --- a/InnerEye/ML/visualizers/metrics_scatterplot.py +++ b/InnerEye/ML/visualizers/metrics_scatterplot.py @@ -207,6 +207,7 @@ def to_dict(data: pd.DataFrame) -> Dict[str, Dict[str, float]]: def write_to_scatterplot_directory(root_folder: Path, plots: Dict[str, plt.Figure]) -> None: """ Writes a file root_folder/scatterplots/basename.png for every plot in plots with key "basename". + :param root_folder: path to a folder :param plots: dictionary from plot basenames to plots (plt.Figure objects) """ diff --git a/InnerEye/ML/visualizers/model_hooks.py b/InnerEye/ML/visualizers/model_hooks.py index a47db8cfb..6d3437362 100644 --- a/InnerEye/ML/visualizers/model_hooks.py +++ b/InnerEye/ML/visualizers/model_hooks.py @@ -40,6 +40,7 @@ def __init__(self, model: Module, layer_name: Optional[List[str]]): def _verify_layer_name(self, model: Module, layer_name: List[str]) -> None: """ Recursively traverses the model and verifies if the layer name is valid + :param model: the model :param layer_name: hierarchical list of layer names to index within model :return: @@ -56,6 +57,7 @@ def _verify_layer_name(self, model: Module, layer_name: List[str]) -> None: def forward_hook_fn(self, module: Module, input: Any, output: Any) -> None: """ Registers a forward hook inside module + :param module: :param input: :param output: diff --git a/InnerEye/ML/visualizers/model_summary.py b/InnerEye/ML/visualizers/model_summary.py index b77e837a7..63fcd9557 100644 --- a/InnerEye/ML/visualizers/model_summary.py +++ b/InnerEye/ML/visualizers/model_summary.py @@ -41,6 +41,7 @@ def __init__(self, model: DeviceAwareModule) -> None: """ Class to summarise the detail of neural network including (I) intermediate tensor shapes, (II) number of trainable and non-trainable parameters, and (III) total GPU/CPU memory requirements. + :param model: BaseModel object containing the computation graph. """ # Need a local import here to avoid circular dependency @@ -63,8 +64,10 @@ def generate_summary(self, """ Produces a human readable summary of the model, and prints it via logging.info. The summary is computed by doing forward propagation through the model, with tensors of a given size or a given list of tensors. + :param input_sizes: The list of sizes of the input tensors to the model. These sizes must be specifies without the leading batch dimension. + :param input_tensors: The tensors to use in model forward propagation. :param log_summaries_to_files: if True, write the summary to a new file under logs/models instead of stdout :return: @@ -116,6 +119,7 @@ def _get_device(module: torch.nn.Module) -> Optional[torch.device]: def compute_tensor_memory_megabytes(input_size: Union[List[torch.Size], Sequence[Tuple]]) -> float: """Returns memory requirement of a tensor by deriving from its size. The returned values are in megabytes + :param input_size: List of input tensor sizes """ # check for the case where the input is not a list of tuples, in which case make it a singleton instance @@ -130,6 +134,7 @@ def _register_hook(self, submodule: torch.nn.Module) -> None: Adds forward pass hooks to each submodule, module that has no nested modules/layers, in order to collect network summary information. Hook handles are stored in a list which are later removed outside the scope. + :param submodule: Children module of the main neural network model. """ @@ -158,6 +163,7 @@ def _generate_summary(self, input_tensors: List[torch.Tensor]) -> None: Creates a list of input torch tensors and registers forward pass hooks to the model, passes the inputs through the model, and collects model information such num of parameters and intermediate tensor size. + :param input_tensors: A list of tensors which are fed into the torch model. """ @@ -210,6 +216,7 @@ def forward_preserve_state(module: DeviceAwareModule, inputs: List[torch.Tensor] Perform forward pass on input module with given list of torch tensors. The function preserves the random state of the backend libraries to avoid reproducibility issues. Additionally, it temporarily sets the model in evaluation mode for inference and then restores its previous state. + :param module: Callable torch module :param inputs: List of input torch tensors :return output: Output torch tensors diff --git a/InnerEye/ML/visualizers/patch_sampling.py b/InnerEye/ML/visualizers/patch_sampling.py index ed23389fd..93d7a2b6f 100644 --- a/InnerEye/ML/visualizers/patch_sampling.py +++ b/InnerEye/ML/visualizers/patch_sampling.py @@ -43,6 +43,7 @@ def visualize_random_crops(sample: Sample, Simulate the effect of sampling random crops (as is done for trainig segmentation models), and store the results as a Nifti heatmap and as 3 axial/sagittal/coronal slices. The heatmap and the slices are stored in the given output folder, with filenames that contain the patient ID as the prefix. + :param sample: The patient information from the dataset, with scans and ground truth labels. :param config: The model configuration. :param output_folder: The folder into which the heatmap and thumbnails should be written. @@ -111,6 +112,7 @@ def visualize_random_crops_for_dataset(config: SegmentationModelBase, output_fol """ For segmentation models only: This function generates visualizations of the effect of sampling random patches for training. Visualizations are stored in both Nifti format, and as 3 PNG thumbnail files, in the output folder. + :param config: The model configuration. :param output_folder: The folder in which the visualizations should be written. If not provided, use a subfolder "patch_sampling" in the model's default output folder diff --git a/InnerEye/ML/visualizers/plot_cross_validation.py b/InnerEye/ML/visualizers/plot_cross_validation.py index b1a2324b6..e9d35ecb6 100644 --- a/InnerEye/ML/visualizers/plot_cross_validation.py +++ b/InnerEye/ML/visualizers/plot_cross_validation.py @@ -198,6 +198,7 @@ def download_or_get_local_file(self, If the blobs_path contains folders, the same folder structure will be created inside the destination folder. For example, downloading "foo.txt" to "/c/temp" will create "/c/temp/foo.txt". Downloading "foo/bar.txt" to "/c/temp" will create "/c/temp/foo/bar.txt" + :param blob_to_download: path of data to download within the run :param destination: directory to write to :param run: The AzureML run to download from. @@ -280,6 +281,7 @@ def get_split_id(tags: Dict[str, Any], is_zero_index: bool = True) -> str: """ Extracts the split index from the tags. If it's negative, this isn't a cross-validation run; gets it from the run_recovery_id instead. + :param tags: Tags associated with a run to get the split id for :param is_zero_index: If True, start cross validation split indices from 0 otherwise 1 :return: @@ -294,6 +296,7 @@ def get_split_id(tags: Dict[str, Any], is_zero_index: bool = True) -> str: def run_recovery_id_suffix(tags: Dict[str, Any]) -> str: """ Returns the part of run_recovery_id after the colon if any. + :param tags: the tags of a run """ run_rec_id = tags[RUN_RECOVERY_ID_KEY] @@ -307,6 +310,7 @@ def download_metrics_file(config: PlotCrossValidationConfig, """ Downloads a metrics.csv file from an Azure run (or local results), and stores it in a local folder. The metrics.csv file will be written into a subfolder named after the model execution mode. + :param config: The cross validation configuration. :param run: The AzureML run to download from. :param destination: The folder to download into. @@ -346,11 +350,13 @@ def download_crossval_result_files(config: PlotCrossValidationConfig, It will download the metrics.csv file for each dataset split (Train,Test, Val) and all of the run's children. When running in segmentation mode, it also downloads the dataset.csv and adds the institutionId and seriesId information for each subject found in the metrics files. + :param config: PlotCrossValidationConfig :param run_recovery_id: run recovery ID, if different from the one in config :param download_to_folder: The root folder in which all downloaded files should be stored. Point to an existing folder with downloaded files for use in unit tests. If not provided, the files will be downloaded to a new folder inside the config.outputs_directory, with the name taken from the run ID. + :param splits_to_evaluate: If supplied, use these values as the split indices to download. Use only for unit testing. :return: The dataframe with all of the downloaded results grouped by execution mode (Test or Val) @@ -431,6 +437,7 @@ def crossval_config_from_model_config(train_config: DeepLearningConfig) -> PlotC """ Creates a configuration for plotting cross validation results that populates some key fields from the given model training configuration. + :param train_config: :return: """ @@ -450,6 +457,7 @@ def get_config_and_results_for_offline_runs(train_config: DeepLearningConfig) -> """ Creates a configuration for cross validation analysis for the given model training configuration, and gets the input files required for cross validation analysis. + :param train_config: The model configuration to work with. """ plot_crossval_config = crossval_config_from_model_config(train_config) @@ -544,6 +552,7 @@ def convert_rows_for_comparisons(split_column_value: Optional[str], """ Given a dataframe resulting from a metrics.csv form, return (a subset of) the same information in the format required for a multi-split metrics file as required for statistical testing. + :param split_column_value: name of this split, for putting in the "Split" column of the result :param dataset_df: dataframe read from a dataset.csv file :param df: dataframe read from a metrics.csv file @@ -578,6 +587,7 @@ def shorten_split_names(config: PlotCrossValidationConfig, metrics: pd.DataFrame """ Replaces values in metrics[COL_SPLIT] by shortened versions consisting of the first 3 and last 3 characters, separated by "..", when that string is shorter. + :param config: for finding short names :param metrics: data frame with a column named COL_SPLIT """ @@ -606,6 +616,7 @@ def plot_metrics(config: PlotCrossValidationConfig, """ Given the dataframe for the downloaded metrics aggregate them into box plots corresponding to the results per split. + :param config: PlotCrossValidationConfig :param dataset_split_metrics: Mapping between model execution mode and a dataframe containing all metrics for it :param root: Root directory to the results for Train/Test and Val datasets @@ -660,6 +671,7 @@ def save_outliers(config: PlotCrossValidationConfig, """ Given the dataframe for the downloaded metrics identifies outliers (score < mean - 3sd) across the splits and saves them in a file outlier.csv in the provided root. + :param config: PlotCrossValidationConfig :param dataset_split_metrics: Mapping between model execution mode and a dataframe containing all metrics for it :param root: Root directory to the results for Train/Test and Val datasets @@ -706,6 +718,7 @@ def create_results_breakdown(df: pd.DataFrame, root_folder: Path) -> Tuple[Path, Creates a breakdown of Dice per execution mode (train/test/val) and structure name, and one of Dice per execution mode. The summaries are saved to files in the root_folder, via dataframe's describe function. + :param df: A data frame that contains columns COL_DICE, COL_MODE and COL_STRUCTURE :param root_folder: The folder into which the result files should be written. :return: The paths to the two files. @@ -728,6 +741,7 @@ def _describe(_df: pd.DataFrame, group_by: List[str], file: Path) -> None: def may_write_lines_to_file(lines: List[str], path: Path) -> None: """ Prints lines to file path if there are any lines; reports what it's doing. + :param lines: list of lines to write (without final newlines) :param path: csv file path to write to """ @@ -748,6 +762,7 @@ def run_statistical_tests_on_file(root_folder: Path, full_csv_file: Path, option same split. Empty results are not printed; this can happen for Wilcoxon when different splits do not intersect (e.g. when processing validation sets on a cross-val run) and for Mann-Whitney when only one institution occurs, or at most one institution has five or more subjects. + :param root_folder: folder to write to :param full_csv_file: MetricsAcrossAllRuns.csv file to read :param options: config options. @@ -771,6 +786,7 @@ def check_result_file_counts(config_and_files: OfflineCrossvalConfigAndFiles, is """ Check that for every ModelExecutionMode appearing in config_and_files.files, the number of files of that mode is equal to the number of cross-validation splits. Throw a ValueError if not. + :param is_ensemble_run: If True, assume that this run of cross validation analysis is for an ensemble model and assert that there are N+1 data files available. If false, this analysis only concerns the cross validation runs, and check that the number of files is N. @@ -804,6 +820,7 @@ def check_result_file_counts(config_and_files: OfflineCrossvalConfigAndFiles, is def plot_cross_validation_from_files(config_and_files: OfflineCrossvalConfigAndFiles, root_folder: Path) -> None: """ Runs various plots for the results of a cross validation run, and writes them to a given folder. + :param config_and_files: The setup for plotting results and the set of data files to analyse. :param root_folder: The folder into which the results should be written. """ @@ -865,6 +882,7 @@ def get_metrics_columns(df: pd.DataFrame) -> Set[str]: """ Get all columns of the data frame that appear to be metrics. A column is considered a metric if it appears as a dictionary value in INTERNAL_TO_LOGGING_COLUMN_NAMES. + :param df: A dataframe to analyze. :return: The set of data frame columns that is also contained in INTERNAL_TO_LOGGING_COLUMN_NAMES. """ @@ -877,6 +895,7 @@ def unroll_aggregate_metrics(df: pd.DataFrame) -> List[EpochMetricValues]: Consumes a dataframe that is read from an aggregate metrics file for classification or segmentation, and converts all entries for execution mode "Val" to (metric_name, metric_value) pairs, sorted by epoch ascendingly. + :param df: :return: """ @@ -902,6 +921,7 @@ def plot_cross_validation(config: PlotCrossValidationConfig) -> Path: Collects results from an AzureML cross validation run, and writes aggregate metrics files. and assert that there are N+1 data files available. If false, this analysis only concerns the cross validation runs, and check that the number of files is N. + :param config: The settings for plotting cross validation results. :return: The path with all cross validation result files. """ diff --git a/InnerEye/ML/visualizers/regression_visualization.py b/InnerEye/ML/visualizers/regression_visualization.py index d60715b4b..fed211daf 100644 --- a/InnerEye/ML/visualizers/regression_visualization.py +++ b/InnerEye/ML/visualizers/regression_visualization.py @@ -15,6 +15,7 @@ def plot_variation_error_prediction( """ Plots the absolute prediction errors as well as the predicted values against the ground truth values. + :param labels: ground truth labels :param predictions: model outputs :param filename: location to save the plot to. If None show the plot instead. diff --git a/InnerEye/ML/visualizers/reliability_curve.py b/InnerEye/ML/visualizers/reliability_curve.py index 261879b20..6658b3554 100644 --- a/InnerEye/ML/visualizers/reliability_curve.py +++ b/InnerEye/ML/visualizers/reliability_curve.py @@ -19,9 +19,11 @@ def plot_reliability_curve( Plots reliability curves for multiple models to observe model calibration errors. Inputs can be either 1-D or a list of 1-D arrays depending on the use case. List elements are intended to be used for different model types, e.g. y_predict: (num_samples, num_models) + :param y_predict: Model predictions, either a 1D array (num_samples) or list of 1D arrays (num_samples, num_models) :param y_true: Target values {0, 1} either a 1D array (num_samples) or list of 1D arrays (num_samples, num_models) Assuming a binary classification case + :param num_bins: Number of bins used for model prediction probabilities. :param normalise: If set to true, predictions are normalised to range [0, 1] diff --git a/InnerEye/Scripts/download_model_and_run_scoring.py b/InnerEye/Scripts/download_model_and_run_scoring.py index 8ddd0316c..4426b2258 100644 --- a/InnerEye/Scripts/download_model_and_run_scoring.py +++ b/InnerEye/Scripts/download_model_and_run_scoring.py @@ -16,6 +16,7 @@ def spawn_and_monitor_subprocess(process: str, args: List[str], env: Dict[str, str]) -> Tuple[int, List[str]]: """ Helper function to spawn and monitor subprocesses. + :param process: The name or path of the process to spawn. :param args: The args to the process. :param env: The environment variables for the process (default is the environment variables of the parent). diff --git a/InnerEye/Scripts/move_model.py b/InnerEye/Scripts/move_model.py index 24b69832e..bc9d1e512 100644 --- a/InnerEye/Scripts/move_model.py +++ b/InnerEye/Scripts/move_model.py @@ -30,6 +30,7 @@ class MoveModelConfig: def get_paths(self) -> Tuple[Path, Path]: """ Gets paths and creates folders if necessary + :param path: Base path :param model_id: The model ID :return: model_path, environment_path @@ -46,6 +47,7 @@ def get_paths(self) -> Tuple[Path, Path]: def download_model(ws: Workspace, config: MoveModelConfig) -> Model: """ Downloads an InnerEye model from an AzureML workspace + :param ws: The AzureML workspace :param config: move config :return: the exported Model @@ -64,6 +66,7 @@ def download_model(ws: Workspace, config: MoveModelConfig) -> Model: def upload_model(ws: Workspace, config: MoveModelConfig) -> Model: """ Uploads an InnerEye model to an AzureML workspace + :param ws: The AzureML workspace :param config: move config :return: imported Model @@ -88,6 +91,7 @@ def upload_model(ws: Workspace, config: MoveModelConfig) -> Model: def get_workspace(config: MoveModelConfig) -> Workspace: """ Get workspace based on command line input config + :param config: MoveModelConfig :return: an Azure ML workspace """ @@ -121,6 +125,7 @@ def main() -> None: def move(ws: Workspace, config: MoveModelConfig) -> Model: """ Moves a model: downloads or uploads the model depending on the configs + :param config: the move model config :param ws: The Azure ML workspace :return: the download or upload model diff --git a/InnerEye/Scripts/prepare_fastmri.py b/InnerEye/Scripts/prepare_fastmri.py index 95f1594ea..13a053300 100644 --- a/InnerEye/Scripts/prepare_fastmri.py +++ b/InnerEye/Scripts/prepare_fastmri.py @@ -81,6 +81,7 @@ def get_azure_auth(azure_config: AzureConfig) -> Union[DefaultAzureCredential, C Returns the authentication object for the azure.identity library, based on either the chosen Service Principal (if set, and if the password was found), or the interactive browser authentication if not all Service Principal information is available. + :param azure_config: The object containing all Azure-related information. :return: An azure.identity authentication object. """ @@ -107,10 +108,12 @@ def create_datafactory_and_run(files_and_tokens: Dict[str, str], is_unittest: bool = False) -> None: """ Builds an Azure Data Factory to download the FastMRI dataset from AWS, and places them in Azure Blob Storage. + :param location: The Azure location in which the Data Factory should be created (for example, "westeurope") :param files_and_tokens: A mapping from file name (like knee.tar.gz) to AWS access token. :param is_unittest: If True, download a small tar.gz file from github. If False, download the "real" fastMRI datafiles from AWS. + :param connection_string: The connection string of the Azure storage where the downloaded data should be stored. """ @@ -167,9 +170,11 @@ def download_and_uncompress(source_file_or_tuple: Union[str, Tuple[str, str]], t From the compressed file in blob storage, it is then uncompressed, and written to a new folder in blob storage. For example, if 'target_folder' is 'foo', the uncompressed file will be written to folder 'foo', and the compressed raw data will be written to 'foo_compressed'. + :param source_file_or_tuple: The name of the .tar.gz or .tar file to download, without any access tokens. If the name is a Tuple[str, str], the second tuple element is the "real" extension, for files where the extension is misleading. + :param target_folder: The folder prefix in the target storage account. :return: A list of pipelines that this method created. """ @@ -329,6 +334,7 @@ def extract_access_tokens(text: str) -> Dict[str, str]: """ Parses the given text for https URLs with an attached access token. Returns a dictionary mapping from file to file with access token, like `knee.tar.gz` -> `?AWSAccessKeyId=...` + :param text: The text with https URLs :return: """ diff --git a/InnerEye/Scripts/submit_for_inference.py b/InnerEye/Scripts/submit_for_inference.py index 21b8acbc0..7589cc072 100755 --- a/InnerEye/Scripts/submit_for_inference.py +++ b/InnerEye/Scripts/submit_for_inference.py @@ -77,8 +77,10 @@ def validate(self) -> None: def copy_image_file(image: Path, destination_folder: Path, use_dicom: bool) -> Path: """ Copy the source image file into the given folder destination_folder. + :param image: image file, must be Gzipped Nifti format with name ending .nii.gz if use_dicom=False or .zip otherwise. + :param destination_folder: top-level directory to copy image into (as test.nii.gz or test.zip) :param use_dicom: True to treat as a zip file. :return: The full path of the image in the destination_folder @@ -94,8 +96,10 @@ def download_files_from_model(model_sas_urls: Dict[str, str], base_name: str, di """ Identifies all the files in an AzureML model that have a given file name (ignoring path), and downloads them to a folder. + :param model_sas_urls: The files making up the model, as a mapping from file name to a URL with an SAS token. + :param base_name: The file name of the files to download. :param dir_path: The folder into which the files will be written. All downloaded files will keep the relative path that they also have in the model. @@ -143,6 +147,7 @@ def choose_download_path(result_image_name: str, download_folder: Path) -> Path: def submit_for_inference(args: SubmitForInferenceConfig, azure_config: AzureConfig) -> Optional[Path]: """ Create and submit an inference to AzureML, and optionally download the resulting segmentation. + :param azure_config: An object with all necessary information for accessing Azure. :param args: configuration, see SubmitForInferenceConfig :return: path to downloaded segmentation on local disc, or None if none. diff --git a/Tests/AfterTraining/test_after_training.py b/Tests/AfterTraining/test_after_training.py index f4f23ffb8..8fc137858 100644 --- a/Tests/AfterTraining/test_after_training.py +++ b/Tests/AfterTraining/test_after_training.py @@ -69,6 +69,7 @@ def get_most_recent_run_id(fallback_run_id_for_local_execution: str = FALLBACK_S file when running on the cloud. For execution on the local dev box, use a hardcoded run ID. Consequently, local execution of tests that use this run may fail, while executing in the cloud passes. In this case, modify the run here to something more recent. + :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code on a local box, outside of Azure build agents. :return: @@ -89,6 +90,7 @@ def get_most_recent_run_id(fallback_run_id_for_local_execution: str = FALLBACK_S def get_most_recent_run(fallback_run_id_for_local_execution: str = FALLBACK_SINGLE_RUN) -> Run: """ Gets the name of the most recently executed AzureML run, instantiates that Run object and returns it. + :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code on a local box, outside of Azure build agents. """ @@ -100,6 +102,7 @@ def get_most_recent_model_id(fallback_run_id_for_local_execution: str = FALLBACK """ Gets the string name of the most recently executed AzureML run, extracts which model that run had registered, and return the model id. + :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code on a local box, outside of Azure build agents. """ @@ -118,6 +121,7 @@ def get_most_recent_model(fallback_run_id_for_local_execution: str = FALLBACK_SI """ Gets the string name of the most recently executed AzureML run, extracts which model that run had registered, and return the instantiated model object. + :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code on a local box, outside of Azure build agents. """ @@ -255,6 +259,7 @@ def test_submit_for_inference(test_output_dirs: OutputFolderForTests) -> None: """ Execute the submit_for_inference script on the model that was recently trained. This starts an AzureML job, and downloads the segmentation. Then check if the segmentation was actually produced. + :param test_output_dirs: Test output directories. """ model = get_most_recent_model(fallback_run_id_for_local_execution=FALLBACK_SINGLE_RUN) diff --git a/Tests/Azure/test_secrets_handling.py b/Tests/Azure/test_secrets_handling.py index ed661fc79..1fe920c74 100644 --- a/Tests/Azure/test_secrets_handling.py +++ b/Tests/Azure/test_secrets_handling.py @@ -20,6 +20,7 @@ def set_environment_variables(variables: Dict[str, str]) -> None: """ Creates an environment variable for each entry in the given dictionary. The dictionary key is the variable name, it will be converted to uppercase before setting. + :param variables: The variable names and their associated values that should be set. """ for name, value in variables.items(): diff --git a/Tests/Common/test_segmentation_configs.py b/Tests/Common/test_segmentation_configs.py index f8f37d8e2..df08497f4 100644 --- a/Tests/Common/test_segmentation_configs.py +++ b/Tests/Common/test_segmentation_configs.py @@ -24,6 +24,7 @@ def generate_random_string(size: int) -> str: """ Generate a random string (upper case or digits) of length size + :param size: length of string to generate. """ return ''.join(random.choices(string.ascii_uppercase + string.digits, k=size)) @@ -32,6 +33,7 @@ def generate_random_string(size: int) -> str: def generate_random_display_ids(size: int) -> List[str]: """ Generate a list of random display ids of length size. + :param size: length of list of random display ids to generate. """ return [generate_random_string(6) for i in range(size)] @@ -40,6 +42,7 @@ def generate_random_display_ids(size: int) -> List[str]: def generate_random_fill_holes(size: int) -> List[bool]: """ Generate a list of random bools of length size. + :param size: length of list of random booleans to generate. """ return [bool(random.getrandbits(1)) for i in range(size)] diff --git a/Tests/ML/models/architectures/DummyScalarModel.py b/Tests/ML/models/architectures/DummyScalarModel.py index 5e0735a2f..078a9bfa5 100644 --- a/Tests/ML/models/architectures/DummyScalarModel.py +++ b/Tests/ML/models/architectures/DummyScalarModel.py @@ -38,6 +38,7 @@ def get_last_encoder_layer_names(self) -> List[str]: def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into images + :param item: ClassificationItem :return: Tensor """ diff --git a/Tests/ML/models/architectures/DummyScalarModel2D.py b/Tests/ML/models/architectures/DummyScalarModel2D.py index 878505fa9..0494b81d0 100644 --- a/Tests/ML/models/architectures/DummyScalarModel2D.py +++ b/Tests/ML/models/architectures/DummyScalarModel2D.py @@ -35,6 +35,7 @@ def get_last_encoder_layer_names(self) -> List[str]: def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into images + :param item: ClassificationItem :return: Tensor """ diff --git a/Tests/ML/pipelines/test_scalar_inference.py b/Tests/ML/pipelines/test_scalar_inference.py index d5fa0e8f2..3c43a1b1d 100644 --- a/Tests/ML/pipelines/test_scalar_inference.py +++ b/Tests/ML/pipelines/test_scalar_inference.py @@ -90,6 +90,7 @@ def __init__(self, expected_image_size_zyx: TupleInt3, scalar_to_return: float) def get_input_tensors(self, item: ScalarItem) -> List[torch.Tensor]: """ Transforms a classification item into images + :param item: ClassificationItem :return: Tensor """ diff --git a/Tests/ML/test_lightning_containers.py b/Tests/ML/test_lightning_containers.py index b7ff3b5bf..933d2f163 100644 --- a/Tests/ML/test_lightning_containers.py +++ b/Tests/ML/test_lightning_containers.py @@ -324,6 +324,7 @@ def test_innereyecontainer_setup_passes_on_allow_incomplete_labels( """ Test that InnerEyeContainer.setup passes on the correct value of allow_incomplete_labels to full_image_dataset.convert_channels_to_file_paths + :param test_output_dirs: Test fixture. :param allow_partial_ground_truth: The value to set allow_incomplete_labels to and check it is passed through. diff --git a/Tests/ML/test_model_testing.py b/Tests/ML/test_model_testing.py index 9bc4c7516..cca114558 100644 --- a/Tests/ML/test_model_testing.py +++ b/Tests/ML/test_model_testing.py @@ -37,6 +37,7 @@ def test_model_test( allow_partial_ground_truth: bool) -> None: """ Check the CSVs (and image files) output by InnerEye.ML.model_testing.segmentation_model_test + :param test_output_dirs: The fixture in conftest.py :param use_partial_ground_truth: Whether to remove some ground truth labels from some test users :param allow_partial_ground_truth: What to set the allow_incomplete_labels flag to diff --git a/Tests/ML/test_regression_tests.py b/Tests/ML/test_regression_tests.py index cbcbf921b..bff3b6ab2 100644 --- a/Tests/ML/test_regression_tests.py +++ b/Tests/ML/test_regression_tests.py @@ -50,6 +50,7 @@ def test_regression_test(test_output_dirs: OutputFolderForTests) -> None: def test_compare_files_text(test_output_dirs: OutputFolderForTests, file_extension: str) -> None: """ Checks the basic code to compare the contents of two text files. + :param test_output_dirs: :param file_extension: The extension of the file to create. """ @@ -102,6 +103,7 @@ def test_compare_files_empty_csv(test_output_dirs: OutputFolderForTests) -> None def test_compare_files_binary(test_output_dirs: OutputFolderForTests, file_extension: str) -> None: """ Checks the comparison of files that are not recognized as text files, for example images. + :param test_output_dirs: :param file_extension: The extension of the file to create. """ diff --git a/Tests/ML/util.py b/Tests/ML/util.py index c4ab0983c..b38c55fec 100644 --- a/Tests/ML/util.py +++ b/Tests/ML/util.py @@ -93,6 +93,7 @@ def load_train_and_test_data_channels(patient_ids: List[int], def assert_file_contains_string(full_file: Union[str, Path], expected: Any = None) -> None: """ Checks if the given file contains an expected string + :param full_file: The path to the file. :param expected: The expected contents of the file, as a string. """ @@ -106,6 +107,7 @@ def assert_file_contains_string(full_file: Union[str, Path], expected: Any = Non def assert_text_files_match(full_file: Path, expected_file: Path) -> None: """ Checks line by line (ignoring leading and trailing spaces) if the given two files contains the exact same strings + :param full_file: The path to the file. :param expected_file: The expected file. """ @@ -151,6 +153,7 @@ def assert_nifti_content(full_file: PathOrString, expected_type: type) -> None: """ Checks if the given nifti file contains the expected unique values, and has the expected type and shape. + :param full_file: The path to the file. :param expected_shape: The expected shape of the image in the nifti file. :param expected_header: the expected image header @@ -214,6 +217,7 @@ def csv_column_contains_value( contains_only_value: bool = True) -> bool: """ Checks that the column in the csv file contains the given value (and perhaps only contains that value) + :param csv_file_path: The path to the CSV :param column_name: The name of the column in which we look for the value :param value: The value to look for @@ -284,6 +288,7 @@ def model_train_unittest(config: Optional[DeepLearningConfig], """ A shortcut for running model training in the unit test suite. It runs training for the given config, with the default checkpoint handler initialized to point to the test output folder specified in dirs. + :param config: The configuration of the model to train. :param output_folder: The test fixture that provides an output folder for the test. :param lightning_container: An optional LightningContainer object that will be pass through to the training routine. diff --git a/Tests/ML/utils/test_checkpoint_handling.py b/Tests/ML/utils/test_checkpoint_handling.py index b4202f830..6451dd1b1 100644 --- a/Tests/ML/utils/test_checkpoint_handling.py +++ b/Tests/ML/utils/test_checkpoint_handling.py @@ -28,6 +28,7 @@ def create_checkpoint_file(file: Path) -> None: """ Creates a very simple torch checkpoint file. The only requirement is that it can safely pass torch.load. + :param file: The path of the checkpoint file that should be written. """ weights = {'state_dict': {'foo': torch.ones((2, 2))}} diff --git a/Tests/ML/utils/test_io_util.py b/Tests/ML/utils/test_io_util.py index 7abf53849..2ba473ae5 100644 --- a/Tests/ML/utils/test_io_util.py +++ b/Tests/ML/utils/test_io_util.py @@ -674,6 +674,7 @@ def test_create_dicom_series(test_output_dirs: OutputFolderForTests) -> None: def test_zip_random_dicom_series(test_output_dirs: OutputFolderForTests) -> None: """ Test that a DICOM series can be created. + :param test_output_dirs: Test output directories. :return: None. """ diff --git a/Tests/ML/utils/test_model_util.py b/Tests/ML/utils/test_model_util.py index f4a2782a7..778d737ec 100644 --- a/Tests/ML/utils/test_model_util.py +++ b/Tests/ML/utils/test_model_util.py @@ -33,6 +33,7 @@ def create_model_and_store_checkpoint(config: ModelConfigBase, checkpoint_path: Creates a Lightning model for the given model configuration, and stores it as a checkpoint file. If a GPU is available, the model is moved to the GPU before storing. The trainer properties `current_epoch` and `global_step` are set to fixed non-default values. + :param config: The model configuration. :param checkpoint_path: The path and filename of the checkpoint file. """ diff --git a/Tests/ML/visualizers/test_plot_cross_validation.py b/Tests/ML/visualizers/test_plot_cross_validation.py index af3fe0b34..a379630e1 100644 --- a/Tests/ML/visualizers/test_plot_cross_validation.py +++ b/Tests/ML/visualizers/test_plot_cross_validation.py @@ -57,6 +57,7 @@ def download_metrics(config: PlotCrossValidationConfig) -> \ def create_run_result_file_list(config: PlotCrossValidationConfig, folder: str) -> List[RunResultFiles]: """ Creates a list of input files for cross validation analysis, from files stored inside of the test data folder. + :param config: The overall cross validation config :param folder: The folder to read from, inside of test_data/plot_cross_validation. :return: @@ -143,6 +144,7 @@ def test_metrics_preparation_for_segmentation(drop_column: Optional[str], def drop_csv_column(path: Path) -> None: """ Load a csv file, drop a column, and save the csv file. + :param path: Path to csv file. """ df = pd.read_csv(path) diff --git a/Tests/ML/visualizers/test_visualize_patches.py b/Tests/ML/visualizers/test_visualize_patches.py index ef27fdcdf..85478963d 100644 --- a/Tests/ML/visualizers/test_visualize_patches.py +++ b/Tests/ML/visualizers/test_visualize_patches.py @@ -29,6 +29,7 @@ def test_visualize_patch_sampling(test_output_dirs: OutputFolderForTests, labels_to_boundary: bool) -> None: """ Tests if patch sampling and producing diagnostic images works as expected. + :param test_output_dirs: :param labels_to_boundary: If true, the ground truth labels are placed close to the image boundary, so that crops have to be adjusted inwards. If false, ground truth labels are all far from the image boundaries. @@ -96,6 +97,7 @@ def test_visualize_patch_sampling(test_output_dirs: OutputFolderForTests, def test_visualize_patch_sampling_2d(test_output_dirs: OutputFolderForTests) -> None: """ Tests if patch sampling works for 2D images. + :param test_output_dirs: """ set_random_seed(0) diff --git a/Tests/SSL/test_ssl_containers.py b/Tests/SSL/test_ssl_containers.py index 0b495deef..14333d4df 100644 --- a/Tests/SSL/test_ssl_containers.py +++ b/Tests/SSL/test_ssl_containers.py @@ -49,6 +49,7 @@ def create_cxr_test_dataset(path_to_test_dataset: Path, """ Creates fake datasets dataframe and dicom images mimicking the expected structure of the datasets of NIHCXR and RSNAKaggleCXR + :param path_to_test_dataset: folder to which we want to save the mock data. :param num_encoder_images: The number of unlabelled images that the dataset should contain (for encoder training) :param num_labelled_images: The number of labelled images that the dataset should contain (for the linear head). @@ -87,6 +88,7 @@ def _compare_stored_metrics(runner: Runner, expected_metrics: Dict[str, float], """ Checks if the StoringLogger in the given runner holds all the expected metrics as results of training epoch 0, up to a given absolute precision. + :param runner: The Innereye runner. :param expected_metrics: A dictionary with all metrics that are expected to be present. """ diff --git a/TestsOutsidePackage/test_register_model.py b/TestsOutsidePackage/test_register_model.py index 10bce66ae..8eaf2dd55 100644 --- a/TestsOutsidePackage/test_register_model.py +++ b/TestsOutsidePackage/test_register_model.py @@ -20,6 +20,7 @@ def create_checkpoints(model_config: SegmentationModelBase, is_ensemble: bool) - """ Creates 1 or 2 empty checkpoint files in the model's checkpoint folder, and returns the paths of those files, both absolute paths and paths relative to the checkpoint folder. + :param model_config: The model configuration, where a correct output folder must be set. :param is_ensemble: If true, 2 checkpoints (simulating an ensemble run) will be created. If false, only a single checkpoint will be created. diff --git a/mypy_runner.py b/mypy_runner.py index 3ecbdac81..6160b1f7e 100644 --- a/mypy_runner.py +++ b/mypy_runner.py @@ -18,6 +18,7 @@ def run_mypy(files: List[str], mypy_executable_path: str) -> int: stderr. We intercept these, and assume that any files mentioned in them have been processed. We run mypy repeatedly on the files that were not mentioned until there are none remaining, or until no further files are mentioned in the logs. + :param files: list of .py files to check :param mypy_executable_path: path to mypy executable :return: maximum return code from any of the mypy runs diff --git a/rewrite_params.py b/rewrite_params.py new file mode 100644 index 000000000..35d7d526c --- /dev/null +++ b/rewrite_params.py @@ -0,0 +1,20 @@ +import os +import re + +for root, _, files in os.walk('./'): + for filename in files: + if filename.endswith('.py') and filename != 'rewrite_params.py': + if root == "./": + filepath = filename + else: + filepath = root + "/" + filename + + with open(filepath, 'r') as file_reader: + lines = file_reader.readlines() + for i, line in enumerate(lines): + if ':param' in line: + prev_line = lines[i-1] + if prev_line != "\n" and '"""' not in prev_line and ":param" not in prev_line: + lines[i] = "\n" + lines[i] + with open(filepath, 'w') as file_reader: + file_reader.writelines(lines) diff --git a/score.py b/score.py index 13d50816b..d4f6d6dd0 100644 --- a/score.py +++ b/score.py @@ -84,6 +84,7 @@ def create_inference_pipeline(model_config: SegmentationModelBase, """ Create pipeline for inference, this can be a single model inference pipeline or an ensemble, if multiple checkpoints provided. + :param model_config: Model config to use to create the pipeline. :param full_path_to_checkpoints: Checkpoints to use for model inference. :param use_gpu: If GPU should be used or not. @@ -115,6 +116,7 @@ def run_inference(images_with_header: List[ImageWithHeader], config: SegmentationModelBase) -> np.ndarray: """ Runs inference on a list of channels and given a config and inference pipeline + :param images_with_header: :param inference_pipeline: :param config: @@ -261,6 +263,7 @@ def score_image(args: ScorePipelineConfig) -> Path: 2) Instantiate an inference pipeline based on the provided model_inference.json in the snapshot 3) Store the segmentation file in the current directory 4) Upload the segmentation to AML + :param args: :return: """ From 6371548ac1093873b9d38a7d2e94541315ef65b6 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 15:13:50 +0100 Subject: [PATCH 08/27] Remove script --- rewrite_params.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 rewrite_params.py diff --git a/rewrite_params.py b/rewrite_params.py deleted file mode 100644 index 35d7d526c..000000000 --- a/rewrite_params.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -import re - -for root, _, files in os.walk('./'): - for filename in files: - if filename.endswith('.py') and filename != 'rewrite_params.py': - if root == "./": - filepath = filename - else: - filepath = root + "/" + filename - - with open(filepath, 'r') as file_reader: - lines = file_reader.readlines() - for i, line in enumerate(lines): - if ':param' in line: - prev_line = lines[i-1] - if prev_line != "\n" and '"""' not in prev_line and ":param" not in prev_line: - lines[i] = "\n" + lines[i] - with open(filepath, 'w') as file_reader: - file_reader.writelines(lines) From 27e778446afc8da05bbe3448e2393143422991e4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 8 Aug 2022 15:33:58 +0100 Subject: [PATCH 09/27] =?UTF-8?q?=F0=9F=93=9D=20Finish=20ML/models=20API?= =?UTF-8?q?=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/ML/SSL/index.rst | 2 ++ .../rst/api/ML/models/architectures/index.rst | 29 +++++++++++++++++++ .../source/rst/api/ML/models/blocks/index.rst | 4 +++ docs/source/rst/api/ML/models/index.rst | 1 - .../source/rst/api/ML/models/layers/index.rst | 10 +++++++ .../source/rst/api/ML/models/losses/index.rst | 10 +++++++ .../rst/api/ML/models/parallel/index.rst | 0 docs/source/rst/api/ML/pipelines/index.rst | 2 ++ docs/source/rst/api/ML/utils/index.rst | 2 ++ docs/source/rst/api/ML/visualizers/index.rst | 2 ++ 10 files changed, 61 insertions(+), 1 deletion(-) delete mode 100644 docs/source/rst/api/ML/models/parallel/index.rst diff --git a/docs/source/rst/api/ML/SSL/index.rst b/docs/source/rst/api/ML/SSL/index.rst index e69de29bb..7c1c12772 100644 --- a/docs/source/rst/api/ML/SSL/index.rst +++ b/docs/source/rst/api/ML/SSL/index.rst @@ -0,0 +1,2 @@ +SSL +=== diff --git a/docs/source/rst/api/ML/models/architectures/index.rst b/docs/source/rst/api/ML/models/architectures/index.rst index e69de29bb..a464da4c7 100644 --- a/docs/source/rst/api/ML/models/architectures/index.rst +++ b/docs/source/rst/api/ML/models/architectures/index.rst @@ -0,0 +1,29 @@ +Architectures +============== + +Base Model +----------- + +.. automodule:: InnerEye.ML.models.architectures.base_model + +U-Nets +-------- +.. automodule:: InnerEye.ML.models.architectures.unet_3d + +.. automodule:: InnerEye.ML.models.architectures.unet_2d + +Classificiation +--------------- + +.. automodule:: InnerEye.ML.models.architectures.classification.bit + +.. automodule:: InnerEye.ML.models.architectures.classification.image_encoder_with_mlp + +.. automodule:: InnerEye.ML.models.architectures.classification.segmentation_encoder + +Others +------- + +.. automodule:: InnerEye.ML.models.architectures.complex + +.. automodule:: InnerEye.ML.models.architectures.mlp diff --git a/docs/source/rst/api/ML/models/blocks/index.rst b/docs/source/rst/api/ML/models/blocks/index.rst index e69de29bb..df34da115 100644 --- a/docs/source/rst/api/ML/models/blocks/index.rst +++ b/docs/source/rst/api/ML/models/blocks/index.rst @@ -0,0 +1,4 @@ +Blocks +====== + +.. automodule:: InnerEye.ML.models.blocks.residual diff --git a/docs/source/rst/api/ML/models/index.rst b/docs/source/rst/api/ML/models/index.rst index d5432a1d3..bf40a00e4 100644 --- a/docs/source/rst/api/ML/models/index.rst +++ b/docs/source/rst/api/ML/models/index.rst @@ -8,4 +8,3 @@ Below is a list of models and various deep learning components that are availabl blocks/index layers/index losses/index - parallel/index diff --git a/docs/source/rst/api/ML/models/layers/index.rst b/docs/source/rst/api/ML/models/layers/index.rst index e69de29bb..d37ef144d 100644 --- a/docs/source/rst/api/ML/models/layers/index.rst +++ b/docs/source/rst/api/ML/models/layers/index.rst @@ -0,0 +1,10 @@ +Layers +====== + +.. automodule:: InnerEye.ML.models.layers.basic + +.. automodule:: InnerEye.ML.models.layers.identity + +.. automodule:: InnerEye.ML.models.layers.pooling_layers + +.. automodule:: InnerEye.ML.models.layers.weight_standardization diff --git a/docs/source/rst/api/ML/models/losses/index.rst b/docs/source/rst/api/ML/models/losses/index.rst index e69de29bb..9b19ab6ef 100644 --- a/docs/source/rst/api/ML/models/losses/index.rst +++ b/docs/source/rst/api/ML/models/losses/index.rst @@ -0,0 +1,10 @@ +Losses +====== + +.. automodule:: InnerEye.ML.models.losses.cross_entropy + +.. automodule:: InnerEye.ML.models.losses.ece + +.. automodule:: InnerEye.ML.models.losses.mixture + +.. automodule:: InnerEye.ML.models.losses.soft_dice diff --git a/docs/source/rst/api/ML/models/parallel/index.rst b/docs/source/rst/api/ML/models/parallel/index.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/source/rst/api/ML/pipelines/index.rst b/docs/source/rst/api/ML/pipelines/index.rst index e69de29bb..0eb3cab54 100644 --- a/docs/source/rst/api/ML/pipelines/index.rst +++ b/docs/source/rst/api/ML/pipelines/index.rst @@ -0,0 +1,2 @@ +Pipelines +========= diff --git a/docs/source/rst/api/ML/utils/index.rst b/docs/source/rst/api/ML/utils/index.rst index e69de29bb..f2c4c3a20 100644 --- a/docs/source/rst/api/ML/utils/index.rst +++ b/docs/source/rst/api/ML/utils/index.rst @@ -0,0 +1,2 @@ +Utils +===== diff --git a/docs/source/rst/api/ML/visualizers/index.rst b/docs/source/rst/api/ML/visualizers/index.rst index e69de29bb..5f77689dc 100644 --- a/docs/source/rst/api/ML/visualizers/index.rst +++ b/docs/source/rst/api/ML/visualizers/index.rst @@ -0,0 +1,2 @@ +Visualizers +=========== From 6a1273f2a7abb60a97a5bc6964a505e997a9e4ce Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 09:07:13 +0100 Subject: [PATCH 10/27] =?UTF-8?q?=F0=9F=93=9D=20Start=20ML/SSL=20API.=20Fi?= =?UTF-8?q?x=20some=20formatting=20issues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transforms_utils.py | 6 ++--- .../ML/models/architectures/base_model.py | 3 +-- InnerEye/ML/models/architectures/unet_3d.py | 3 ++- InnerEye/ML/pipelines/inference.py | 8 +++--- docs/source/conf.py | 2 +- .../ML/SSL/datamodules_and_datasets/index.rst | 26 +++++++++++++++++++ docs/source/rst/api/ML/SSL/index.rst | 10 +++++-- .../api/ML/SSL/lightning_containers/index.rst | 2 ++ .../api/ML/SSL/lightning_modules/index.rst | 2 ++ docs/source/rst/api/ML/SSL/utils/index.rst | 2 ++ docs/source/rst/api/ML/configs.rst | 7 ----- docs/source/rst/api/ML/index.rst | 4 ++- docs/source/rst/api/ML/pipelines.rst | 8 ------ docs/source/rst/api/ML/pipelines/index.rst | 11 ++++++++ 14 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst create mode 100644 docs/source/rst/api/ML/SSL/lightning_containers/index.rst create mode 100644 docs/source/rst/api/ML/SSL/lightning_modules/index.rst create mode 100644 docs/source/rst/api/ML/SSL/utils/index.rst delete mode 100644 docs/source/rst/api/ML/configs.rst delete mode 100644 docs/source/rst/api/ML/pipelines.rst diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py index 778d4df94..30d6914b8 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py @@ -33,11 +33,11 @@ def get_ssl_transforms_from_config(config: CfgNode, are called on. If False, simply return one transformed version of the sample centered and cropped. :param use_training_augmentations_for_validation: If True, use augmentation at validation time too. - This is required for SSL validation loss to be meaningful. If False, only apply basic processing step - (no augmentations) + This is required for SSL validation loss to be meaningful. If False, only apply basic processing step + (no augmentations) :param expand_channels: if True the expand channel transformation from InnerEye.ML.augmentations.image_transforms - will be added to the transformation passed through the config. This is needed for single channel images as CXR. + will be added to the transformation passed through the config. This is needed for single channel images as CXR. """ train_transforms = create_transforms_from_config(config, apply_augmentations=True, expand_channels=expand_channels) diff --git a/InnerEye/ML/models/architectures/base_model.py b/InnerEye/ML/models/architectures/base_model.py index b2970e3ef..4dfc1e21f 100644 --- a/InnerEye/ML/models/architectures/base_model.py +++ b/InnerEye/ML/models/architectures/base_model.py @@ -186,8 +186,7 @@ def generate_model_summary(self, crop_size: Optional[TupleInt3] = None, When called again with the same crop_size, the summary is not created again. :param crop_size: The crop size for which the summary should be created. If not provided, - the minimum allowed crop size is used. - + the minimum allowed crop size is used. :param log_summaries_to_files: whether to write the summary to a file """ if crop_size is None: diff --git a/InnerEye/ML/models/architectures/unet_3d.py b/InnerEye/ML/models/architectures/unet_3d.py index 6b103f313..bc99eefb3 100644 --- a/InnerEye/ML/models/architectures/unet_3d.py +++ b/InnerEye/ML/models/architectures/unet_3d.py @@ -43,7 +43,8 @@ class UNetDecodeBlock(torch.nn.Module): :param channels: A tuple containing the number of input and output channels :param upsample_kernel_size: Spatial support of upsampling kernels. If an integer is provided, the same value - will be repeated for all three dimensions. For non-cubic kernels please pass a list or tuple with three elements + will be repeated for all three dimensions. For non-cubic kernels please pass a list or tuple with three + elements. :param upsampling_stride: Upsamling factor used in deconvolutional layer. Similar to the `upsample_kernel_size` parameter, if an integer is passed, the same upsampling factor will be used for all three dimensions. diff --git a/InnerEye/ML/pipelines/inference.py b/InnerEye/ML/pipelines/inference.py index 1d0edda5e..fc4dcbfdc 100644 --- a/InnerEye/ML/pipelines/inference.py +++ b/InnerEye/ML/pipelines/inference.py @@ -202,12 +202,12 @@ def create_from_checkpoint(path_to_checkpoint: Path, If there is no checkpoint file for the given epoch, return None. :param path_to_checkpoint: The path to the checkpoint that we want to load - model_config.checkpoint_folder - + model_config.checkpoint_folder :param model_config: Model related configurations. :param pipeline_id: Numeric identifier for the pipeline (useful for logging when ensembling) - :return InferencePipeline: an instantiated inference pipeline instance, or None if there was no checkpoint - file for this epoch. + + :returns InferencePipeline: an instantiated inference pipeline instance, or None if there was no checkpoint + file for this epoch. """ if not path_to_checkpoint.is_file(): # not raising a value error here: This is used to create individual pipelines for ensembles, diff --git a/docs/source/conf.py b/docs/source/conf.py index 81f768a4b..efdbb4ba5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -61,7 +61,7 @@ # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for +# The theme to use for HTML and HTML Help pages. See the documentation fora # a list of builtin themes. # html_theme = 'furo' diff --git a/docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst b/docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst new file mode 100644 index 000000000..2181f81a4 --- /dev/null +++ b/docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst @@ -0,0 +1,26 @@ +SSL Datamodules and Datasets +============================ + +Datasets +---------- + +.. automodule:: InnerEye.ML.SSL.datamodules_and_datasets.cifar_datasets + +.. automodule:: InnerEye.ML.SSL.datamodules_and_datasets.cxr_datasets + + +Dataset Class Utils +-------------------- + +.. automodule:: InnerEye.ML.SSL.datamodules_and_datasets.dataset_cls_utils + +Datamodules +----------- + +.. automodule:: InnerEye.ML.SSL.datamodules_and_datasets.datamodules + + +Transformation Utils +-------------------- + +.. automodule:: InnerEye.ML.SSL.datamodules_and_datasets.transforms_utils diff --git a/docs/source/rst/api/ML/SSL/index.rst b/docs/source/rst/api/ML/SSL/index.rst index 7c1c12772..3cd8812a7 100644 --- a/docs/source/rst/api/ML/SSL/index.rst +++ b/docs/source/rst/api/ML/SSL/index.rst @@ -1,2 +1,8 @@ -SSL -=== +Self-Supervised Learning (SSL) +============================== + +.. toctree:: + lightning_containers/index + lightning_modules/index + datamodules_and_datasets/index + utils/index diff --git a/docs/source/rst/api/ML/SSL/lightning_containers/index.rst b/docs/source/rst/api/ML/SSL/lightning_containers/index.rst new file mode 100644 index 000000000..4d6e58bee --- /dev/null +++ b/docs/source/rst/api/ML/SSL/lightning_containers/index.rst @@ -0,0 +1,2 @@ +SSL Lightning Containers +======================== diff --git a/docs/source/rst/api/ML/SSL/lightning_modules/index.rst b/docs/source/rst/api/ML/SSL/lightning_modules/index.rst new file mode 100644 index 000000000..3c106f982 --- /dev/null +++ b/docs/source/rst/api/ML/SSL/lightning_modules/index.rst @@ -0,0 +1,2 @@ +SSL Lightning Modules +===================== diff --git a/docs/source/rst/api/ML/SSL/utils/index.rst b/docs/source/rst/api/ML/SSL/utils/index.rst new file mode 100644 index 000000000..fe9f8c4a1 --- /dev/null +++ b/docs/source/rst/api/ML/SSL/utils/index.rst @@ -0,0 +1,2 @@ +SSL Utils +========== diff --git a/docs/source/rst/api/ML/configs.rst b/docs/source/rst/api/ML/configs.rst deleted file mode 100644 index 18bedb629..000000000 --- a/docs/source/rst/api/ML/configs.rst +++ /dev/null @@ -1,7 +0,0 @@ -Segmentation Model Configuration -================================ - -.. autoclass:: InnerEye.ML.config.SegmentationModelBase - -.. automodule:: InnerEye.ML.config - :exclude-members: SegmentationModelBase diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst index 4c5d53a7c..e9c052f5f 100644 --- a/docs/source/rst/api/ML/index.rst +++ b/docs/source/rst/api/ML/index.rst @@ -7,6 +7,8 @@ Machine learning models/index dataset/index augmentations/index + pipelines/index + SSL/index + utils/index runner photometric_normalization - pipelines diff --git a/docs/source/rst/api/ML/pipelines.rst b/docs/source/rst/api/ML/pipelines.rst deleted file mode 100644 index 509d2c748..000000000 --- a/docs/source/rst/api/ML/pipelines.rst +++ /dev/null @@ -1,8 +0,0 @@ -Pipelines -========= - -.. automodule:: InnerEye.ML.pipelines.inference - -.. automodule:: InnerEye.ML.pipelines.ensemble - -.. automodule:: InnerEye.ML.pipelines.scalar_inference diff --git a/docs/source/rst/api/ML/pipelines/index.rst b/docs/source/rst/api/ML/pipelines/index.rst index 0eb3cab54..b65e299a9 100644 --- a/docs/source/rst/api/ML/pipelines/index.rst +++ b/docs/source/rst/api/ML/pipelines/index.rst @@ -1,2 +1,13 @@ Pipelines ========= + +Inference +--------- + +.. automodule:: InnerEye.ML.pipelines.inference + +.. automodule:: InnerEye.ML.pipelines.scalar_inference + +Enesemble +--------- +.. automodule:: InnerEye.ML.pipelines.ensemble From 562173b8b0708135efae1022a041b22e67f193e0 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 09:26:11 +0100 Subject: [PATCH 11/27] =?UTF-8?q?=F0=9F=93=9D=20Correct=20whitespace=20iss?= =?UTF-8?q?ues=20in=20`:param`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Azure/azure_runner.py | 4 +-- InnerEye/Azure/azure_util.py | 4 +-- InnerEye/Azure/secrets_handling.py | 6 ++--- InnerEye/Azure/tensorboard_monitor.py | 2 +- .../Common/Statistics/mann_whitney_test.py | 2 +- .../Statistics/report_structure_extremes.py | 2 +- InnerEye/Common/common_util.py | 4 +-- InnerEye/Common/resource_monitor.py | 4 +-- InnerEye/Common/spawn_subprocess.py | 2 +- .../datamodules_and_datasets/datamodules.py | 4 +-- .../transforms_utils.py | 2 +- .../SSL/lightning_containers/ssl_container.py | 6 ++--- .../SSL/lightning_modules/byol/byol_module.py | 4 +-- InnerEye/ML/SSL/utils.py | 2 +- InnerEye/ML/augmentations/image_transforms.py | 2 +- .../ML/augmentations/transform_pipeline.py | 8 +++--- InnerEye/ML/baselines_util.py | 10 +++---- .../ML/configs/classification/CovidModel.py | 2 +- InnerEye/ML/configs/other/HelloContainer.py | 2 +- .../configs/segmentation/HeadAndNeckBase.py | 10 +++---- .../ML/configs/segmentation/ProstateBase.py | 10 +++---- .../configs/unit_testing/passthrough_model.py | 2 +- InnerEye/ML/dataset/full_image_dataset.py | 2 +- InnerEye/ML/dataset/scalar_dataset.py | 14 +++++----- InnerEye/ML/deep_learning_config.py | 6 ++--- InnerEye/ML/lightning_base.py | 10 +++---- InnerEye/ML/lightning_container.py | 2 +- InnerEye/ML/lightning_loggers.py | 6 ++--- InnerEye/ML/lightning_metrics.py | 4 +-- InnerEye/ML/lightning_models.py | 10 +++---- InnerEye/ML/metrics.py | 4 +-- InnerEye/ML/metrics_dict.py | 20 +++++++------- InnerEye/ML/model_config_base.py | 2 +- InnerEye/ML/model_testing.py | 6 ++--- InnerEye/ML/model_training.py | 2 +- .../ML/models/architectures/base_model.py | 4 +-- .../classification/image_encoder_with_mlp.py | 26 +++++++++---------- .../classification/segmentation_encoder.py | 7 +++-- InnerEye/ML/models/architectures/complex.py | 2 +- InnerEye/ML/models/architectures/unet_2d.py | 4 +-- InnerEye/ML/models/architectures/unet_3d.py | 12 ++++----- InnerEye/ML/models/layers/basic.py | 4 +-- InnerEye/ML/models/losses/soft_dice.py | 4 +-- InnerEye/ML/photometric_normalization.py | 2 +- InnerEye/ML/pipelines/ensemble.py | 2 +- InnerEye/ML/pipelines/scalar_inference.py | 2 +- InnerEye/ML/plotting.py | 14 +++++----- .../classification_multilabel_report.py | 6 ++--- InnerEye/ML/reports/classification_report.py | 14 +++++----- InnerEye/ML/reports/segmentation_report.py | 4 +-- InnerEye/ML/run_ml.py | 18 ++++++------- InnerEye/ML/runner.py | 6 ++--- InnerEye/ML/scalar_config.py | 6 ++--- InnerEye/ML/surface_distance_heatmaps.py | 2 +- InnerEye/ML/utils/checkpoint_handling.py | 6 ++--- InnerEye/ML/utils/csv_util.py | 10 +++---- InnerEye/ML/utils/dataset_util.py | 4 +-- InnerEye/ML/utils/image_util.py | 18 ++++++------- InnerEye/ML/utils/io_util.py | 8 +++--- InnerEye/ML/utils/layer_util.py | 10 +++---- InnerEye/ML/utils/metrics_util.py | 4 +-- InnerEye/ML/utils/ml_util.py | 12 ++++----- InnerEye/ML/utils/model_util.py | 2 +- InnerEye/ML/utils/surface_distance_utils.py | 2 +- .../ML/visualizers/metrics_scatterplot.py | 2 +- InnerEye/ML/visualizers/model_hooks.py | 2 +- InnerEye/ML/visualizers/model_summary.py | 2 +- InnerEye/ML/visualizers/patch_sampling.py | 2 +- .../ML/visualizers/plot_cross_validation.py | 8 +++--- InnerEye/Scripts/prepare_fastmri.py | 4 +-- InnerEye/Scripts/submit_for_inference.py | 6 ++--- Tests/AfterTraining/test_after_training.py | 8 +++--- Tests/ML/test_lightning_containers.py | 2 +- Tests/ML/util.py | 2 +- .../ML/visualizers/test_visualize_patches.py | 2 +- TestsOutsidePackage/test_register_model.py | 2 +- 76 files changed, 218 insertions(+), 221 deletions(-) diff --git a/InnerEye/Azure/azure_runner.py b/InnerEye/Azure/azure_runner.py index 3a6623a9e..0f3362bc4 100644 --- a/InnerEye/Azure/azure_runner.py +++ b/InnerEye/Azure/azure_runner.py @@ -176,7 +176,7 @@ def parse_args_and_add_yaml_variables(parser: ArgumentParser, :param parser: The parser to use. :param yaml_config_file: The path to the YAML file that contains values to supply into sys.argv. :param fail_on_unknown_args: If True, raise an exception if the parser encounters an argument that it does not - recognize. If False, unrecognized arguments will be ignored, and added to the "unknown" field of the parser result. + recognize. If False, unrecognized arguments will be ignored, and added to the "unknown" field of the parser result. :return: The parsed arguments, and overrides """ settings_from_yaml = read_all_settings(yaml_config_file, project_root=project_root) @@ -217,7 +217,7 @@ def parse_arguments(parser: ArgumentParser, :param parser: The parser to use :param settings_from_yaml: A dictionary of settings read from a YAML config file. :param fail_on_unknown_args: If True, raise an exception if the parser encounters an argument that it does not - recognize. If False, unrecognized arguments will be ignored, and added to the "unknown" field of the parser result. + recognize. If False, unrecognized arguments will be ignored, and added to the "unknown" field of the parser result. :param args: Arguments to parse. If not given, use those in sys.argv :return: The parsed arguments, and overrides diff --git a/InnerEye/Azure/azure_util.py b/InnerEye/Azure/azure_util.py index 18b516a0e..2ffb80128 100644 --- a/InnerEye/Azure/azure_util.py +++ b/InnerEye/Azure/azure_util.py @@ -78,7 +78,7 @@ def fetch_run(workspace: Workspace, run_recovery_id: str) -> Run: :param workspace: the configured AzureML workspace to search for the experiment. :param run_recovery_id: The Run to find. Either in the full recovery ID format, experiment_name:run_id - or just the run_id + or just the run_id :return: The AzureML run. """ return get_aml_run_from_run_id(aml_workspace=workspace, run_id=run_recovery_id) @@ -114,7 +114,7 @@ def fetch_child_runs( :param run: parent run to fetch child run from :param status: if provided, returns only child runs with this status :param expected_number_cross_validation_splits: when recovering child runs from AML hyperdrive - sometimes the get_children function fails to retrieve all children. If the number of child runs + sometimes the get_children function fails to retrieve all children. If the number of child runs retrieved by AML is lower than the expected number of splits, we try to retrieve them manually. """ if is_ensemble_run(run): diff --git a/InnerEye/Azure/secrets_handling.py b/InnerEye/Azure/secrets_handling.py index 20a2f7e62..cc043ba85 100644 --- a/InnerEye/Azure/secrets_handling.py +++ b/InnerEye/Azure/secrets_handling.py @@ -39,7 +39,7 @@ def read_secrets_from_file(self, secrets_to_read: List[str]) -> Optional[Dict[st in the project root directory. :param secrets_to_read: The list of secret names to read from the YAML file. These will be converted to - uppercase. + uppercase. :return: A dictionary with secrets, or None if the file does not exist. """ secrets_file = self.project_root / fixed_paths.PROJECT_SECRETS_FILE @@ -61,7 +61,7 @@ def get_secrets_from_environment_or_file(self, secrets_to_read: List[str]) -> Di secret is not found, its value will be None. :param secrets_to_read: The list of secret names to read from the YAML file. These will be converted to - uppercase. + uppercase. """ # Read all secrets from a local file if present, and sets the matching environment variables. # If no secrets file is present, no environment variable is modified or created. @@ -75,7 +75,7 @@ def get_secret_from_environment(self, name: str, allow_missing: bool = False) -> :param name: The name of the environment variable to read. It will be converted to uppercase. :param allow_missing: If true, the function returns None if there is no entry of the given name in - any of the places searched. If false, missing entries will raise a ValueError. + any of the places searched. If false, missing entries will raise a ValueError. :return: Value of the secret. None, if there is no value and allow_missing is True. """ diff --git a/InnerEye/Azure/tensorboard_monitor.py b/InnerEye/Azure/tensorboard_monitor.py index 9d14cc0b1..76adc055b 100644 --- a/InnerEye/Azure/tensorboard_monitor.py +++ b/InnerEye/Azure/tensorboard_monitor.py @@ -97,7 +97,7 @@ def main(settings_yaml_file: Optional[Path] = None, :param settings_yaml_file: The YAML file that contains all information for accessing Azure. :param project_root: The root folder that contains all code for the present run. This is only used to locate - a private settings file InnerEyePrivateSettings.yml. + a private settings file InnerEyePrivateSettings.yml. """ monitor_config = AMLTensorBoardMonitorConfig.parse_args() settings_yaml_file = settings_yaml_file or monitor_config.settings diff --git a/InnerEye/Common/Statistics/mann_whitney_test.py b/InnerEye/Common/Statistics/mann_whitney_test.py index 3eec955f0..050debafc 100644 --- a/InnerEye/Common/Statistics/mann_whitney_test.py +++ b/InnerEye/Common/Statistics/mann_whitney_test.py @@ -90,7 +90,7 @@ def compose_distribution_comparisons(file_contents: List[List[List[str]]]) -> Li Composes comparisons as detailed above. :param file_contents: two or more lists of rows, where each "rows" is returned by read_csv_file on - (typically) a statistics.csv file + (typically) a statistics.csv file :return a list of lines to print """ value_lists: List[Dict[str, List[float]]] = [parse_values(rows) for rows in file_contents] diff --git a/InnerEye/Common/Statistics/report_structure_extremes.py b/InnerEye/Common/Statistics/report_structure_extremes.py index 61450d3ae..5395821d0 100644 --- a/InnerEye/Common/Statistics/report_structure_extremes.py +++ b/InnerEye/Common/Statistics/report_structure_extremes.py @@ -130,7 +130,7 @@ def report_structure_extremes_for_subject(subj_dir: str, series_id: str) -> Iter """ :param subj_dir: subject directory, containing .nii.gz files :param series_id: series identifier for the subject - Yields a line for every .nii.gz file in the directory. + Yields a line for every .nii.gz file in the directory. """ subject = os.path.basename(subj_dir) series_prefix = "" if series_id is None else series_id[:8] diff --git a/InnerEye/Common/common_util.py b/InnerEye/Common/common_util.py index 383d31ce5..08442b950 100644 --- a/InnerEye/Common/common_util.py +++ b/InnerEye/Common/common_util.py @@ -77,7 +77,7 @@ def get_best_epoch_results_path(mode: ModelExecutionMode, :param mode: model execution mode :param model_proc: whether this is for an ensemble or single model. If ensemble, we return a different path - to avoid colliding with the results from the single model that may have been created earlier in the same run. + to avoid colliding with the results from the single model that may have been created earlier in the same run. """ subpath = Path(BEST_EPOCH_FOLDER_NAME) / mode.value if model_proc == ModelProcessing.ENSEMBLE_CREATION: @@ -140,7 +140,7 @@ def logging_to_stdout(log_level: Union[int, str] = logging.INFO) -> None: Logging will use a timestamp as the prefix, using UTC. :param log_level: The logging level. All logging message with a level at or above this level will be written to - stdout. log_level can be numeric, or one of the pre-defined logging strings (INFO, DEBUG, ...). + stdout. log_level can be numeric, or one of the pre-defined logging strings (INFO, DEBUG, ...). """ log_level = standardize_log_level(log_level) logger = logging.getLogger() diff --git a/InnerEye/Common/resource_monitor.py b/InnerEye/Common/resource_monitor.py index bed251e2b..b9437de78 100644 --- a/InnerEye/Common/resource_monitor.py +++ b/InnerEye/Common/resource_monitor.py @@ -107,7 +107,7 @@ def enumerate(self, prefix: str = "") -> List[Tuple[str, float]]: Tensorboard. :param prefix: If provided, this string as used as an additional prefix for the metric name itself. If prefix - is "max", the metric would look like "maxLoad_Percent" + is "max", the metric would look like "maxLoad_Percent" :return: A list of (name, value) tuples. """ return [ @@ -153,7 +153,7 @@ def __init__(self, :param interval_seconds: The interval in seconds at which usage statistics should be written. :param tensorboard_folder: The path in which to create a tensorboard logfile. :param csv_results_folder: The path in which the CSV file with aggregate metrics will be created. - When running in AzureML, this should NOT reside inside the /logs folder. + When running in AzureML, this should NOT reside inside the /logs folder. """ super().__init__(name="Resource Monitor", daemon=True) self._interval_seconds = interval_seconds diff --git a/InnerEye/Common/spawn_subprocess.py b/InnerEye/Common/spawn_subprocess.py index 8b37aa57f..f2622825e 100644 --- a/InnerEye/Common/spawn_subprocess.py +++ b/InnerEye/Common/spawn_subprocess.py @@ -18,7 +18,7 @@ def spawn_and_monitor_subprocess(process: str, :param process: The name and path of the executable to spawn. :param args: The args to the process. :param env: The environment variables that the new process will run with. If not provided, copy the - environment from the current process. + environment from the current process. :return: Return code after the process has finished, and the list of lines that were written to stdout by the subprocess. """ diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py b/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py index 45a175045..2c5faee8d 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/datamodules.py @@ -39,7 +39,7 @@ def __init__(self, BEWARE VisionDataModule expects the first positional argument of your class to be the data directory. :param return_index: whether the return the index in __get_item__, the dataset_cls is expected to implement - this logic. + this logic. :param train_transforms: transforms to use at training time :param val_transforms: transforms to use at validation time @@ -115,7 +115,7 @@ def __init__(self, :param encoder_module: datamodule to use for training of SSL. :param linear_head_module: datamodule to use for training of linear head on top of frozen encoder. Can use a - different batch size than the encoder module. CombinedDataModule logic will take care of aggregation. + different batch size than the encoder module. CombinedDataModule logic will take care of aggregation. """ super().__init__(*args, **kwargs) self.encoder_module = encoder_module diff --git a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py index 30d6914b8..7e80e84e9 100644 --- a/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py +++ b/InnerEye/ML/SSL/datamodules_and_datasets/transforms_utils.py @@ -30,7 +30,7 @@ def get_ssl_transforms_from_config(config: CfgNode, :param config: configuration defining which augmentations to apply as well as their intensities. :param return_two_views_per_sample: if True the resulting transforms will return two versions of each sample they - are called on. If False, simply return one transformed version of the sample centered and cropped. + are called on. If False, simply return one transformed version of the sample centered and cropped. :param use_training_augmentations_for_validation: If True, use augmentation at validation time too. This is required for SSL validation loss to be meaningful. If False, only apply basic processing step diff --git a/InnerEye/ML/SSL/lightning_containers/ssl_container.py b/InnerEye/ML/SSL/lightning_containers/ssl_container.py index 5bd07ba23..fe25760a0 100644 --- a/InnerEye/ML/SSL/lightning_containers/ssl_container.py +++ b/InnerEye/ML/SSL/lightning_containers/ssl_container.py @@ -239,13 +239,13 @@ def _get_transforms(self, augmentation_config: Optional[CfgNode], Returns the transformation pipeline for training and validation. :param augmentation_config: optional yaml config defining strength of augmenentations. Ignored for CIFAR - examples. + examples. :param dataset_name: name of the dataset, value has to be in SSLDatasetName, determines which transformation - pipeline to return. + pipeline to return. :param is_ssl_encoder_module: if True the transformation pipeline will yield two versions of the image it is - applied on and it applies the training transformations also at validation time. Note that if your + applied on and it applies the training transformations also at validation time. Note that if your transformation does not contain any randomness, the pipeline will return two identical copies. If False, it will return only one transformation. :return: training transformation pipeline and validation transformation pipeline. diff --git a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py index 8201730bb..c10420f80 100644 --- a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py +++ b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py @@ -49,7 +49,7 @@ def __init__(self, :param warmup_epochs: Number of epochs for scheduler warm up (linear increase from 0 to base_lr). :param use_7x7_first_conv_in_resnet: If True, use a 7x7 kernel (default) in the first layer of resnet. - If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not + If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not shrink them. :param weight_decay: L2-norm weight decay. @@ -83,7 +83,7 @@ def shared_step(self, batch: BatchType, batch_idx: int) -> T: and training step. :param batch: assumed to be a batch a Tuple(List[tensor, tensor, tensor], tensor) to match lightning-bolts - SimCLRTrainDataTransform API; the first tuple element contains a list of three tensor where the two first + SimCLRTrainDataTransform API; the first tuple element contains a list of three tensor where the two first elements contain two are two strong augmented versions of the original images in the batch and the last is a milder augmentation (ignored here). diff --git a/InnerEye/ML/SSL/utils.py b/InnerEye/ML/SSL/utils.py index a185365da..f0af4ddb6 100644 --- a/InnerEye/ML/SSL/utils.py +++ b/InnerEye/ML/SSL/utils.py @@ -41,7 +41,7 @@ def create_ssl_encoder(encoder_name: str, use_7x7_first_conv_in_resnet: bool = T :param encoder_name: available choices: resnet18, resnet50, resnet101 and densenet121. :param use_7x7_first_conv_in_resnet: If True, use a 7x7 kernel (default) in the first layer of resnet. - If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not shrink them. + If False, replace first layer by a 3x3 kernel. This is required for small CIFAR 32x32 images to not shrink them. """ from pl_bolts.models.self_supervised.resnets import resnet18, resnet50, resnet101 from InnerEye.ML.SSL.encoders import DenseNet121Encoder diff --git a/InnerEye/ML/augmentations/image_transforms.py b/InnerEye/ML/augmentations/image_transforms.py index b0e0dc0d6..556db9064 100644 --- a/InnerEye/ML/augmentations/image_transforms.py +++ b/InnerEye/ML/augmentations/image_transforms.py @@ -23,7 +23,7 @@ class RandomGamma: def __init__(self, scale: Tuple[float, float]) -> None: """ :param scale: a tuple (min_gamma, max_gamma) that specifies the range of possible values to sample the gamma - value from when the transformation is called. + value from when the transformation is called. """ self.scale = scale diff --git a/InnerEye/ML/augmentations/transform_pipeline.py b/InnerEye/ML/augmentations/transform_pipeline.py index 269a93483..9f1309ca9 100644 --- a/InnerEye/ML/augmentations/transform_pipeline.py +++ b/InnerEye/ML/augmentations/transform_pipeline.py @@ -34,11 +34,11 @@ def __init__(self, use_different_transformation_per_channel: bool = False): """ :param transforms: List of transformations to apply to images. Supports out of the boxes torchvision transforms - as they accept data of arbitrary dimension. You can also define your own transform class but be aware that you + as they accept data of arbitrary dimension. You can also define your own transform class but be aware that you function should expect input of shape [C, Z, H, W] and apply the same transformation to each Z slice. :param use_different_transformation_per_channel: if True, apply a different version of the augmentation pipeline - for each channel. If False, applies the same transformation to each channel, separately. + for each channel. If False, applies the same transformation to each channel, separately. """ self.use_different_transformation_per_channel = use_different_transformation_per_channel self.pipeline = Compose(transforms) if isinstance(transforms, List) else transforms @@ -97,10 +97,10 @@ def create_transforms_from_config(config: CfgNode, :param config: config yaml file fixing strength and type of augmentation to apply :param apply_augmentations: if True return transformation pipeline with augmentations. Else, - disable augmentations i.e. only resize and center crop the image. + disable augmentations i.e. only resize and center crop the image. :param expand_channels: if True the expand channel transformation from InnerEye.ML.augmentations.image_transforms - will be added to the transformation passed through the config. This is needed for single channel images as CXR. + will be added to the transformation passed through the config. This is needed for single channel images as CXR. """ transforms: List[Any] = [] if expand_channels: diff --git a/InnerEye/ML/baselines_util.py b/InnerEye/ML/baselines_util.py index e53e38cd7..74f2e0dce 100755 --- a/InnerEye/ML/baselines_util.py +++ b/InnerEye/ML/baselines_util.py @@ -111,7 +111,7 @@ def download_and_compare_scores(outputs_folder: Path, azure_config: AzureConfig, """ :param azure_config: Azure configuration to use for downloading data :param comparison_blob_storage_paths: list of paths to directories containing metrics.csv and dataset.csv files, - each of the form run_recovery_id/rest_of_path + each of the form run_recovery_id/rest_of_path :param model_dataset_df: dataframe containing contents of dataset.csv for the current model :param model_metrics_df: dataframe containing contents of metrics.csv for the current model @@ -196,11 +196,11 @@ def compare_files(expected: Path, actual: Path, csv_relative_tolerance: float = basis. :param expected: A file that contains the expected contents. The type of comparison (text or binary) is chosen - based on the extension of this file. + based on the extension of this file. :param actual: A file that contains the actual contents. :param csv_relative_tolerance: When comparing CSV files, use this as the maximum allowed relative discrepancy. - If 0.0, do not allow any discrepancy. + If 0.0, do not allow any discrepancy. :return: An empty string if the files appear identical, or otherwise an error message with details. """ @@ -259,7 +259,7 @@ def compare_folder_contents(expected_folder: Path, :param actual_folder: The output folder with the actually produced files. :param run: An AzureML run :param csv_relative_tolerance: When comparing CSV files, use this as the maximum allowed relative discrepancy. - If 0.0, do not allow any discrepancy. + If 0.0, do not allow any discrepancy. :return: A list of human readable error messages, with message and file path. If no errors are found, the list is empty. """ @@ -304,7 +304,7 @@ def compare_folders_and_run_outputs(expected: Path, actual: Path, csv_relative_t :param expected: A folder with files that are expected to be present. :param actual: The output folder with the actually produced files. :param csv_relative_tolerance: When comparing CSV files, use this as the maximum allowed relative discrepancy. - If 0.0, do not allow any discrepancy. + If 0.0, do not allow any discrepancy. """ if not expected.is_dir(): raise ValueError(f"Folder with expected files does not exist: {expected}") diff --git a/InnerEye/ML/configs/classification/CovidModel.py b/InnerEye/ML/configs/classification/CovidModel.py index d15f94a06..b86b8150e 100644 --- a/InnerEye/ML/configs/classification/CovidModel.py +++ b/InnerEye/ML/configs/classification/CovidModel.py @@ -245,7 +245,7 @@ def generate_custom_report(self, report_dir: Path, model_proc: ModelProcessing) :param report_dir: Directory report is to be written to :param model_proc: Whether this is a single or ensemble model (model_output.csv will be located in different - paths for single vs ensemble runs.) + paths for single vs ensemble runs.) """ diff --git a/InnerEye/ML/configs/other/HelloContainer.py b/InnerEye/ML/configs/other/HelloContainer.py index 9c872a105..3aa5862ef 100644 --- a/InnerEye/ML/configs/other/HelloContainer.py +++ b/InnerEye/ML/configs/other/HelloContainer.py @@ -37,7 +37,7 @@ def __init__(self, raw_data: List[List[float]]) -> None: Creates the 1-dim regression dataset. :param raw_data: The raw data, e.g. from a cross validation split or loaded from file. This - must be numeric data which can be converted into a tensor. See the static method + must be numeric data which can be converted into a tensor. See the static method from_path_and_indexes for an example call. """ super().__init__() diff --git a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py index 68b117eb6..52b854066 100644 --- a/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py +++ b/InnerEye/ML/configs/segmentation/HeadAndNeckBase.py @@ -50,19 +50,19 @@ def __init__(self, :param ground_truth_ids: List of ground truth ids. :param ground_truth_ids_display_names: Optional list of ground truth id display names. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param colours: Optional list of colours. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param fill_holes: Optional list of fill hole flags. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param roi_interpreted_types: Optional list of roi_interpreted_types. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param class_weights: Optional list of class weights. If - present then must be of the same length as ground_truth_ids + 1. + present then must be of the same length as ground_truth_ids + 1. :param slice_exclusion_rules: Optional list of SliceExclusionRules. :param summed_probability_rules: Optional list of SummedProbabilityRule. diff --git a/InnerEye/ML/configs/segmentation/ProstateBase.py b/InnerEye/ML/configs/segmentation/ProstateBase.py index cee8dcd7f..4150b9990 100644 --- a/InnerEye/ML/configs/segmentation/ProstateBase.py +++ b/InnerEye/ML/configs/segmentation/ProstateBase.py @@ -31,19 +31,19 @@ def __init__(self, :param ground_truth_ids: List of ground truth ids. :param ground_truth_ids_display_names: Optional list of ground truth id display names. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param colours: Optional list of colours. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param fill_holes: Optional list of fill hole flags. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param interpreted_types: Optional list of interpreted_types. If - present then must be of the same length as ground_truth_ids. + present then must be of the same length as ground_truth_ids. :param class_weights: Optional list of class weights. If - present then must be of the same length as ground_truth_ids + 1. + present then must be of the same length as ground_truth_ids + 1. :param kwargs: Additional arguments that will be passed through to the SegmentationModelBase constructor. """ diff --git a/InnerEye/ML/configs/unit_testing/passthrough_model.py b/InnerEye/ML/configs/unit_testing/passthrough_model.py index 1849448bb..00ed4031b 100644 --- a/InnerEye/ML/configs/unit_testing/passthrough_model.py +++ b/InnerEye/ML/configs/unit_testing/passthrough_model.py @@ -91,7 +91,7 @@ def forward(self, patches: torch.Tensor) -> torch.Tensor: Ignore the actual patches and return a fixed segmentation, explained in make_nesting_rectangles. :param patches: Set of patches, of shape (#patches, #image_channels, Z, Y, X). Only the shape - is used. + is used. :return: Fixed tensor of shape (#patches, number_of_classes, Z, Y, Z). """ output_size: TupleInt3 = (patches.shape[2], patches.shape[3], patches.shape[4]) diff --git a/InnerEye/ML/dataset/full_image_dataset.py b/InnerEye/ML/dataset/full_image_dataset.py index cab2136e7..7005f0963 100644 --- a/InnerEye/ML/dataset/full_image_dataset.py +++ b/InnerEye/ML/dataset/full_image_dataset.py @@ -85,7 +85,7 @@ def __init__(self, dataset: Any, num_samples: int = None) -> None: """ :param dataset: a dataset - :num_samples: number of samples to draw. If None the number of samples + :num_samples: number of samples to draw. If None the number of samples corresponds to the length of the dataset. """ self.dataset = dataset diff --git a/InnerEye/ML/dataset/scalar_dataset.py b/InnerEye/ML/dataset/scalar_dataset.py index eff40adee..6b65c227c 100644 --- a/InnerEye/ML/dataset/scalar_dataset.py +++ b/InnerEye/ML/dataset/scalar_dataset.py @@ -123,7 +123,7 @@ def _get_single_channel_row(subject_rows: pd.DataFrame, :param subject_rows: A set of rows all belonging to the same subject. :param channel: The value to look for in the `channel_column` column. This can be null. If it is null, - the input `subject_rows` is expected to have exactly 1 row. + the input `subject_rows` is expected to have exactly 1 row. :param subject_id: A string describing the presently processed subject. This is only used for error reporting. :return: A dictionary mapping from column names to values, created from the unique row that was found. @@ -360,14 +360,14 @@ def __init__(self, :param image_channels: The names of all channels (stored in the CSV_CHANNEL_HEADER column of the dataframe) :param label_channels: The name of the channel where the label scalar or vector is read from. :param transform_labels: a label transformation or a list of label transformation to apply to the labels. - If a list is provided, the transformations are applied in order from left to right. + If a list is provided, the transformations are applied in order from left to right. :param non_image_feature_channels: non_image_feature_channels: A dictionary of the names of all channels where - additional scalar values should be read from. The keys should map each feature to its channels. + additional scalar values should be read from. The keys should map each feature to its channels. :param numerical_columns: The names of all columns where additional scalar values should be read from. :param sequence_column: The name of the column that contains the sequence index, that will be stored in - metadata.sequence_position. If this column name is not provided, the sequence_position will be 0. + metadata.sequence_position. If this column name is not provided, the sequence_position will be 0. :param subject_column: The name of the column that contains the subject identifier :param channel_column: The name of the column that contains the row identifier ("channels") @@ -655,10 +655,10 @@ def __init__(self, segmentation_transform: Optional[Callable] = None) -> None: """ :param image_transform: transformation function to apply to images field. If None, images field is unchanged by - call. + call. :param segmentation_transform: transformation function to apply to segmentations field. If None segmentations - field is unchanged by call. + field is unchanged by call. """ self.image_transform = image_transform self.segmentation_transform = segmentation_transform @@ -784,7 +784,7 @@ def __init__(self, args: ScalarModelBase, :param args: The model configuration object. :param data_frame: The dataframe to read from. :param feature_statistics: If given, the normalization factor for the non-image features is taken - from the values provided. If None, the normalization factor is computed from the data in the present dataset. + from the values provided. If None, the normalization factor is computed from the data in the present dataset. :param sample_transform: Sample transforms that should be applied. :param name: Name of the dataset, used for diagnostics logging diff --git a/InnerEye/ML/deep_learning_config.py b/InnerEye/ML/deep_learning_config.py index eef5eab97..10dbeaa76 100644 --- a/InnerEye/ML/deep_learning_config.py +++ b/InnerEye/ML/deep_learning_config.py @@ -119,15 +119,15 @@ def create(project_root: Path, will be created for all outputs and logs. :param project_root: The root folder that contains the code that submitted the present training run. - When running inside the InnerEye repository, it is the git repo root. When consuming InnerEye as a package, + When running inside the InnerEye repository, it is the git repo root. When consuming InnerEye as a package, this should be the root of the source code that calls the package. :param is_offline_run: If true, this is a run outside AzureML. If False, it is inside AzureML. :param model_name: The name of the model that is trained. This is used to generate a run-specific output - folder. + folder. :param output_to: If provided, the output folders will be created as a subfolder of this argument. If not - given, the output folders will be created inside of the project root. + given, the output folders will be created inside of the project root. """ if not project_root.is_absolute(): raise ValueError(f"The project root is required to be an absolute path, but got {project_root}") diff --git a/InnerEye/ML/lightning_base.py b/InnerEye/ML/lightning_base.py index 7e2c6a690..0dbe6af2a 100644 --- a/InnerEye/ML/lightning_base.py +++ b/InnerEye/ML/lightning_base.py @@ -316,13 +316,13 @@ def log_on_epoch(self, floating point, it is converted to a Tensor on the current device to enable synchronization. :param sync_dist_override: If not None, use this value for the sync_dist argument to self.log. If None, - set it automatically depending on the use of DDP. + set it automatically depending on the use of DDP. :param name: The name of the metric to log :param value: The value of the metric. This can be a tensor, floating point value, or a Metric class. :param is_training: If true, give the metric a "train/" prefix, otherwise a "val/" prefix. :param reduce_fx: The reduce function to use when synchronizing the tensors across GPUs. This must be - a value recognized by sync_ddp: "sum", "mean" + a value recognized by sync_ddp: "sum", "mean" """ metric_name = name if isinstance(name, str) else name.value prefix = TRAIN_PREFIX if is_training else VALIDATION_PREFIX @@ -340,7 +340,7 @@ def store_epoch_results(self, metrics: DictStrFloat, epoch: int, is_training: bo :param metrics: A dictionary with all the metrics to write, as key/value pairs. :param epoch: The epoch to which the metrics belong. :param is_training: If true, write the metrics to the logger for training metrics, if False, write to the logger - for validation metrics. + for validation metrics. """ file_logger = self.train_epoch_metrics_logger if is_training else self.val_epoch_metrics_logger store_epoch_metrics(metrics, epoch, file_logger=file_logger) @@ -379,7 +379,7 @@ def training_or_validation_step(self, :param sample: The minibatch of data that should be processed. :param batch_index: The index of the current minibatch. :param is_training: If true, this has been called from `training_step`, otherwise it has been called from - `validation_step`. + `validation_step`. """ raise NotImplementedError("This method must be overwritten in a derived class.") @@ -390,7 +390,7 @@ def write_loss(self, is_training: bool, loss: torch.Tensor) -> None: :param loss: The loss value that should be logged. :param is_training: If True, the logged metric will be called "train/Loss". If False, the metric will - be called "val/Loss" + be called "val/Loss" """ assert isinstance(self.trainer, Trainer) self.log_on_epoch(MetricType.LOSS, loss, is_training) diff --git a/InnerEye/ML/lightning_container.py b/InnerEye/ML/lightning_container.py index d55a92a79..327f7b303 100644 --- a/InnerEye/ML/lightning_container.py +++ b/InnerEye/ML/lightning_container.py @@ -58,7 +58,7 @@ def on_inference_epoch_start(self, dataset_split: ModelExecutionMode, is_ensembl :param dataset_split: Indicates whether the item comes from the training, validation or test set. :param is_ensemble_model: If False, the model_outputs come from an individual model. If True, the model - outputs come from multiple models. + outputs come from multiple models. """ pass diff --git a/InnerEye/ML/lightning_loggers.py b/InnerEye/ML/lightning_loggers.py index 24df64095..a3d8b90b8 100644 --- a/InnerEye/ML/lightning_loggers.py +++ b/InnerEye/ML/lightning_loggers.py @@ -83,7 +83,7 @@ def extract_by_prefix(self, epoch: int, prefix_filter: str = "") -> DictStrFloat :param epoch: The epoch for which results should be read. :param prefix_filter: If empty string, return all metrics. If not empty, return only those metrics that - have a name starting with `prefix`, and strip off the prefix. + have a name starting with `prefix`, and strip off the prefix. :return: A metrics dictionary. """ epoch_results = self.results_per_epoch.get(epoch, None) @@ -106,7 +106,7 @@ def to_metrics_dicts(self, prefix_filter: str = "") -> Dict[int, DictStrFloat]: prefix is stripped off in the result. :param prefix_filter: If empty string, return all metrics. If not empty, return only those metrics that - have a name starting with `prefix`, and strip off the prefix. + have a name starting with `prefix`, and strip off the prefix. :return: A dictionary mapping from epoch number to metric name to metric value. """ return {epoch: self.extract_by_prefix(epoch, prefix_filter) for epoch in self.epochs} @@ -117,7 +117,7 @@ def get_metric(self, is_training: bool, metric_type: str) -> List[float]: the value that a specific metric attains in all of the epochs. :param is_training: If True, read metrics that have a "train/" prefix, otherwise those that have a "val/" - prefix. + prefix. :param metric_type: The metric to extract. :return: A list of floating point numbers, with one entry per entry in the the training or validation results. diff --git a/InnerEye/ML/lightning_metrics.py b/InnerEye/ML/lightning_metrics.py index 934015df0..0b1d46e5f 100644 --- a/InnerEye/ML/lightning_metrics.py +++ b/InnerEye/ML/lightning_metrics.py @@ -291,11 +291,11 @@ def __init__(self, ground_truth_ids: List[str], is_training: bool, :param ground_truth_ids: The list of anatomical structures that should be stored. :param metric_name: The name of the metric that should be stored. This is used in the names of the individual - metrics. + metrics. :param is_training: If true, use "train/" as the prefix for all metric names, otherwise "val/" :param use_average_across_structures: If True, keep track of the average metric value across structures, - while skipping NaNs. If false, only store the per-structure metric values. + while skipping NaNs. If false, only store the per-structure metric values. """ super().__init__() prefix = (TRAIN_PREFIX if is_training else VALIDATION_PREFIX) + metric_name + "/" diff --git a/InnerEye/ML/lightning_models.py b/InnerEye/ML/lightning_models.py index a3f0743b0..f7a8b690c 100644 --- a/InnerEye/ML/lightning_models.py +++ b/InnerEye/ML/lightning_models.py @@ -70,7 +70,7 @@ def training_or_validation_step(self, Runs training for a single minibatch of training or validation data, and computes all metrics. :param is_training: If true, the method is called from `training_step`, otherwise it is called from - `validation_step`. + `validation_step`. :param sample: The batched sample on which the model should be trained. :param batch_index: The index of the present batch (supplied only for diagnostics). @@ -110,7 +110,7 @@ def compute_metrics(self, cropped_sample: CroppedSample, segmentation: torch.Ten :param cropped_sample: The batched image crops used for training or validation. :param segmentation: The segmentation that was produced by the model. :param is_training: If true, the method is called from `training_step`, otherwise it is called from - `validation_step`. + `validation_step`. """ # dice_per_crop_and_class has one row per crop, with background class removed # Dice NaN means that both ground truth and prediction are empty. @@ -235,11 +235,11 @@ def training_or_validation_step(self, Runs training for a single minibatch of training or validation data, and computes all metrics. :param is_training: If true, the method is called from `training_step`, otherwise it is called from - `validation_step`. + `validation_step`. :param sample: The batched sample on which the model should be trained. :param batch_index: The index of the present batch (supplied only for diagnostics). - Runs a minibatch of training or validation data through the model. + Runs a minibatch of training or validation data through the model. """ model_inputs_and_labels = get_scalar_model_inputs_and_labels(self.model, sample) labels = model_inputs_and_labels.labels @@ -325,7 +325,7 @@ def create_lightning_model(config: ModelConfigBase, set_optimizer_and_scheduler: The `optimizer` and `l_rate_scheduler` object of the Lightning model will also be populated. :param set_optimizer_and_scheduler: If True (default), initialize the optimizer and LR scheduler of the model. - If False, skip that step (this is only meant to be used for unit tests.) + If False, skip that step (this is only meant to be used for unit tests.) :param config: An InnerEye model configuration object :return: A PyTorch Lightning model object. diff --git a/InnerEye/ML/metrics.py b/InnerEye/ML/metrics.py index 53384069f..e4c283be8 100644 --- a/InnerEye/ML/metrics.py +++ b/InnerEye/ML/metrics.py @@ -225,7 +225,7 @@ def compute_dice_across_patches(segmentation: torch.Tensor, :param segmentation: Tensor containing class ids predicted by a model. :param ground_truth: One-hot encoded torch tensor containing ground-truth label ids. :param allow_multiple_classes_for_each_pixel: If set to False, ground-truth tensor has - to contain only one foreground label for each pixel. + to contain only one foreground label for each pixel. :return A torch tensor of size (Patches, Classes) with the Dice scores. Dice scores are computed for all classes including the background class at index 0. """ @@ -303,7 +303,7 @@ def compute_scalar_metrics(metrics_dict: ScalarMetricsDict, :param model_output: A tensor containing model outputs. :param labels: A tensor containing class labels. :param loss_type: The type of loss that the model uses. This is required to optionally convert 2-dim model output - to probabilities. + to probabilities. """ _model_output_channels = model_output.shape[1] model_output_hues = metrics_dict.get_hue_names(include_default=len(metrics_dict.hues_without_default) == 0) diff --git a/InnerEye/ML/metrics_dict.py b/InnerEye/ML/metrics_dict.py index a4735068c..ef5aa0c57 100644 --- a/InnerEye/ML/metrics_dict.py +++ b/InnerEye/ML/metrics_dict.py @@ -38,7 +38,7 @@ def average_metric_values(values: List[float], skip_nan_when_averaging: bool) -> :param values: The individual values that should be averaged. :param skip_nan_when_averaging: If True, compute mean with any NaN values. If False, any NaN value present - in the argument will make the function return NaN. + in the argument will make the function return NaN. :return: The average of the provided values. If the argument is an empty list, NaN will be returned. """ if skip_nan_when_averaging: @@ -109,10 +109,10 @@ def add_predictions(self, :param subject_ids: Subject ids associated with the predictions and labels. :param predictions: A numpy array with model predictions, of size [N x C] for N samples in C classes, or size - [N x 1] or size [N] for binary. + [N x 1] or size [N] for binary. :param labels: A numpy array with labels, of size [N x C] for N samples in C classes, or size - [N x 1] or size [N] for binary. + [N x 1] or size [N] for binary. """ if predictions.ndim == 1: predictions = np.expand_dims(predictions, axis=1) @@ -197,7 +197,7 @@ class MetricsDict: def __init__(self, hues: Optional[List[str]] = None, is_classification_metrics: bool = True) -> None: """ :param hues: Supported hues for this metrics dict, otherwise all records will belong to the - default hue. + default hue. :param is_classification_metrics: If this is a classification metrics dict """ @@ -218,7 +218,7 @@ def subject_ids(self, hue: str = DEFAULT_HUE_KEY) -> List[str]: Return the subject ids that have metrics associated with them in this dictionary. :param hue: If provided then subject ids belonging to this hue only will be returned. - Otherwise subject ids for the default hue will be returned. + Otherwise subject ids for the default hue will be returned. """ return self._get_hue(hue=hue).subject_ids @@ -270,7 +270,7 @@ def values(self, hue: str = DEFAULT_HUE_KEY) -> Dict[str, Any]: Returns values held currently in the dict :param hue: will be used to restrict values for the provided hue otherwise values in the default - hue will be returned. + hue will be returned. :return: Dictionary of values for this object. """ return self._get_hue(hue).values @@ -309,7 +309,7 @@ def add_metric(self, :param metric_name: The name of the metric to add. This can be a string or a value in the MetricType enum. :param metric_value: The values of the metric, as a float or integer. :param skip_nan_when_averaging: If True, averaging this metric will skip any NaN (not a number) values. - If False, NaN will propagate through the mean computation. + If False, NaN will propagate through the mean computation. :param hue: The hue for which this record belongs to, default hue will be used if None provided. """ @@ -346,10 +346,10 @@ def add_predictions(self, subject_ids: Sequence[str], :param subject_ids: Subject ids associated with the predictions and labels. :param predictions: A numpy array with model predictions, of size [N x C] for N samples in C classes, or size - [N x 1] or size [N] for binary. + [N x 1] or size [N] for binary. :param labels: A numpy array with labels, of size [N x C] for N samples in C classes, or size - [N x 1] or size [N] for binary. + [N x 1] or size [N] for binary. :param hue: The hue this prediction belongs to, default hue will be used if None provided. """ @@ -378,7 +378,7 @@ def average(self, :param add_metrics_from_entries: average existing metrics in the dict. :param across_hues: If True then same metric types will be averaged regardless of hues, otherwise - separate averages for each metric type for each hue will be computed, Default is True. + separate averages for each metric type for each hue will be computed, Default is True. :return: A MetricsDict object with a single-item list for each of the metrics. """ diff --git a/InnerEye/ML/model_config_base.py b/InnerEye/ML/model_config_base.py index 713063389..8a72da9c8 100644 --- a/InnerEye/ML/model_config_base.py +++ b/InnerEye/ML/model_config_base.py @@ -276,7 +276,7 @@ def __init__(self, train: Optional[Callable] = None, """ :param train: the transformation(s) to apply to the training set. - Should be a function that takes a sample as input and outputs sample. + Should be a function that takes a sample as input and outputs sample. :param val: the transformation(s) to apply to the validation set :param test: the transformation(s) to apply to the test set diff --git a/InnerEye/ML/model_testing.py b/InnerEye/ML/model_testing.py index 3a46609f7..46b0f6a59 100644 --- a/InnerEye/ML/model_testing.py +++ b/InnerEye/ML/model_testing.py @@ -129,7 +129,7 @@ def segmentation_model_test_epoch(config: SegmentationModelBase, :param execution_mode: Is the model evaluated on train, test, or validation set? :param results_folder: The folder where to store the results. :param epoch_and_split: A string that should uniquely identify the epoch and the data split (train/val/test). - :raises TypeError: If the arguments are of the wrong type. + :raises TypeError: If the arguments are of the wrong type. :raises ValueError: When there are issues loading the model. :return A list with the mean dice score (across all structures apart from background) for each image. """ @@ -241,7 +241,7 @@ def populate_metrics_writer( Populate a MetricsPerPatientWriter with the metrics for each patient :param model_prediction_evaluations: The list of PatientMetadata/MetricsDict tuples obtained - from evaluate_model_predictions + from evaluate_model_predictions :param config: The SegmentationModelBase config from which we read the ground_truth_ids :returns: A new MetricsPerPatientWriter and a list of foreground DICE score averages @@ -285,7 +285,7 @@ def store_inference_results(inference_result: InferencePipeline.Result, Store the segmentation, posteriors, and binary predictions into Nifti files. :param inference_result: The inference result for a given patient_id and epoch. Posteriors must be in - (Classes x Z x Y x X) shape, segmentation in (Z, Y, X) + (Classes x Z x Y x X) shape, segmentation in (Z, Y, X) :param config: The test configurations. :param results_folder: The folder where the prediction should be stored. diff --git a/InnerEye/ML/model_training.py b/InnerEye/ML/model_training.py index 77331ddb8..01108ba09 100644 --- a/InnerEye/ML/model_training.py +++ b/InnerEye/ML/model_training.py @@ -210,7 +210,7 @@ def model_train(checkpoint_path: Optional[Path], :param checkpoint_path: Checkpoint path for model initialization :param num_nodes: The number of nodes to use in distributed training. :param container: A container object that holds the training data in PyTorch Lightning format - and the model to train. + and the model to train. :return: A tuple of [Trainer, StoringLogger]. Trainer is the Lightning Trainer object that was used for fitting the model. The StoringLogger object is returned when training an InnerEye built-in model, this is None when fitting other models. diff --git a/InnerEye/ML/models/architectures/base_model.py b/InnerEye/ML/models/architectures/base_model.py index 4dfc1e21f..c3a065ba0 100644 --- a/InnerEye/ML/models/architectures/base_model.py +++ b/InnerEye/ML/models/architectures/base_model.py @@ -23,7 +23,7 @@ def __init__(self, multiple_of: Optional[IntOrTuple3] = None, """ :param multiple_of: Stores minimum size and other conditions that a training crop size must satisfy. :param minimum_size: Training crops must have a size that is a multiple of this value, along each dimension. - For example, if set to (1, 16, 16), the crop size has to be a multiple of 16 along X and Y, and a + For example, if set to (1, 16, 16), the crop size has to be a multiple of 16 along X and Y, and a multiple of 1 (i.e., any number) along the Z dimension. :param num_dimensions: Training crops must have a size that is at least this value. @@ -128,7 +128,7 @@ def __init__(self, :param name: A human readable name of the model. :param input_channels: The number of image input channels. :param crop_size_constraints: The size constraints for the training crop size. If not provided, - a minimum crop size of 1 is assumed. + a minimum crop size of 1 is assumed. """ super().__init__() self.num_dimensions = 3 diff --git a/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py b/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py index 6f1af363a..6e4be2f98 100644 --- a/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py +++ b/InnerEye/ML/models/architectures/classification/image_encoder_with_mlp.py @@ -66,27 +66,27 @@ def __init__(self, is fed through average pooling and an MLP. :param encode_channels_jointly: If False, create a UNet encoder structure separately for each channel. If True, - encode all channels jointly (convolution will run over all channels). + encode all channels jointly (convolution will run over all channels). :param num_encoder_blocks: Number of UNet encoder blocks. :param initial_feature_channels: Number of feature channels in the first UNet encoder. :param num_image_channels: Number of channels of the input. Input is expected to be of size - B x num_image_channels x Z x Y x X, where B is the batch dimension. + B x num_image_channels x Z x Y x X, where B is the batch dimension. :param num_non_image_features: Number of non imaging features will be used in the model. :param kernel_size_per_encoding_block: The size of the kernels per encoding block, assumed to be the same - if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default + if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default performs convolutions only in X and Y. :param stride_size_per_encoding_block: The stride size for the encoding block, assumed to be the same - if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default + if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default reduces spatial dimensions only in X and Y. :param encoder_dimensionality_reduction_factor: how to reduce the dimensionality of the image features in the - combined model to balance with non imaging features. + combined model to balance with non imaging features. :param scan_size: should be a tuple representing 3D tensor shape and if specified it's usedd in initializing - gated pooling or z-adaptive. The first element should be representing the z-direction for classification images + gated pooling or z-adaptive. The first element should be representing the z-direction for classification images """ super().__init__() self.num_non_image_features = num_non_image_features @@ -300,29 +300,29 @@ def __init__(self, layer. :param encode_channels_jointly: If False, create a UNet encoder structure separately for each channel. If True, - encode all channels jointly (convolution will run over all channels). + encode all channels jointly (convolution will run over all channels). :param num_encoder_blocks: Number of UNet encoder blocks. :param initial_feature_channels: Number of feature channels in the first UNet encoder. :param num_image_channels: Number of channels of the input. Input is expected to be of size - B x num_image_channels x Z x Y x X, where B is the batch dimension. + B x num_image_channels x Z x Y x X, where B is the batch dimension. :param mlp_dropout: The amount of dropout that should be applied between the two layers of the classifier MLP. :param final_activation: Activation function to normalize the logits default is Identity. :param num_non_image_features: Number of non imaging features will be used in the model. :param kernel_size_per_encoding_block: The size of the kernels per encoding block, assumed to be the same - if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default + if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default performs convolutions only in X and Y. :param stride_size_per_encoding_block: The stride size for the encoding block, assumed to be the same - if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default + if a single tuple is provided. Otherwise the list of tuples must match num_encoder_blocks. Default reduces spatial dimensions only in X and Y. :param encoder_dimensionality_reduction_factor: how to reduce the dimensionality of the image features in the - combined model to balance with non imaging features. + combined model to balance with non imaging features. :param scan_size: should be a tuple representing 3D tensor shape and if specified it's usedd in initializing - gated pooling or z-adaptive. The first element should be representing the z-direction for classification images + gated pooling or z-adaptive. The first element should be representing the z-direction for classification images """ super().__init__(imaging_feature_type=imaging_feature_type, encode_channels_jointly=encode_channels_jointly, @@ -389,7 +389,7 @@ def create_mlp(input_num_feature_channels: int, :param final_output_channels: if provided, the final number of output channels. :param final_layer: if provided, the final (activation) layer to apply :param hidden_layer_num_feature_channels: if provided, will be used to create hidden layers, If None then - input_num_feature_channels // 2 will be used to create the hidden layer. + input_num_feature_channels // 2 will be used to create the hidden layer. :return: """ layers: List[torch.nn.Module] = [] diff --git a/InnerEye/ML/models/architectures/classification/segmentation_encoder.py b/InnerEye/ML/models/architectures/classification/segmentation_encoder.py index 40cda17d5..0f6d5b458 100644 --- a/InnerEye/ML/models/architectures/classification/segmentation_encoder.py +++ b/InnerEye/ML/models/architectures/classification/segmentation_encoder.py @@ -160,13 +160,12 @@ def __init__(self, ) -> None: """ :param encode_channels_jointly: If False, create an encoder structure separately for each channel. If True, - encode all channels jointly (convolution will run over all channels). + encode all channels jointly (convolution will run over all channels). :param num_image_channels: Number of channels of the input. Input is expected to be of size - B x (num_image_channels * 10) x Z x Y x X, where B is the batch dimension. - + B x (num_image_channels * 10) x Z x Y x X, where B is the batch dimension. :param use_mixed_precision: If True, assume that training happens with mixed precision. Segmentations will - be converted to float16 tensors right away. If False, segmentations will be converted to float32 tensors. + be converted to float16 tensors right away. If False, segmentations will be converted to float32 tensors. """ super().__init__() self.encoder_input_channels = \ diff --git a/InnerEye/ML/models/architectures/complex.py b/InnerEye/ML/models/architectures/complex.py index 5755cb8de..8437232ef 100644 --- a/InnerEye/ML/models/architectures/complex.py +++ b/InnerEye/ML/models/architectures/complex.py @@ -30,7 +30,7 @@ def __init__(self, :param args: The full model configuration. :param full_channels_list: A vector of channel sizes. First entry is the number of image channels, - then all feature channels, then the number of classes. + then all feature channels, then the number of classes. :param network_definition: :param crop_size_constraints: The size constraints for the training crop size. diff --git a/InnerEye/ML/models/architectures/unet_2d.py b/InnerEye/ML/models/architectures/unet_2d.py index a3d7ae7ce..854067e29 100644 --- a/InnerEye/ML/models/architectures/unet_2d.py +++ b/InnerEye/ML/models/architectures/unet_2d.py @@ -29,13 +29,13 @@ def __init__(self, :param input_image_channels: The number of image channels that the model should consume. :param initial_feature_channels: The number of feature maps used in the model in the first convolution layer. - Subsequent layers will contain number of feature maps that are multiples of `initial_channels` + Subsequent layers will contain number of feature maps that are multiples of `initial_channels` (2^(image_level) * initial_channels) :param num_classes: Number of output classes :param num_downsampling_paths: Number of image levels used in Unet (in encoding and decoding paths) :param downsampling_dilation: An additional dilation that is used in the second convolution layer in each - of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good + of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good choice is (1, 2, 2), to increase the receptive field only in X and Y. :param padding_mode: The type of padding that should be applied. diff --git a/InnerEye/ML/models/architectures/unet_3d.py b/InnerEye/ML/models/architectures/unet_3d.py index bc99eefb3..ac80dd9ad 100644 --- a/InnerEye/ML/models/architectures/unet_3d.py +++ b/InnerEye/ML/models/architectures/unet_3d.py @@ -47,7 +47,7 @@ class UNetDecodeBlock(torch.nn.Module): elements. :param upsampling_stride: Upsamling factor used in deconvolutional layer. Similar to the `upsample_kernel_size` - parameter, if an integer is passed, the same upsampling factor will be used for all three dimensions. + parameter, if an integer is passed, the same upsampling factor will be used for all three dimensions. :param activation: Linear/Non-linear activation function that is used after linear deconv/conv mappings. :param depth: The depth inside the UNet at which the layer operates. This is only for diagnostic purposes. @@ -128,10 +128,10 @@ class UNetEncodeBlock(torch.nn.Module): :param channels: A list containing two elements representing the number of input and output channels :param kernel_size: Spatial support of convolution kernels. If an integer is provided, the same value will - be repeated for all three dimensions. For non-cubic kernels please pass a tuple with three elements. + be repeated for all three dimensions. For non-cubic kernels please pass a tuple with three elements. :param downsampling_stride: Downsampling factor used in the first convolutional layer. If an integer is - passed, the same downsampling factor will be used for all three dimensions. + passed, the same downsampling factor will be used for all three dimensions. :param dilation: Dilation of convolution kernels - If set to > 1, kernels capture content from wider range. :param activation: Linear/Non-linear activation function that is used after linear convolution mappings. @@ -193,17 +193,17 @@ def __init__(self, :param input_image_channels: Number of image channels (scans) that are fed into the model. :param initial_feature_channels: Number of feature-maps used in the model - Subsequent layers will contain - number + number of featuremaps that is multiples of `initial_feature_channels` (e.g. 2^(image_level) * initial_feature_channels) :param num_classes: Number of output classes :param kernel_size: Spatial support of conv kernels in each spatial axis. :param num_downsampling_paths: Number of image levels used in Unet (in encoding and decoding paths) :param downsampling_factor: Spatial downsampling factor for each tensor axis (depth, width, height). This will - be used as the stride for the first convolution layer in each encoder block. + be used as the stride for the first convolution layer in each encoder block. :param downsampling_dilation: An additional dilation that is used in the second convolution layer in each - of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good + of the encoding blocks of the UNet. This can be used to increase the receptive field of the network. A good choice is (1, 2, 2), to increase the receptive field only in X and Y. :param crop_size: The size of the crop that should be used for training. diff --git a/InnerEye/ML/models/layers/basic.py b/InnerEye/ML/models/layers/basic.py index dbc589309..202901b7f 100644 --- a/InnerEye/ML/models/layers/basic.py +++ b/InnerEye/ML/models/layers/basic.py @@ -19,12 +19,12 @@ class BasicLayer(torch.nn.Module): :param kernel_size: Spatial support of convolution kernels :param stride: Kernel stride lenght for convolution op :param padding: Feature map padding after convolution op {"constant/zero", "no_padding"}. When it is set to - "no_padding", no padding is applied. For "constant", feature-map tensor size is kept the same at the output by + "no_padding", no padding is applied. For "constant", feature-map tensor size is kept the same at the output by padding with zeros. :param dilation: Kernel dilation used in convolution layer :param use_bias: If set to True, a bias parameter will be added to the layer. Default is set to False as - batch normalisation layer has an affine parameter which are used applied after the bias term is added. + batch normalisation layer has an affine parameter which are used applied after the bias term is added. :param activation: Activation layer (e.g. nonlinearity) to be used after the convolution and batch norm operations. """ diff --git a/InnerEye/ML/models/losses/soft_dice.py b/InnerEye/ML/models/losses/soft_dice.py index 305d1d8ee..c58901700 100644 --- a/InnerEye/ML/models/losses/soft_dice.py +++ b/InnerEye/ML/models/losses/soft_dice.py @@ -24,10 +24,10 @@ def __init__(self, """ :param eps: A small constant to smooth Sorensen-Dice Loss function. Additionally, it avoids division by zero. :param apply_softmax: If true, the input to the loss function will be first fed through a Softmax operation. - If false, the input to the loss function will be used as is. + If false, the input to the loss function will be used as is. :param class_weight_power: power to raise 1/C to, where C is the number of voxels in each class. Should be - non-negative to help increase accuracy on small structures. + non-negative to help increase accuracy on small structures. """ super().__init__() #: Small value to avoid division by zero errors. diff --git a/InnerEye/ML/photometric_normalization.py b/InnerEye/ML/photometric_normalization.py index c05cebc3b..bd0b8696d 100644 --- a/InnerEye/ML/photometric_normalization.py +++ b/InnerEye/ML/photometric_normalization.py @@ -276,7 +276,7 @@ def mri_window(image_in: np.ndarray, :param image_in: The image to normalize. :param mask: Consider only pixel values of the input image for which the mask is non-zero. If None the whole - image is considered. + image is considered. :param output_range: The desired value range of the result image. :param sharpen: number of standard deviation either side of mean to include in the window diff --git a/InnerEye/ML/pipelines/ensemble.py b/InnerEye/ML/pipelines/ensemble.py index c1d062947..a12d1bdcf 100644 --- a/InnerEye/ML/pipelines/ensemble.py +++ b/InnerEye/ML/pipelines/ensemble.py @@ -51,7 +51,7 @@ def aggregate_results(results: Iterable[InferencePipeline.Result], Helper method to aggregate results from multiple inference pipelines, based on the aggregation type provided. :param results: inference pipeline results to aggregate. This may be a Generator to prevent multiple large - posterior arrays being held at the same time. The first element of the sequence is modified in place to + posterior arrays being held at the same time. The first element of the sequence is modified in place to minimize memory use. :param aggregation_type: aggregation function to use to combine the results. diff --git a/InnerEye/ML/pipelines/scalar_inference.py b/InnerEye/ML/pipelines/scalar_inference.py index ff5cc70a8..e09520ae4 100644 --- a/InnerEye/ML/pipelines/scalar_inference.py +++ b/InnerEye/ML/pipelines/scalar_inference.py @@ -182,7 +182,7 @@ def aggregate_model_outputs(self, model_outputs: torch.Tensor) -> torch.Tensor: Aggregates the forward pass results from the individual models in the ensemble. :param model_outputs: List of model outputs for every model in the ensemble. - (Number of ensembles) x (batch_size) x 1 + (Number of ensembles) x (batch_size) x 1 """ # aggregate model outputs if self.aggregation_type == EnsembleAggregationType.Average: diff --git a/InnerEye/ML/plotting.py b/InnerEye/ML/plotting.py index 37cec242a..ebed2368b 100644 --- a/InnerEye/ML/plotting.py +++ b/InnerEye/ML/plotting.py @@ -122,13 +122,13 @@ def plot_image_and_label_contour(image: np.ndarray, :param image: A 2D image :param labels: A binary 2D image, or a list of binary 2D images. A contour will be plotted for each of those - binary images. + binary images. :param contour_arguments: A dictionary of keyword arguments that will be passed directly into matplotlib's - contour function. Can also be a list of dictionaries, with one dict per entry in the 'labels' argument. + contour function. Can also be a list of dictionaries, with one dict per entry in the 'labels' argument. :param image_range: If provided, the image will be plotted using the given range for the color limits. - If None, the minimum and maximum image values will be mapped to the endpoints of the color map. + If None, the minimum and maximum image values will be mapped to the endpoints of the color map. :param plot_file_name: The file name that should be used to save the plot. """ @@ -250,10 +250,10 @@ def plot_normalization_result(loaded_images: Sample, at the slice where the foreground has most pixels. :param loaded_images: An instance of Sample with the image and the labels. The first channel of the image will - be plotted. + be plotted. :param image_range: The image value range that will be mapped to the color map. If None, the full image range - will be mapped to the colormap. + will be mapped to the colormap. :param normalizer: The photometric normalization that should be applied. :param result_folder: The folder into which the resulting PNG files should be written. @@ -304,7 +304,7 @@ def plot_contours_for_all_classes(sample: Sample, :param result_folder: The folder into which the resulting plot PNG files should be written. :param result_prefix: A string prefix that will be used for all plots. :param image_range: The minimum and maximum image values that will be mapped to the color map ranges. - If None, use the actual min and max values. + If None, use the actual min and max values. :param channel_index: The index of the image channel that should be plotted. :return: The paths to all generated PNG files. @@ -493,7 +493,7 @@ def scan_with_transparent_overlay(scan: np.ndarray, :param scan: A 3-dimensional image in (Z, Y, X) ordering :param overlay: A 3-dimensional image in (Z, Y, X) ordering, with values between 0 and 1. :param dimension: The array dimension along with the plot should be created. dimension=0 will generate - an axial slice. + an axial slice. :param position: The index in the chosen dimension where the plot should be created. :param spacing: The tuple of voxel spacings, in (Z, Y, X) order. diff --git a/InnerEye/ML/reports/classification_multilabel_report.py b/InnerEye/ML/reports/classification_multilabel_report.py index ad915813c..a38779d4b 100644 --- a/InnerEye/ML/reports/classification_multilabel_report.py +++ b/InnerEye/ML/reports/classification_multilabel_report.py @@ -140,11 +140,9 @@ def print_metrics_for_thresholded_output_for_all_prediction_targets(csv_to_set_o prediction targets that exist in the dataset (i.e. for every subset of classes that occur in the dataset). :param csv_to_set_optimal_threshold: Csv written during inference time for the val set. This is used to determine - the optimal threshold for classification. - + the optimal threshold for classification. :param csv_to_compute_metrics: Csv written during inference time for the test set. Metrics are calculated for - this csv. - + this csv. :param config: Model config """ diff --git a/InnerEye/ML/reports/classification_report.py b/InnerEye/ML/reports/classification_report.py index 7de18a93a..b9156da07 100644 --- a/InnerEye/ML/reports/classification_report.py +++ b/InnerEye/ML/reports/classification_report.py @@ -179,7 +179,7 @@ def plot_pr_and_roc_curves(labels_and_model_outputs: LabelsAndPredictions, axs: :param labels_and_model_outputs: :param axs: Pair of axes objects onto which to plot the ROC and PR curves, respectively. New axes are created by - default. + default. :param plot_kwargs: Plotting options to be passed to both `ax.plot(...)` calls. """ @@ -211,11 +211,11 @@ def plot_scores_and_summary(all_labels_and_model_outputs: Sequence[LabelsAndPred at each horizontal location. :param all_labels_and_model_outputs: Collection of ground-truth labels and model predictions (e.g. for various - cross-validation runs). + cross-validation runs). :param scoring_fn: A scoring function mapping a `LabelsAndPredictions` object to X and Y coordinates for plotting. :param interval_width: A value in [0, 1] representing what fraction of the data should be contained in - the shaded area. The edges of the interval are `median +/- interval_width/2`. + the shaded area. The edges of the interval are `median +/- interval_width/2`. :param ax: Axes object onto which to plot (default: use current axes). :return: A tuple of `(line_handles, summary_handle)` to use in setting a legend for the plot: `line_handles` is a @@ -248,10 +248,10 @@ def plot_pr_and_roc_curves_crossval(all_labels_and_model_outputs: Sequence[Label shaded 80% confidence interval (computed over TPRs and precisions for each fixed FPR and recall value). :param all_labels_and_model_outputs: Collection of ground-truth labels and model predictions (e.g. for various - cross-validation runs). + cross-validation runs). :param axs: Pair of axes objects onto which to plot the ROC and PR curves, respectively. New axes are created by - default. + default. """ if axs is None: _, axs = plt.subplots(1, 2) @@ -293,7 +293,7 @@ def plot_pr_and_roc_curves_from_csv(metrics_csv: Path, config: ScalarModelBase, :param config: Model config. :param data_split: Whether to filter the CSV file for Train, Val, or Test results (default: no filtering). :param is_crossval_report: If True, assumes CSV contains results for multiple cross-validation runs and plots the - curves with median and confidence intervals. Otherwise, plots curves for a single run. + curves with median and confidence intervals. Otherwise, plots curves for a single run. """ for prediction_target in config.target_names: print_header(f"Class: {prediction_target}", level=3) @@ -315,7 +315,7 @@ def get_metric(predictions_to_set_optimal_threshold: LabelsAndPredictions, Given LabelsAndPredictions objects for the validation and test sets, return the specified metric. :param predictions_to_set_optimal_threshold: This set of ground truth labels and model predictions is used to - determine the optimal threshold for classification. + determine the optimal threshold for classification. :param predictions_to_compute_metrics: The set of labels and model outputs to calculate metrics for. :param metric: The name of the metric to calculate. diff --git a/InnerEye/ML/reports/segmentation_report.py b/InnerEye/ML/reports/segmentation_report.py index 37c420570..0ff7c29b6 100644 --- a/InnerEye/ML/reports/segmentation_report.py +++ b/InnerEye/ML/reports/segmentation_report.py @@ -59,7 +59,7 @@ def display_metric(df: pd.DataFrame, :param metric_name: The metric to sort by. :param outlier_range: The standard deviation range data points must fall outside of to be considered an outlier :param high_values_are_good: If true, high values of the metric indicate good performance. If false, low - values indicate good performance. + values indicate good performance. """ print_header(metric_name, level=2) # Display with best structures first. @@ -87,7 +87,7 @@ def worst_patients_and_outliers(df: pd.DataFrame, :param outlier_range: The multiplier for standard deviation when constructing the interval for outliers. :param metric_name: The metric for which the "worst" patients should be computed. :param high_values_are_good: If True, high values for the metric are considered good, and hence low values - are marked as outliers. If False, low values are considered good, and high values are marked as outliers. + are marked as outliers. If False, low values are considered good, and high values are marked as outliers. :param max_row_count: The maximum number of rows to print. :return: diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py index 5d2bde49a..8b61355d6 100644 --- a/InnerEye/ML/run_ml.py +++ b/InnerEye/ML/run_ml.py @@ -115,24 +115,24 @@ def __init__(self, as a package! :param model_config: If None, run the training as per the `container` argument (bring-your-own-model). If not - None, this is the model configuration for a built-in InnerEye model. + None, this is the model configuration for a built-in InnerEye model. :param container: The LightningContainer object to use for training. If None, assume that the training is - for a built-in InnerEye model. + for a built-in InnerEye model. :param azure_config: Azure related configurations :param project_root: Project root. This should only be omitted if calling run_ml from the test suite. Supplying - it is crucial when using InnerEye as a package or submodule! + it is crucial when using InnerEye as a package or submodule! :param post_cross_validation_hook: A function to call after waiting for completion of cross validation runs. - The function is called with the model configuration and the path to the downloaded and merged metrics files. + The function is called with the model configuration and the path to the downloaded and merged metrics files. :param model_deployment_hook: an optional function for deploying a model in an application-specific way. - If present, it should take a LightningContainer, an AzureConfig, an AzureML Model and a ModelProcessing object + If present, it should take a LightningContainer, an AzureConfig, an AzureML Model and a ModelProcessing object as arguments, and return an object of any type. :param output_subfolder: If provided, the output folder structure will have an additional subfolder, - when running outside AzureML. + when running outside AzureML. """ self.model_config = model_config if container is None: @@ -155,7 +155,7 @@ def setup(self, azure_run_info: Optional[AzureRunInfo] = None) -> None: and call the setup method. It sets the random seeds, and then creates the actual Lightning modules. :param azure_run_info: When running in AzureML or on a local VM, this contains the paths to the datasets. - This can be missing when running in unit tests, where the local dataset paths are already populated. + This can be missing when running in unit tests, where the local dataset paths are already populated. """ if self._has_setup_run: return @@ -474,7 +474,7 @@ def run_inference(self, checkpoint_paths: List[Path], :param checkpoint_paths: Checkpoint paths to initialize model :param model_proc: whether we are running an ensemble model from within a child run with index 0. If we are, - then outputs will be written to OTHER_RUNS/ENSEMBLE under the main outputs directory. + then outputs will be written to OTHER_RUNS/ENSEMBLE under the main outputs directory. """ # run full image inference on existing or newly trained model on the training, and testing set @@ -616,7 +616,7 @@ def copy_child_paths_to_folder(self, :param model_folder: The folder into which all files should be copied. :param checkpoint_paths: A list with absolute paths to checkpoint files. They are expected to be - inside of the model's checkpoint folder. + inside of the model's checkpoint folder. :param python_environment: The Python environment that is used in the present AzureML run. """ diff --git a/InnerEye/ML/runner.py b/InnerEye/ML/runner.py index 81a00f231..39eabab91 100755 --- a/InnerEye/ML/runner.py +++ b/InnerEye/ML/runner.py @@ -114,10 +114,10 @@ class Runner: :param project_root: The root folder that contains all of the source code that should be executed. :param yaml_config_file: The path to the YAML file that contains values to supply into sys.argv. :param post_cross_validation_hook: A function to call after waiting for completion of cross validation runs. - The function is called with the model configuration and the path to the downloaded and merged metrics files. + The function is called with the model configuration and the path to the downloaded and merged metrics files. :param model_deployment_hook: an optional function for deploying a model in an application-specific way. - If present, it should take a model config (SegmentationModelBase), an AzureConfig, and an AzureML + If present, it should take a model config (SegmentationModelBase), an AzureConfig, and an AzureML Model as arguments, and return an optional Path and a further object of any type. """ @@ -382,7 +382,7 @@ def run_in_situ(self, azure_run_info: AzureRunInfo) -> None: Actually run the AzureML job; this method will typically run on an Azure VM. :param azure_run_info: Contains all information about the present run in AzureML, in particular where the - datasets are mounted. + datasets are mounted. """ # Only set the logging level now. Usually, when we set logging to DEBUG, we want diagnostics about the model # build itself, but not the tons of debug information that AzureML submissions create. diff --git a/InnerEye/ML/scalar_config.py b/InnerEye/ML/scalar_config.py index 8f6abd5a1..e3f3bd111 100644 --- a/InnerEye/ML/scalar_config.py +++ b/InnerEye/ML/scalar_config.py @@ -93,7 +93,7 @@ def get_scaling_transform(max_value: int = 100, min_value: int = 0, last_in_pipe :param max_value: :param min_value: :param last_in_pipeline: if the transformation is the last - in the pipeline it should expect a single label as an argument. + in the pipeline it should expect a single label as an argument. Else if returns a list of scaled labels for further transforms. :return: The scaling function """ @@ -593,10 +593,10 @@ def get_non_image_features_dict(default_channels: List[str], Returns the channels dictionary for non-imaging features. :param default_channels: the channels to use for all features except the features specified - in specific_channels + in specific_channels :param specific_channels: a dictionary mapping feature names to channels for all features that do - not use the default channels + not use the default channels """ non_imaging_features_dict = {DEFAULT_KEY: default_channels} if specific_channels is not None: diff --git a/InnerEye/ML/surface_distance_heatmaps.py b/InnerEye/ML/surface_distance_heatmaps.py index a653d167e..db12b2a68 100644 --- a/InnerEye/ML/surface_distance_heatmaps.py +++ b/InnerEye/ML/surface_distance_heatmaps.py @@ -50,7 +50,7 @@ def load_predictions(run_type: SurfaceDistanceRunType, azure_config: AzureConfig :param execution_mode: ModelExecutionMode: Either Test, Train or Val :param extended_annotators: List of annotators plus model_name to load segmentations for :param outlier_range: The standard deviation from the mean which the points have to be below - to be considered an outlier. + to be considered an outlier. :return: list of [(subject_id, structure name and dice_scores)] """ predictions = [] diff --git a/InnerEye/ML/utils/checkpoint_handling.py b/InnerEye/ML/utils/checkpoint_handling.py index d54c82269..ab9de7a33 100644 --- a/InnerEye/ML/utils/checkpoint_handling.py +++ b/InnerEye/ML/utils/checkpoint_handling.py @@ -76,7 +76,7 @@ def download_recovery_checkpoints_or_weights(self, only_return_path: bool = Fals This is called at the start of training. :param: only_return_path: if True, return a RunRecovery object with the path to the checkpoint without actually - downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple + downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple nodes. If False, return the RunRecovery object and download the checkpoint to disk. """ if self.azure_config.run_recovery_id: @@ -403,10 +403,10 @@ def download_all_checkpoints_from_run(config: OutputParams, run: Run, :param config: Model related configs. :param run: Run whose checkpoints should be recovered :param subfolder: optional subfolder name, if provided the checkpoints will be downloaded to - CHECKPOINT_FOLDER / subfolder. If None, the checkpoint are downloaded to CHECKPOINT_FOLDER of the current run. + CHECKPOINT_FOLDER / subfolder. If None, the checkpoint are downloaded to CHECKPOINT_FOLDER of the current run. :param: only_return_path: if True, return a RunRecovery object with the path to the checkpoint without actually - downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple + downloading the checkpoints. This is useful to avoid duplicating checkpoint download when running on multiple nodes. If False, return the RunRecovery object and download the checkpoint to disk. :return: run recovery information """ diff --git a/InnerEye/ML/utils/csv_util.py b/InnerEye/ML/utils/csv_util.py index e0a686db9..7aceeec02 100644 --- a/InnerEye/ML/utils/csv_util.py +++ b/InnerEye/ML/utils/csv_util.py @@ -39,7 +39,7 @@ def load_csv(csv_path: Path, expected_cols: List[str], col_type_converters: Opti :param csv_path: Path to file :param expected_cols: A list of the columns which must, as a minimum, be present. :param col_type_converters: Dictionary of column: type, which ensures certain DataFrame columns are parsed with - specific types + specific types :return: Loaded pandas DataFrame """ if not expected_cols: @@ -84,7 +84,7 @@ def extract_outliers(df: pd.DataFrame, outlier_range: float, outlier_col: str = :param df: DataFrame from which to extract the outliers :param outlier_range: The number of standard deviation from the mean which the points have to be apart - to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above + to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above mean + outlier_range * std (if outlier_type is HIGH) or below mean - outlier_range * std (if outlier_type is LOW). @@ -113,13 +113,13 @@ def mark_outliers(df: pd.DataFrame, :param df: DataFrame from which to extract the outliers :param outlier_range: The number of standard deviation from the mean which the points have to be apart - to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above + to be considered an outlier i.e. a point is considered an outlier if its outlier_col value is above mean + outlier_range * std (if outlier_type is HIGH) or below mean - outlier_range * std (if outlier_type is LOW). :param outlier_col: The column from which to calculate outliers, e.g. Dice :param high_values_are_good: If True, high values for the metric are considered good, and hence low values - are marked as outliers. If False, low values are considered good, and high values are marked as outliers. + are marked as outliers. If False, low values are considered good, and high values are marked as outliers. :return: DataFrame with an additional column `is_outlier` """ if outlier_range < 0: @@ -145,7 +145,7 @@ def get_worst_performing_outliers(df: pd.DataFrame, :param df: Metrics DataFrame :param outlier_col_name: The column by which to determine outliers :param outlier_range: The standard deviation from the mean which the points have to be below - to be considered an outlier. + to be considered an outlier. :param max_n_outliers: the number of (worst performing) outlier IDs to return. :return: a sorted list (worst to best) of all the worst performing outliers diff --git a/InnerEye/ML/utils/dataset_util.py b/InnerEye/ML/utils/dataset_util.py index e71ed1e31..7871a7dba 100644 --- a/InnerEye/ML/utils/dataset_util.py +++ b/InnerEye/ML/utils/dataset_util.py @@ -31,7 +31,7 @@ class CategoricalToOneHotEncoder(OneHotEncoderBase): def __init__(self, columns_and_possible_categories: OrderedDict[str, List[str]]): """ :param columns_and_possible_categories: Mapping between dataset column names - to their possible values. eg: {'Inject': ['True', 'False']}. This is required + to their possible values. eg: {'Inject': ['True', 'False']}. This is required to establish the one-hot encoding each of the possible values. """ super().__init__() @@ -86,7 +86,7 @@ def get_feature_length(self, feature_name: str) -> int: of length 3. :param feature_name: the name of the column for which to compute the feature - length. + length. :returns the feature length i.e. number of possible values for this feature. """ return self._feature_length[feature_name] diff --git a/InnerEye/ML/utils/image_util.py b/InnerEye/ML/utils/image_util.py index 99dbaddf8..b92fa744a 100644 --- a/InnerEye/ML/utils/image_util.py +++ b/InnerEye/ML/utils/image_util.py @@ -185,7 +185,7 @@ def _pad_images(images: np.ndarray, :param images: the image(s) to be padded, in shape: Z x Y x X or batched in shape: Batches x Z x Y x X. :param padding_vector: padding before and after in each dimension eg: ((2,2), (3,3), (2,0)) - will pad 4 pixels in Z (2 on each side), 6 pixels in Y (3 on each side) + will pad 4 pixels in Z (2 on each side), 6 pixels in Y (3 on each side) and 2 in X (2 on the left and 0 on the right). :param padding_mode: a valid numpy padding mode. @@ -210,7 +210,7 @@ def posteriors_to_segmentation(posteriors: NumpyOrTorch) -> NumpyOrTorch: Perform argmax on the class dimension. :param posteriors: Confidence maps [0,1] for each patch per class in format: Batches x Class x Z x Y x X - or Class x Z x Y x X for non-batched input + or Class x Z x Y x X for non-batched input :returns segmentation: argmaxed posteriors with each voxel belonging to a single class: Batches x Z x Y x X or Z x Y x X for non-batched input """ @@ -246,7 +246,7 @@ def largest_connected_components(img: np.ndarray, :param img: np.ndarray :param deletion_limit: if set, a component is deleted only if its voxel count as a proportion of all the - True voxels in the input is less than deletion_limit. + True voxels in the input is less than deletion_limit. :param class_index: Optional. Can be used to provide a class index for logging purposes if the image contains only pixels from a specific class. @@ -288,7 +288,7 @@ def extract_largest_foreground_connected_component( :param multi_label_array: An array of class assignments, i.e. value c at (z, y, x) is a class c. :param restrictions: restrict processing to a subset of the classes (if provided). Each element is a - pair (class_index, threshold) where threshold may be None. + pair (class_index, threshold) where threshold may be None. :return: An array of class assignments """ if restrictions is None: @@ -352,7 +352,7 @@ def multi_label_array_to_binary(array: np.ndarray, num_classes_including_backgro :param array: An array of class assignments. :param num_classes_including_background: The number of class assignments to search for. If 3 classes, - the class assignments to search for will be 0, 1, and 2. + the class assignments to search for will be 0, 1, and 2. :return: an array of size (num_classes_including_background, array.shape) """ return np.stack(list(binaries_from_multi_label_array(array, num_classes_including_background))) @@ -505,7 +505,7 @@ def compute_uncertainty_map_from_posteriors(posteriors: np.ndarray) -> np.ndarra Normalized Shannon Entropy: https://en.wiktionary.org/wiki/Shannon_entropy :param posteriors: Normalized probability distribution in range [0, 1] for each class, - in shape: Class x Z x Y x X + in shape: Class x Z x Y x X :return: Shannon Entropy for each voxel, shape: Z x Y x X expected range is [0,1] where 1 represents low confidence or uniform posterior distribution across classes. """ @@ -520,11 +520,11 @@ def gaussian_smooth_posteriors(posteriors: np.ndarray, kernel_size_mm: TupleFloa Performs Gaussian smoothing on posteriors :param posteriors: Normalized probability distribution in range [0, 1] for each class, - in shape: Class x Z x Y x X + in shape: Class x Z x Y x X :param kernel_size_mm: The size of the smoothing kernel in mm to be used in each dimension (Z, Y, X) :param voxel_spacing_mm: Voxel spacing to use to map from mm space to pixel space for the - Gaussian sigma parameter for each dimension in (Z x Y x X) order. + Gaussian sigma parameter for each dimension in (Z x Y x X) order. :return: """ check_if_posterior_array(posteriors) @@ -565,7 +565,7 @@ def segmentation_to_one_hot(segmentation: torch.Tensor, :param segmentation: A segmentation as a multi-label map of shape [B, C, Z, Y, X] :param use_gpu: If true, and the input is not yet on the GPU, move the intermediate tensors to the GPU. The result - will be on the same device as the argument `segmentation` + will be on the same device as the argument `segmentation` :param result_dtype: The torch data type that the result tensor should have. This would be either float16 or float32 :return: A torch tensor with one-hot encoding of the segmentation of shape diff --git a/InnerEye/ML/utils/io_util.py b/InnerEye/ML/utils/io_util.py index 52ab589d6..129a37f70 100644 --- a/InnerEye/ML/utils/io_util.py +++ b/InnerEye/ML/utils/io_util.py @@ -209,7 +209,7 @@ def load_nifti_image(path: PathOrString, image_type: Optional[Type] = float) -> :return: A numpy array of the image and header data if applicable. :param image_type: The type to load the image in, set to None to not cast, default is float - :raises ValueError: If the path is invalid or the image is not 3D. + :raises ValueError: If the path is invalid or the image is not 3D. """ def _is_valid_image_path(_path: Path) -> bool: @@ -300,7 +300,7 @@ def load_hdf5_file(path_str: Union[str, Path], load_segmentation: bool = False) :param path_str: The path of the HDF5 file that should be loaded. :param load_segmentation: If True, the `segmentation` field of the result object will be populated. If - False, the field will be set to None. + False, the field will be set to None. :return: HDF5Object """ @@ -338,10 +338,10 @@ def load_images_and_stack(files: Iterable[Path], :param files: The paths of the files to load. :param load_segmentation: If True it loads segmentation if present on the same file as the image. This is only - supported for loading from HDF5 files. + supported for loading from HDF5 files. :param center_crop_size: If supplied, all loaded images will be cropped to the size given here. The crop will be - taken from the center of the image. + taken from the center of the image. :param image_size: If supplied, all loaded images will be resized immediately after loading. :return: A wrapper class that contains the loaded images, and if load_segmentation is True, also the segmentations diff --git a/InnerEye/ML/utils/layer_util.py b/InnerEye/ML/utils/layer_util.py index 5847c7202..77cea8a63 100644 --- a/InnerEye/ML/utils/layer_util.py +++ b/InnerEye/ML/utils/layer_util.py @@ -38,16 +38,16 @@ def get_padding_from_kernel_size(padding: PaddingMode, Returns padding value required for convolution layers based on input kernel size and dilation. :param padding: Padding type (Enum) {`zero`, `no_padding`}. Option `zero` is intended to preserve the tensor shape. - In `no_padding` option, padding is not applied and the function returns only zeros. + In `no_padding` option, padding is not applied and the function returns only zeros. :param kernel_size: Spatial support of the convolution kernel. It is used to determine the padding size. This can be - a scalar, tuple or array. + a scalar, tuple or array. :param dilation: Dilation of convolution kernel. It is used to determine the padding size. This can be a scalar, - tuple or array. + tuple or array. :param num_dimensions: The number of dimensions that the returned padding tuple should have, if both - kernel_size and dilation are scalars. + kernel_size and dilation are scalars. :return padding value required for convolution layers based on input kernel size and dilation. """ if isinstance(kernel_size, Sized): @@ -73,7 +73,7 @@ def get_upsampling_kernel_size(downsampling_factor: IntOrTuple3, num_dimensions: https://distill.pub/2016/deconv-checkerboard/ :param downsampling_factor: downsampling factor use for each dimension of the kernel. Can be - either a list of len(num_dimension) with one factor per dimension or an int in which case the + either a list of len(num_dimension) with one factor per dimension or an int in which case the same factor will be applied for all dimension. :param num_dimensions: number of dimensions of the kernel diff --git a/InnerEye/ML/utils/metrics_util.py b/InnerEye/ML/utils/metrics_util.py index fd635a4ca..084bba399 100644 --- a/InnerEye/ML/utils/metrics_util.py +++ b/InnerEye/ML/utils/metrics_util.py @@ -71,7 +71,7 @@ def save_aggregates_to_csv(self, file_path: Path, allow_incomplete_labels: bool :param file_path: The name of the file to write to. :param allow_incomplete_labels: boolean flag. If false, all ground truth files must be provided. - If true, ground truth files are optional and we add a total_patients count column for easy + If true, ground truth files are optional and we add a total_patients count column for easy comparison. (Defaults to False.) """ @@ -210,7 +210,7 @@ def binary_classification_accuracy(model_output: Union[torch.Tensor, np.ndarray] :param model_output: A tensor containing model outputs. :param label: A tensor containing class labels. :param threshold: the cut-off probability threshold for predictions. If model_ouput is > threshold, the predicted - class is 1 else 0. + class is 1 else 0. :return: 1.0 if all predicted classes match the expected classes given in 'labels'. 0.0 if no predicted classes match their labels. """ diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index 4da218dad..c49118625 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -83,7 +83,7 @@ def validate_dataset_paths( :param dataset_path: The base path :param custom_dataset_csv : The name of the dataset csv file - :raise ValueError if the dataset does not exist. + :raise ValueError if the dataset does not exist. """ if not dataset_path.is_dir(): raise ValueError("The dataset_path argument should be the path to the base directory of the data " @@ -107,17 +107,17 @@ def check_size_matches(arg1: Union[np.ndarray, torch.Tensor], :param arg1: The first array to check. :param arg2: The second array to check. :param dim1: The expected number of dimensions of arg1. If zero, no check for number of dimensions will be - conducted. + conducted. :param dim2: The expected number of dimensions of arg2. If zero, no check for number of dimensions will be - conducted. + conducted. :param matching_dimensions: The dimensions along which the two arguments have to match. For example, if - arg1.ndim==4 and arg2.ndim==5, matching_dimensions==[3] checks if arg1.shape[3] == arg2.shape[3]. + arg1.ndim==4 and arg2.ndim==5, matching_dimensions==[3] checks if arg1.shape[3] == arg2.shape[3]. :param arg1_name: If provided, all error messages will use that string to instead of "arg1" :param arg2_name: If provided, all error messages will use that string to instead of "arg2" - :raise ValueError if shapes don't match + :raise ValueError if shapes don't match """ if arg1 is None or arg2 is None: raise Exception("arg1 and arg2 cannot be None.") @@ -132,7 +132,7 @@ def check_dim(expected: int, actual_shape: Any, name: str) -> None: :param expected: expected shape :param actual_shape: :param name: variable name - :raise ValueError if not the same shape + :raise ValueError if not the same shape """ if len(actual_shape) != expected: raise ValueError("'{}' was expected to have ndim == {}, but is {}. Shape is {}" diff --git a/InnerEye/ML/utils/model_util.py b/InnerEye/ML/utils/model_util.py index dd8fc445d..efe23f64d 100644 --- a/InnerEye/ML/utils/model_util.py +++ b/InnerEye/ML/utils/model_util.py @@ -279,7 +279,7 @@ def get_scalar_model_inputs_and_labels(model: torch.nn.Module, :param model: The instantiated PyTorch model. :param target_indices: If this list is non-empty, assume that the model is a sequence model, and build the - model inputs and labels for a model that predicts those specific positions in the sequence. If the list is empty, + model inputs and labels for a model that predicts those specific positions in the sequence. If the list is empty, assume that the model is a normal (non-sequence) model. :param sample: A training sample, as returned by a PyTorch data loader (dictionary mapping from field name to value) diff --git a/InnerEye/ML/utils/surface_distance_utils.py b/InnerEye/ML/utils/surface_distance_utils.py index d8398131e..a634465a4 100644 --- a/InnerEye/ML/utils/surface_distance_utils.py +++ b/InnerEye/ML/utils/surface_distance_utils.py @@ -211,7 +211,7 @@ def extract_border(img: np.ndarray, connectivity: int = 1) -> np.ndarray: :param img: Array containing structure from which to extract the border :param connectivity: integer determining which pixels are considered neighbours of the central element, - ranging from 1 = no diagonal elements and rank = all elements + ranging from 1 = no diagonal elements and rank = all elements :return: """ if not np.unique(img.astype(int)).tolist() == [0, 1]: diff --git a/InnerEye/ML/visualizers/metrics_scatterplot.py b/InnerEye/ML/visualizers/metrics_scatterplot.py index 2ffa4f0a4..acbabbde4 100644 --- a/InnerEye/ML/visualizers/metrics_scatterplot.py +++ b/InnerEye/ML/visualizers/metrics_scatterplot.py @@ -34,7 +34,7 @@ def create_scatterplots(data: Dict[str, Dict[str, Dict[str, float]]], against: O :param data: dictionary such that data[run][structure][seriesId] = dice_score :param against: run names to plot against (as y axis); if None or empty, do all against all :param max_dice: maximum Dice score to expect; if None, either 1.0 or 100.0 will be inferred from the data - in code called from here. + in code called from here. """ runs = sorted(data.keys()) result = {} diff --git a/InnerEye/ML/visualizers/model_hooks.py b/InnerEye/ML/visualizers/model_hooks.py index 6d3437362..faf7f156b 100644 --- a/InnerEye/ML/visualizers/model_hooks.py +++ b/InnerEye/ML/visualizers/model_hooks.py @@ -16,7 +16,7 @@ def __init__(self, model: Module, layer_name: Optional[List[str]]): """ :param model: pytorch model :param layer_name: name of direct submodule of modle, or nested module names structure as a list - layer_name = [_2D_encoder] , layer_name = [_2D_encoder, conv_layers, 0] + layer_name = [_2D_encoder] , layer_name = [_2D_encoder, conv_layers, 0] """ super().__init__() diff --git a/InnerEye/ML/visualizers/model_summary.py b/InnerEye/ML/visualizers/model_summary.py index 63fcd9557..a91f91290 100644 --- a/InnerEye/ML/visualizers/model_summary.py +++ b/InnerEye/ML/visualizers/model_summary.py @@ -66,7 +66,7 @@ def generate_summary(self, doing forward propagation through the model, with tensors of a given size or a given list of tensors. :param input_sizes: The list of sizes of the input tensors to the model. These sizes must be specifies - without the leading batch dimension. + without the leading batch dimension. :param input_tensors: The tensors to use in model forward propagation. :param log_summaries_to_files: if True, write the summary to a new file under logs/models instead of stdout diff --git a/InnerEye/ML/visualizers/patch_sampling.py b/InnerEye/ML/visualizers/patch_sampling.py index 93d7a2b6f..2cc3c238a 100644 --- a/InnerEye/ML/visualizers/patch_sampling.py +++ b/InnerEye/ML/visualizers/patch_sampling.py @@ -115,7 +115,7 @@ def visualize_random_crops_for_dataset(config: SegmentationModelBase, output_fol :param config: The model configuration. :param output_folder: The folder in which the visualizations should be written. If not provided, use a subfolder - "patch_sampling" in the model's default output folder + "patch_sampling" in the model's default output folder """ dataset_splits = config.get_dataset_splits() # Load a sample using the full image data loader diff --git a/InnerEye/ML/visualizers/plot_cross_validation.py b/InnerEye/ML/visualizers/plot_cross_validation.py index e9d35ecb6..ae41416bc 100644 --- a/InnerEye/ML/visualizers/plot_cross_validation.py +++ b/InnerEye/ML/visualizers/plot_cross_validation.py @@ -203,7 +203,7 @@ def download_or_get_local_file(self, :param destination: directory to write to :param run: The AzureML run to download from. :param local_src_subdir: if not None, then if we copy from a local results folder, that folder is - self.outputs_directory/local_src_subdir/blob_to_download instead of self.outputs_directory/blob_to_download + self.outputs_directory/local_src_subdir/blob_to_download instead of self.outputs_directory/blob_to_download :return: The path to the downloaded file, or None if the file was not found. """ blob_path = Path(blob_to_download) @@ -354,11 +354,11 @@ def download_crossval_result_files(config: PlotCrossValidationConfig, :param config: PlotCrossValidationConfig :param run_recovery_id: run recovery ID, if different from the one in config :param download_to_folder: The root folder in which all downloaded files should be stored. Point to an existing - folder with downloaded files for use in unit tests. If not provided, the files will be downloaded to a new folder + folder with downloaded files for use in unit tests. If not provided, the files will be downloaded to a new folder inside the config.outputs_directory, with the name taken from the run ID. :param splits_to_evaluate: If supplied, use these values as the split indices to download. Use only for - unit testing. + unit testing. :return: The dataframe with all of the downloaded results grouped by execution mode (Test or Val) and directory where the epoch results were downloaded to. """ @@ -788,7 +788,7 @@ def check_result_file_counts(config_and_files: OfflineCrossvalConfigAndFiles, is that mode is equal to the number of cross-validation splits. Throw a ValueError if not. :param is_ensemble_run: If True, assume that this run of cross validation analysis is for an ensemble model - and assert that there are N+1 data files available. If false, this analysis only concerns the cross + and assert that there are N+1 data files available. If false, this analysis only concerns the cross validation runs, and check that the number of files is N. """ result_files_by_mode = defaultdict(list) diff --git a/InnerEye/Scripts/prepare_fastmri.py b/InnerEye/Scripts/prepare_fastmri.py index 13a053300..0d4aef164 100644 --- a/InnerEye/Scripts/prepare_fastmri.py +++ b/InnerEye/Scripts/prepare_fastmri.py @@ -112,7 +112,7 @@ def create_datafactory_and_run(files_and_tokens: Dict[str, str], :param location: The Azure location in which the Data Factory should be created (for example, "westeurope") :param files_and_tokens: A mapping from file name (like knee.tar.gz) to AWS access token. :param is_unittest: If True, download a small tar.gz file from github. If False, download the "real" fastMRI - datafiles from AWS. + datafiles from AWS. :param connection_string: The connection string of the Azure storage where the downloaded data should be stored. """ @@ -172,7 +172,7 @@ def download_and_uncompress(source_file_or_tuple: Union[str, Tuple[str, str]], t compressed raw data will be written to 'foo_compressed'. :param source_file_or_tuple: The name of the .tar.gz or .tar file to download, without any access tokens. - If the name is a Tuple[str, str], the second tuple element is the "real" extension, for files where the + If the name is a Tuple[str, str], the second tuple element is the "real" extension, for files where the extension is misleading. :param target_folder: The folder prefix in the target storage account. diff --git a/InnerEye/Scripts/submit_for_inference.py b/InnerEye/Scripts/submit_for_inference.py index 7589cc072..aea8eb9bf 100755 --- a/InnerEye/Scripts/submit_for_inference.py +++ b/InnerEye/Scripts/submit_for_inference.py @@ -79,7 +79,7 @@ def copy_image_file(image: Path, destination_folder: Path, use_dicom: bool) -> P Copy the source image file into the given folder destination_folder. :param image: image file, must be Gzipped Nifti format with name ending .nii.gz if use_dicom=False or .zip - otherwise. + otherwise. :param destination_folder: top-level directory to copy image into (as test.nii.gz or test.zip) :param use_dicom: True to treat as a zip file. @@ -98,11 +98,11 @@ def download_files_from_model(model_sas_urls: Dict[str, str], base_name: str, di to a folder. :param model_sas_urls: The files making up the model, as a mapping from file name to a URL with - an SAS token. + an SAS token. :param base_name: The file name of the files to download. :param dir_path: The folder into which the files will be written. All downloaded files will keep the relative - path that they also have in the model. + path that they also have in the model. :return: a list of the files that were downloaded. """ downloaded: List[Path] = [] diff --git a/Tests/AfterTraining/test_after_training.py b/Tests/AfterTraining/test_after_training.py index 8fc137858..5f61f4b04 100644 --- a/Tests/AfterTraining/test_after_training.py +++ b/Tests/AfterTraining/test_after_training.py @@ -71,7 +71,7 @@ def get_most_recent_run_id(fallback_run_id_for_local_execution: str = FALLBACK_S In this case, modify the run here to something more recent. :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code - on a local box, outside of Azure build agents. + on a local box, outside of Azure build agents. :return: """ run_recovery_file = Path(RUN_RECOVERY_FILE) @@ -92,7 +92,7 @@ def get_most_recent_run(fallback_run_id_for_local_execution: str = FALLBACK_SING Gets the name of the most recently executed AzureML run, instantiates that Run object and returns it. :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code - on a local box, outside of Azure build agents. + on a local box, outside of Azure build agents. """ run_recovery_id = get_most_recent_run_id(fallback_run_id_for_local_execution=fallback_run_id_for_local_execution) return get_default_azure_config().fetch_run(run_recovery_id=run_recovery_id) @@ -104,7 +104,7 @@ def get_most_recent_model_id(fallback_run_id_for_local_execution: str = FALLBACK and return the model id. :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code - on a local box, outside of Azure build agents. + on a local box, outside of Azure build agents. """ most_recent_run = get_most_recent_run_id(fallback_run_id_for_local_execution=fallback_run_id_for_local_execution) azure_config = AzureConfig.from_yaml(fixed_paths.SETTINGS_YAML_FILE, @@ -123,7 +123,7 @@ def get_most_recent_model(fallback_run_id_for_local_execution: str = FALLBACK_SI and return the instantiated model object. :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code - on a local box, outside of Azure build agents. + on a local box, outside of Azure build agents. """ model_id = get_most_recent_model_id(fallback_run_id_for_local_execution=fallback_run_id_for_local_execution) return Model(workspace=get_default_workspace(), id=model_id) diff --git a/Tests/ML/test_lightning_containers.py b/Tests/ML/test_lightning_containers.py index 933d2f163..e6c208e58 100644 --- a/Tests/ML/test_lightning_containers.py +++ b/Tests/ML/test_lightning_containers.py @@ -327,7 +327,7 @@ def test_innereyecontainer_setup_passes_on_allow_incomplete_labels( :param test_output_dirs: Test fixture. :param allow_partial_ground_truth: The value to set allow_incomplete_labels to and check it is - passed through. + passed through. """ config = DummyModel() config.set_output_to(test_output_dirs.root_dir) diff --git a/Tests/ML/util.py b/Tests/ML/util.py index b38c55fec..456d28e81 100644 --- a/Tests/ML/util.py +++ b/Tests/ML/util.py @@ -293,7 +293,7 @@ def model_train_unittest(config: Optional[DeepLearningConfig], :param output_folder: The test fixture that provides an output folder for the test. :param lightning_container: An optional LightningContainer object that will be pass through to the training routine. :param checkpoint_handler: The checkpoint handler that should be used for training. If not provided, it will be - created via get_default_checkpoint_handler. + created via get_default_checkpoint_handler. :return: Tuple[StoringLogger, CheckpointHandler] """ runner = MLRunner(model_config=config, container=lightning_container) diff --git a/Tests/ML/visualizers/test_visualize_patches.py b/Tests/ML/visualizers/test_visualize_patches.py index 85478963d..57b87e637 100644 --- a/Tests/ML/visualizers/test_visualize_patches.py +++ b/Tests/ML/visualizers/test_visualize_patches.py @@ -32,7 +32,7 @@ def test_visualize_patch_sampling(test_output_dirs: OutputFolderForTests, :param test_output_dirs: :param labels_to_boundary: If true, the ground truth labels are placed close to the image boundary, so that - crops have to be adjusted inwards. If false, ground truth labels are all far from the image boundaries. + crops have to be adjusted inwards. If false, ground truth labels are all far from the image boundaries. """ set_random_seed(0) shape = (10, 30, 30) diff --git a/TestsOutsidePackage/test_register_model.py b/TestsOutsidePackage/test_register_model.py index 8eaf2dd55..34afb8af4 100644 --- a/TestsOutsidePackage/test_register_model.py +++ b/TestsOutsidePackage/test_register_model.py @@ -23,7 +23,7 @@ def create_checkpoints(model_config: SegmentationModelBase, is_ensemble: bool) - :param model_config: The model configuration, where a correct output folder must be set. :param is_ensemble: If true, 2 checkpoints (simulating an ensemble run) will be created. If false, only a - single checkpoint will be created. + single checkpoint will be created. :return: Tuple[absolute checkpoint paths, relative checkpoint paths] """ # To simulate ensemble models, there are two checkpoints, one in the root dir and one in a folder From 416e90735e45947c991223ad7f5c9747fc076125 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 09:35:06 +0100 Subject: [PATCH 12/27] =?UTF-8?q?=F0=9F=93=9D=20Fix=20whitespace=20errors?= =?UTF-8?q?=20on=20`:return`=20statements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Azure/azure_runner.py | 2 +- InnerEye/Azure/run_pytest.py | 2 +- InnerEye/Azure/secrets_handling.py | 4 ++-- InnerEye/Common/Statistics/mann_whitney_test.py | 6 +++--- InnerEye/Common/Statistics/report_structure_extremes.py | 2 +- InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py | 2 +- InnerEye/Common/spawn_subprocess.py | 2 +- InnerEye/ML/SSL/lightning_containers/ssl_container.py | 6 +++--- InnerEye/ML/baselines_util.py | 6 +++--- InnerEye/ML/dataset/scalar_dataset.py | 2 +- InnerEye/ML/dataset/scalar_sample.py | 4 ++-- InnerEye/ML/deep_learning_config.py | 8 ++++---- InnerEye/ML/metrics.py | 2 +- InnerEye/ML/model_config_base.py | 4 ++-- InnerEye/ML/model_training.py | 4 ++-- InnerEye/ML/pipelines/ensemble.py | 4 ++-- InnerEye/ML/plotting.py | 2 +- InnerEye/ML/reports/classification_report.py | 4 ++-- InnerEye/ML/run_ml.py | 4 ++-- InnerEye/ML/runner.py | 4 ++-- InnerEye/ML/utils/device_aware_module.py | 2 +- InnerEye/ML/utils/features_util.py | 2 +- InnerEye/ML/utils/hdf5_util.py | 2 +- InnerEye/ML/utils/image_util.py | 6 +++--- InnerEye/ML/utils/io_util.py | 2 +- InnerEye/ML/utils/metrics_util.py | 2 +- InnerEye/ML/utils/ml_util.py | 4 ++-- InnerEye/ML/utils/model_util.py | 2 +- InnerEye/ML/utils/sequence_utils.py | 2 +- InnerEye/ML/visualizers/plot_cross_validation.py | 8 ++++---- InnerEye/Scripts/download_model_and_run_scoring.py | 2 +- 31 files changed, 54 insertions(+), 54 deletions(-) diff --git a/InnerEye/Azure/azure_runner.py b/InnerEye/Azure/azure_runner.py index 0f3362bc4..c64bcb182 100644 --- a/InnerEye/Azure/azure_runner.py +++ b/InnerEye/Azure/azure_runner.py @@ -107,7 +107,7 @@ def create_dataset_configs(azure_config: AzureConfig, :param all_dataset_mountpoints: When using the datasets in AzureML, these are the per-dataset mount points. :param all_local_datasets: The paths for all local versions of the datasets. :return: A list of DatasetConfig objects, in the same order as datasets were provided in all_azure_dataset_ids, - omitting datasets with an empty name. + omitting datasets with an empty name. """ datasets: List[DatasetConfig] = [] num_local = len(all_local_datasets) diff --git a/InnerEye/Azure/run_pytest.py b/InnerEye/Azure/run_pytest.py index 271877208..ed55d2d91 100644 --- a/InnerEye/Azure/run_pytest.py +++ b/InnerEye/Azure/run_pytest.py @@ -23,7 +23,7 @@ def run_pytest(pytest_mark: str, outputs_folder: Path) -> Tuple[bool, Path]: :param pytest_mark: The PyTest mark to use for filtering out the tests to run. :param outputs_folder: The folder into which the test result XML file should be written. :return: True if PyTest found tests to execute and completed successfully, False otherwise. - Also returns the path to the generated PyTest results file. + Also returns the path to the generated PyTest results file. """ from _pytest.main import ExitCode _outputs_file = outputs_folder / PYTEST_RESULTS_FILE diff --git a/InnerEye/Azure/secrets_handling.py b/InnerEye/Azure/secrets_handling.py index cc043ba85..15ce4fd62 100644 --- a/InnerEye/Azure/secrets_handling.py +++ b/InnerEye/Azure/secrets_handling.py @@ -107,7 +107,7 @@ def read_all_settings(project_settings_file: Optional[Path] = None, :param project_settings_file: The first YAML settings file to read. :param project_root: The folder that can contain a 'InnerEyePrivateSettings.yml' file. :return: A dictionary mapping from string to variable value. The dictionary key is the union of variable names - found in the two settings files. + found in the two settings files. """ private_settings_file = None if project_root and project_root.is_dir(): @@ -126,7 +126,7 @@ def read_settings_and_merge(project_settings_file: Optional[Path] = None, :param project_settings_file: The first YAML settings file to read. :param private_settings_file: The second YAML settings file to read. Settings in this file has higher priority. :return: A dictionary mapping from string to variable value. The dictionary key is the union of variable names - found in the two settings files. + found in the two settings files. """ result = dict() if project_settings_file: diff --git a/InnerEye/Common/Statistics/mann_whitney_test.py b/InnerEye/Common/Statistics/mann_whitney_test.py index 050debafc..3544969da 100644 --- a/InnerEye/Common/Statistics/mann_whitney_test.py +++ b/InnerEye/Common/Statistics/mann_whitney_test.py @@ -187,7 +187,7 @@ def roc_value(lst1: List[float], lst2: List[float]) -> float: :param lst1: a list of numbers :param lst2: another list of numbers :return: the proportion of pairs (x, y), where x is from lst1 and y is from lst2, for which - x < y, with x == y counting as half an instance. + x < y, with x == y counting as half an instance. """ if len(lst1) == 0 or len(lst2) == 0: return 0.5 @@ -273,7 +273,7 @@ def compare_scores_across_institutions(metrics_file: str, splits_to_use: str = " :param splits_to_use: a comma-separated list of split names :param mode_to_use: test, validation etc :return: a list of comparison lines between pairs of splits. If splits_to_use is non empty, - only pairs involving at least one split from that set are compared. + only pairs involving at least one split from that set are compared. """ valid_splits = set(splits_to_use.split(",")) if splits_to_use else None metrics = pd.read_csv(metrics_file) @@ -330,7 +330,7 @@ def get_arguments(arglist: List[str] = None) -> Tuple[Optional[argparse.Namespac The value of the "-a" switch is one or more split names; pairs of splits not including these will not be compared. :return: parsed arguments and identifier for pattern (1, 2, 3 as above), or None, None if none of the - patterns are followed + patterns are followed """ # Use argparse because we want to have mandatory non-switch arguments, which GenericConfig doesn't support. parser = argparse.ArgumentParser("Run Mann-Whitney tests") diff --git a/InnerEye/Common/Statistics/report_structure_extremes.py b/InnerEye/Common/Statistics/report_structure_extremes.py index 5395821d0..0d7809a35 100644 --- a/InnerEye/Common/Statistics/report_structure_extremes.py +++ b/InnerEye/Common/Statistics/report_structure_extremes.py @@ -175,7 +175,7 @@ def extent_list(presence: np.array, max_value: int) -> Tuple[List[int], List[str :param presence: a 1-D array of distinct integers in increasing order. :param max_value: any integer, not necessarily related to presence :return: two tuples: (1) a list of the minimum and maximum values of presence, and max_value; - (2) a list of strings, each denoting a missing range of values within "presence". + (2) a list of strings, each denoting a missing range of values within "presence". """ if len(presence) == 0: return [-1, -1, max_value], [] diff --git a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py index a9cb0f074..64888a8cd 100644 --- a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py +++ b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py @@ -136,7 +136,7 @@ def difference_counts(values1: List[float], values2: List[float]) -> Tuple[int, :param values1: list of values :param values2: list of values, same length as values1 :return: number of pairs in which first value is greater than second, and number of pairs - in which second is greater than first + in which second is greater than first """ n1 = 0 n2 = 0 diff --git a/InnerEye/Common/spawn_subprocess.py b/InnerEye/Common/spawn_subprocess.py index f2622825e..864dec80b 100644 --- a/InnerEye/Common/spawn_subprocess.py +++ b/InnerEye/Common/spawn_subprocess.py @@ -20,7 +20,7 @@ def spawn_and_monitor_subprocess(process: str, :param env: The environment variables that the new process will run with. If not provided, copy the environment from the current process. :return: Return code after the process has finished, and the list of lines that were written to stdout by the - subprocess. + subprocess. """ if env is None: env = dict(os.environ.items()) diff --git a/InnerEye/ML/SSL/lightning_containers/ssl_container.py b/InnerEye/ML/SSL/lightning_containers/ssl_container.py index fe25760a0..af73c47f1 100644 --- a/InnerEye/ML/SSL/lightning_containers/ssl_container.py +++ b/InnerEye/ML/SSL/lightning_containers/ssl_container.py @@ -204,9 +204,9 @@ def _create_ssl_data_modules(self, is_ssl_encoder_module: bool) -> InnerEyeVisio :param is_ssl_encoder_module: whether to return the data module for SSL training or for linear head. If true, :return transforms with two views per sample (batch like (img_v1, img_v2, label)). If False, return only one - view per sample but also return the index of the sample in the dataset (to make sure we don't use twice the same - batch in one training epoch (batch like (index, img_v1, label), as classifier dataloader expected to be shorter - than SSL training, hence CombinedDataloader might loop over data several times per epoch). + view per sample but also return the index of the sample in the dataset (to make sure we don't use twice the same + batch in one training epoch (batch like (index, img_v1, label), as classifier dataloader expected to be shorter + than SSL training, hence CombinedDataloader might loop over data several times per epoch). """ datamodule_args = self.datamodule_args[SSLDataModuleType.ENCODER] if is_ssl_encoder_module else \ self.datamodule_args[SSLDataModuleType.LINEAR_HEAD] diff --git a/InnerEye/ML/baselines_util.py b/InnerEye/ML/baselines_util.py index 74f2e0dce..59c78b27a 100755 --- a/InnerEye/ML/baselines_util.py +++ b/InnerEye/ML/baselines_util.py @@ -116,8 +116,8 @@ def download_and_compare_scores(outputs_folder: Path, azure_config: AzureConfig, :param model_dataset_df: dataframe containing contents of dataset.csv for the current model :param model_metrics_df: dataframe containing contents of metrics.csv for the current model :return: a dataframe for all the data (current model and all baselines); whether any comparisons were - done, i.e. whether a valid baseline was found; and the text lines to be written to the Wilcoxon results - file. + done, i.e. whether a valid baseline was found; and the text lines to be written to the Wilcoxon results + file. """ comparison_baselines = get_comparison_baselines(outputs_folder, azure_config, comparison_blob_storage_paths) result = perform_score_comparisons(model_dataset_df, model_metrics_df, comparison_baselines) @@ -261,7 +261,7 @@ def compare_folder_contents(expected_folder: Path, :param csv_relative_tolerance: When comparing CSV files, use this as the maximum allowed relative discrepancy. If 0.0, do not allow any discrepancy. :return: A list of human readable error messages, with message and file path. If no errors are found, the list is - empty. + empty. """ messages = [] if run and is_offline_run_context(run): diff --git a/InnerEye/ML/dataset/scalar_dataset.py b/InnerEye/ML/dataset/scalar_dataset.py index 6b65c227c..2d3fd8d61 100644 --- a/InnerEye/ML/dataset/scalar_dataset.py +++ b/InnerEye/ML/dataset/scalar_dataset.py @@ -746,7 +746,7 @@ def load_item(self, item: ScalarDataSource) -> ScalarItem: :param item: The item to load. :return: A ClassificationItem instances with the loaded images, and the labels and non-image features copied - from the argument. + from the argument. """ sample = item.load_images( root_path=self.args.local_dataset, diff --git a/InnerEye/ML/dataset/scalar_sample.py b/InnerEye/ML/dataset/scalar_sample.py index fe16c48d8..2d7fbc4f7 100644 --- a/InnerEye/ML/dataset/scalar_sample.py +++ b/InnerEye/ML/dataset/scalar_sample.py @@ -135,7 +135,7 @@ def load_images(self, :param image_size: If given, all loaded images will be reshaped to the size given here, prior to the center crop. :return: An instance of ClassificationItem, with the same label and numerical_non_image_features fields, - and all images loaded. + and all images loaded. """ full_channel_files = self.get_all_image_filepaths(root_path=root_path, file_mapping=file_mapping) @@ -160,7 +160,7 @@ def is_valid(self) -> bool: be not None, and none of the non imaging features may be NaN or infinity. :return: True if channel files is a list with not-None entries, and all non imaging features are finite - floating point numbers. + floating point numbers. """ return self.files_valid() and super().is_valid() diff --git a/InnerEye/ML/deep_learning_config.py b/InnerEye/ML/deep_learning_config.py index 10dbeaa76..729e4eea6 100644 --- a/InnerEye/ML/deep_learning_config.py +++ b/InnerEye/ML/deep_learning_config.py @@ -855,10 +855,10 @@ def load_checkpoint_and_modify(self, path_to_checkpoint: Path) -> Dict[str, Any] :param path_to_checkpoint: Path to the checkpoint file. :return: Dictionary with model and optimizer state dicts. The dict should have at least the following keys: - 1. Key ModelAndInfo.MODEL_STATE_DICT_KEY and value set to the model state dict. - 2. Key ModelAndInfo.EPOCH_KEY and value set to the checkpoint epoch. - Other (optional) entries corresponding to keys ModelAndInfo.OPTIMIZER_STATE_DICT_KEY and - ModelAndInfo.MEAN_TEACHER_STATE_DICT_KEY are also supported. + 1. Key ModelAndInfo.MODEL_STATE_DICT_KEY and value set to the model state dict. + 2. Key ModelAndInfo.EPOCH_KEY and value set to the checkpoint epoch. + Other (optional) entries corresponding to keys ModelAndInfo.OPTIMIZER_STATE_DICT_KEY and + ModelAndInfo.MEAN_TEACHER_STATE_DICT_KEY are also supported. """ return load_checkpoint(path_to_checkpoint=path_to_checkpoint, use_gpu=self.use_gpu) diff --git a/InnerEye/ML/metrics.py b/InnerEye/ML/metrics.py index e4c283be8..75919f4b5 100644 --- a/InnerEye/ML/metrics.py +++ b/InnerEye/ML/metrics.py @@ -227,7 +227,7 @@ def compute_dice_across_patches(segmentation: torch.Tensor, :param allow_multiple_classes_for_each_pixel: If set to False, ground-truth tensor has to contain only one foreground label for each pixel. :return A torch tensor of size (Patches, Classes) with the Dice scores. Dice scores are computed for - all classes including the background class at index 0. + all classes including the background class at index 0. """ check_size_matches(segmentation, ground_truth, 4, 5, [0, -3, -2, -1], arg1_name="segmentation", arg2_name="ground_truth") diff --git a/InnerEye/ML/model_config_base.py b/InnerEye/ML/model_config_base.py index 8a72da9c8..13f8b9a6f 100644 --- a/InnerEye/ML/model_config_base.py +++ b/InnerEye/ML/model_config_base.py @@ -119,7 +119,7 @@ def create_data_loaders(self) -> Dict[ModelExecutionMode, Any]: """ Creates the torch DataLoaders that supply the training and the validation set during training only. :return: A dictionary, with keys ModelExecutionMode.TRAIN and ModelExecutionMode.VAL, and their respective - data loaders. + data loaders. """ logging.info("Starting to read and parse the datasets.") if self._datasets_for_training is None: @@ -185,7 +185,7 @@ def get_cross_validation_dataset_splits(self, dataset_split: DatasetSplits) -> D :param dataset_split: The full dataset, split into training, validation and test section. :return: The dataset split with training and validation sections shuffled according to the current - cross validation index. + cross validation index. """ splits = dataset_split.get_k_fold_cross_validation_splits(self.number_of_cross_validation_splits) return splits[self.cross_validation_split_index] diff --git a/InnerEye/ML/model_training.py b/InnerEye/ML/model_training.py index 01108ba09..9c31460d5 100644 --- a/InnerEye/ML/model_training.py +++ b/InnerEye/ML/model_training.py @@ -212,8 +212,8 @@ def model_train(checkpoint_path: Optional[Path], :param container: A container object that holds the training data in PyTorch Lightning format and the model to train. :return: A tuple of [Trainer, StoringLogger]. Trainer is the Lightning Trainer object that was used for fitting - the model. The StoringLogger object is returned when training an InnerEye built-in model, this is None when - fitting other models. + the model. The StoringLogger object is returned when training an InnerEye built-in model, this is None when + fitting other models. """ lightning_model = container.model diff --git a/InnerEye/ML/pipelines/ensemble.py b/InnerEye/ML/pipelines/ensemble.py index a12d1bdcf..849c5488b 100644 --- a/InnerEye/ML/pipelines/ensemble.py +++ b/InnerEye/ML/pipelines/ensemble.py @@ -56,7 +56,7 @@ def aggregate_results(results: Iterable[InferencePipeline.Result], :param aggregation_type: aggregation function to use to combine the results. :return: InferenceResult: contains a Segmentation for each of the classes and their posterior - probabilities. + probabilities. """ if aggregation_type != EnsembleAggregationType.Average: raise NotImplementedError(f"Ensembling is not implemented for aggregation type: {aggregation_type}") @@ -86,7 +86,7 @@ def predict_whole_image(self, image_channels: np.ndarray, :param mask: A binary image used to ignore results outside it in format: Z x Y x X. :param patient_id: The identifier of the patient this image belongs to. :return InferenceResult: that contains Segmentation for each of the classes and their posterior - probabilities. + probabilities. """ logging.info(f"Ensembling inference pipelines ({self._get_pipeline_ids()}) " f"predictions for patient: {patient_id}, " diff --git a/InnerEye/ML/plotting.py b/InnerEye/ML/plotting.py index ebed2368b..ff0eb1eb3 100644 --- a/InnerEye/ML/plotting.py +++ b/InnerEye/ML/plotting.py @@ -70,7 +70,7 @@ def plot_val_dice_per_epoch(metrics: Dict[str, Any]) -> int: :param metrics: :return: The number of series that were plotted in the graph. Can return 0 if the metrics dictionary - does not contain any validation Dice score. + does not contain any validation Dice score. """ plt.clf() series_count = 0 diff --git a/InnerEye/ML/reports/classification_report.py b/InnerEye/ML/reports/classification_report.py index b9156da07..82f443e11 100644 --- a/InnerEye/ML/reports/classification_report.py +++ b/InnerEye/ML/reports/classification_report.py @@ -219,8 +219,8 @@ def plot_scores_and_summary(all_labels_and_model_outputs: Sequence[LabelsAndPred :param ax: Axes object onto which to plot (default: use current axes). :return: A tuple of `(line_handles, summary_handle)` to use in setting a legend for the plot: `line_handles` is a - list corresponding to the curves for each `LabelsAndPredictions`, and `summary_handle` references the median line - and shaded CI area. + list corresponding to the curves for each `LabelsAndPredictions`, and `summary_handle` references the median line + and shaded CI area. """ if ax is None: ax = plt.gca() diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py index 8b61355d6..7382326f1 100644 --- a/InnerEye/ML/run_ml.py +++ b/InnerEye/ML/run_ml.py @@ -530,7 +530,7 @@ def register_model(self, :param checkpoint_paths: Checkpoint paths to register. :param model_proc: whether it's a single or ensemble model. :returns Tuple element 1: AML model object, or None if no model could be registered. - Tuple element 2: The result of running the model_deployment_hook, or None if no hook was supplied. + Tuple element 2: The result of running the model_deployment_hook, or None if no hook was supplied. """ if self.is_offline_run: raise ValueError("Cannot register models when InnerEye is running outside of AzureML.") @@ -731,7 +731,7 @@ def are_sibling_runs_finished(self) -> bool: or cancelled. :return: True if all sibling runs of the current run have finished (they either completed successfully, - or failed). False if any of them is still pending (running or queued). + or failed). False if any of them is still pending (running or queued). """ if (not self.is_offline_run) \ and (azure_util.is_cross_validation_child_run(RUN_CONTEXT)): diff --git a/InnerEye/ML/runner.py b/InnerEye/ML/runner.py index 39eabab91..5f50fa7eb 100755 --- a/InnerEye/ML/runner.py +++ b/InnerEye/ML/runner.py @@ -205,7 +205,7 @@ def run(self) -> Tuple[Optional[DeepLearningConfig], AzureRunInfo]: The main entry point for training and testing models from the commandline. This chooses a model to train via a commandline argument, runs training or testing, and writes all required info to disk and logs. :return: If submitting to AzureML, returns the model configuration that was used for training, - including commandline overrides applied (if any). + including commandline overrides applied (if any). """ # Usually, when we set logging to DEBUG, we want diagnostics about the model # build itself, but not the tons of debug information that AzureML submissions create. @@ -450,7 +450,7 @@ def run(project_root: Path, The main entry point for training and testing models from the commandline. This chooses a model to train via a commandline argument, runs training or testing, and writes all required info to disk and logs. :return: If submitting to AzureML, returns the model configuration that was used for training, - including commandline overrides applied (if any). For details on the arguments, see the constructor of Runner. + including commandline overrides applied (if any). For details on the arguments, see the constructor of Runner. """ runner = Runner(project_root, yaml_config_file, post_cross_validation_hook, model_deployment_hook) return runner.run() diff --git a/InnerEye/ML/utils/device_aware_module.py b/InnerEye/ML/utils/device_aware_module.py index 51718166d..3293e01d9 100644 --- a/InnerEye/ML/utils/device_aware_module.py +++ b/InnerEye/ML/utils/device_aware_module.py @@ -25,7 +25,7 @@ def __init__(self) -> None: def get_devices(self) -> List[torch.device]: """ :return a list of device ids on which this module - is deployed. + is deployed. """ return list({x.device for x in self.parameters()}) diff --git a/InnerEye/ML/utils/features_util.py b/InnerEye/ML/utils/features_util.py index b38a955d8..ddafeabb7 100644 --- a/InnerEye/ML/utils/features_util.py +++ b/InnerEye/ML/utils/features_util.py @@ -32,7 +32,7 @@ def from_data_sources(sources: List[ScalarDataSource]) -> FeatureStatistics: :param sources: list of data sources :return: a Feature Statistics object storing mean and standard deviation for each non-imaging feature of - the dataset. + the dataset. """ if len(sources) == 0: raise ValueError("sources must have a length greater than 0") diff --git a/InnerEye/ML/utils/hdf5_util.py b/InnerEye/ML/utils/hdf5_util.py index 9f4788b57..d8f2d2175 100644 --- a/InnerEye/ML/utils/hdf5_util.py +++ b/InnerEye/ML/utils/hdf5_util.py @@ -75,7 +75,7 @@ def parse_acquisition_date(date: str) -> Optional[datetime]: :param date: string representing a date :return: converted date, None if the string is invalid for - date conversion. + date conversion. """ try: return datetime.strptime(date, DATE_FORMAT) diff --git a/InnerEye/ML/utils/image_util.py b/InnerEye/ML/utils/image_util.py index b92fa744a..4df71cf48 100644 --- a/InnerEye/ML/utils/image_util.py +++ b/InnerEye/ML/utils/image_util.py @@ -212,7 +212,7 @@ def posteriors_to_segmentation(posteriors: NumpyOrTorch) -> NumpyOrTorch: :param posteriors: Confidence maps [0,1] for each patch per class in format: Batches x Class x Z x Y x X or Class x Z x Y x X for non-batched input :returns segmentation: argmaxed posteriors with each voxel belonging to a single class: Batches x Z x Y x X - or Z x Y x X for non-batched input + or Z x Y x X for non-batched input """ if posteriors is None: @@ -507,7 +507,7 @@ def compute_uncertainty_map_from_posteriors(posteriors: np.ndarray) -> np.ndarra :param posteriors: Normalized probability distribution in range [0, 1] for each class, in shape: Class x Z x Y x X :return: Shannon Entropy for each voxel, shape: Z x Y x X expected range is [0,1] where 1 represents - low confidence or uniform posterior distribution across classes. + low confidence or uniform posterior distribution across classes. """ check_if_posterior_array(posteriors) @@ -569,7 +569,7 @@ def segmentation_to_one_hot(segmentation: torch.Tensor, :param result_dtype: The torch data type that the result tensor should have. This would be either float16 or float32 :return: A torch tensor with one-hot encoding of the segmentation of shape - [B, C*HDF5_NUM_SEGMENTATION_CLASSES, Z, Y, X] + [B, C*HDF5_NUM_SEGMENTATION_CLASSES, Z, Y, X] """ def to_cuda(x: torch.Tensor) -> torch.Tensor: diff --git a/InnerEye/ML/utils/io_util.py b/InnerEye/ML/utils/io_util.py index 129a37f70..543ef5537 100644 --- a/InnerEye/ML/utils/io_util.py +++ b/InnerEye/ML/utils/io_util.py @@ -345,7 +345,7 @@ def load_images_and_stack(files: Iterable[Path], :param image_size: If supplied, all loaded images will be resized immediately after loading. :return: A wrapper class that contains the loaded images, and if load_segmentation is True, also the segmentations - that were present in the files. + that were present in the files. """ images = [] segmentations = [] diff --git a/InnerEye/ML/utils/metrics_util.py b/InnerEye/ML/utils/metrics_util.py index 084bba399..830fb88e4 100644 --- a/InnerEye/ML/utils/metrics_util.py +++ b/InnerEye/ML/utils/metrics_util.py @@ -212,7 +212,7 @@ def binary_classification_accuracy(model_output: Union[torch.Tensor, np.ndarray] :param threshold: the cut-off probability threshold for predictions. If model_ouput is > threshold, the predicted class is 1 else 0. :return: 1.0 if all predicted classes match the expected classes given in 'labels'. 0.0 if no predicted classes - match their labels. + match their labels. """ model_output, label = convert_input_and_label(model_output, label) predicted_class = model_output > threshold diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index c49118625..d4328819d 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -176,7 +176,7 @@ def is_test_from_execution_mode(execution_mode: ModelExecutionMode) -> bool: Returns a boolean by checking the execution type. The output is used to determine the properties of the forward pass, e.g. model gradient updates or metric computation. :return True if execution mode is VAL or TEST, False if TRAIN - :raise ValueError if the execution mode is invalid + :raise ValueError if the execution mode is invalid """ if execution_mode == ModelExecutionMode.TRAIN: return False @@ -212,6 +212,6 @@ def is_tensor_nan(tensor: torch.Tensor) -> bool: :param tensor: The tensor to check. :return: True if any of the tensor elements is Not a Number, False if all entries are valid numbers. - If the tensor is empty, the function returns False. + If the tensor is empty, the function returns False. """ return bool(torch.isnan(tensor).any().item()) diff --git a/InnerEye/ML/utils/model_util.py b/InnerEye/ML/utils/model_util.py index efe23f64d..63c88ae8d 100644 --- a/InnerEye/ML/utils/model_util.py +++ b/InnerEye/ML/utils/model_util.py @@ -284,7 +284,7 @@ def get_scalar_model_inputs_and_labels(model: torch.nn.Module, :param sample: A training sample, as returned by a PyTorch data loader (dictionary mapping from field name to value) :return: An instance of ScalarModelInputsAndLabels, containing the list of model input tensors, - label tensor, subject IDs, and the data item reconstructed from the data loader output + label tensor, subject IDs, and the data item reconstructed from the data loader output """ scalar_model: DeviceAwareModule[ScalarItem, torch.Tensor] = model # type: ignore scalar_item = ScalarItem.from_dict(sample) diff --git a/InnerEye/ML/utils/sequence_utils.py b/InnerEye/ML/utils/sequence_utils.py index f70793f43..6870bd0a3 100644 --- a/InnerEye/ML/utils/sequence_utils.py +++ b/InnerEye/ML/utils/sequence_utils.py @@ -54,7 +54,7 @@ def sequences_to_padded_tensor(sequences: List[torch.Tensor], :param sequences: List of Tensors to pad :param padding_value: Padding value to use, default is 0.0 :return: Output tensor with shape B x * where * is the max dimensions from the list of provided tensors. - And B is the number of tensors in the list of sequences provided. + And B is the number of tensors in the list of sequences provided. """ return pad_sequence(sequences, batch_first=True, padding_value=padding_value) diff --git a/InnerEye/ML/visualizers/plot_cross_validation.py b/InnerEye/ML/visualizers/plot_cross_validation.py index ae41416bc..cf8762a00 100644 --- a/InnerEye/ML/visualizers/plot_cross_validation.py +++ b/InnerEye/ML/visualizers/plot_cross_validation.py @@ -487,8 +487,8 @@ def load_dataframes(result_files: List[RunResultFiles], config: PlotCrossValidat :param result_files: The list of files to read. :param config: The overall configuration for the cross validation analysis. :return: A dictionary that maps from model execution mode (TEST, VAL) to concatenated metrics. - For segmentation models, the dataset dataframe is joined to the metrics, and the portal series - ID and institution ID are added. + For segmentation models, the dataset dataframe is joined to the metrics, and the portal series + ID and institution ID are added. """ dataset_split_metrics: Dict[ModelExecutionMode, List[pd.DataFrame]] = \ {mode: [] for mode in config.execution_modes_to_download()} @@ -940,8 +940,8 @@ def add_comparison_data(config: PlotCrossValidationConfig, metrics: pd.DataFrame :param config: configuration of this plotting run :param metrics: on entry, metrics for just the focus (target) run :return: if there are comparison runs, an extended version of metrics including statistics for those runs, - and a list of focus (target) split names to compare them against; otherwise, "metrics" is returned, with - focus_splits as None, so we will get an all-against-all statistical comparison. + and a list of focus (target) split names to compare them against; otherwise, "metrics" is returned, with + focus_splits as None, so we will get an all-against-all statistical comparison. """ if config.comparison_run_recovery_ids is None: return metrics, None diff --git a/InnerEye/Scripts/download_model_and_run_scoring.py b/InnerEye/Scripts/download_model_and_run_scoring.py index 4426b2258..308804f0b 100644 --- a/InnerEye/Scripts/download_model_and_run_scoring.py +++ b/InnerEye/Scripts/download_model_and_run_scoring.py @@ -21,7 +21,7 @@ def spawn_and_monitor_subprocess(process: str, args: List[str], env: Dict[str, s :param args: The args to the process. :param env: The environment variables for the process (default is the environment variables of the parent). :return: Return code after the process has finished, and the list of lines that were written to stdout by the - subprocess. + subprocess. """ p = subprocess.Popen( [process] + args, From a778dacd84f7b42d3f2023c949a2fbe6442c5583 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 10:24:39 +0100 Subject: [PATCH 13/27] =?UTF-8?q?=F0=9F=93=9D=20Fix=20:return:=20statement?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Common/Statistics/mann_whitney_test.py | 2 +- InnerEye/ML/SSL/lightning_containers/ssl_container.py | 2 +- .../ML/SSL/lightning_containers/ssl_image_classifier.py | 6 +++--- InnerEye/ML/SSL/lightning_modules/byol/byol_module.py | 5 ++++- InnerEye/ML/lightning_metrics.py | 2 +- InnerEye/ML/metrics.py | 2 +- InnerEye/ML/metrics_dict.py | 2 +- InnerEye/ML/model_testing.py | 6 +++--- InnerEye/ML/pipelines/ensemble.py | 2 +- InnerEye/ML/pipelines/inference.py | 4 ++-- InnerEye/ML/run_ml.py | 2 +- InnerEye/ML/utils/dataset_util.py | 4 ++-- InnerEye/ML/utils/device_aware_module.py | 6 +++--- InnerEye/ML/utils/features_util.py | 2 +- InnerEye/ML/utils/image_util.py | 6 +++--- InnerEye/ML/utils/layer_util.py | 2 +- InnerEye/ML/utils/metrics_util.py | 4 ++-- InnerEye/ML/utils/ml_util.py | 2 +- InnerEye/ML/utils/surface_distance_utils.py | 4 ++-- InnerEye/ML/utils/temperature_scaling.py | 2 +- InnerEye/ML/visualizers/model_summary.py | 2 +- Tests/ML/util.py | 2 +- docs/source/conf.py | 2 +- 23 files changed, 38 insertions(+), 35 deletions(-) diff --git a/InnerEye/Common/Statistics/mann_whitney_test.py b/InnerEye/Common/Statistics/mann_whitney_test.py index 3544969da..955d80222 100644 --- a/InnerEye/Common/Statistics/mann_whitney_test.py +++ b/InnerEye/Common/Statistics/mann_whitney_test.py @@ -91,7 +91,7 @@ def compose_distribution_comparisons(file_contents: List[List[List[str]]]) -> Li :param file_contents: two or more lists of rows, where each "rows" is returned by read_csv_file on (typically) a statistics.csv file - :return a list of lines to print + :return: a list of lines to print """ value_lists: List[Dict[str, List[float]]] = [parse_values(rows) for rows in file_contents] return compose_distribution_comparisons_on_lists(value_lists) diff --git a/InnerEye/ML/SSL/lightning_containers/ssl_container.py b/InnerEye/ML/SSL/lightning_containers/ssl_container.py index af73c47f1..94189eaf6 100644 --- a/InnerEye/ML/SSL/lightning_containers/ssl_container.py +++ b/InnerEye/ML/SSL/lightning_containers/ssl_container.py @@ -203,7 +203,7 @@ def _create_ssl_data_modules(self, is_ssl_encoder_module: bool) -> InnerEyeVisio Returns torch lightning data module for encoder or linear head :param is_ssl_encoder_module: whether to return the data module for SSL training or for linear head. If true, - :return transforms with two views per sample (batch like (img_v1, img_v2, label)). If False, return only one + :return: transforms with two views per sample (batch like (img_v1, img_v2, label)). If False, return only one view per sample but also return the index of the sample in the dataset (to make sure we don't use twice the same batch in one training epoch (batch like (index, img_v1, label), as classifier dataloader expected to be shorter than SSL training, hence CombinedDataloader might loop over data several times per epoch). diff --git a/InnerEye/ML/SSL/lightning_containers/ssl_image_classifier.py b/InnerEye/ML/SSL/lightning_containers/ssl_image_classifier.py index 66a7d32fe..216340575 100644 --- a/InnerEye/ML/SSL/lightning_containers/ssl_image_classifier.py +++ b/InnerEye/ML/SSL/lightning_containers/ssl_image_classifier.py @@ -19,11 +19,11 @@ class SSLClassifierContainer(SSLContainer): """ This module is used to train a linear classifier on top of a frozen (or not) encoder. - If you are running on AML, you can specify the SSL training run id via the --pretraining_run_recovery_id flag. This - will automatically download the checkpoints for you and take the latest one as the starting weights of your + If you are running on AML, you can specify the SSL training run id via the ``--pretraining_run_recovery_id`` flag. + This will automatically download the checkpoints for you and take the latest one as the starting weights of your classifier. - If you are running locally, you can specify the path to your SSL weights via the --local_ssl_weights_path parameter. + If you are running locally, you can specify the path to your SSL weights via the ``--local_ssl_weights_path`` flag. See docs/self_supervised_models.md for more details. """ diff --git a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py index c10420f80..5998b8e3f 100644 --- a/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py +++ b/InnerEye/ML/SSL/lightning_modules/byol/byol_module.py @@ -120,7 +120,10 @@ def setup(self, *args: Any, **kwargs: Any) -> None: self.train_iters_per_epoch = self.hparams.num_samples // global_batch_size # type: ignore def configure_optimizers(self) -> Any: - # exclude certain parameters + """Testing this out + + :return: _description_ + """ parameters = self.exclude_from_wt_decay(self.online_network.named_parameters(), weight_decay=self.hparams.weight_decay) # type: ignore optimizer = Adam(parameters, diff --git a/InnerEye/ML/lightning_metrics.py b/InnerEye/ML/lightning_metrics.py index 0b1d46e5f..59dd3e3b3 100644 --- a/InnerEye/ML/lightning_metrics.py +++ b/InnerEye/ML/lightning_metrics.py @@ -165,7 +165,7 @@ def _get_metrics_at_optimal_cutoff(self) -> Tuple[torch.Tensor, torch.Tensor, to difference between true positive rate and false positive rate is smallest. Then, computes the false positive rate, false negative rate and accuracy at this threshold (i.e. when the predicted probability is higher than the threshold the predicted label is 1 otherwise 0). - :returns: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) + :return: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) """ preds, targets = self._get_preds_and_targets() if torch.unique(targets).numel() == 1: diff --git a/InnerEye/ML/metrics.py b/InnerEye/ML/metrics.py index 75919f4b5..ec2c7a854 100644 --- a/InnerEye/ML/metrics.py +++ b/InnerEye/ML/metrics.py @@ -226,7 +226,7 @@ def compute_dice_across_patches(segmentation: torch.Tensor, :param ground_truth: One-hot encoded torch tensor containing ground-truth label ids. :param allow_multiple_classes_for_each_pixel: If set to False, ground-truth tensor has to contain only one foreground label for each pixel. - :return A torch tensor of size (Patches, Classes) with the Dice scores. Dice scores are computed for + :return: A torch tensor of size (Patches, Classes) with the Dice scores. Dice scores are computed for all classes including the background class at index 0. """ check_size_matches(segmentation, ground_truth, 4, 5, [0, -3, -2, -1], diff --git a/InnerEye/ML/metrics_dict.py b/InnerEye/ML/metrics_dict.py index ef5aa0c57..0415c2c8a 100644 --- a/InnerEye/ML/metrics_dict.py +++ b/InnerEye/ML/metrics_dict.py @@ -457,7 +457,7 @@ def get_metrics_at_optimal_cutoff(self, hue: str = DEFAULT_HUE_KEY) -> Tuple: predicted probability is higher than the threshold the predicted label is 1 otherwise 0). :param hue: The hue to restrict the values used for computation, otherwise all values will be used. - :returns: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) + :return: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy) """ fpr, tpr, thresholds = roc_curve(self.get_labels(hue=hue), self.get_predictions(hue=hue)) optimal_idx = MetricsDict.get_optimal_idx(fpr=fpr, tpr=tpr) diff --git a/InnerEye/ML/model_testing.py b/InnerEye/ML/model_testing.py index 46b0f6a59..9ee4574d3 100644 --- a/InnerEye/ML/model_testing.py +++ b/InnerEye/ML/model_testing.py @@ -131,7 +131,7 @@ def segmentation_model_test_epoch(config: SegmentationModelBase, :param epoch_and_split: A string that should uniquely identify the epoch and the data split (train/val/test). :raises TypeError: If the arguments are of the wrong type. :raises ValueError: When there are issues loading the model. - :return A list with the mean dice score (across all structures apart from background) for each image. + :return: A list with the mean dice score (across all structures apart from background) for each image. """ ml_util.set_random_seed(config.get_effective_random_seed(), "Model testing") results_folder.mkdir(exist_ok=True) @@ -212,7 +212,7 @@ def evaluate_model_predictions(process_id: int, :param config: Segmentation model config object :param dataset: Dataset object, it is used to load intensity image, labels, and patient metadata. :param results_folder: Path to results folder - :returns [PatientMetadata, list[list]]: Patient metadata and list of computed metrics for each image. + :return: [PatientMetadata, list[list]]: Patient metadata and list of computed metrics for each image. """ sample = dataset.get_samples_at_index(index=process_id)[0] logging.info(f"Evaluating predictions for patient {sample.patient_id}") @@ -244,7 +244,7 @@ def populate_metrics_writer( from evaluate_model_predictions :param config: The SegmentationModelBase config from which we read the ground_truth_ids - :returns: A new MetricsPerPatientWriter and a list of foreground DICE score averages + :return: A new MetricsPerPatientWriter and a list of foreground DICE score averages """ average_dice: List[FloatOrInt] = [] metrics_writer = MetricsPerPatientWriter() diff --git a/InnerEye/ML/pipelines/ensemble.py b/InnerEye/ML/pipelines/ensemble.py index 849c5488b..e652ef9a2 100644 --- a/InnerEye/ML/pipelines/ensemble.py +++ b/InnerEye/ML/pipelines/ensemble.py @@ -85,7 +85,7 @@ def predict_whole_image(self, image_channels: np.ndarray, :param voxel_spacing_mm: Voxel spacing to use for each dimension in (Z x Y x X) order :param mask: A binary image used to ignore results outside it in format: Z x Y x X. :param patient_id: The identifier of the patient this image belongs to. - :return InferenceResult: that contains Segmentation for each of the classes and their posterior + :return: InferenceResult: that contains Segmentation for each of the classes and their posterior probabilities. """ logging.info(f"Ensembling inference pipelines ({self._get_pipeline_ids()}) " diff --git a/InnerEye/ML/pipelines/inference.py b/InnerEye/ML/pipelines/inference.py index fc4dcbfdc..048adb81d 100644 --- a/InnerEye/ML/pipelines/inference.py +++ b/InnerEye/ML/pipelines/inference.py @@ -206,7 +206,7 @@ def create_from_checkpoint(path_to_checkpoint: Path, :param model_config: Model related configurations. :param pipeline_id: Numeric identifier for the pipeline (useful for logging when ensembling) - :returns InferencePipeline: an instantiated inference pipeline instance, or None if there was no checkpoint + :return: InferencePipeline: an instantiated inference pipeline instance, or None if there was no checkpoint file for this epoch. """ if not path_to_checkpoint.is_file(): @@ -247,7 +247,7 @@ def predict_whole_image(self, image_channels: np.ndarray, :param voxel_spacing_mm: Voxel spacing to use for each dimension in (Z x Y x X) order :param mask: A binary image used to ignore results outside it in format: Z x Y x X. :param patient_id: The identifier of the patient this image belongs to (defaults to 0 if None provided). - :return InferenceResult: that contains Segmentation for each of the classes and their posterior probabilities. + :return: InferenceResult: that contains Segmentation for each of the classes and their posterior probabilities. """ torch.cuda.empty_cache() if image_channels is None: diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py index 7382326f1..cf14611ec 100644 --- a/InnerEye/ML/run_ml.py +++ b/InnerEye/ML/run_ml.py @@ -529,7 +529,7 @@ def register_model(self, :param checkpoint_paths: Checkpoint paths to register. :param model_proc: whether it's a single or ensemble model. - :returns Tuple element 1: AML model object, or None if no model could be registered. + :return: Tuple element 1: AML model object, or None if no model could be registered. Tuple element 2: The result of running the model_deployment_hook, or None if no hook was supplied. """ if self.is_offline_run: diff --git a/InnerEye/ML/utils/dataset_util.py b/InnerEye/ML/utils/dataset_util.py index 7871a7dba..7a3ec857d 100644 --- a/InnerEye/ML/utils/dataset_util.py +++ b/InnerEye/ML/utils/dataset_util.py @@ -75,7 +75,7 @@ def encode(self, x: Dict[str, List[str]]) -> torch.Tensor: def get_supported_dataset_column_names(self) -> List[str]: """ - :returns list of categorical columns that are supported by this encoder + :return: list of categorical columns that are supported by this encoder """ return list(self._columns_and_possible_categories.keys()) @@ -87,7 +87,7 @@ def get_feature_length(self, feature_name: str) -> int: :param feature_name: the name of the column for which to compute the feature length. - :returns the feature length i.e. number of possible values for this feature. + :return: the feature length i.e. number of possible values for this feature. """ return self._feature_length[feature_name] diff --git a/InnerEye/ML/utils/device_aware_module.py b/InnerEye/ML/utils/device_aware_module.py index 3293e01d9..1a907f174 100644 --- a/InnerEye/ML/utils/device_aware_module.py +++ b/InnerEye/ML/utils/device_aware_module.py @@ -24,21 +24,21 @@ def __init__(self) -> None: def get_devices(self) -> List[torch.device]: """ - :return a list of device ids on which this module + :return: a list of device ids on which this module is deployed. """ return list({x.device for x in self.parameters()}) def get_number_trainable_parameters(self) -> int: """ - :return the number of trainable parameters in the module. + :return: the number of trainable parameters in the module. """ return sum(p.numel() for p in self.parameters() if p.requires_grad) def is_model_on_gpu(self) -> bool: """ Checks if the model is cuda activated or not - :return True if the model is running on the GPU. + :return: True if the model is running on the GPU. """ try: cuda_activated = next(self.parameters()).is_cuda diff --git a/InnerEye/ML/utils/features_util.py b/InnerEye/ML/utils/features_util.py index ddafeabb7..2417c6a42 100644 --- a/InnerEye/ML/utils/features_util.py +++ b/InnerEye/ML/utils/features_util.py @@ -88,7 +88,7 @@ def standardize(self, sources: List[ScalarDataSource]) -> List[ScalarDataSource] All features that have zero standard deviation (constant features) are left untouched. :param sources: list of datasources. - :return list of data sources where all non-imaging features are standardized. + :return: list of data sources where all non-imaging features are standardized. """ def apply_source(source: ScalarDataSource) -> ScalarDataSource: diff --git a/InnerEye/ML/utils/image_util.py b/InnerEye/ML/utils/image_util.py index 4df71cf48..883087af0 100644 --- a/InnerEye/ML/utils/image_util.py +++ b/InnerEye/ML/utils/image_util.py @@ -76,7 +76,7 @@ def apply_mask_to_posteriors(posteriors: NumpyOrTorch, mask: NumpyOrTorch) -> Nu :param posteriors: image tensors in shape: Batches (optional) x Classes x Z x Y x X :param mask: image tensor in shape: Batches (optional) x Z x Y x X - :return posteriors with mask applied + :return: posteriors with mask applied """ ml_util.check_size_matches(posteriors, mask, matching_dimensions=[-1, -2, -3]) @@ -211,7 +211,7 @@ def posteriors_to_segmentation(posteriors: NumpyOrTorch) -> NumpyOrTorch: :param posteriors: Confidence maps [0,1] for each patch per class in format: Batches x Class x Z x Y x X or Class x Z x Y x X for non-batched input - :returns segmentation: argmaxed posteriors with each voxel belonging to a single class: Batches x Z x Y x X + :return: segmentation: argmaxed posteriors with each voxel belonging to a single class: Batches x Z x Y x X or Z x Y x X for non-batched input """ @@ -374,7 +374,7 @@ def get_center_crop(image: NumpyOrTorch, crop_shape: TupleInt3) -> NumpyOrTorch: :param image: The original image to extract crop from :param crop_shape: The shape of the center crop to extract - :return the center region as specified by the crop_shape argument. + :return: the center region as specified by the crop_shape argument. """ if image is None or crop_shape is None: raise Exception("image and crop_shape must not be None") diff --git a/InnerEye/ML/utils/layer_util.py b/InnerEye/ML/utils/layer_util.py index 77cea8a63..24cb10c2b 100644 --- a/InnerEye/ML/utils/layer_util.py +++ b/InnerEye/ML/utils/layer_util.py @@ -48,7 +48,7 @@ def get_padding_from_kernel_size(padding: PaddingMode, :param num_dimensions: The number of dimensions that the returned padding tuple should have, if both kernel_size and dilation are scalars. - :return padding value required for convolution layers based on input kernel size and dilation. + :return: padding value required for convolution layers based on input kernel size and dilation. """ if isinstance(kernel_size, Sized): num_dimensions = len(kernel_size) diff --git a/InnerEye/ML/utils/metrics_util.py b/InnerEye/ML/utils/metrics_util.py index 830fb88e4..67694c1a5 100644 --- a/InnerEye/ML/utils/metrics_util.py +++ b/InnerEye/ML/utils/metrics_util.py @@ -256,7 +256,7 @@ def convert_input_and_label(model_output: Union[torch.Tensor, np.ndarray], label: Union[torch.Tensor, np.ndarray]) -> Tuple[torch.Tensor, torch.Tensor]: """ Ensures that both model_output and label are tensors of dtype float32. - :return a Tuple with model_output, label as float tensors. + :return: a Tuple with model_output, label as float tensors. """ if not torch.is_tensor(model_output): model_output = torch.tensor(model_output) @@ -273,6 +273,6 @@ def is_missing_ground_truth(ground_truth: np.ndarray) -> bool: :param ground_truth: ground truth binary array with dimensions: [Z x Y x X]. :param label_id: Integer index of the label to check. - :returns: True if the label is missing (signified by NaN), False otherwise. + :return: True if the label is missing (signified by NaN), False otherwise. """ return np.isnan(ground_truth[0, 0, 0]) diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index d4328819d..0dc35a3d9 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -175,7 +175,7 @@ def is_test_from_execution_mode(execution_mode: ModelExecutionMode) -> bool: """ Returns a boolean by checking the execution type. The output is used to determine the properties of the forward pass, e.g. model gradient updates or metric computation. - :return True if execution mode is VAL or TEST, False if TRAIN + :return: True if execution mode is VAL or TEST, False if TRAIN :raise ValueError if the execution mode is invalid """ if execution_mode == ModelExecutionMode.TRAIN: diff --git a/InnerEye/ML/utils/surface_distance_utils.py b/InnerEye/ML/utils/surface_distance_utils.py index a634465a4..a3d185eb3 100644 --- a/InnerEye/ML/utils/surface_distance_utils.py +++ b/InnerEye/ML/utils/surface_distance_utils.py @@ -97,7 +97,7 @@ def get_run_output_dir(azure_config: AzureConfig, model_config: SegmentationMode :param azure_config: :param model_config: - :return output_dir: directory that all artifact paths use as a prefix + :return: output_dir: directory that all artifact paths use as a prefix """ if not azure_config.run_recovery_id: raise ValueError("azure_config.run_recovery_id is not provided") @@ -150,7 +150,7 @@ def get_subject_prefix(model_config: SegmentationModelBase, train_mode: ModelExe :param model_config: Config :param train_mode: Model execution mode -i.e. train, test or val :param subject_id: ID of the subject - :return prefix: the filepath prefix within the container from which to download all artifacts + :return: prefix: the filepath prefix within the container from which to download all artifacts """ prefix = model_config.outputs_folder / BEST_EPOCH_FOLDER_NAME / train_mode.value / "{0:03d}".format(subject_id) return prefix diff --git a/InnerEye/ML/utils/temperature_scaling.py b/InnerEye/ML/utils/temperature_scaling.py index 0b6f49424..a42db9ba9 100644 --- a/InnerEye/ML/utils/temperature_scaling.py +++ b/InnerEye/ML/utils/temperature_scaling.py @@ -63,7 +63,7 @@ def set_temperature(self, :param labels: Labels to use to learn the temperature parameter :param criterion_fn: A criterion function s.t: (logits, labels) => (loss, ECE) :param use_gpu: If True then GPU will be used otherwise CPU will be used. - :return Optimal temperature value + :return: Optimal temperature value """ if use_gpu: logits = logits.cuda() diff --git a/InnerEye/ML/visualizers/model_summary.py b/InnerEye/ML/visualizers/model_summary.py index a91f91290..85c7bd95f 100644 --- a/InnerEye/ML/visualizers/model_summary.py +++ b/InnerEye/ML/visualizers/model_summary.py @@ -219,7 +219,7 @@ def forward_preserve_state(module: DeviceAwareModule, inputs: List[torch.Tensor] :param module: Callable torch module :param inputs: List of input torch tensors - :return output: Output torch tensors + :return: output: Output torch tensors """ if not isinstance(inputs, list): raise RuntimeError("Inputs object has to be a list of torch tensors") diff --git a/Tests/ML/util.py b/Tests/ML/util.py index 456d28e81..003b49a11 100644 --- a/Tests/ML/util.py +++ b/Tests/ML/util.py @@ -222,7 +222,7 @@ def csv_column_contains_value( :param column_name: The name of the column in which we look for the value :param value: The value to look for :param contains_only_value: Check that this is the only value in the column (default True) - :returns: Boolean, whether the CSV column contains the value (and perhaps only the value) + :return: Boolean, whether the CSV column contains the value (and perhaps only the value) """ result = True if not csv_file_path.exists: diff --git a/docs/source/conf.py b/docs/source/conf.py index efdbb4ba5..81f768a4b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -61,7 +61,7 @@ # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation fora +# The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'furo' From 33b557c31ae031cb2ebb88827d1df995b76d2fa5 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 10:24:56 +0100 Subject: [PATCH 14/27] =?UTF-8?q?=F0=9F=93=9D=20Finish=20ML/SSL=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/ML/SSL/index.rst | 8 ++++++ .../api/ML/SSL/lightning_containers/index.rst | 10 +++++++ .../api/ML/SSL/lightning_modules/index.rst | 26 +++++++++++++++++++ docs/source/rst/api/ML/SSL/utils/index.rst | 2 ++ 4 files changed, 46 insertions(+) diff --git a/docs/source/rst/api/ML/SSL/index.rst b/docs/source/rst/api/ML/SSL/index.rst index 3cd8812a7..cd317dad4 100644 --- a/docs/source/rst/api/ML/SSL/index.rst +++ b/docs/source/rst/api/ML/SSL/index.rst @@ -1,6 +1,14 @@ Self-Supervised Learning (SSL) ============================== +Encoder +------- + +.. automodule:: InnerEye.ML.SSL.encoders + +All Other SSL Components +------------------------ + .. toctree:: lightning_containers/index lightning_modules/index diff --git a/docs/source/rst/api/ML/SSL/lightning_containers/index.rst b/docs/source/rst/api/ML/SSL/lightning_containers/index.rst index 4d6e58bee..f08061187 100644 --- a/docs/source/rst/api/ML/SSL/lightning_containers/index.rst +++ b/docs/source/rst/api/ML/SSL/lightning_containers/index.rst @@ -1,2 +1,12 @@ SSL Lightning Containers ======================== + +Container +--------- + +.. automodule:: InnerEye.ML.SSL.lightning_containers.ssl_container + +Image Classifier +---------------- + +.. automodule:: InnerEye.ML.SSL.lightning_containers.ssl_image_classifier diff --git a/docs/source/rst/api/ML/SSL/lightning_modules/index.rst b/docs/source/rst/api/ML/SSL/lightning_modules/index.rst index 3c106f982..eb7ec8cb8 100644 --- a/docs/source/rst/api/ML/SSL/lightning_modules/index.rst +++ b/docs/source/rst/api/ML/SSL/lightning_modules/index.rst @@ -1,2 +1,28 @@ SSL Lightning Modules ===================== + +Bring Your Own Latent (BYOL) +------------------------------ + +.. automodule:: InnerEye.ML.SSL.lightning_modules.byol.byol_models + +.. automodule:: InnerEye.ML.SSL.lightning_modules.byol.byol_module + +.. automodule:: InnerEye.ML.SSL.lightning_modules.byol.byol_moving_average + +SimCLR +------- + +.. automodule:: InnerEye.ML.SSL.lightning_modules.simclr_module + + +SSL Classifier +----------------- + +.. automodule:: InnerEye.ML.SSL.lightning_modules.ssl_classifier_module + + +Online Evaluator +----------------- + +.. automodule:: InnerEye.ML.SSL.lightning_modules.ssl_online_evaluator diff --git a/docs/source/rst/api/ML/SSL/utils/index.rst b/docs/source/rst/api/ML/SSL/utils/index.rst index fe9f8c4a1..62266ea8a 100644 --- a/docs/source/rst/api/ML/SSL/utils/index.rst +++ b/docs/source/rst/api/ML/SSL/utils/index.rst @@ -1,2 +1,4 @@ SSL Utils ========== + +.. automodule:: InnerEye.ML.SSL.utils From 7d4f466290b27f4aed22cf11cf05937cf9c9ce9f Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 11:29:57 +0100 Subject: [PATCH 15/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20ML/utils=20API=20doc?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/ML/utils/index.rst | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/source/rst/api/ML/utils/index.rst b/docs/source/rst/api/ML/utils/index.rst index f2c4c3a20..239fffaa7 100644 --- a/docs/source/rst/api/ML/utils/index.rst +++ b/docs/source/rst/api/ML/utils/index.rst @@ -1,2 +1,58 @@ Utils ===== + +Data Utils +----------- +.. automodule:: InnerEye.ML.utils.csv_util + +.. automodule:: InnerEye.ML.utils.dataset_util + +.. automodule:: InnerEye.ML.utils.hdf5_util + +.. automodule:: InnerEye.ML.utils.image_util + +.. automodule:: InnerEye.ML.utils.io_util + +.. automodule:: InnerEye.ML.utils.split_dataset + +.. automodule:: InnerEye.ML.utils.features_util + +.. automodule:: InnerEye.ML.utils.transforms + +Model Utils +----------- + +.. automodule:: InnerEye.ML.utils.checkpoint_handling + +.. automodule:: InnerEye.ML.utils.config_loader + +.. automodule:: InnerEye.ML.utils.model_util + +.. automodule:: InnerEye.ML.utils.layer_util + +.. automodule:: InnerEye.ML.utils.model_metadata_util + +Training Utils +-------------- + +.. automodule:: InnerEye.ML.utils.lr_scheduler + +.. automodule:: InnerEye.ML.utils.metrics_util + +.. automodule:: InnerEye.ML.utils.sequence_utils + +.. automodule:: InnerEye.ML.utils.supervised_criterion + +.. automodule:: InnerEye.ML.utils.run_recovery + +.. automodule:: InnerEye.ML.utils.surface_distance_utils + +.. automodule:: InnerEye.ML.utils.temperature_scaling + +.. automodule:: InnerEye.ML.utils.device_aware_module + +Other / Misc Utils +------------------ +.. automodule:: InnerEye.ML.utils.ml_util + +.. automodule:: InnerEye.ML.utils.plotting_util From 19ab5b21def58efa377e200264043ef9e36a7eb8 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 11:39:11 +0100 Subject: [PATCH 16/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20visualizer=20docs,?= =?UTF-8?q?=20fix=20`:raise`=20indents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/model_testing.py | 2 +- InnerEye/ML/utils/io_util.py | 2 +- InnerEye/ML/utils/ml_util.py | 10 +++++----- docs/source/rst/api/ML/visualizers/index.rst | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/InnerEye/ML/model_testing.py b/InnerEye/ML/model_testing.py index 9ee4574d3..2e4766092 100644 --- a/InnerEye/ML/model_testing.py +++ b/InnerEye/ML/model_testing.py @@ -129,7 +129,7 @@ def segmentation_model_test_epoch(config: SegmentationModelBase, :param execution_mode: Is the model evaluated on train, test, or validation set? :param results_folder: The folder where to store the results. :param epoch_and_split: A string that should uniquely identify the epoch and the data split (train/val/test). - :raises TypeError: If the arguments are of the wrong type. + :raises TypeError: If the arguments are of the wrong type. :raises ValueError: When there are issues loading the model. :return: A list with the mean dice score (across all structures apart from background) for each image. """ diff --git a/InnerEye/ML/utils/io_util.py b/InnerEye/ML/utils/io_util.py index 543ef5537..b2c814079 100644 --- a/InnerEye/ML/utils/io_util.py +++ b/InnerEye/ML/utils/io_util.py @@ -209,7 +209,7 @@ def load_nifti_image(path: PathOrString, image_type: Optional[Type] = float) -> :return: A numpy array of the image and header data if applicable. :param image_type: The type to load the image in, set to None to not cast, default is float - :raises ValueError: If the path is invalid or the image is not 3D. + :raises ValueError: If the path is invalid or the image is not 3D. """ def _is_valid_image_path(_path: Path) -> bool: diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index 0dc35a3d9..17e2aa83c 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -82,8 +82,8 @@ def validate_dataset_paths( Validates that the required dataset csv file exists in the given path. :param dataset_path: The base path - :param custom_dataset_csv : The name of the dataset csv file - :raise ValueError if the dataset does not exist. + :param custom_dataset_csv: The name of the dataset csv file + :raise ValueError if the dataset does not exist. """ if not dataset_path.is_dir(): raise ValueError("The dataset_path argument should be the path to the base directory of the data " @@ -117,7 +117,7 @@ def check_size_matches(arg1: Union[np.ndarray, torch.Tensor], :param arg1_name: If provided, all error messages will use that string to instead of "arg1" :param arg2_name: If provided, all error messages will use that string to instead of "arg2" - :raise ValueError if shapes don't match + :raise ValueError if shapes don't match """ if arg1 is None or arg2 is None: raise Exception("arg1 and arg2 cannot be None.") @@ -132,7 +132,7 @@ def check_dim(expected: int, actual_shape: Any, name: str) -> None: :param expected: expected shape :param actual_shape: :param name: variable name - :raise ValueError if not the same shape + :raise ValueError if not the same shape """ if len(actual_shape) != expected: raise ValueError("'{}' was expected to have ndim == {}, but is {}. Shape is {}" @@ -176,7 +176,7 @@ def is_test_from_execution_mode(execution_mode: ModelExecutionMode) -> bool: Returns a boolean by checking the execution type. The output is used to determine the properties of the forward pass, e.g. model gradient updates or metric computation. :return: True if execution mode is VAL or TEST, False if TRAIN - :raise ValueError if the execution mode is invalid + :raise ValueError if the execution mode is invalid """ if execution_mode == ModelExecutionMode.TRAIN: return False diff --git a/docs/source/rst/api/ML/visualizers/index.rst b/docs/source/rst/api/ML/visualizers/index.rst index 5f77689dc..65356a06a 100644 --- a/docs/source/rst/api/ML/visualizers/index.rst +++ b/docs/source/rst/api/ML/visualizers/index.rst @@ -1,2 +1,20 @@ Visualizers =========== + +Below you will find all classes and functions related to the visualization of models and outputs. + +.. automodule:: InnerEye.ML.visualizers.activation_maps + +.. automodule:: InnerEye.ML.visualizers.metrics_scatterplot + +.. automodule:: InnerEye.ML.visualizers.model_hooks + +.. automodule:: InnerEye.ML.visualizers.model_summary + +.. automodule:: InnerEye.ML.visualizers.patch_sampling + +.. automodule:: InnerEye.ML.visualizers.plot_cross_validation + +.. automodule:: InnerEye.ML.visualizers.regression_visualization + +.. automodule:: InnerEye.ML.visualizers.reliability_curve From 67169af21c5161bd4416555965eb5f9aee3ae6f7 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 11:45:13 +0100 Subject: [PATCH 17/27] =?UTF-8?q?=F0=9F=93=9D=20Fix=20more=20issues=20with?= =?UTF-8?q?=20the=20`:raises:`=20formatting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/utils/io_util.py | 2 +- InnerEye/ML/utils/ml_util.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/InnerEye/ML/utils/io_util.py b/InnerEye/ML/utils/io_util.py index b2c814079..e17ea1d4c 100644 --- a/InnerEye/ML/utils/io_util.py +++ b/InnerEye/ML/utils/io_util.py @@ -636,7 +636,7 @@ def store_binary_mask_as_nifti(image: np.ndarray, header: ImageHeader, file_name :param header: The image header :param file_name: The name of the file for this image. :return: the path to the saved image - :raises: when image is not binary + :raises Exception: when image is not binary """ if not is_binary_array(image): raise Exception("Array values must be binary.") diff --git a/InnerEye/ML/utils/ml_util.py b/InnerEye/ML/utils/ml_util.py index 17e2aa83c..3310da1d7 100644 --- a/InnerEye/ML/utils/ml_util.py +++ b/InnerEye/ML/utils/ml_util.py @@ -83,7 +83,7 @@ def validate_dataset_paths( :param dataset_path: The base path :param custom_dataset_csv: The name of the dataset csv file - :raise ValueError if the dataset does not exist. + :raises: ValueError if the dataset does not exist. """ if not dataset_path.is_dir(): raise ValueError("The dataset_path argument should be the path to the base directory of the data " @@ -117,7 +117,7 @@ def check_size_matches(arg1: Union[np.ndarray, torch.Tensor], :param arg1_name: If provided, all error messages will use that string to instead of "arg1" :param arg2_name: If provided, all error messages will use that string to instead of "arg2" - :raise ValueError if shapes don't match + :raises: ValueError if shapes don't match """ if arg1 is None or arg2 is None: raise Exception("arg1 and arg2 cannot be None.") @@ -132,7 +132,7 @@ def check_dim(expected: int, actual_shape: Any, name: str) -> None: :param expected: expected shape :param actual_shape: :param name: variable name - :raise ValueError if not the same shape + :raises: ValueError if not the same shape """ if len(actual_shape) != expected: raise ValueError("'{}' was expected to have ndim == {}, but is {}. Shape is {}" @@ -175,8 +175,9 @@ def is_test_from_execution_mode(execution_mode: ModelExecutionMode) -> bool: """ Returns a boolean by checking the execution type. The output is used to determine the properties of the forward pass, e.g. model gradient updates or metric computation. + :return: True if execution mode is VAL or TEST, False if TRAIN - :raise ValueError if the execution mode is invalid + :raises ValueError: if the execution mode is invalid """ if execution_mode == ModelExecutionMode.TRAIN: return False From 7619004c9f606b9e0268f17dfe4762184cfccfc3 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 12:03:33 +0100 Subject: [PATCH 18/27] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Restructuring=20fold?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/conf.py | 2 +- docs/source/rst/api/ML/ML.rst | 14 ++++++++++++++ docs/source/rst/api/ML/{SSL/index.rst => SSL.rst} | 8 ++++---- .../index.rst => datamodules_and_datasets.rst} | 0 .../index.rst => lightning_containers.rst} | 0 .../index.rst => lightning_modules.rst} | 0 .../rst/api/ML/SSL/{utils/index.rst => utils.rst} | 0 .../{augmentations/index.rst => augmentations.rst} | 0 .../rst/api/ML/{configs/index.rst => configs.rst} | 10 +++++----- .../index.rst => classification.rst} | 0 .../api/ML/configs/{other/index.rst => other.rst} | 0 .../{regression/index.rst => regression.rst} | 0 .../{segmentation/index.rst => segmentation.rst} | 0 .../rst/api/ML/configs/{ssl/index.rst => ssl.rst} | 0 .../rst/api/ML/{dataset/index.rst => dataset.rst} | 0 docs/source/rst/api/ML/index.rst | 14 -------------- .../rst/api/ML/{models/index.rst => models.rst} | 8 ++++---- .../{architectures/index.rst => architectures.rst} | 0 .../api/ML/models/{blocks/index.rst => blocks.rst} | 0 .../api/ML/models/{layers/index.rst => layers.rst} | 0 .../api/ML/models/{losses/index.rst => losses.rst} | 0 .../api/ML/{pipelines/index.rst => pipelines.rst} | 0 .../rst/api/ML/{utils/index.rst => utils.rst} | 0 .../ML/{visualizers/index.rst => visualizers.rst} | 0 docs/source/rst/api/index.rst | 2 +- 25 files changed, 29 insertions(+), 29 deletions(-) create mode 100644 docs/source/rst/api/ML/ML.rst rename docs/source/rst/api/ML/{SSL/index.rst => SSL.rst} (62%) rename docs/source/rst/api/ML/SSL/{datamodules_and_datasets/index.rst => datamodules_and_datasets.rst} (100%) rename docs/source/rst/api/ML/SSL/{lightning_containers/index.rst => lightning_containers.rst} (100%) rename docs/source/rst/api/ML/SSL/{lightning_modules/index.rst => lightning_modules.rst} (100%) rename docs/source/rst/api/ML/SSL/{utils/index.rst => utils.rst} (100%) rename docs/source/rst/api/ML/{augmentations/index.rst => augmentations.rst} (100%) rename docs/source/rst/api/ML/{configs/index.rst => configs.rst} (61%) rename docs/source/rst/api/ML/configs/{classification/index.rst => classification.rst} (100%) rename docs/source/rst/api/ML/configs/{other/index.rst => other.rst} (100%) rename docs/source/rst/api/ML/configs/{regression/index.rst => regression.rst} (100%) rename docs/source/rst/api/ML/configs/{segmentation/index.rst => segmentation.rst} (100%) rename docs/source/rst/api/ML/configs/{ssl/index.rst => ssl.rst} (100%) rename docs/source/rst/api/ML/{dataset/index.rst => dataset.rst} (100%) delete mode 100644 docs/source/rst/api/ML/index.rst rename docs/source/rst/api/ML/{models/index.rst => models.rst} (64%) rename docs/source/rst/api/ML/models/{architectures/index.rst => architectures.rst} (100%) rename docs/source/rst/api/ML/models/{blocks/index.rst => blocks.rst} (100%) rename docs/source/rst/api/ML/models/{layers/index.rst => layers.rst} (100%) rename docs/source/rst/api/ML/models/{losses/index.rst => losses.rst} (100%) rename docs/source/rst/api/ML/{pipelines/index.rst => pipelines.rst} (100%) rename docs/source/rst/api/ML/{utils/index.rst => utils.rst} (100%) rename docs/source/rst/api/ML/{visualizers/index.rst => visualizers.rst} (100%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 81f768a4b..01cda6f0b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -34,7 +34,7 @@ author = 'Microsoft' # The full version, including alpha/beta/rc tags -release = '1.0.0' +release = 'v0.4' # -- General configuration --------------------------------------------------- diff --git a/docs/source/rst/api/ML/ML.rst b/docs/source/rst/api/ML/ML.rst new file mode 100644 index 000000000..7bb25fac7 --- /dev/null +++ b/docs/source/rst/api/ML/ML.rst @@ -0,0 +1,14 @@ +Machine learning +================ + +.. toctree:: + + configs + models + dataset + augmentations + pipelines + SSL + utils + runner + photometric_normalization diff --git a/docs/source/rst/api/ML/SSL/index.rst b/docs/source/rst/api/ML/SSL.rst similarity index 62% rename from docs/source/rst/api/ML/SSL/index.rst rename to docs/source/rst/api/ML/SSL.rst index cd317dad4..807e64181 100644 --- a/docs/source/rst/api/ML/SSL/index.rst +++ b/docs/source/rst/api/ML/SSL.rst @@ -10,7 +10,7 @@ All Other SSL Components ------------------------ .. toctree:: - lightning_containers/index - lightning_modules/index - datamodules_and_datasets/index - utils/index + SSL/lightning_containers + SSL/lightning_modules + SSL/datamodules_and_datasets + SSL/utils diff --git a/docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst b/docs/source/rst/api/ML/SSL/datamodules_and_datasets.rst similarity index 100% rename from docs/source/rst/api/ML/SSL/datamodules_and_datasets/index.rst rename to docs/source/rst/api/ML/SSL/datamodules_and_datasets.rst diff --git a/docs/source/rst/api/ML/SSL/lightning_containers/index.rst b/docs/source/rst/api/ML/SSL/lightning_containers.rst similarity index 100% rename from docs/source/rst/api/ML/SSL/lightning_containers/index.rst rename to docs/source/rst/api/ML/SSL/lightning_containers.rst diff --git a/docs/source/rst/api/ML/SSL/lightning_modules/index.rst b/docs/source/rst/api/ML/SSL/lightning_modules.rst similarity index 100% rename from docs/source/rst/api/ML/SSL/lightning_modules/index.rst rename to docs/source/rst/api/ML/SSL/lightning_modules.rst diff --git a/docs/source/rst/api/ML/SSL/utils/index.rst b/docs/source/rst/api/ML/SSL/utils.rst similarity index 100% rename from docs/source/rst/api/ML/SSL/utils/index.rst rename to docs/source/rst/api/ML/SSL/utils.rst diff --git a/docs/source/rst/api/ML/augmentations/index.rst b/docs/source/rst/api/ML/augmentations.rst similarity index 100% rename from docs/source/rst/api/ML/augmentations/index.rst rename to docs/source/rst/api/ML/augmentations.rst diff --git a/docs/source/rst/api/ML/configs/index.rst b/docs/source/rst/api/ML/configs.rst similarity index 61% rename from docs/source/rst/api/ML/configs/index.rst rename to docs/source/rst/api/ML/configs.rst index e592f92a1..e8701309d 100644 --- a/docs/source/rst/api/ML/configs/index.rst +++ b/docs/source/rst/api/ML/configs.rst @@ -5,8 +5,8 @@ The following is a list of the pre-defined configurations that can be used or in .. toctree:: - classification/index - regression/index - segmentation/index - ssl/index - other/index + configs/classification + configs/regression + configs/segmentation + configs/ssl + configs/other diff --git a/docs/source/rst/api/ML/configs/classification/index.rst b/docs/source/rst/api/ML/configs/classification.rst similarity index 100% rename from docs/source/rst/api/ML/configs/classification/index.rst rename to docs/source/rst/api/ML/configs/classification.rst diff --git a/docs/source/rst/api/ML/configs/other/index.rst b/docs/source/rst/api/ML/configs/other.rst similarity index 100% rename from docs/source/rst/api/ML/configs/other/index.rst rename to docs/source/rst/api/ML/configs/other.rst diff --git a/docs/source/rst/api/ML/configs/regression/index.rst b/docs/source/rst/api/ML/configs/regression.rst similarity index 100% rename from docs/source/rst/api/ML/configs/regression/index.rst rename to docs/source/rst/api/ML/configs/regression.rst diff --git a/docs/source/rst/api/ML/configs/segmentation/index.rst b/docs/source/rst/api/ML/configs/segmentation.rst similarity index 100% rename from docs/source/rst/api/ML/configs/segmentation/index.rst rename to docs/source/rst/api/ML/configs/segmentation.rst diff --git a/docs/source/rst/api/ML/configs/ssl/index.rst b/docs/source/rst/api/ML/configs/ssl.rst similarity index 100% rename from docs/source/rst/api/ML/configs/ssl/index.rst rename to docs/source/rst/api/ML/configs/ssl.rst diff --git a/docs/source/rst/api/ML/dataset/index.rst b/docs/source/rst/api/ML/dataset.rst similarity index 100% rename from docs/source/rst/api/ML/dataset/index.rst rename to docs/source/rst/api/ML/dataset.rst diff --git a/docs/source/rst/api/ML/index.rst b/docs/source/rst/api/ML/index.rst deleted file mode 100644 index e9c052f5f..000000000 --- a/docs/source/rst/api/ML/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -Machine learning -================ - -.. toctree:: - - configs/index - models/index - dataset/index - augmentations/index - pipelines/index - SSL/index - utils/index - runner - photometric_normalization diff --git a/docs/source/rst/api/ML/models/index.rst b/docs/source/rst/api/ML/models.rst similarity index 64% rename from docs/source/rst/api/ML/models/index.rst rename to docs/source/rst/api/ML/models.rst index bf40a00e4..5ef742a43 100644 --- a/docs/source/rst/api/ML/models/index.rst +++ b/docs/source/rst/api/ML/models.rst @@ -4,7 +4,7 @@ Models Below is a list of models and various deep learning components that are available within InnerEye-DeepLearning. .. toctree:: - architectures/index - blocks/index - layers/index - losses/index + models/architectures + models/blocks + models/layers + models/losses diff --git a/docs/source/rst/api/ML/models/architectures/index.rst b/docs/source/rst/api/ML/models/architectures.rst similarity index 100% rename from docs/source/rst/api/ML/models/architectures/index.rst rename to docs/source/rst/api/ML/models/architectures.rst diff --git a/docs/source/rst/api/ML/models/blocks/index.rst b/docs/source/rst/api/ML/models/blocks.rst similarity index 100% rename from docs/source/rst/api/ML/models/blocks/index.rst rename to docs/source/rst/api/ML/models/blocks.rst diff --git a/docs/source/rst/api/ML/models/layers/index.rst b/docs/source/rst/api/ML/models/layers.rst similarity index 100% rename from docs/source/rst/api/ML/models/layers/index.rst rename to docs/source/rst/api/ML/models/layers.rst diff --git a/docs/source/rst/api/ML/models/losses/index.rst b/docs/source/rst/api/ML/models/losses.rst similarity index 100% rename from docs/source/rst/api/ML/models/losses/index.rst rename to docs/source/rst/api/ML/models/losses.rst diff --git a/docs/source/rst/api/ML/pipelines/index.rst b/docs/source/rst/api/ML/pipelines.rst similarity index 100% rename from docs/source/rst/api/ML/pipelines/index.rst rename to docs/source/rst/api/ML/pipelines.rst diff --git a/docs/source/rst/api/ML/utils/index.rst b/docs/source/rst/api/ML/utils.rst similarity index 100% rename from docs/source/rst/api/ML/utils/index.rst rename to docs/source/rst/api/ML/utils.rst diff --git a/docs/source/rst/api/ML/visualizers/index.rst b/docs/source/rst/api/ML/visualizers.rst similarity index 100% rename from docs/source/rst/api/ML/visualizers/index.rst rename to docs/source/rst/api/ML/visualizers.rst diff --git a/docs/source/rst/api/index.rst b/docs/source/rst/api/index.rst index 341507bba..746bcb34e 100644 --- a/docs/source/rst/api/index.rst +++ b/docs/source/rst/api/index.rst @@ -1,3 +1,3 @@ .. toctree:: - ML/index + ML/ML From bdc2a516af9cb8446fc8b97143da30d185fcbc26 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 12:06:09 +0100 Subject: [PATCH 19/27] =?UTF-8?q?=F0=9F=93=9D=20Limit=20API=20`toctree`=20?= =?UTF-8?q?depth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 4ab41f9d5..7d9631cb9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,6 +40,7 @@ InnerEye-DeepLearning Documentation .. toctree:: :caption: API documentation (🚧 Work In Progress 🚧) + :maxdepth: 2 rst/api/index From 56c3a5273dc32b8280fec11f8dc9a82cc07f9c11 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 13:26:59 +0100 Subject: [PATCH 20/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20primary=20InnerEye/M?= =?UTF-8?q?L=20files=20API=20to=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/baselines_util.py | 1 - docs/source/rst/api/ML/ML.rst | 9 +++++---- docs/source/rst/api/ML/core.rst | 13 +++++++++++++ docs/source/rst/api/ML/core/common.rst | 10 ++++++++++ docs/source/rst/api/ML/core/configs.rst | 12 ++++++++++++ docs/source/rst/api/ML/core/lightning.rst | 14 ++++++++++++++ docs/source/rst/api/ML/core/metrics.rst | 6 ++++++ docs/source/rst/api/ML/core/testing.rst | 4 ++++ docs/source/rst/api/ML/core/training.rst | 6 ++++++ docs/source/rst/api/ML/core/visualization.rst | 8 ++++++++ .../rst/api/ML/photometric_normalization.rst | 4 ---- docs/source/rst/api/ML/runner.rst | 4 ---- 12 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 docs/source/rst/api/ML/core.rst create mode 100644 docs/source/rst/api/ML/core/common.rst create mode 100644 docs/source/rst/api/ML/core/configs.rst create mode 100644 docs/source/rst/api/ML/core/lightning.rst create mode 100644 docs/source/rst/api/ML/core/metrics.rst create mode 100644 docs/source/rst/api/ML/core/testing.rst create mode 100644 docs/source/rst/api/ML/core/training.rst create mode 100644 docs/source/rst/api/ML/core/visualization.rst delete mode 100644 docs/source/rst/api/ML/photometric_normalization.rst delete mode 100644 docs/source/rst/api/ML/runner.rst diff --git a/InnerEye/ML/baselines_util.py b/InnerEye/ML/baselines_util.py index 59c78b27a..c42e54d23 100755 --- a/InnerEye/ML/baselines_util.py +++ b/InnerEye/ML/baselines_util.py @@ -112,7 +112,6 @@ def download_and_compare_scores(outputs_folder: Path, azure_config: AzureConfig, :param azure_config: Azure configuration to use for downloading data :param comparison_blob_storage_paths: list of paths to directories containing metrics.csv and dataset.csv files, each of the form run_recovery_id/rest_of_path - :param model_dataset_df: dataframe containing contents of dataset.csv for the current model :param model_metrics_df: dataframe containing contents of metrics.csv for the current model :return: a dataframe for all the data (current model and all baselines); whether any comparisons were diff --git a/docs/source/rst/api/ML/ML.rst b/docs/source/rst/api/ML/ML.rst index 7bb25fac7..48ec3f2db 100644 --- a/docs/source/rst/api/ML/ML.rst +++ b/docs/source/rst/api/ML/ML.rst @@ -2,13 +2,14 @@ Machine learning ================ .. toctree:: + :caption: Docuemntation for InnerEye/ML - configs + core models dataset augmentations - pipelines SSL + configs utils - runner - photometric_normalization + pipelines + visualizers diff --git a/docs/source/rst/api/ML/core.rst b/docs/source/rst/api/ML/core.rst new file mode 100644 index 000000000..499e07918 --- /dev/null +++ b/docs/source/rst/api/ML/core.rst @@ -0,0 +1,13 @@ +InnerEye/ML Core +================ + +.. toctree:: + :caption: Key Components in running InnerEye-DeepLearning + + core/common + core/lightning + core/training + core/testing + core/configs + core/metrics + core/visualization diff --git a/docs/source/rst/api/ML/core/common.rst b/docs/source/rst/api/ML/core/common.rst new file mode 100644 index 000000000..78a620518 --- /dev/null +++ b/docs/source/rst/api/ML/core/common.rst @@ -0,0 +1,10 @@ +Common +======= + +.. automodule:: InnerEye.ML.run_ml + +.. automodule:: InnerEye.ML.runner + +.. automodule:: InnerEye.ML.common + +.. automodule:: InnerEye.ML.baselines_util diff --git a/docs/source/rst/api/ML/core/configs.rst b/docs/source/rst/api/ML/core/configs.rst new file mode 100644 index 000000000..a8a29c9b8 --- /dev/null +++ b/docs/source/rst/api/ML/core/configs.rst @@ -0,0 +1,12 @@ +Configs +======= + +.. automodule:: InnerEye.ML.config + +.. automodule:: InnerEye.ML.model_config_base + +.. automodule:: InnerEye.ML.deep_learning_config + +.. automodule:: InnerEye.ML.scalar_config + +.. automodule:: InnerEye.ML.model_inference_config diff --git a/docs/source/rst/api/ML/core/lightning.rst b/docs/source/rst/api/ML/core/lightning.rst new file mode 100644 index 000000000..c7a09d759 --- /dev/null +++ b/docs/source/rst/api/ML/core/lightning.rst @@ -0,0 +1,14 @@ +Lightning Components +================================ + +.. automodule:: InnerEye.ML.lightning_base + +.. automodule:: InnerEye.ML.lightning_container + +.. automodule:: InnerEye.ML.lightning_models + +.. automodule:: InnerEye.ML.lightning_helpers + +.. automodule:: InnerEye.ML.lightning_loggers + +.. automodule:: InnerEye.ML.lightning_metrics diff --git a/docs/source/rst/api/ML/core/metrics.rst b/docs/source/rst/api/ML/core/metrics.rst new file mode 100644 index 000000000..d53493235 --- /dev/null +++ b/docs/source/rst/api/ML/core/metrics.rst @@ -0,0 +1,6 @@ +Metrics +======== + +.. automodule:: InnerEye.ML.metrics + +.. automodule:: InnerEye.ML.metrics_dict diff --git a/docs/source/rst/api/ML/core/testing.rst b/docs/source/rst/api/ML/core/testing.rst new file mode 100644 index 000000000..039206826 --- /dev/null +++ b/docs/source/rst/api/ML/core/testing.rst @@ -0,0 +1,4 @@ +Testing Models +============== + +.. automodule:: InnerEye.ML.model_testing diff --git a/docs/source/rst/api/ML/core/training.rst b/docs/source/rst/api/ML/core/training.rst new file mode 100644 index 000000000..add905979 --- /dev/null +++ b/docs/source/rst/api/ML/core/training.rst @@ -0,0 +1,6 @@ +Training Models +================ + +.. automodule:: InnerEye.ML.model_training + +.. automodule:: InnerEye.ML.photometric_normalization diff --git a/docs/source/rst/api/ML/core/visualization.rst b/docs/source/rst/api/ML/core/visualization.rst new file mode 100644 index 000000000..aba7d6502 --- /dev/null +++ b/docs/source/rst/api/ML/core/visualization.rst @@ -0,0 +1,8 @@ +Visualization +============== + +.. automodule:: InnerEye.ML.plotting + +.. automodule:: InnerEye.ML.normalize_and_visualize_dataset + +.. automodule:: InnerEye.ML.surface_distance_heatmaps diff --git a/docs/source/rst/api/ML/photometric_normalization.rst b/docs/source/rst/api/ML/photometric_normalization.rst deleted file mode 100644 index d8ed901a4..000000000 --- a/docs/source/rst/api/ML/photometric_normalization.rst +++ /dev/null @@ -1,4 +0,0 @@ -Photometric normalization -========================= - -.. automodule:: InnerEye.ML.photometric_normalization diff --git a/docs/source/rst/api/ML/runner.rst b/docs/source/rst/api/ML/runner.rst deleted file mode 100644 index 46e0793a4..000000000 --- a/docs/source/rst/api/ML/runner.rst +++ /dev/null @@ -1,4 +0,0 @@ -Runner -====== - -.. automodule:: InnerEye.ML.runner From e9e5ee897c034e2648b395499a400bb5ed223a1c Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 14:07:38 +0100 Subject: [PATCH 21/27] =?UTF-8?q?=F0=9F=93=9D=20Fix=20and=20add=20`InnerEy?= =?UTF-8?q?e/ML/*.py`=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Common/common_util.py | 23 +++++++++++-------- InnerEye/ML/deep_learning_config.py | 4 +--- InnerEye/ML/lightning_container.py | 22 ++++++++++-------- InnerEye/ML/metrics_dict.py | 5 ++-- InnerEye/ML/model_training.py | 2 +- .../ML/normalize_and_visualize_dataset.py | 13 +++++++---- docs/source/rst/api/ML/core.rst | 2 +- docs/source/rst/api/ML/core/lightning.rst | 1 + 8 files changed, 40 insertions(+), 32 deletions(-) diff --git a/InnerEye/Common/common_util.py b/InnerEye/Common/common_util.py index 08442b950..cbfdc1840 100644 --- a/InnerEye/Common/common_util.py +++ b/InnerEye/Common/common_util.py @@ -307,16 +307,19 @@ def check_properties_are_not_none(obj: Any, ignore: Optional[List[str]] = None) def initialize_instance_variables(func: Callable) -> Callable: """ - Automatically assigns the input parameters. - - >>> class process: - ... @initialize_instance_variables - ... def __init__(self, cmd, reachable=False, user='root'): - ... pass - >>> p = process('halt', True) - >>> # noinspection PyUnresolvedReferences - >>> p.cmd, p.reachable, p.user - ('halt', True, 'root') + Automatically assigns the input parameters. Example usage:: + + class process: + @initialize_instance_variables + def __init__(self, cmd, reachable=False, user='root'): + pass + p = process('halt', True) + print(p.cmd, p.reachable, p.user) + + Outputs:: + + ('halt', True, 'root') + """ names, varargs, keywords, defaults, _, _, _ = inspect.getfullargspec(func) diff --git a/InnerEye/ML/deep_learning_config.py b/InnerEye/ML/deep_learning_config.py index 729e4eea6..413ca7f5b 100644 --- a/InnerEye/ML/deep_learning_config.py +++ b/InnerEye/ML/deep_learning_config.py @@ -120,12 +120,10 @@ def create(project_root: Path, :param project_root: The root folder that contains the code that submitted the present training run. When running inside the InnerEye repository, it is the git repo root. When consuming InnerEye as a package, - this should be the root of the source code that calls the package. - + this should be the root of the source code that calls the package. :param is_offline_run: If true, this is a run outside AzureML. If False, it is inside AzureML. :param model_name: The name of the model that is trained. This is used to generate a run-specific output folder. - :param output_to: If provided, the output folders will be created as a subfolder of this argument. If not given, the output folders will be created inside of the project root. """ diff --git a/InnerEye/ML/lightning_container.py b/InnerEye/ML/lightning_container.py index 327f7b303..6a8153b00 100644 --- a/InnerEye/ML/lightning_container.py +++ b/InnerEye/ML/lightning_container.py @@ -31,16 +31,18 @@ class InnerEyeInference(abc.ABC): form of inference is slightly different from what PyTorch Lightning does in its `Trainer.test` method. In particular, this inference can be executed on any of the training, validation, or test set. - The inference code calls the methods in this order: - - model.on_inference_start() - for dataset_split in [Train, Val, Test] - model.on_inference_epoch_start(dataset_split, is_ensemble_model=False) - for batch_idx, item in enumerate(dataloader[dataset_split])): - model_outputs = model.forward(item) - model.inference_step(item, batch_idx, model_outputs) - model.on_inference_epoch_end() - model.on_inference_end() + The inference code calls the methods in this order:: + + model.on_inference_start() + + for dataset_split in [Train, Val, Test] + model.on_inference_epoch_start(dataset_split, is_ensemble_model=False) + for batch_idx, item in enumerate(dataloader[dataset_split])): + model_outputs = model.forward(item) + model.inference_step(item, batch_idx, model_outputs) + model.on_inference_epoch_end() + model.on_inference_end() + """ def on_inference_start(self) -> None: diff --git a/InnerEye/ML/metrics_dict.py b/InnerEye/ML/metrics_dict.py index 0415c2c8a..514db8704 100644 --- a/InnerEye/ML/metrics_dict.py +++ b/InnerEye/ML/metrics_dict.py @@ -713,8 +713,9 @@ def load_execution_mode_metrics_from_df(df: pd.DataFrame, """ Helper function to create BinaryClassificationMetricsDict grouped by ModelExecutionMode and epoch from a given dataframe. The following columns must exist in the provided data frame: - >>> LoggingColumns.DataSplit - >>> LoggingColumns.Epoch + + * LoggingColumns.DataSplit + * LoggingColumns.Epoch :param df: DataFrame to use for creating the metrics dict. :param is_classification_metrics: If the current metrics are for classification or not. diff --git a/InnerEye/ML/model_training.py b/InnerEye/ML/model_training.py index 9c31460d5..d2824f15a 100644 --- a/InnerEye/ML/model_training.py +++ b/InnerEye/ML/model_training.py @@ -326,7 +326,7 @@ def model_train(checkpoint_path: Optional[Path], def aggregate_and_create_subject_metrics_file(outputs_folder: Path) -> None: """ This functions takes all the subject metrics file written by each GPU (one file per GPU) and aggregates them into - one single metrics file. Results is saved in config.outputs_folder / mode.value / SUBJECT_METRICS_FILE_NAME. + one single metrics file. Results is saved in ``config.outputs_folder / mode.value / SUBJECT_METRICS_FILE_NAME``. This is done for the metrics files for training and for validation data separately. :param config: model config diff --git a/InnerEye/ML/normalize_and_visualize_dataset.py b/InnerEye/ML/normalize_and_visualize_dataset.py index 820decf58..34ddaed77 100644 --- a/InnerEye/ML/normalize_and_visualize_dataset.py +++ b/InnerEye/ML/normalize_and_visualize_dataset.py @@ -45,8 +45,9 @@ def create_parser(yaml_file_path: Path) -> ParserResult: Create a parser for all runner arguments, even though we are only using a subset of the arguments. This way, we can get secrets handling in a consistent way. In particular, this will create arguments for - --local_dataset - --azure_dataset_id + + * ``--local_dataset`` + * ``--azure_dataset_id`` """ parser = create_runner_parser(SegmentationModelBase) NormalizeAndVisualizeConfig.add_args(parser) @@ -67,9 +68,11 @@ def get_configs(default_model_config: SegmentationModelBase, def main(yaml_file_path: Path) -> None: """ Invoke either by - * specifying a model, '--model Lung' - * or specifying dataset and normalization parameters separately: --azure_dataset_id=foo --norm_method=None - In addition, the arguments '--image_channel' and '--gt_channel' must be specified (see below). + + * specifying a model, ``--model Lung`` + * or specifying dataset and normalization parameters separately: ``--azure_dataset_id=foo --norm_method=None`` + + In addition, the arguments ``--image_channel`` and ``--gt_channel`` must be specified. """ config, runner_config, args = get_configs(SegmentationModelBase(should_validate=False), yaml_file_path) dataset_config = DatasetConfig(name=config.azure_dataset_id, diff --git a/docs/source/rst/api/ML/core.rst b/docs/source/rst/api/ML/core.rst index 499e07918..efed6e292 100644 --- a/docs/source/rst/api/ML/core.rst +++ b/docs/source/rst/api/ML/core.rst @@ -1,4 +1,4 @@ -InnerEye/ML Core +ML Core ================ .. toctree:: diff --git a/docs/source/rst/api/ML/core/lightning.rst b/docs/source/rst/api/ML/core/lightning.rst index c7a09d759..2cce730f0 100644 --- a/docs/source/rst/api/ML/core/lightning.rst +++ b/docs/source/rst/api/ML/core/lightning.rst @@ -2,6 +2,7 @@ Lightning Components ================================ .. automodule:: InnerEye.ML.lightning_base + :exclude-members: configure_optimizers .. automodule:: InnerEye.ML.lightning_container From c1d84a89e16516771b0fb4c1e0e9c00c6a67f49f Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 9 Aug 2022 14:25:42 +0100 Subject: [PATCH 22/27] =?UTF-8?q?=E2=9A=B0=EF=B8=8F=20Remove=20weird=20`se?= =?UTF-8?q?ttings.json`=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 958b52a90..7d3a9471b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,5 +13,4 @@ "files.trimTrailingWhitespace": true, "files.trimFinalNewlines": true, "files.insertFinalNewline": true, - "esbonio.sphinx.confDir": "" } From 933bc7bae043dfe646b37e1fcd89c283428071b6 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 15 Aug 2022 12:57:01 +0100 Subject: [PATCH 23/27] =?UTF-8?q?=F0=9F=93=8C=20Switch=20recommonmark=20to?= =?UTF-8?q?=20MyST-parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/requirements.txt | 2 +- docs/source/conf.py | 9 +-------- docs/source/md/hippocampus_model.md | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ba20548e1..3e3d8c84c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -- recommonmark +- myst-parser - readthedocs-sphinx-ext diff --git a/docs/source/conf.py b/docs/source/conf.py index 01cda6f0b..f90fc65bc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,9 +24,6 @@ fixed_paths.add_submodules_to_path() -# -- Imports ----------------------------------------------------------------- -from recommonmark.parser import CommonMarkParser - # -- Project information ----------------------------------------------------- project = 'InnerEye-DeepLearning' @@ -46,7 +43,7 @@ 'sphinx.ext.autodoc', "sphinx.ext.autosummary", 'sphinx_rtd_theme', - 'recommonmark', + 'myst_parser', 'sphinx.ext.viewcode', ] @@ -71,10 +68,6 @@ # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] -source_parsers = { - '.md': CommonMarkParser, -} - source_suffix = { '.rst': 'restructuredtext', '.md': 'markdown', diff --git a/docs/source/md/hippocampus_model.md b/docs/source/md/hippocampus_model.md index a288f1e75..e254fbc71 100644 --- a/docs/source/md/hippocampus_model.md +++ b/docs/source/md/hippocampus_model.md @@ -117,4 +117,3 @@ Dice Score for Left and Right hippocampus respectively on a held-out test set of | hippocampus_L | 252 | 0.918 | 0.022 | 0.819 | 0.953 | 2.206 | 0.812 | 1.206 | 6.964 | 0.168 | 0.054 | 0.096 | 0.399 | | hippocampus_R | 252 | 0.918 | 0.024 | 0.816 | 0.954 | 2.185 | 0.706 | 1.206 | 4.966 | 0.168 | 0.055 | 0.094 | 0.433 | | | | | | | | | | | | | | | | ---- From 6e610b0ccb165b04c0044e63351179a4a19e3c9b Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 16 Aug 2022 10:55:31 +0100 Subject: [PATCH 24/27] =?UTF-8?q?=F0=9F=93=8C=20Add=20myst-parser=20to=20`?= =?UTF-8?q?environment.yml`,=20lock=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TestSubmodule/environment.yml | 45 +++++++++++++++++++---------------- docs/README.md | 14 ++++------- docs/requirements.txt | 2 -- environment.yml | 45 +++++++++++++++++++---------------- primary_deps.yml | 1 + 5 files changed, 55 insertions(+), 52 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/TestSubmodule/environment.yml b/TestSubmodule/environment.yml index 62f7d7683..8f5b5c7de 100644 --- a/TestSubmodule/environment.yml +++ b/TestSubmodule/environment.yml @@ -53,10 +53,10 @@ dependencies: - pytorch-mutex=1.0=cuda - readline=8.1.2=h7f8727e_1 - setuptools=61.2.0=py38h06a4308_0 - - sqlite=3.39.0=h5082296_0 + - sqlite=3.39.2=h5082296_0 - tk=8.6.12=h1ccaba5_0 - torchvision=0.11.1=py38_cu113 - - typing_extensions=4.1.1=pyh06a4308_0 + - typing_extensions=4.3.0=py38h06a4308_0 - wheel=0.37.1=pyhd3eb1b0_0 - x264=1!157.20191217=h7b6447c_0 - xz=5.2.5=h7f8727e_1 @@ -76,12 +76,12 @@ dependencies: - async-timeout==4.0.2 - attrs==22.1.0 - azure-common==1.1.28 - - azure-core==1.24.2 + - azure-core==1.25.0 - azure-graphrbac==0.61.1 - azure-identity==1.7.0 - azure-mgmt-authorization==0.61.0 - azure-mgmt-containerregistry==10.0.0 - - azure-mgmt-core==1.3.1 + - azure-mgmt-core==1.3.2 - azure-mgmt-datafactory==1.1.0 - azure-mgmt-keyvault==9.3.0 - azure-mgmt-resource==12.1.0 @@ -118,24 +118,24 @@ dependencies: - commonmark==0.9.1 - conda-merge==0.1.5 - contextlib2==21.6.0 - - coverage==6.4.2 + - coverage==6.4.3 - cryptography==3.3.2 - cycler==0.11.0 - - databricks-cli==0.17.0 + - databricks-cli==0.17.1 - dataclasses-json==0.5.2 - - debugpy==1.6.2 + - debugpy==1.6.3 - defusedxml==0.7.1 - deprecated==1.2.13 - distro==1.7.0 - docker==4.3.1 - - docutils==0.17.1 + - docutils==0.18.1 - dotnetcore2==2.1.23 - entrypoints==0.4 - execnet==1.9.0 - fastjsonschema==2.16.1 - fastmri==0.2.0 - flake8==3.8.3 - - flask==2.2.0 + - flask==2.2.2 - frozenlist==1.3.1 - fsspec==2022.7.1 - furo==2022.6.21 @@ -152,7 +152,7 @@ dependencies: - h5py==2.10.0 - hi-ml==0.2.2 - hi-ml-azure==0.2.2 - - humanize==4.2.3 + - humanize==4.3.0 - idna==3.3 - imageio==2.15.0 - imagesize==1.4.1 @@ -181,18 +181,22 @@ dependencies: - kiwisolver==1.4.4 - lightning-bolts==0.4.0 - llvmlite==0.34.0 + - lxml==4.9.1 - mako==1.2.1 - markdown==3.4.1 + - markdown-it-py==2.1.0 - markupsafe==2.1.1 - marshmallow==3.17.0 - marshmallow-enum==1.5.1 - matplotlib==3.3.0 - mccabe==0.6.1 + - mdit-py-plugins==0.3.0 + - mdurl==0.1.2 - mistune==0.8.4 - mlflow==1.23.1 - - mlflow-skinny==1.27.0 + - mlflow-skinny==1.28.0 - monai==0.6.0 - - more-itertools==8.13.0 + - more-itertools==8.14.0 - msal==1.18.0 - msal-extensions==0.3.1 - msrest==0.7.1 @@ -200,8 +204,9 @@ dependencies: - multidict==6.0.2 - mypy==0.910 - mypy-extensions==0.4.3 + - myst-parser==0.18.0 - nbclient==0.6.6 - - nbconvert==6.5.0 + - nbconvert==6.5.3 - nbformat==5.4.0 - ndg-httpsclient==0.5.1 - nest-asyncio==1.5.5 @@ -250,12 +255,12 @@ dependencies: - pytest-xdist==1.34.0 - python-dateutil==2.8.2 - pytorch-lightning==1.5.5 - - pytz==2022.1 + - pytz==2022.2.1 - pywavelets==1.3.0 - pyyaml==6.0 - - pyzmq==23.2.0 + - pyzmq==23.2.1 - qtconsole==5.3.1 - - qtpy==2.1.0 + - qtpy==2.2.0 - querystring-parser==1.2.4 - requests==2.28.1 - requests-oauthlib==1.3.1 @@ -269,7 +274,7 @@ dependencies: - scikit-learn==0.23.2 - scipy==1.5.2 - seaborn==0.10.1 - - secretstorage==3.3.2 + - secretstorage==3.3.3 - send2trash==1.8.0 - simpleitk==1.2.4 - six==1.15.0 @@ -285,7 +290,7 @@ dependencies: - sphinxcontrib-jsmath==1.0.1 - sphinxcontrib-qthelp==1.0.3 - sphinxcontrib-serializinghtml==1.1.5 - - sqlalchemy==1.4.39 + - sqlalchemy==1.4.40 - sqlparse==0.4.2 - stopit==1.1.2 - stringcase==1.2.0 @@ -297,7 +302,7 @@ dependencies: - terminado==0.15.0 - textwrap3==0.9.2 - threadpoolctl==3.1.0 - - tifffile==2022.7.31 + - tifffile==2022.8.12 - tinycss2==1.1.1 - toml==0.10.2 - tomli==2.0.1 @@ -310,7 +315,7 @@ dependencies: - urllib3==1.26.7 - webencodings==0.5.1 - websocket-client==1.3.3 - - werkzeug==2.2.1 + - werkzeug==2.2.2 - widgetsnbextension==3.6.1 - wrapt==1.14.1 - yacs==0.1.8 diff --git a/docs/README.md b/docs/README.md index dc12dfd4e..69e344b85 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,12 +1,6 @@ # Building docs for InnerEye-DeepLearning -1. First, make sure you have all the packages necessary for InnerEye. -2. Install pip dependencies from docs/requirements.txt: - -```shell -pip install -r requirements.txt -``` - -1. Run `make html` from the `docs` folder. This will create html files under docs/build/html. -1. From the `docs/build/html` folder, run `python -m http.server 8080` to host the docs locally. -1. From your browser, navigate to `http://localhost:8080` to view the documentation. +1. First, make sure you have set up your conda environment as described in the [Quick Setup Guide](../README.md#quick-setup). +2. Run `make html` from the `docs` folder. This will create html files under docs/build/html. +3. From the `docs/build/html` folder, run `python -m http.server 8080` to host the docs locally. +4. From your browser, navigate to `http://localhost:8080` to view the documentation. diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 3e3d8c84c..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -- myst-parser -- readthedocs-sphinx-ext diff --git a/environment.yml b/environment.yml index 62f7d7683..8f5b5c7de 100644 --- a/environment.yml +++ b/environment.yml @@ -53,10 +53,10 @@ dependencies: - pytorch-mutex=1.0=cuda - readline=8.1.2=h7f8727e_1 - setuptools=61.2.0=py38h06a4308_0 - - sqlite=3.39.0=h5082296_0 + - sqlite=3.39.2=h5082296_0 - tk=8.6.12=h1ccaba5_0 - torchvision=0.11.1=py38_cu113 - - typing_extensions=4.1.1=pyh06a4308_0 + - typing_extensions=4.3.0=py38h06a4308_0 - wheel=0.37.1=pyhd3eb1b0_0 - x264=1!157.20191217=h7b6447c_0 - xz=5.2.5=h7f8727e_1 @@ -76,12 +76,12 @@ dependencies: - async-timeout==4.0.2 - attrs==22.1.0 - azure-common==1.1.28 - - azure-core==1.24.2 + - azure-core==1.25.0 - azure-graphrbac==0.61.1 - azure-identity==1.7.0 - azure-mgmt-authorization==0.61.0 - azure-mgmt-containerregistry==10.0.0 - - azure-mgmt-core==1.3.1 + - azure-mgmt-core==1.3.2 - azure-mgmt-datafactory==1.1.0 - azure-mgmt-keyvault==9.3.0 - azure-mgmt-resource==12.1.0 @@ -118,24 +118,24 @@ dependencies: - commonmark==0.9.1 - conda-merge==0.1.5 - contextlib2==21.6.0 - - coverage==6.4.2 + - coverage==6.4.3 - cryptography==3.3.2 - cycler==0.11.0 - - databricks-cli==0.17.0 + - databricks-cli==0.17.1 - dataclasses-json==0.5.2 - - debugpy==1.6.2 + - debugpy==1.6.3 - defusedxml==0.7.1 - deprecated==1.2.13 - distro==1.7.0 - docker==4.3.1 - - docutils==0.17.1 + - docutils==0.18.1 - dotnetcore2==2.1.23 - entrypoints==0.4 - execnet==1.9.0 - fastjsonschema==2.16.1 - fastmri==0.2.0 - flake8==3.8.3 - - flask==2.2.0 + - flask==2.2.2 - frozenlist==1.3.1 - fsspec==2022.7.1 - furo==2022.6.21 @@ -152,7 +152,7 @@ dependencies: - h5py==2.10.0 - hi-ml==0.2.2 - hi-ml-azure==0.2.2 - - humanize==4.2.3 + - humanize==4.3.0 - idna==3.3 - imageio==2.15.0 - imagesize==1.4.1 @@ -181,18 +181,22 @@ dependencies: - kiwisolver==1.4.4 - lightning-bolts==0.4.0 - llvmlite==0.34.0 + - lxml==4.9.1 - mako==1.2.1 - markdown==3.4.1 + - markdown-it-py==2.1.0 - markupsafe==2.1.1 - marshmallow==3.17.0 - marshmallow-enum==1.5.1 - matplotlib==3.3.0 - mccabe==0.6.1 + - mdit-py-plugins==0.3.0 + - mdurl==0.1.2 - mistune==0.8.4 - mlflow==1.23.1 - - mlflow-skinny==1.27.0 + - mlflow-skinny==1.28.0 - monai==0.6.0 - - more-itertools==8.13.0 + - more-itertools==8.14.0 - msal==1.18.0 - msal-extensions==0.3.1 - msrest==0.7.1 @@ -200,8 +204,9 @@ dependencies: - multidict==6.0.2 - mypy==0.910 - mypy-extensions==0.4.3 + - myst-parser==0.18.0 - nbclient==0.6.6 - - nbconvert==6.5.0 + - nbconvert==6.5.3 - nbformat==5.4.0 - ndg-httpsclient==0.5.1 - nest-asyncio==1.5.5 @@ -250,12 +255,12 @@ dependencies: - pytest-xdist==1.34.0 - python-dateutil==2.8.2 - pytorch-lightning==1.5.5 - - pytz==2022.1 + - pytz==2022.2.1 - pywavelets==1.3.0 - pyyaml==6.0 - - pyzmq==23.2.0 + - pyzmq==23.2.1 - qtconsole==5.3.1 - - qtpy==2.1.0 + - qtpy==2.2.0 - querystring-parser==1.2.4 - requests==2.28.1 - requests-oauthlib==1.3.1 @@ -269,7 +274,7 @@ dependencies: - scikit-learn==0.23.2 - scipy==1.5.2 - seaborn==0.10.1 - - secretstorage==3.3.2 + - secretstorage==3.3.3 - send2trash==1.8.0 - simpleitk==1.2.4 - six==1.15.0 @@ -285,7 +290,7 @@ dependencies: - sphinxcontrib-jsmath==1.0.1 - sphinxcontrib-qthelp==1.0.3 - sphinxcontrib-serializinghtml==1.1.5 - - sqlalchemy==1.4.39 + - sqlalchemy==1.4.40 - sqlparse==0.4.2 - stopit==1.1.2 - stringcase==1.2.0 @@ -297,7 +302,7 @@ dependencies: - terminado==0.15.0 - textwrap3==0.9.2 - threadpoolctl==3.1.0 - - tifffile==2022.7.31 + - tifffile==2022.8.12 - tinycss2==1.1.1 - toml==0.10.2 - tomli==2.0.1 @@ -310,7 +315,7 @@ dependencies: - urllib3==1.26.7 - webencodings==0.5.1 - websocket-client==1.3.3 - - werkzeug==2.2.1 + - werkzeug==2.2.2 - widgetsnbextension==3.6.1 - wrapt==1.14.1 - yacs==0.1.8 diff --git a/primary_deps.yml b/primary_deps.yml index b662b86fb..aa8ff1918 100644 --- a/primary_deps.yml +++ b/primary_deps.yml @@ -42,6 +42,7 @@ dependencies: - monai==0.6.0 - mypy-extensions==0.4.3 - mypy==0.910 + - myst-parser==0.18.0 - numba==0.51.2 - numba==0.51.2 - numpy==1.19.1 From e4e86df601d6c5b88da3905dfa14ac70b39508d1 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 17 Aug 2022 11:02:00 +0100 Subject: [PATCH 25/27] =?UTF-8?q?=F0=9F=9A=A7=20Add=20Common=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/Common/common.rst | 20 ++++++++++++++++++++ docs/source/rst/api/Common/index.rst | 0 2 files changed, 20 insertions(+) create mode 100644 docs/source/rst/api/Common/common.rst delete mode 100644 docs/source/rst/api/Common/index.rst diff --git a/docs/source/rst/api/Common/common.rst b/docs/source/rst/api/Common/common.rst new file mode 100644 index 000000000..28731b5ea --- /dev/null +++ b/docs/source/rst/api/Common/common.rst @@ -0,0 +1,20 @@ +Common +======= + +This page contains the docs for all shared components that can be found under `InnerEye/Common/ `_. + +.. automodule:: InnerEye.Common.common_util + +.. automodule:: InnerEye.Common.fixed_paths_for_tests + +.. automodule:: InnerEye.Common.fixed_paths + +.. automodule:: InnerEye.Common.generic_parsing + +.. automodule:: InnerEye.Common.metrics_constants + +.. automodule:: InnerEye.Common.output_directories + +.. automodule:: InnerEye.Common.resource_monitor + +.. automodule:: InnerEye.Common.spawn_subprocess diff --git a/docs/source/rst/api/Common/index.rst b/docs/source/rst/api/Common/index.rst deleted file mode 100644 index e69de29bb..000000000 From 58535d30d9d528be274547f31430ea8ba9a0ade5 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 18 Aug 2022 10:42:12 +0100 Subject: [PATCH 26/27] =?UTF-8?q?=F0=9F=93=9D=20Add=20common=20docstrings?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/Common/common_util.py | 52 ++++++++++++++++------------- InnerEye/Common/generic_parsing.py | 1 + InnerEye/Common/resource_monitor.py | 14 ++++---- docs/source/rst/api/index.rst | 1 + 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/InnerEye/Common/common_util.py b/InnerEye/Common/common_util.py index cbfdc1840..ca80730d8 100644 --- a/InnerEye/Common/common_util.py +++ b/InnerEye/Common/common_util.py @@ -43,27 +43,34 @@ class ModelProcessing(Enum): """ Enum used in model training and inference, used to decide where to put files and what logging messages to print. The meanings of the values are: - ENSEMBLE_CREATION: we are creating and processing an ensemble model from within the child run with + + * ``ENSEMBLE_CREATION``: we are creating and processing an ensemble model from within the child run with cross-validation index 0 of the HyperDrive run that created this model. - DEFAULT: any other situation, *including* where the model is an ensemble model created by an earlier run + * ``DEFAULT``: any other situation, *including* where the model is an ensemble model created by an earlier run (so the current run is standalone, not part of a HyperDrive run). - There are four scenarios, only one of which uses ModelProcessing.ENSEMBLE_CREATION. - (1) Training and inference on a single model in a single (non-HyperDrive) run. - (2) Training and inference on a single model that is part of an ensemble, in HyperDrive child run. - (3) Inference on an ensemble model taking place in a HyperDrive child run that trained one of the component - models of the ensemble and whose cross validation index is 0. - (4) Inference on a single or ensemble model created in an another run specified by the value of run_recovery_id. - * Scenario (1) happens when we train a model (train=True) with number_of_cross_validation_splits=0. In this - case, the value of ModelProcessing passed around is DEFAULT. - * Scenario (2) happens when we train a model (train=True) with number_of_cross_validation_splits>0. In this - case, the value of ModelProcessing passed around is DEFAULT in each of the child runs while training and running - inference on its own single model. However, the child run whose cross validation index is 0 then goes on to - carry out Scenario (3), and does more processing with ModelProcessing value ENSEMBLE_CREATION, to create and - register the ensemble model, run inference on it, and upload information about the ensemble model to the parent run. - * Scenario (4) happens when we do an inference-only run (train=False), and specify an existing model with - run_recovery_id (and necessarily number_of_cross_validation_splits=0, even if the recovered run was a HyperDrive - one). This model may be either a single one or an ensemble one; in both cases, a ModelProcessing value of DEFAULT is - used. + + There are four scenarios, only one of which uses ``ModelProcessing.ENSEMBLE_CREATION``: + + 1. Training and inference on a single model in a single (non-HyperDrive) run. + 2. Training and inference on a single model that is part of an ensemble, in HyperDrive child run. + 3. Inference on an ensemble model taking place in a HyperDrive child run that trained one of the component + models of the ensemble and whose cross validation index is 0. + 4. Inference on a single or ensemble model created in an another run specified by the value of run_recovery_id. + + The scenarios occur under the following conditions: + + * Scenario 1 happens when we train a model (``train=True``) with ``number_of_cross_validation_splits=0``. In + this case, the value of ModelProcessing passed around is DEFAULT. + * Scenario 2 happens when we train a model (``train=True``) with ``number_of_cross_validation_splits > 0``. In + this case, the value of ModelProcessing passed around is DEFAULT in each of the child runs while training and + running inference on its own single model. However, the child run whose cross validation index is 0 then goes + on to carry out Scenario 3, and does more processing with ModelProcessing value ``ENSEMBLE_CREATION``, to + create and register the ensemble model, run inference on it, and upload information about the ensemble model + to the parent run. + * Scenario 4 happens when we do an inference-only run (``train=False``), and specify an existing model with + ``run_recovery_id`` (and necessarily ``number_of_cross_validation_splits=0``, even if the recovered run was a + HyperDrive one). This model may be either a single one or an ensemble one; in both cases, a ModelProcessing + value of ``DEFAULT`` is used. """ DEFAULT = 'default' ENSEMBLE_CREATION = 'ensemble_creation' @@ -113,7 +120,6 @@ def check_is_any_of(message: str, actual: Optional[str], valid: Iterable[Optiona :param message: The prefix for the error message. :param actual: The actual value. :param valid: The set of valid strings that 'actual' is allowed to take on. - :return: """ if actual not in valid: all_valid = ", ".join(["" if v is None else v for v in valid]) @@ -140,7 +146,8 @@ def logging_to_stdout(log_level: Union[int, str] = logging.INFO) -> None: Logging will use a timestamp as the prefix, using UTC. :param log_level: The logging level. All logging message with a level at or above this level will be written to - stdout. log_level can be numeric, or one of the pre-defined logging strings (INFO, DEBUG, ...). + stdout. log_level can be numeric, or one of the pre-defined logging strings (``loging.INFO``, ``logging.DEBUG``, + etc.). """ log_level = standardize_log_level(log_level) logger = logging.getLogger() @@ -379,7 +386,6 @@ def namespace_to_path(namespace: str, root: PathOrString = repository_root_direc :param namespace: Namespace to convert to path :param root: Path to prefix (default is project root) - :return: """"" return Path(root, *namespace.split(".")) @@ -391,7 +397,7 @@ def path_to_namespace(path: Path, root: PathOrString = repository_root_directory :param path: Path to convert to namespace :param root: Path prefix to remove from namespace (default is project root) - :return: + :return: String representation to path of namespace """ return ".".join([Path(x).stem for x in path.relative_to(root).parts]) diff --git a/InnerEye/Common/generic_parsing.py b/InnerEye/Common/generic_parsing.py index b4a97f2ed..f9a4d3e6a 100644 --- a/InnerEye/Common/generic_parsing.py +++ b/InnerEye/Common/generic_parsing.py @@ -269,6 +269,7 @@ def parse_args(cls: Type[T], args: Optional[List[str]] = None) -> T: def get_overridable_parameters(cls: Type[GenericConfig]) -> Dict[str, param.Parameter]: """ Get properties that are not constant, readonly or private (eg: prefixed with an underscore). + :return: A dictionary of parameter names and their definitions. """ return dict((k, v) for k, v in cls.params().items() diff --git a/InnerEye/Common/resource_monitor.py b/InnerEye/Common/resource_monitor.py index b9437de78..e28e29c57 100644 --- a/InnerEye/Common/resource_monitor.py +++ b/InnerEye/Common/resource_monitor.py @@ -29,8 +29,8 @@ def memory_in_gb(bytes: int) -> float: """ Converts a memory amount in bytes to gigabytes. - :param bytes: - :return: + :param bytes: Number of bytes + :return: Equivalent memory amount in gigabytes """ gb = 2 ** 30 return bytes / gb @@ -65,8 +65,8 @@ def max(self, other: GpuUtilization) -> GpuUtilization: """ Computes the metric-wise maximum of the two GpuUtilization objects. - :param other: - :return: + :param other: The other GpuUtilization object. + :return: The metric-wise maximum of the two GpuUtilization objects. """ return GpuUtilization( # Effectively ignore ID. We could enforce consistent IDs, but then we could not compute overall max. @@ -83,7 +83,7 @@ def average(self) -> GpuUtilization: """ Returns a GPU utilization object that contains all metrics of the present object, divided by the number of observations. - :return: + :return: The GPU utilization object. """ return GpuUtilization( id=self.id, @@ -120,10 +120,10 @@ def enumerate(self, prefix: str = "") -> List[Tuple[str, float]]: @staticmethod def from_gpu(gpu: GPU) -> GpuUtilization: """ - Creates a GpuUtilization object from data coming from the gputil library. + Creates a ``GpuUtilization`` object from data coming from the gputil library. :param gpu: GPU diagnostic data from gputil. - :return: + :return: ``GpuUtilization`` object """ return GpuUtilization( id=gpu.id, diff --git a/docs/source/rst/api/index.rst b/docs/source/rst/api/index.rst index 746bcb34e..0ae59bde1 100644 --- a/docs/source/rst/api/index.rst +++ b/docs/source/rst/api/index.rst @@ -1,3 +1,4 @@ .. toctree:: ML/ML + Common/common From d3c618de4e63c6fb46598c43bc7bd8bc455becf0 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 12 Sep 2022 11:48:46 +0100 Subject: [PATCH 27/27] =?UTF-8?q?=F0=9F=90=9B=20Remove=20git=20artifacts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/api/index.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/source/rst/api/index.rst b/docs/source/rst/api/index.rst index 32b5c0c98..84e000802 100644 --- a/docs/source/rst/api/index.rst +++ b/docs/source/rst/api/index.rst @@ -1,8 +1,5 @@ .. toctree:: ML/ML -<<<<<<< HEAD Common/common -======= Scripts/scripts ->>>>>>> main