feat(AutoML): implemented block function evaluation

aucmedi/automl/block_eval.py

@@ -1,2 +1,92 @@
-# take input image and ground truth
-# take input pred csv
+#==============================================================================#
+#  Author:       Dominik Müller                                                #
+#  Copyright:    2022 IT-Infrastructure for Translational Medical Research,    #
+#                University of Augsburg                                        #
+#                                                                              #
+#  This program is free software: you can redistribute it and/or modify        #
+#  it under the terms of the GNU General Public License as published by        #
+#  the Free Software Foundation, either version 3 of the License, or           #
+#  (at your option) any later version.                                         #
+#                                                                              #
+#  This program is distributed in the hope that it will be useful,             #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+#  GNU General Public License for more details.                                #
+#                                                                              #
+#  You should have received a copy of the GNU General Public License           #
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#==============================================================================#
+#-----------------------------------------------------#
+#                   Library imports                   #
+#-----------------------------------------------------#
+# External libraries
+import os
+import pandas as pd
+import numpy as np
+# Internal libraries
+from aucmedi import *
+from aucmedi.evaluation import evaluate_performance
+
+#-----------------------------------------------------#
+#            Building Blocks for Evaluation           #
+#-----------------------------------------------------#
+def block_evaluate(config):
+    """ Internal code block for AutoML evaluation.
+
+    This function is called by the Command-Line-Interface (CLI) of AUCMEDI.
+
+    Args:
+        config (dict):                      Configuration dictionary containing all required
+                                            parameters for performing an AutoML evaluation.
+
+    The following attributes are stored in the `config` dictionary:
+
+    Attributes:
+        interface (str):                    String defining format interface for loading/storing data (`csv` or `dictionary`).
+        path_imagedir (str):                Path to the directory containing the images.
+        path_data (str):                    Path to the index/class annotation file if required. (csv/json).
+        input (str):                        Path to the input file in which predicted csv file is stored.
+        output (str):                       Path to the directory in which evaluation figures and tables should be stored.
+        ohe (bool):                         Boolean option whether annotation data is sparse categorical or one-hot encoded.
+    """
+    # Peak into the dataset via the input interface
+    ds = input_interface(config["interface"],
+                         config["path_imagedir"],
+                         path_data=config["path_data"],
+                         training=True,
+                         ohe=config["ohe"],
+                         image_format=None)
+    (index_list, class_ohe, class_n, class_names, image_format) = ds
+
+    # Create output directory
+    if not os.path.exists(config["output"]) : os.mkdir(config["output"])
+
+    # Read prediction csv
+    df_pred = pd.read_csv(config["input"])
+
+    # Create ground truth pandas dataframe
+    df_index = pd.DataFrame(data={"SAMPLE": index_list})
+    df_gt_data = pd.DataFrame(data=class_ohe, columns=class_names)
+    df_gt = pd.concat([df_index, df_gt_data], axis=1, sort=False)
+
+    # Merge dataframes to verify correct order
+    df_merged = df_pred.merge(df_gt, on="SAMPLE", suffixes=("_pd", "_gt"))
+    # Extract pd and gt again to NumPy
+    data_pd = df_merged.iloc[:, 1:(class_n+1)].to_numpy()
+    data_gt = df_merged.iloc[:, (class_n+1):9].to_numpy()
+
+    # Identify task (multi-class vs multi-label)
+    if np.sum(data_pd) > (class_ohe.shape[0] + 1.5) : multi_label = True
+    else : multi_label = False
+
+    # Evaluate performance via AUCMEDI evaluation submodule
+    evaluate_performance(data_pd, data_gt,
+                         out_path=config["output"],
+                         class_names=class_names,
+                         multi_label=multi_label,
+                         metrics_threshold=0.5,
+                         suffix=None,
+                         store_csv=True,
+                         plot_barplot=True,
+                         plot_confusion_matrix=True,
+                         plot_roc_curve=True)