From 3566791270ff2030af5c2a75522dd367bd55da74 Mon Sep 17 00:00:00 2001 From: muellerdo Date: Mon, 4 Jul 2022 18:37:35 +0200 Subject: [PATCH] feat(AutoML): reworked AutoML CLI arguments - closes #151 --- aucmedi/automl/block_eval.py | 19 ++-- aucmedi/automl/block_pred.py | 21 ++-- aucmedi/automl/block_train.py | 28 +++-- aucmedi/automl/cli.py | 183 ++++++++++++++++--------------- aucmedi/automl/parser_cli.py | 7 -- docs/getstarted/usage_automl.md | 11 ++ tests/test_automl_block_eval.py | 18 ++- tests/test_automl_block_pred.py | 88 +++++++-------- tests/test_automl_block_train.py | 36 +++--- tests/test_automl_cli.py | 93 ++++++++-------- 10 files changed, 255 insertions(+), 249 deletions(-) diff --git a/aucmedi/automl/block_eval.py b/aucmedi/automl/block_eval.py index f16836ef..05ef51c4 100644 --- a/aucmedi/automl/block_eval.py +++ b/aucmedi/automl/block_eval.py @@ -43,27 +43,30 @@ def block_evaluate(config): The following attributes are stored in the `config` dictionary: Attributes: - interface (str): String defining format interface for loading/storing data (`csv` or `dictionary`). path_imagedir (str): Path to the directory containing the ground truth images. - path_data (str): Path to the index/class annotation file if required. (csv/json). - input (str): Path to the input file in which predicted csv file is stored. - output (str): Path to the directory in which evaluation figures and tables should be stored. + path_gt (str): Path to the index/class annotation file if required. (only for 'csv' interface). + path_pred (str): Path to the input file in which predicted csv file is stored. + path_evaldir (str): Path to the directory in which evaluation figures and tables should be stored. ohe (bool): Boolean option whether annotation data is sparse categorical or one-hot encoded. """ + # Obtain interface + if config["path_gt"] is None : config["interface"] = "directory" + else : config["interface"] = "csv" # Peak into the dataset via the input interface ds = input_interface(config["interface"], config["path_imagedir"], - path_data=config["path_data"], + path_data=config["path_gt"], training=True, ohe=config["ohe"], image_format=None) (index_list, class_ohe, class_n, class_names, image_format) = ds # Create output directory - if not os.path.exists(config["output"]) : os.mkdir(config["output"]) + if not os.path.exists(config["path_evaldir"]): + os.mkdir(config["path_evaldir"]) # Read prediction csv - df_pred = pd.read_csv(config["input"]) + df_pred = pd.read_csv(config["path_pred"]) # Create ground truth pandas dataframe df_index = pd.DataFrame(data={"SAMPLE": index_list}) @@ -96,7 +99,7 @@ def block_evaluate(config): # Evaluate performance via AUCMEDI evaluation submodule evaluate_performance(data_pd, data_gt, - out_path=config["output"], + out_path=config["path_evaldir"], class_names=class_names, multi_label=multi_label, metrics_threshold=0.5, diff --git a/aucmedi/automl/block_pred.py b/aucmedi/automl/block_pred.py index 66a3f804..76399940 100644 --- a/aucmedi/automl/block_pred.py +++ b/aucmedi/automl/block_pred.py @@ -45,9 +45,9 @@ def block_predict(config): The following attributes are stored in the `config` dictionary: Attributes: - path_imagedir (str): Path to the directory containing the images. - input (str): Path to the input directory in which fitted models and metadata are stored. - output (str): Path to the output file in which predicted csv file should be stored. + path_imagedir (str): Path to the directory containing the images for prediction. + path_modeldir (str): Path to the model directory in which fitted model weights and metadata are stored. + path_pred (str): Path to the output file in which predicted csv file should be stored. xai_method (str or None): Key for XAI method. xai_directory (str or None): Path to the output directory in which predicted image xai heatmaps should be stored. batch_size (int): Number of samples inside a single batch. @@ -63,11 +63,11 @@ def block_predict(config): (index_list, _, _, _, image_format) = ds # Verify existence of input directory - if not os.path.exists(config["input"]): - raise FileNotFoundError(config["input"]) + if not os.path.exists(config["path_modeldir"]): + raise FileNotFoundError(config["path_modeldir"]) # Load metadata from training - path_meta = os.path.join(config["input"], "meta.training.json") + path_meta = os.path.join(config["path_modeldir"], "meta.training.json") with open(path_meta, "r") as json_file: meta_training = json.load(json_file) @@ -123,7 +123,7 @@ def block_predict(config): standardize_mode=model.meta_standardize, **paras_datagen) # Load model - path_model = os.path.join(config["input"], "model.last.hdf5") + path_model = os.path.join(config["path_modeldir"], "model.last.hdf5") model.load(path_model) # Start model inference preds = model.predict(prediction_generator=pred_gen) @@ -141,7 +141,8 @@ def block_predict(config): standardize_mode=model.meta_standardize, **paras_datagen) # Load model - path_model = os.path.join(config["input"], "model.best_loss.hdf5") + path_model = os.path.join(config["path_modeldir"], + "model.best_loss.hdf5") model.load(path_model) # Start model inference via Augmenting preds = predict_augmenting(model, pred_gen) @@ -163,7 +164,7 @@ def block_predict(config): standardize_mode=None, **paras_datagen) # Load composite model directory - el.load(config["input"]) + el.load(config["path_modeldir"]) # Start model inference via ensemble learning preds = el.predict(pred_gen) @@ -173,7 +174,7 @@ def block_predict(config): df_merged = pd.concat([df_index, df_pd], axis=1, sort=False) df_merged.sort_values(by=["SAMPLE"], inplace=True) # Store predictions to disk - df_merged.to_csv(config["output"], index=False) + df_merged.to_csv(config["path_pred"], index=False) # Create XAI heatmaps if config["xai_method"] is not None and config["xai_directory"] is not None: diff --git a/aucmedi/automl/block_train.py b/aucmedi/automl/block_train.py index 64438412..b16daf87 100644 --- a/aucmedi/automl/block_train.py +++ b/aucmedi/automl/block_train.py @@ -52,10 +52,9 @@ def block_train(config): The following attributes are stored in the `config` dictionary: Attributes: - interface (str): String defining format interface for loading/storing data (`csv` or `dictionary`). path_imagedir (str): Path to the directory containing the images. - path_data (str): Path to the index/class annotation file if required. (csv/json). - output (str): Path to the output directory in which fitted models and metadata are stored. + path_modeldir (str): Path to the output directory in which fitted models and metadata are stored. + path_gt (str): Path to the index/class annotation file if required. (only for 'csv' interface). analysis (str): Analysis mode for the AutoML training. Options: `["minimal", "standard", "advanced"]`. ohe (bool): Boolean option whether annotation data is sparse categorical or one-hot encoded. three_dim (bool): Boolean, whether data is 2D or 3D. @@ -67,17 +66,21 @@ def block_train(config): metalearner (str): Key for Metalearner or Aggregate function. architecture (str or list of str): Key (str) of a neural network model Architecture class instance. """ + # Obtain interface + if config["path_gt"] is None : config["interface"] = "directory" + else : config["interface"] = "csv" # Peak into the dataset via the input interface ds = input_interface(config["interface"], config["path_imagedir"], - path_data=config["path_data"], + path_data=config["path_gt"], training=True, ohe=config["ohe"], image_format=None) (index_list, class_ohe, class_n, class_names, image_format) = ds # Create output directory - if not os.path.exists(config["output"]) : os.mkdir(config["output"]) + if not os.path.exists(config["path_modeldir"]): + os.mkdir(config["path_modeldir"]) # Identify task (multi-class vs multi-label) if np.sum(class_ohe) > class_ohe.shape[0] : config["multi_label"] = True @@ -95,20 +98,21 @@ def block_train(config): # Store meta information config["class_names"] = class_names - path_meta = os.path.join(config["output"], "meta.training.json") + path_meta = os.path.join(config["path_modeldir"], "meta.training.json") with open(path_meta, "w") as json_io: json.dump(config, json_io) # Define Callbacks callbacks = [] if config["analysis"] == "standard": - cb_loss = ModelCheckpoint(os.path.join(config["output"], + cb_loss = ModelCheckpoint(os.path.join(config["path_modeldir"], "model.best_loss.hdf5"), monitor="val_loss", verbose=1, save_best_only=True) callbacks.append(cb_loss) if config["analysis"] in ["minimal", "standard"]: - cb_cl = CSVLogger(os.path.join(config["output"], "logs.training.csv"), + cb_cl = CSVLogger(os.path.join(config["path_modeldir"], + "logs.training.csv"), separator=',', append=True) callbacks.append(cb_cl) if config["analysis"] != "minimal": @@ -215,7 +219,7 @@ def block_train(config): # Start model training hist = model.train(training_generator=train_gen, **paras_train) # Store model - path_model = os.path.join(config["output"], "model.last.hdf5") + path_model = os.path.join(config["path_modeldir"], "model.last.hdf5") model.dump(path_model) elif config["analysis"] == "standard": # Setup neural network @@ -248,7 +252,7 @@ def block_train(config): validation_generator=val_gen, **paras_train) # Store model - path_model = os.path.join(config["output"], "model.last.hdf5") + path_model = os.path.join(config["path_modeldir"], "model.last.hdf5") model.dump(path_model) else: # Sanity check of architecutre config @@ -275,7 +279,7 @@ def block_train(config): # Start model training hist = el.train(training_generator=train_gen, **paras_train) # Store model directory - el.dump(config["output"]) + el.dump(config["path_modeldir"]) # Plot fitting history - evaluate_fitting(train_history=hist, out_path=config["output"]) + evaluate_fitting(train_history=hist, out_path=config["path_modeldir"]) diff --git a/aucmedi/automl/cli.py b/aucmedi/automl/cli.py index 1c829f2b..0e8b3a8a 100644 --- a/aucmedi/automl/cli.py +++ b/aucmedi/automl/cli.py @@ -82,42 +82,16 @@ def cli_training(subparsers): # Setup SubParser parser_train = subparsers.add_parser("training", help=desc, add_help=False) - # Add required configuration arguments - ra = parser_train.add_argument_group("required arguments") - ra.add_argument("--interface", - type=str, - required=True, - choices=["csv", "directory"], - help="String defining format interface for loading/storing"\ - + " data", - ) - ra.add_argument("--path_imagedir", - type=str, - required=True, - help="Path to the directory containing the images", - ) - - # Add optional configuration arguments - oa = parser_train.add_argument_group("optional arguments") - oa.add_argument("-h", - "--help", - action="help", - help="show this help message and exit") - oa.add_argument("--analysis", + # Add IO arguments + od = parser_train.add_argument_group("Arguments - I/O") + od.add_argument("--path_imagedir", type=str, required=False, - default="standard", - choices=["minimal", "standard", "advanced"], - help="Analysis mode for the AutoML training " + \ + default="training", + help="Path to the directory containing the images " + \ "(default: '%(default)s')", ) - oa.add_argument("--path_data", - type=str, - required=False, - help="Path to the index/class annotation CSV file " + \ - "(only required for interface 'csv')", - ) - oa.add_argument("--path_output", + od.add_argument("--path_modeldir", type=str, required=False, default="model", @@ -125,23 +99,43 @@ def cli_training(subparsers): "models and metadata are stored " + \ "(default: '%(default)s')", ) - oa.add_argument("--ohe", + od.add_argument("--path_gt", + type=str, + required=False, + help="Path to the index/class annotation CSV file " + \ + "(only required for defining the ground truth via " + \ + "'csv' instead of 'directory' interface)", + ) + + od.add_argument("--ohe", action="store_true", required=False, default=False, help="Boolean option whether annotation data is sparse " + \ "categorical or one-hot encoded " + \ - "(only required for interface 'csv', " + \ + "(only required for interface 'csv' and multi-" + \ + "label data, " + \ "default: '%(default)s')", ) - oa.add_argument("--three_dim", + + # Add configuration arguments + oc = parser_train.add_argument_group("Arguments - Configuration") + oc.add_argument("--analysis", + type=str, + required=False, + default="standard", + choices=["minimal", "standard", "advanced"], + help="Analysis mode for the AutoML training " + \ + "(default: '%(default)s')", + ) + oc.add_argument("--three_dim", action="store_true", required=False, default=False, help="Boolean, whether imaging data is 2D or 3D " + \ "(default: '%(default)s')", ) - oa.add_argument("--shape_3D", + oc.add_argument("--shape_3D", type=str, required=False, default="128x128x128", @@ -150,7 +144,7 @@ def cli_training(subparsers): "format: '1x2x3', " + \ "default: '%(default)s')", ) - oa.add_argument("--epochs", + oc.add_argument("--epochs", type=int, required=False, default=500, @@ -158,14 +152,14 @@ def cli_training(subparsers): "one iteration through the complete data set " + \ "(default: '%(default)s')", ) - oa.add_argument("--batch_size", + oc.add_argument("--batch_size", type=int, required=False, default=24, help="Number of samples inside a single batch " + \ "(default: '%(default)s')", ) - oa.add_argument("--workers", + oc.add_argument("--workers", type=int, required=False, default=1, @@ -173,14 +167,14 @@ def cli_training(subparsers): "batches during runtime " + \ "(default: '%(default)s')", ) - oa.add_argument("--metalearner", + oc.add_argument("--metalearner", type=str, required=False, default="mean", help="Key for Metalearner or Aggregate function "+ \ "(default: '%(default)s')", ) - oa.add_argument("--architecture", + oc.add_argument("--architecture", type=str, required=False, default="DenseNet121", @@ -191,6 +185,13 @@ def cli_training(subparsers): "default: '%(default)s')", ) + # Add other arguments + oo = parser_train.add_argument_group("Arguments - Other") + oo.add_argument("-h", + "--help", + action="help", + help="show this help message and exit") + # Help page hook for passing no parameters if len(sys.argv)==2 and sys.argv[1] == "training": parser_train.print_help(sys.stderr) @@ -200,50 +201,48 @@ def cli_training(subparsers): # CLI - Prediction # #-----------------------------------------------------# def cli_prediction(subparsers): - # Set description for cli training + # Set description for cli prediction desc = """ Pipeline hub for Inference via AUCMEDI AutoML """ # Setup SubParser parser_predict = subparsers.add_parser("prediction", help=desc, add_help=False) - # Add required configuration arguments - ra = parser_predict.add_argument_group("required arguments") - ra.add_argument("--path_imagedir", + # Add IO arguments + od = parser_predict.add_argument_group("Arguments - I/O") + od.add_argument("--path_imagedir", type=str, - required=True, - help="Path to the directory containing the images", + required=False, + default="test", + help="Path to the directory containing the images " + \ + "(default: '%(default)s')", ) - - # Add optional configuration arguments - oa = parser_predict.add_argument_group("optional arguments") - oa.add_argument("-h", - "--help", - action="help", - help="show this help message and exit") - oa.add_argument("--path_input", + od.add_argument("--path_modeldir", type=str, required=False, default="model", - help="Path to the input directory in which fitted " + \ - "models and metadata are stored " + \ + help="Path to the model directory in which fitted " + \ + "model weights and metadata are stored " + \ "(default: '%(default)s')", ) - oa.add_argument("--path_output", + od.add_argument("--path_pred", type=str, required=False, - default="predictions.csv", + default="preds.csv", help="Path to the output file in which predicted csv " + \ "file should be stored " + \ "(default: '%(default)s')", ) - oa.add_argument("--xai_method", + + # Add configuration arguments + oc = parser_predict.add_argument_group("Arguments - Configuration") + oc.add_argument("--xai_method", type=str, required=False, help="Key for XAI method " + \ "(default: '%(default)s')", ) - oa.add_argument("--xai_directory", + oc.add_argument("--xai_directory", type=str, required=False, default="xai", @@ -251,14 +250,14 @@ def cli_prediction(subparsers): "image xai heatmaps should be stored " + \ "(default: '%(default)s')", ) - oa.add_argument("--batch_size", + oc.add_argument("--batch_size", type=int, required=False, default=12, help="Number of samples inside a single batch " + \ "(default: '%(default)s')", ) - oa.add_argument("--workers", + oc.add_argument("--workers", type=int, required=False, default=1, @@ -267,6 +266,13 @@ def cli_prediction(subparsers): "(default: '%(default)s')", ) + # Add other arguments + oo = parser_predict.add_argument_group("Arguments - Other") + oo.add_argument("-h", + "--help", + action="help", + help="show this help message and exit") + # Help page hook for passing no parameters if len(sys.argv)==2 and sys.argv[1] == "prediction": parser_predict.print_help(sys.stderr) @@ -276,59 +282,49 @@ def cli_prediction(subparsers): # CLI - Evaluation # #-----------------------------------------------------# def cli_evaluation(subparsers): - # Set description for cli training + # Set description for cli evaluation desc = """ Pipeline hub for Evaluation via AUCMEDI AutoML """ # Setup SubParser parser_evaluate = subparsers.add_parser("evaluation", help=desc, add_help=False) - # Add required configuration arguments - ra = parser_evaluate.add_argument_group("required arguments") - ra.add_argument("--interface", - type=str, - required=True, - choices=["csv", "directory"], - help="String defining format interface for loading/storing"\ - + " data", - ) - ra.add_argument("--path_imagedir", + # Add IO arguments + od = parser_evaluate.add_argument_group("Arguments - I/O") + od.add_argument("--path_imagedir", type=str, - required=True, + required=False, + default="training", help="Path to the directory containing the ground truth" + \ - " images", + " images " + \ + "(default: '%(default)s')", ) - - # Add optional configuration arguments - oa = parser_evaluate.add_argument_group("optional arguments") - oa.add_argument("-h", - "--help", - action="help", - help="show this help message and exit") - oa.add_argument("--path_data", + od.add_argument("--path_gt", type=str, required=False, help="Path to the index/class annotation CSV file " + \ - "(only required for interface 'csv')", + "(only required for defining the ground truth via " + \ + "'csv' instead of 'directory' interface)", ) - oa.add_argument("--ohe", + od.add_argument("--ohe", action="store_true", required=False, default=False, help="Boolean option whether annotation data is sparse " + \ "categorical or one-hot encoded " + \ - "(only required for interface 'csv', " + \ + "(only required for interface 'csv' and multi-" + \ + "label data, " + \ "default: '%(default)s')", ) - oa.add_argument("--path_input", + od.add_argument("--path_pred", type=str, required=False, - default="predictions.csv", + default="preds.csv", help="Path to the output file in which predicted csv " + \ "file are stored " + \ "(default: '%(default)s')", ) - oa.add_argument("--path_output", + od.add_argument("--path_evaldir", type=str, required=False, default="evaluation", @@ -337,6 +333,13 @@ def cli_evaluation(subparsers): "(default: '%(default)s')", ) + # Add other arguments + oo = parser_evaluate.add_argument_group("Arguments - Other") + oo.add_argument("-h", + "--help", + action="help", + help="show this help message and exit") + # Help page hook for passing no parameters if len(sys.argv)==2 and sys.argv[1] == "evaluation": parser_evaluate.print_help(sys.stderr) diff --git a/aucmedi/automl/parser_cli.py b/aucmedi/automl/parser_cli.py index a5b86b2e..1fd0e9a2 100644 --- a/aucmedi/automl/parser_cli.py +++ b/aucmedi/automl/parser_cli.py @@ -38,13 +38,6 @@ def parse_cli(args): # Handle architecture list if "," in config["architecture"]: config["architecture"] = config["architecture"].split(",") - else: - # Handle input path - config["input"] = config["path_input"] - del config["path_input"] - # Handle output path - config["output"] = config["path_output"] - del config["path_output"] # Return valid configs return config diff --git a/docs/getstarted/usage_automl.md b/docs/getstarted/usage_automl.md index 55e4215e..6bb23f53 100644 --- a/docs/getstarted/usage_automl.md +++ b/docs/getstarted/usage_automl.md @@ -1,3 +1,14 @@ # work in progress Coming soon. + + +``` +working_dir/ + training/ + test/ + model/ + evaluation/ + metadata.json + preds.csv +``` diff --git a/tests/test_automl_block_eval.py b/tests/test_automl_block_eval.py index e0a77b6f..40cb1e14 100644 --- a/tests/test_automl_block_eval.py +++ b/tests/test_automl_block_eval.py @@ -68,13 +68,12 @@ def setUpClass(self): suffix=".train") # Define config config = { - "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": self.model_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": self.model_dir.name, "analysis": "minimal", "ohe": False, - "two_dim": True, + "three_dim": False, "epochs": 8, "batch_size": 4, "workers": 1, @@ -89,8 +88,8 @@ def setUpClass(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": self.model_dir.name, - "output": self.pred_path.name, + "path_modeldir": self.model_dir.name, + "path_pred": self.pred_path.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -105,11 +104,10 @@ def setUpClass(self): def test_eval_performance(self): # Define config config = { - "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "input": self.pred_path.name, - "output": self.model_dir.name, + "path_gt": self.tmp_csv.name, + "path_pred": self.pred_path.name, + "path_evaldir": self.model_dir.name, "ohe": False, } # Run evaluation block diff --git a/tests/test_automl_block_pred.py b/tests/test_automl_block_pred.py index a41eb952..c08ad502 100644 --- a/tests/test_automl_block_pred.py +++ b/tests/test_automl_block_pred.py @@ -102,8 +102,8 @@ def test_minimal(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "minimal", "ohe": False, "three_dim": False, @@ -123,8 +123,8 @@ def test_minimal(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -151,8 +151,8 @@ def test_minimal_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": input_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": input_dir.name, "analysis": "minimal", "ohe": True, "three_dim": False, @@ -171,8 +171,8 @@ def test_minimal_multilabel(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -195,8 +195,8 @@ def test_minimal_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "minimal", "ohe": False, "three_dim": True, @@ -216,8 +216,8 @@ def test_minimal_3D(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data3D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -240,8 +240,8 @@ def test_minimal_xai(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "minimal", "ohe": False, "three_dim": False, @@ -263,8 +263,8 @@ def test_minimal_xai(self): suffix=".xai") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": "gradcam", @@ -296,8 +296,8 @@ def test_standard(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "standard", "ohe": False, "three_dim": False, @@ -316,8 +316,8 @@ def test_standard(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -340,8 +340,8 @@ def test_standard_xai(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "standard", "ohe": False, "three_dim": False, @@ -362,8 +362,8 @@ def test_standard_xai(self): suffix=".xai") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": "gradcam", @@ -388,8 +388,8 @@ def test_standard_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": input_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": input_dir.name, "analysis": "standard", "ohe": True, "three_dim": False, @@ -407,8 +407,8 @@ def test_standard_multilabel(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -431,8 +431,8 @@ def test_standard_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "standard", "ohe": False, "three_dim": True, @@ -452,8 +452,8 @@ def test_standard_3D(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data3D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -479,8 +479,8 @@ def test_composite(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "advanced", "ohe": False, "three_dim": False, @@ -499,8 +499,8 @@ def test_composite(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -523,8 +523,8 @@ def test_composite_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": input_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": input_dir.name, "analysis": "advanced", "ohe": True, "three_dim": False, @@ -543,8 +543,8 @@ def test_composite_multilabel(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data2D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, @@ -567,8 +567,8 @@ def test_composite_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": input_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": input_dir.name, "analysis": "advanced", "ohe": False, "three_dim": True, @@ -588,8 +588,8 @@ def test_composite_3D(self): suffix=".pred.csv") config = { "path_imagedir": self.tmp_data3D.name, - "input": input_dir.name, - "output": tmp_output.name, + "path_modeldir": input_dir.name, + "path_pred": tmp_output.name, "batch_size": 4, "workers": 1, "xai_method": None, diff --git a/tests/test_automl_block_train.py b/tests/test_automl_block_train.py index fbfff8c6..eb1ff66d 100644 --- a/tests/test_automl_block_train.py +++ b/tests/test_automl_block_train.py @@ -101,8 +101,8 @@ def test_minimal(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "minimal", "ohe": False, "three_dim": False, @@ -129,8 +129,8 @@ def test_minimal_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": output_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": output_dir.name, "analysis": "minimal", "ohe": True, "three_dim": False, @@ -157,8 +157,8 @@ def test_minimal_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "minimal", "ohe": False, "three_dim": True, @@ -188,8 +188,8 @@ def test_standard(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "standard", "ohe": False, "three_dim": False, @@ -216,8 +216,8 @@ def test_standard_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": output_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": output_dir.name, "analysis": "standard", "ohe": True, "three_dim": False, @@ -244,8 +244,8 @@ def test_standard_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "standard", "ohe": False, "three_dim": True, @@ -276,8 +276,8 @@ def test_composite(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "advanced", "ohe": False, "three_dim": False, @@ -300,8 +300,8 @@ def test_composite_multilabel(self): config = { "interface": "csv", "path_imagedir": self.tmp_data2D.name, - "path_data": self.tmp_csv_ohe.name, - "output": output_dir.name, + "path_gt": self.tmp_csv_ohe.name, + "path_modeldir": output_dir.name, "analysis": "advanced", "ohe": True, "three_dim": False, @@ -324,8 +324,8 @@ def test_composite_3D(self): config = { "interface": "csv", "path_imagedir": self.tmp_data3D.name, - "path_data": self.tmp_csv.name, - "output": output_dir.name, + "path_gt": self.tmp_csv.name, + "path_modeldir": output_dir.name, "analysis": "advanced", "ohe": False, "three_dim": True, diff --git a/tests/test_automl_cli.py b/tests/test_automl_cli.py index b9c96a40..f6d4b6ca 100644 --- a/tests/test_automl_cli.py +++ b/tests/test_automl_cli.py @@ -95,29 +95,26 @@ def test_training_empty(self): def test_training_functionality(self): args = ["aucmedi", "training"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] - args_optional = ["--epochs", "1", - "--architecture", "Vanilla", - "--path_output", self.tmp_model.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", self.tmp_data.name, + "--epochs", "1", + "--architecture", "Vanilla", + "--path_modeldir", self.tmp_model.name] + with patch.object(sys, "argv", args + args_config): main() def test_training_args(self): args = ["aucmedi", "training"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] + args_config = ["--path_imagedir", self.tmp_data.name] # Build and run CLI functions - with patch.object(sys, "argv", args + args_required): + with patch.object(sys, "argv", args + args_config): parser, subparsers = cli_core() cli_training(subparsers) args = parser.parse_args() config_cli = parse_cli(args) # Define possible config parameters - config_map = ["interface", - "path_imagedir", - "path_data", - "output", + config_map = ["path_imagedir", + "path_gt", + "path_modeldir", "analysis", "ohe", "three_dim", @@ -145,39 +142,38 @@ def test_prediction_empty(self): def test_prediction_functionality(self): # Training args = ["aucmedi", "training"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] - args_optional = ["--epochs", "1", - "--architecture", "Vanilla", - "--path_output", self.tmp_model.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", self.tmp_data.name, + "--epochs", "1", + "--architecture", "Vanilla", + "--path_modeldir", self.tmp_model.name] + with patch.object(sys, "argv", args + args_config): main() # Prediction tmp_output = tempfile.NamedTemporaryFile(mode="w", prefix="tmp.aucmedi.", suffix=".pred.csv") args = ["aucmedi", "prediction"] - args_required = ["--path_imagedir", os.path.join(self.tmp_data.name, - "class_0")] - args_optional = ["--path_input", self.tmp_model.name, - "--path_output", tmp_output.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", os.path.join(self.tmp_data.name, + "class_0"), + "--path_modeldir", self.tmp_model.name, + "--path_pred", tmp_output.name] + with patch.object(sys, "argv", args + args_config): main() def test_prediction_args(self): args = ["aucmedi", "prediction"] - args_required = ["--path_imagedir", os.path.join(self.tmp_data.name, + args_config = ["--path_imagedir", os.path.join(self.tmp_data.name, "class_0")] # Build and run CLI functions - with patch.object(sys, "argv", args + args_required): + with patch.object(sys, "argv", args + args_config): parser, subparsers = cli_core() cli_prediction(subparsers) args = parser.parse_args() config_cli = parse_cli(args) # Define possible config parameters config_map = ["path_imagedir", - "input", - "output", + "path_modeldir", + "path_pred", "xai_method", "xai_directory", "batch_size", @@ -200,49 +196,46 @@ def test_evaluation_empty(self): def test_evaluation_functionality(self): # Training args = ["aucmedi", "training"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] - args_optional = ["--epochs", "1", - "--architecture", "Vanilla", - "--path_output", self.tmp_model.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", self.tmp_data.name, + "--epochs", "1", + "--architecture", "Vanilla", + "--path_modeldir", self.tmp_model.name] + with patch.object(sys, "argv", args + args_config): main() # Prediction tmp_output = tempfile.NamedTemporaryFile(mode="w", prefix="tmp.aucmedi.", suffix=".pred.csv") args = ["aucmedi", "prediction"] - args_required = ["--path_imagedir", os.path.join(self.tmp_data.name, - "class_0")] - args_optional = ["--path_input", self.tmp_model.name, - "--path_output", tmp_output.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", os.path.join(self.tmp_data.name, + "class_0"), + "--path_modeldir", self.tmp_model.name, + "--path_pred", tmp_output.name] + with patch.object(sys, "argv", args + args_config): main() # Evaluation tmp_eval = tempfile.TemporaryDirectory(prefix="tmp.aucmedi.", suffix=".eval") args = ["aucmedi", "evaluation"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] - args_optional = ["--path_input", tmp_output.name, - "--path_output", tmp_eval.name] - with patch.object(sys, "argv", args + args_required + args_optional): + args_config = ["--path_imagedir", self.tmp_data.name, + "--path_pred", tmp_output.name, + "--path_evaldir", tmp_eval.name] + with patch.object(sys, "argv", args + args_config): main() def test_evaluation_args(self): args = ["aucmedi", "evaluation"] - args_required = ["--interface", "directory", - "--path_imagedir", self.tmp_data.name] + args_config = ["--path_imagedir", self.tmp_data.name] # Build and run CLI functions - with patch.object(sys, "argv", args + args_required): + with patch.object(sys, "argv", args + args_config): parser, subparsers = cli_core() cli_evaluation(subparsers) args = parser.parse_args() config_cli = parse_cli(args) # Define possible config parameters - config_map = ["path_data", - "input", - "output", + config_map = ["path_gt", + "path_pred", + "path_evaldir", "ohe", ] # Check existence