Skip to content

Commit

Permalink
Update OTX explain CLI arguments (#2671)
Browse files Browse the repository at this point in the history
* Change int8 to uint8 to XAI tests

* Add probabilities for CLI demo

* Rename arguments for explain

* Fix pre-commit

* Remove extra changes

* Fix integration tests

* Fix integration "explain_all_classes" test for OV
  • Loading branch information
GalyaZalesskaya authored Nov 28, 2023
1 parent 0fed1ac commit de0f5ae
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 51 deletions.
16 changes: 8 additions & 8 deletions docs/source/guide/get_started/cli_commands.rst
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ With the ``--help`` command, you can list additional information, such as its pa
.. code-block::
(otx) ...$ otx explain --help
usage: otx explain [-h] --explain-data-roots EXPLAIN_DATA_ROOTS [--save-explanation-to SAVE_EXPLANATION] --load-weights LOAD_WEIGHTS [--explain-algorithm EXPLAIN_ALGORITHM] [--overlay-weight OVERLAY_WEIGHT] [template] {params} ...
usage: otx explain [-h] --input INPUT [--output OUTPUT] --load-weights LOAD_WEIGHTS [--explain-algorithm EXPLAIN_ALGORITHM] [--overlay-weight OVERLAY_WEIGHT] [template] {params} ...
positional arguments:
template Enter the path or ID or name of the template file.
Expand All @@ -459,9 +459,9 @@ With the ``--help`` command, you can list additional information, such as its pa
optional arguments:
-h, --help show this help message and exit
--explain-data-roots EXPLAIN_DATA_ROOTS
-i INPUT, --input INPUT
Comma-separated paths to explain data folders.
--save-explanation-to SAVE_EXPLANATION_TO
-o OUTPUT, --output OUTPUT
Output path for explanation images.
--load-weights LOAD_WEIGHTS
Load model weights from previously saved checkpoint.
Expand All @@ -475,13 +475,13 @@ With the ``--help`` command, you can list additional information, such as its pa
Weight of the saliency map when overlaying the input image with saliency map.
The command below will generate saliency maps (heatmaps with red colored areas of focus) of the trained model on the provided dataset and save the resulting images to ``save-explanation-to`` path:
The command below will generate saliency maps (heatmaps with red colored areas of focus) of the trained model on the provided dataset and save the resulting images to ``output`` path:

.. code-block::
(otx) ...$ otx explain SSD --explain-data-roots <path/to/explain/root> \
(otx) ...$ otx explain SSD --input <path/to/explain/root> \
--load-weights <path/to/model_weights> \
--save-explanation-to <path/to/output/root> \
--output <path/to/output/root> \
--explain-algorithm classwisesaliencymap \
--overlay-weight 0.5
Expand All @@ -496,9 +496,9 @@ By default, the model is exported to the OpenVINO™ IR format without extra fea
(otx) ...$ otx export SSD --load-weights <path/to/trained/weights.pth> \
--output outputs/openvino/with_features \
--dump-features
(otx) ...$ otx explain SSD --explain-data-roots <path/to/explain/root> \
(otx) ...$ otx explain SSD --input <path/to/explain/root> \
--load-weights outputs/openvino/with_features \
--save-explanation-to <path/to/output/root> \
--output <path/to/output/root> \
--explain-algorithm classwisesaliencymap \
--overlay-weight 0.5
Expand Down
8 changes: 4 additions & 4 deletions docs/source/guide/tutorials/base/explain.rst
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ created in the previous step.
. venv/otx/bin/activate
2. ``otx explain`` returns saliency maps (heatmaps with red colored areas of focus)
at the path specified by ``--save-explanation-to``.
at the path specified by ``--output``.

.. code-block::
otx explain --explain-data-roots otx-workspace-DETECTION/splitted_dataset/val/ \
--save-explanation-to outputs/explanation \
otx explain --input otx-workspace-DETECTION/splitted_dataset/val/ \
--output outputs/explanation \
--load-weights outputs/weights.pth
3. To specify the algorithm of saliency map creation for classification,
Expand All @@ -48,7 +48,7 @@ For detection task, we can choose between the following methods:


4. As a result we will get a folder with a pair of generated
images for each image in ``--explain-data-roots``:
images for each image in ``--input``:

- saliency map - where red color means more attention of the model
- overlay - where the saliency map is combined with the original image:
Expand Down
19 changes: 9 additions & 10 deletions src/otx/cli/tools/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,14 @@ def get_args():
parser, hyper_parameters, params = get_parser_and_hprams_data()

parser.add_argument(
"--explain-data-roots",
"-i",
"--input",
required=True,
help="Comma-separated paths to explain data folders.",
)
parser.add_argument(
"--save-explanation-to",
"-o",
"--output",
default="saliency_dump",
help="Output path for explanation images.",
)
Expand Down Expand Up @@ -123,10 +125,7 @@ def _log_after_saving(explain_predicted_classes, explained_image_counter, args,
"Please adjust training pipeline or use different model-data pair."
)
if explained_image_counter > 0:
logger.info(
f"Saliency maps saved to {args.save_explanation_to} for {explained_image_counter} "
f"out of {num_images} images."
)
logger.info(f"Saliency maps saved to {args.output} for {explained_image_counter} out of {num_images} images.")


def main():
Expand Down Expand Up @@ -169,10 +168,10 @@ def main():
f"{args.explain_algorithm} currently not supported. \
Currently only support {SUPPORTED_EXPLAIN_ALGORITHMS}"
)
if not Path(args.save_explanation_to).exists():
Path(args.save_explanation_to).mkdir(parents=True)
if not Path(args.output).exists():
Path(args.output).mkdir(parents=True)

image_files = get_image_files(args.explain_data_roots)
image_files = get_image_files(args.input)
dataset_to_explain = get_explain_dataset_from_filelist(image_files)
explain_predicted_classes = not args.explain_all_classes
explain_parameters = ExplainParameters(
Expand Down Expand Up @@ -201,7 +200,7 @@ def main():
process_saliency_maps=explain_parameters.process_saliency_maps,
img=explained_data.numpy,
saliency_map=saliency_data.numpy,
save_dir=args.save_explanation_to,
save_dir=args.output,
fname=fname,
weight=args.overlay_weight,
)
Expand Down
6 changes: 3 additions & 3 deletions src/otx/cli/tools/utils/demo/visualization.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def draw_masks(frame: Mat, predictions, put_object_count: bool = False):
cv2.drawContours(frame, contours, -1, color, 1)
rect = cv2.boundingRect(contours[0])
cv2.rectangle(frame, (rect[0], rect[1]), (rect[0] + rect[2], rect[1] + rect[3]), color, 1)
put_text_on_rect_bg(frame, label.name, (rect[0], rect[1]), color=color)
put_text_on_rect_bg(frame, f"{label.name} {label.probability*100:.1f}%", (rect[0], rect[1]), color=color)
cv2.bitwise_or(aggregated_mask, mask, dst=aggregated_mask)
cv2.bitwise_or(
aggregated_colored_mask,
Expand Down Expand Up @@ -110,7 +110,7 @@ def put_labels(frame: Mat, predictions: List[Annotation]):
assert len(predictions[0].get_labels()) == 1
label = predictions[0].get_labels()[0]
color = tuple(getattr(label.color, x) for x in ("blue", "green", "red"))
put_text_on_rect_bg(frame, label.name, (0, 0), color=color)
put_text_on_rect_bg(frame, f"{label.name} {label.probability*100:.1f}%", (0, 0), color=color)
return frame


Expand All @@ -129,7 +129,7 @@ def draw_bounding_boxes(frame: Mat, predictions: List[Annotation], put_object_co
label = prediction.get_labels()[0]
color = tuple(getattr(label.color, x) for x in ("blue", "green", "red"))
cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness=2)
put_text_on_rect_bg(frame, label.name, (x1, y1), color=color)
put_text_on_rect_bg(frame, f"{label.name} {label.probability*100:.1f}%", (x1, y1), color=color)
else:
warn(
f"Predictions called on Annotations with shape {type(prediction.shape)}."
Expand Down
52 changes: 26 additions & 26 deletions tests/test_suite/run_test_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -859,16 +859,16 @@ def otx_explain_testing(template, root, otx_dir, args, trained=False):

save_dir = f"explain_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand Down Expand Up @@ -899,16 +899,16 @@ def otx_explain_testing_all_classes(template, root, otx_dir, args):

save_dir = f"explain_all_classes_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand All @@ -923,7 +923,7 @@ def otx_explain_testing_all_classes(template, root, otx_dir, args):
assert len(os.listdir(output_dir)) == len(os.listdir(output_dir_explain_only_predicted_classes))
else:
assert len(os.listdir(output_dir)) >= len(os.listdir(output_dir_explain_only_predicted_classes))
assert all([os.path.splitext(fname)[1] == ".tiff" for fname in os.listdir(output_dir)])
assert all([os.path.splitext(fname)[1] in [".tiff", ".log"] for fname in os.listdir(output_dir)])


def otx_explain_testing_process_saliency_maps(template, root, otx_dir, args, trained=False):
Expand All @@ -945,16 +945,16 @@ def otx_explain_testing_process_saliency_maps(template, root, otx_dir, args, tra

save_dir = f"explain_process_saliency_maps_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand Down Expand Up @@ -986,16 +986,16 @@ def otx_explain_openvino_testing(template, root, otx_dir, args, trained=False):

save_dir = f"explain_ov_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand Down Expand Up @@ -1027,16 +1027,16 @@ def otx_explain_all_classes_openvino_testing(template, root, otx_dir, args):

save_dir = f"explain_ov_all_classes_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand All @@ -1052,7 +1052,7 @@ def otx_explain_all_classes_openvino_testing(template, root, otx_dir, args):
assert len(os.listdir(output_dir)) == len(os.listdir(output_dir_explain_only_predicted_classes))
else:
assert len(os.listdir(output_dir)) >= len(os.listdir(output_dir_explain_only_predicted_classes))
assert all([os.path.splitext(fname)[1] == ".tiff" for fname in os.listdir(output_dir)])
assert all([os.path.splitext(fname)[1] in [".tiff", ".log"] for fname in os.listdir(output_dir)])


def otx_explain_process_saliency_maps_openvino_testing(template, root, otx_dir, args, trained=False):
Expand All @@ -1074,16 +1074,16 @@ def otx_explain_process_saliency_maps_openvino_testing(template, root, otx_dir,

save_dir = f"explain_ov_process_saliency_maps_{template.model_template_id}/{test_algorithm}/{train_type}/"
output_dir = os.path.join(template_work_dir, save_dir)
explain_data_root = os.path.join(otx_dir, args["--input"])
data_input = os.path.join(otx_dir, args["--input"])
command_line = [
"otx",
"explain",
template.model_template_path,
"--load-weights",
weights_path,
"--explain-data-root",
explain_data_root,
"--save-explanation-to",
"--input",
data_input,
"--output",
output_dir,
"--explain-algorithm",
test_algorithm,
Expand Down

0 comments on commit de0f5ae

Please sign in to comment.