From f3c2ffbece7cc086dc120daa297ee1e60a11672f Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Thu, 9 Mar 2023 01:55:57 +0800 Subject: [PATCH 01/12] ObjectDetectionWOnnx --- examples/build.gradle | 1 + .../examples/inference/MaskDetectionOnnx.java | 93 +++++++++++++++++++ .../examples/inference/ObjectDetection.java | 6 +- 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java diff --git a/examples/build.gradle b/examples/build.gradle index 67e84f09044..52b20567fb0 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -32,6 +32,7 @@ dependencies { runtimeOnly project(":engines:pytorch:pytorch-model-zoo") runtimeOnly project(":engines:tensorflow:tensorflow-model-zoo") runtimeOnly project(":engines:mxnet:mxnet-model-zoo") + runtimeOnly project(":engines:onnxruntime:onnxruntime-engine") testImplementation("org.testng:testng:${testng_version}") { exclude group: "junit", module: "junit" diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java new file mode 100644 index 00000000000..9d1f855df50 --- /dev/null +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java @@ -0,0 +1,93 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.examples.inference; + +import ai.djl.Application; +import ai.djl.ModelException; +import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * An example of inference using an object detection model. + * + *

See this doc + * for information about this example. + */ +public final class MaskDetectionOnnx { + + private static final Logger logger = LoggerFactory.getLogger(MaskDetectionOnnx.class); + + private MaskDetectionOnnx() {} + + public static void main(String[] args) throws IOException, ModelException, TranslateException { + DetectedObjects detection = MaskDetectionOnnx.predict(); + logger.info("{}", detection); + } + + public static DetectedObjects predict() throws IOException, ModelException, TranslateException { + Path imageFile = Paths.get("/Users/fenkexin/Desktop/djl/examples/src/test/resources/dog_bike_car.jpg"); + Image img = ImageFactory.getInstance().fromFile(imageFile); + + String backbone; + if ("TensorFlow".equals(Engine.getDefaultEngineName())) { + backbone = "mobilenet_v2"; + } else { + backbone = "resnet50"; + } + + Criteria criteria = + Criteria.builder() + .optApplication(Application.CV.OBJECT_DETECTION) + .setTypes(Image.class, DetectedObjects.class) +// .optFilter("backbone", backbone) + .optEngine("OnnxRuntime") + .optProgress(new ProgressBar()) + .build(); + + try (ZooModel model = criteria.loadModel()) { + try (Predictor predictor = model.newPredictor()) { + DetectedObjects detection = predictor.predict(img); + saveBoundingBoxImage(img, detection); + return detection; + } + } + } + + private static void saveBoundingBoxImage(Image img, DetectedObjects detection) + throws IOException { + Path outputDir = Paths.get("build/output"); + Files.createDirectories(outputDir); + + img.drawBoundingBoxes(detection); + + Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); + // OpenJDK can't save jpg with alpha channel + img.save(Files.newOutputStream(imagePath), "png"); + logger.info("Detected objects image has been saved in: {}", imagePath); + } +} diff --git a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java index 08643cb3b96..fcbf81d753a 100644 --- a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java @@ -51,7 +51,7 @@ public static void main(String[] args) throws IOException, ModelException, Trans } public static DetectedObjects predict() throws IOException, ModelException, TranslateException { - Path imageFile = Paths.get("src/test/resources/dog_bike_car.jpg"); + Path imageFile = Paths.get("/Users/fenkexin/Desktop/djl/examples/src/test/resources/dog_bike_car.jpg"); Image img = ImageFactory.getInstance().fromFile(imageFile); String backbone; @@ -65,8 +65,8 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran Criteria.builder() .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) - .optFilter("backbone", backbone) - .optEngine(Engine.getDefaultEngineName()) +// .optFilter("backbone", backbone) + .optEngine("OnnxRuntime") .optProgress(new ProgressBar()) .build(); From a2055abd68c6d227249a54eb40435232b8eab608 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sat, 11 Mar 2023 02:44:52 +0800 Subject: [PATCH 02/12] utilities --- .../djl/modality/cv/BufferedImageFactory.java | 24 ++++++--- .../ai/djl/modality/cv/transform/OneHot.java | 2 +- examples/docs/mask_detection.md | 49 +++++++++++++++++++ examples/docs/object_detection.md | 20 ++++++++ examples/docs/train_transfer_fresh_fruit.md | 4 +- 5 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 examples/docs/mask_detection.md diff --git a/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java b/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java index d850b08a143..79f7248572d 100644 --- a/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java +++ b/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java @@ -39,6 +39,8 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import javax.imageio.ImageIO; @@ -302,23 +304,31 @@ public void drawBoundingBoxes(DetectedObjects detections) { g.setStroke(new BasicStroke(stroke)); g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); - int imageWidth = image.getWidth(); - int imageHeight = image.getHeight(); + float widthScale = (float) (image.getWidth() / 640.0); + float heightScale = (float) (image.getHeight() / 640.0); List list = detections.items(); + int k = 10; + Map classNumberTable = new ConcurrentHashMap<>(); for (DetectedObjects.DetectedObject result : list) { String className = result.getClassName(); BoundingBox box = result.getBoundingBox(); - g.setPaint(randomColor().darker()); + if (classNumberTable.containsKey(className)) { + g.setPaint(new Color(classNumberTable.get(className))); + } else { + g.setPaint(new Color(k)); + classNumberTable.put(className, k); + k = (k + 100) % 255; + } Rectangle rectangle = box.getBounds(); - int x = (int) (rectangle.getX() * imageWidth); - int y = (int) (rectangle.getY() * imageHeight); + int x = (int) (rectangle.getX() * widthScale); + int y = (int) (rectangle.getY() * heightScale); g.drawRect( x, y, - (int) (rectangle.getWidth() * imageWidth), - (int) (rectangle.getHeight() * imageHeight)); + (int) (rectangle.getWidth() * widthScale), + (int) (rectangle.getHeight() * heightScale)); drawText(g, className, x, y, stroke, 4); // If we have a mask instead of a plain rectangle, draw tha mask if (box instanceof Mask) { diff --git a/api/src/main/java/ai/djl/modality/cv/transform/OneHot.java b/api/src/main/java/ai/djl/modality/cv/transform/OneHot.java index 0e99c4603d8..e80152ae660 100644 --- a/api/src/main/java/ai/djl/modality/cv/transform/OneHot.java +++ b/api/src/main/java/ai/djl/modality/cv/transform/OneHot.java @@ -21,7 +21,7 @@ public class OneHot implements Transform { private int numClass; /** - * Creates a {@code toOneHot} {@link Transform} that converts the sparse label to one-hot label. + * Creates a {@code OneHot} {@link Transform} that converts the sparse label to one-hot label. * * @param numClass number of classes */ diff --git a/examples/docs/mask_detection.md b/examples/docs/mask_detection.md new file mode 100644 index 00000000000..16c814fcecc --- /dev/null +++ b/examples/docs/mask_detection.md @@ -0,0 +1,49 @@ +# Mask detection with YOLOv5 - training and inference + +[YOLOv5s]() is a powerful model for object detection tasks. With the transfer learning technique, a pre-trained YOLOv5 model can be utilized in various customized object detection tasks with relatively small dataset. + +In this example, we apply it on the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images). We first train the YOLOv5s model in Python, with the help of [ATLearn](), a python transfer learning toolkit. +Then, the model is saved as an ONNX model, which is then imported into DJL for inference. We apply it on the mask wearing detection task. The source code can be found at [MaskDetectionOnnx.java](https://github.com/deepjavalibrary/djl/blob/master/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java) + +## The training part in ATLearn + +We initially attempted to import a pretrained YOLOv5 into DJL, and fine-tune it with the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images), similar to [Train ResNet for Fruit Freshness Classficiation](./train_transfer_fresh_fruit.md). However, YOLOv5 can not be converted to a PyTorch traced model, due to its data-dependent execution flow (see this [ref](https://discuss.pytorch.org/t/yolov5-convert-to-torchscript/150180)), whick blocks the idea of retraining a Yolov5 model in DJL. So the training part is entirely in python. + +The retraining of YOLOv5 can be found in an example in ATLearn [example](). In this example, the YOLOv5 layers near the input are frozen while those near the output are fine-tuned with the customized data. This follows the transfer learning idea. + +In this example, the trained model is exported to ONNX file and is then also used for inference in python, which will serve as a benchmark. + +## Setup guide + +To configure your development environment, follow [setup](../../docs/development/setup.md). + +## Run mask detection example + +### Input image file +You can find the image used in this example in the project test resource folder: `src/test/resources/maksssksksss627.png`. + +![dogs](../src/test/resources/maksssksksss627.png) + +### Build the project and run +Use the following command to run the project: + +``` +cd examples +./gradlew run -Dai.djl.default_engine=PyTorch -Dmain=ai.djl.examples.inference.MaskDetectionOnnx +``` + +Your output should look like the following: + +```text +[INFO ] - Detected objects image has been saved in: /Users/fenkexin/Desktop/djl/examples/build/output/mask-wearing.png +[INFO ] - { + "w/o mask": 0.8998132944107056, + "w/ mask": 0.8930246829986572, + "w/ mask": 0.8708265423774719, + ... +} +``` + +An output image with bounding box will be saved as `build/output/detected-mask-wearing.png`: + +![detected-dogs](img/detected-mask-wearing.png) \ No newline at end of file diff --git a/examples/docs/object_detection.md b/examples/docs/object_detection.md index 8f01702733b..f654aa43ccc 100644 --- a/examples/docs/object_detection.md +++ b/examples/docs/object_detection.md @@ -43,3 +43,23 @@ Your output should look like the following: An output image with bounding box will be saved as build/output/detected-dog_bike_car.png: ![detected-dogs](img/detected-dog_bike_car.png) + +## Run object detection example with other engines +For objection detection application, other than the default model zoo with the default engine, we can also run it with other engines and model zoo. Here, we demonstrate with a pre-trained *YOLOV5s ONNX* model. + +First, the following line needs to be added into the dependencies in `build.gradle`. Otherwise, the ONNX model zoo won't be found. +```gradle + runtimeOnly project(":engines:onnxruntime:onnxruntime-engine") +``` + +Then use the following criteria +```java + Criteria criteria = + Criteria.builder() + .optApplication(Application.CV.OBJECT_DETECTION) + .setTypes(Image.class, DetectedObjects.class) + .optEngine("OnnxRuntime") + .optProgress(new ProgressBar()) + .build(); +``` +where the `optFilter` is removed and `optEngine` is specified. The rest would be the same. \ No newline at end of file diff --git a/examples/docs/train_transfer_fresh_fruit.md b/examples/docs/train_transfer_fresh_fruit.md index 1d6e7823703..7813968fbea 100644 --- a/examples/docs/train_transfer_fresh_fruit.md +++ b/examples/docs/train_transfer_fresh_fruit.md @@ -1,4 +1,4 @@ -# Blazing fast training with small dataset for Java applications +# Train ResNet for Fruit Freshness Classficiation Deep learning has shown its strong power in solving problems in various areas like CV, NLP, reinforcement learning, etc., which generates numerous examples of successful applications. @@ -205,7 +205,7 @@ Here, the data are preprocessed with the normalization and randomization functio commonly used for [image classification](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html). The randomization are for training only. -**Model training and export. ** Finally, we can run the model training with `Easytrain.fit`, +**Model training and export.** Finally, we can run the model training with `Easytrain.fit`, and save the model for prediction. In the end, the `model.close()` and `embedding.close()` are called. In DJL, during the creation of `Model` and `ZooModel`, the native resources (e.g., memories in the assigned in PyTorch) are allocated. These resources are managed From 1a27f8d36f6adf724d77c9e29112c41c3864ebe0 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sat, 11 Mar 2023 02:49:08 +0800 Subject: [PATCH 03/12] main --- ...kDetectionOnnx.java => MaskDetection.java} | 41 ++++++++++--------- .../examples/inference/ObjectDetection.java | 7 ++-- 2 files changed, 24 insertions(+), 24 deletions(-) rename examples/src/main/java/ai/djl/examples/inference/{MaskDetectionOnnx.java => MaskDetection.java} (70%) diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java similarity index 70% rename from examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java rename to examples/src/main/java/ai/djl/examples/inference/MaskDetection.java index 9d1f855df50..651f6527d07 100644 --- a/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance * with the License. A copy of the License is located at @@ -14,11 +14,11 @@ import ai.djl.Application; import ai.djl.ModelException; -import ai.djl.engine.Engine; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.translator.YoloV5TranslatorFactory; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ZooModel; import ai.djl.training.util.ProgressBar; @@ -30,62 +30,63 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * An example of inference using an object detection model. * *

See this doc + * href="https://github.com/deepjavalibrary/djl/blob/master/examples/docs/mask_detection.md">doc * for information about this example. */ -public final class MaskDetectionOnnx { +public final class MaskDetection { - private static final Logger logger = LoggerFactory.getLogger(MaskDetectionOnnx.class); + private static final Logger logger = LoggerFactory.getLogger(MaskDetection.class); - private MaskDetectionOnnx() {} + private MaskDetection() {} public static void main(String[] args) throws IOException, ModelException, TranslateException { - DetectedObjects detection = MaskDetectionOnnx.predict(); + DetectedObjects detection = MaskDetection.predict(); logger.info("{}", detection); } public static DetectedObjects predict() throws IOException, ModelException, TranslateException { - Path imageFile = Paths.get("/Users/fenkexin/Desktop/djl/examples/src/test/resources/dog_bike_car.jpg"); + Path imageFile = Paths.get("src/test/resources/maksssksksss627.png"); Image img = ImageFactory.getInstance().fromFile(imageFile); - String backbone; - if ("TensorFlow".equals(Engine.getDefaultEngineName())) { - backbone = "mobilenet_v2"; - } else { - backbone = "resnet50"; - } + String modelPath = "src/test/resources/mask.onnx"; + Map arguments = new ConcurrentHashMap<>(); + arguments.put("translatorFactory", YoloV5TranslatorFactory.class.getName()); Criteria criteria = Criteria.builder() .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) -// .optFilter("backbone", backbone) + .optModelUrls(modelPath) .optEngine("OnnxRuntime") + .optArguments(arguments) .optProgress(new ProgressBar()) .build(); try (ZooModel model = criteria.loadModel()) { try (Predictor predictor = model.newPredictor()) { DetectedObjects detection = predictor.predict(img); - saveBoundingBoxImage(img, detection); + String outputDir = "/Users/fenkexin/Desktop/djl/examples/build/output"; + saveBoundingBoxImage(img, detection, outputDir); return detection; } } } - private static void saveBoundingBoxImage(Image img, DetectedObjects detection) + private static void saveBoundingBoxImage(Image img, DetectedObjects detection, String outputDir) throws IOException { - Path outputDir = Paths.get("build/output"); - Files.createDirectories(outputDir); + Path outputPath = Paths.get(outputDir); + Files.createDirectories(outputPath); img.drawBoundingBoxes(detection); - Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); + Path imagePath = outputPath.resolve("mask-wearing.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); diff --git a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java index fcbf81d753a..207ea7b1859 100644 --- a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java @@ -51,7 +51,7 @@ public static void main(String[] args) throws IOException, ModelException, Trans } public static DetectedObjects predict() throws IOException, ModelException, TranslateException { - Path imageFile = Paths.get("/Users/fenkexin/Desktop/djl/examples/src/test/resources/dog_bike_car.jpg"); + Path imageFile = Paths.get("src/test/resources/dog_bike_car.jpg"); Image img = ImageFactory.getInstance().fromFile(imageFile); String backbone; @@ -65,8 +65,7 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran Criteria.builder() .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) -// .optFilter("backbone", backbone) - .optEngine("OnnxRuntime") + .optFilter("backbone", backbone) .optProgress(new ProgressBar()) .build(); @@ -86,7 +85,7 @@ private static void saveBoundingBoxImage(Image img, DetectedObjects detection) img.drawBoundingBoxes(detection); - Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); + Path imagePath = outputDir.resolve("dog_bike_car.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); From eb8cf8e9296d9e67cfb12376760cb5fc67825450 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sat, 11 Mar 2023 15:04:59 +0800 Subject: [PATCH 04/12] doc --- examples/docs/mask_detection.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/docs/mask_detection.md b/examples/docs/mask_detection.md index 16c814fcecc..e541dabe983 100644 --- a/examples/docs/mask_detection.md +++ b/examples/docs/mask_detection.md @@ -1,15 +1,15 @@ # Mask detection with YOLOv5 - training and inference -[YOLOv5s]() is a powerful model for object detection tasks. With the transfer learning technique, a pre-trained YOLOv5 model can be utilized in various customized object detection tasks with relatively small dataset. +[YOLOv5]() is a powerful model for object detection tasks. With the transfer learning technique, a pre-trained YOLOv5 model can be utilized in various customized object detection tasks with relatively small dataset. In this example, we apply it on the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images). We first train the YOLOv5s model in Python, with the help of [ATLearn](), a python transfer learning toolkit. Then, the model is saved as an ONNX model, which is then imported into DJL for inference. We apply it on the mask wearing detection task. The source code can be found at [MaskDetectionOnnx.java](https://github.com/deepjavalibrary/djl/blob/master/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java) ## The training part in ATLearn -We initially attempted to import a pretrained YOLOv5 into DJL, and fine-tune it with the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images), similar to [Train ResNet for Fruit Freshness Classficiation](./train_transfer_fresh_fruit.md). However, YOLOv5 can not be converted to a PyTorch traced model, due to its data-dependent execution flow (see this [ref](https://discuss.pytorch.org/t/yolov5-convert-to-torchscript/150180)), whick blocks the idea of retraining a Yolov5 model in DJL. So the training part is entirely in python. +We initially attempted to import a pretrained YOLOv5 into DJL, and fine-tune it with the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images), similar to [Train ResNet for Fruit Freshness Classficiation](./train_transfer_fresh_fruit.md). However, YOLOv5 can not be converted to a PyTorch traced model, due to its data-dependent execution flow (see this [discussion](https://discuss.pytorch.org/t/yolov5-convert-to-torchscript/150180)), whick blocks the idea of retraining a Yolov5 model in DJL. So the training part is entirely in python. -The retraining of YOLOv5 can be found in an example in ATLearn [example](). In this example, the YOLOv5 layers near the input are frozen while those near the output are fine-tuned with the customized data. This follows the transfer learning idea. +The retraining of YOLOv5 can be found in an example in ATLearn: `examples/docs/face_mask_detection.md`. In this example, the YOLOv5 layers near the input are frozen while those near the output are fine-tuned with the customized data. This follows the transfer learning idea. In this example, the trained model is exported to ONNX file and is then also used for inference in python, which will serve as a benchmark. @@ -22,7 +22,7 @@ To configure your development environment, follow [setup](../../docs/development ### Input image file You can find the image used in this example in the project test resource folder: `src/test/resources/maksssksksss627.png`. -![dogs](../src/test/resources/maksssksksss627.png) +![dogs](../src/test/resources/face_mask.png) ### Build the project and run Use the following command to run the project: @@ -46,4 +46,4 @@ Your output should look like the following: An output image with bounding box will be saved as `build/output/detected-mask-wearing.png`: -![detected-dogs](img/detected-mask-wearing.png) \ No newline at end of file +![detected-dogs](../src/test/resources/face_mask_result.png) \ No newline at end of file From 84fe33c50a10f9150faa173474d6be3edab9d46c Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sat, 11 Mar 2023 15:05:15 +0800 Subject: [PATCH 05/12] doc --- examples/docs/mask_detection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/docs/mask_detection.md b/examples/docs/mask_detection.md index e541dabe983..a450899b72d 100644 --- a/examples/docs/mask_detection.md +++ b/examples/docs/mask_detection.md @@ -1,6 +1,6 @@ # Mask detection with YOLOv5 - training and inference -[YOLOv5]() is a powerful model for object detection tasks. With the transfer learning technique, a pre-trained YOLOv5 model can be utilized in various customized object detection tasks with relatively small dataset. +YOLOv5 is a powerful model for object detection tasks. With the transfer learning technique, a pre-trained YOLOv5 model can be utilized in various customized object detection tasks with relatively small dataset. In this example, we apply it on the [Face Mask Detection dataset](https://www.kaggle.com/datasets/andrewmvd/face-mask-detection?select=images). We first train the YOLOv5s model in Python, with the help of [ATLearn](), a python transfer learning toolkit. Then, the model is saved as an ONNX model, which is then imported into DJL for inference. We apply it on the mask wearing detection task. The source code can be found at [MaskDetectionOnnx.java](https://github.com/deepjavalibrary/djl/blob/master/examples/src/main/java/ai/djl/examples/inference/MaskDetectionOnnx.java) From 874aee302af74e634abd62c11ec099aa6cdaf04d Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sun, 12 Mar 2023 11:48:47 +0800 Subject: [PATCH 06/12] doc --- .../src/main/java/ai/djl/examples/inference/MaskDetection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java index 651f6527d07..9992af98e18 100644 --- a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java @@ -23,6 +23,7 @@ import ai.djl.repository.zoo.ZooModel; import ai.djl.training.util.ProgressBar; import ai.djl.translate.TranslateException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 4409139fb5a9e477aefeebe0dd36c8a4df9706b5 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Sun, 12 Mar 2023 12:01:53 +0800 Subject: [PATCH 07/12] doc --- .../main/java/ai/djl/examples/inference/ObjectDetection.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java index 207ea7b1859..08643cb3b96 100644 --- a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java @@ -66,6 +66,7 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) .optFilter("backbone", backbone) + .optEngine(Engine.getDefaultEngineName()) .optProgress(new ProgressBar()) .build(); @@ -85,7 +86,7 @@ private static void saveBoundingBoxImage(Image img, DetectedObjects detection) img.drawBoundingBoxes(detection); - Path imagePath = outputDir.resolve("dog_bike_car.png"); + Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); From ec79e87ab8ad0ed0eff12f98e6fc5d50e094695d Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 14 Mar 2023 23:54:46 -0700 Subject: [PATCH 08/12] Upload artifacts to S3 bucket --- examples/docs/mask_detection.md | 10 ++++----- examples/docs/object_detection.md | 11 +++++++--- .../djl/examples/inference/MaskDetection.java | 21 +++++++------------ 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/examples/docs/mask_detection.md b/examples/docs/mask_detection.md index a450899b72d..0cb92115acf 100644 --- a/examples/docs/mask_detection.md +++ b/examples/docs/mask_detection.md @@ -20,22 +20,22 @@ To configure your development environment, follow [setup](../../docs/development ## Run mask detection example ### Input image file -You can find the image used in this example in the project test resource folder: `src/test/resources/maksssksksss627.png`. +We use the following image as input: -![dogs](../src/test/resources/face_mask.png) +![mask](https://resources.djl.ai/images/face_mask_detection/face_mask.png) ### Build the project and run Use the following command to run the project: ``` cd examples -./gradlew run -Dai.djl.default_engine=PyTorch -Dmain=ai.djl.examples.inference.MaskDetectionOnnx +./gradlew run -Dmain=ai.djl.examples.inference.MaskDetection ``` Your output should look like the following: ```text -[INFO ] - Detected objects image has been saved in: /Users/fenkexin/Desktop/djl/examples/build/output/mask-wearing.png +[INFO ] - Detected objects image has been saved in: build/output/face_mask_result.png [INFO ] - { "w/o mask": 0.8998132944107056, "w/ mask": 0.8930246829986572, @@ -46,4 +46,4 @@ Your output should look like the following: An output image with bounding box will be saved as `build/output/detected-mask-wearing.png`: -![detected-dogs](../src/test/resources/face_mask_result.png) \ No newline at end of file +![detected-result](https://resources.djl.ai/images/face_mask_detection/face_mask_result.png) diff --git a/examples/docs/object_detection.md b/examples/docs/object_detection.md index f654aa43ccc..a5f3ed43c20 100644 --- a/examples/docs/object_detection.md +++ b/examples/docs/object_detection.md @@ -45,14 +45,18 @@ An output image with bounding box will be saved as build/output/detected-dog_bik ![detected-dogs](img/detected-dog_bike_car.png) ## Run object detection example with other engines -For objection detection application, other than the default model zoo with the default engine, we can also run it with other engines and model zoo. Here, we demonstrate with a pre-trained *YOLOV5s ONNX* model. +For objection detection application, other than the default model zoo with the default engine, +we can also run it with other engines and model zoo. Here, we demonstrate with a pre-trained *YOLOV5s ONNX* model. + +First, the following line needs to be added into the dependencies in `build.gradle`. Otherwise, +the ONNX model zoo won't be found. -First, the following line needs to be added into the dependencies in `build.gradle`. Otherwise, the ONNX model zoo won't be found. ```gradle runtimeOnly project(":engines:onnxruntime:onnxruntime-engine") ``` Then use the following criteria + ```java Criteria criteria = Criteria.builder() @@ -62,4 +66,5 @@ Then use the following criteria .optProgress(new ProgressBar()) .build(); ``` -where the `optFilter` is removed and `optEngine` is specified. The rest would be the same. \ No newline at end of file + +where the `optFilter` is removed and `optEngine` is specified. The rest would be the same. diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java index 9992af98e18..2c2a239815b 100644 --- a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java @@ -12,7 +12,6 @@ */ package ai.djl.examples.inference; -import ai.djl.Application; import ai.djl.ModelException; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; @@ -31,8 +30,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; /** * An example of inference using an object detection model. @@ -53,27 +50,23 @@ public static void main(String[] args) throws IOException, ModelException, Trans } public static DetectedObjects predict() throws IOException, ModelException, TranslateException { - Path imageFile = Paths.get("src/test/resources/maksssksksss627.png"); - Image img = ImageFactory.getInstance().fromFile(imageFile); - - String modelPath = "src/test/resources/mask.onnx"; - Map arguments = new ConcurrentHashMap<>(); - arguments.put("translatorFactory", YoloV5TranslatorFactory.class.getName()); + String imageUrl = "https://resources.djl.ai/images/face_mask_detection/face_mask.png"; + Image img = ImageFactory.getInstance().fromUrl(imageUrl); + String modelUrl = "https://resources.djl.ai/demo/onnxruntime/face_mask_detection.zip"; Criteria criteria = Criteria.builder() - .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) - .optModelUrls(modelPath) + .optModelUrls(modelUrl) .optEngine("OnnxRuntime") - .optArguments(arguments) + .optTranslatorFactory(new YoloV5TranslatorFactory()) .optProgress(new ProgressBar()) .build(); try (ZooModel model = criteria.loadModel()) { try (Predictor predictor = model.newPredictor()) { DetectedObjects detection = predictor.predict(img); - String outputDir = "/Users/fenkexin/Desktop/djl/examples/build/output"; + String outputDir = "build/output"; saveBoundingBoxImage(img, detection, outputDir); return detection; } @@ -87,7 +80,7 @@ private static void saveBoundingBoxImage(Image img, DetectedObjects detection, S img.drawBoundingBoxes(detection); - Path imagePath = outputPath.resolve("mask-wearing.png"); + Path imagePath = outputPath.resolve("face_mask_result.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); From e35201108853353dbc30e8d0240e290708705894 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Thu, 9 Mar 2023 01:55:57 +0800 Subject: [PATCH 09/12] ObjectDetectionWOnnx --- .../main/java/ai/djl/examples/inference/ObjectDetection.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java index 08643cb3b96..207ea7b1859 100644 --- a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java @@ -66,7 +66,6 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) .optFilter("backbone", backbone) - .optEngine(Engine.getDefaultEngineName()) .optProgress(new ProgressBar()) .build(); @@ -86,7 +85,7 @@ private static void saveBoundingBoxImage(Image img, DetectedObjects detection) img.drawBoundingBoxes(detection); - Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); + Path imagePath = outputDir.resolve("dog_bike_car.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); From 4def4d5d8fdef1433194f5eb2792a386c22bed9d Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Wed, 15 Mar 2023 16:26:44 +0800 Subject: [PATCH 10/12] Revert "ObjectDetectionWOnnx" This reverts commit e35201108853353dbc30e8d0240e290708705894. --- .../main/java/ai/djl/examples/inference/ObjectDetection.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java index 207ea7b1859..08643cb3b96 100644 --- a/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/ObjectDetection.java @@ -66,6 +66,7 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran .optApplication(Application.CV.OBJECT_DETECTION) .setTypes(Image.class, DetectedObjects.class) .optFilter("backbone", backbone) + .optEngine(Engine.getDefaultEngineName()) .optProgress(new ProgressBar()) .build(); @@ -85,7 +86,7 @@ private static void saveBoundingBoxImage(Image img, DetectedObjects detection) img.drawBoundingBoxes(detection); - Path imagePath = outputDir.resolve("dog_bike_car.png"); + Path imagePath = outputDir.resolve("detected-dog_bike_car.png"); // OpenJDK can't save jpg with alpha channel img.save(Files.newOutputStream(imagePath), "png"); logger.info("Detected objects image has been saved in: {}", imagePath); From e440fdb20457187f1a1d31da3c24c0222b737dc7 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Wed, 15 Mar 2023 18:01:18 +0800 Subject: [PATCH 11/12] rescale image --- .../djl/modality/cv/BufferedImageFactory.java | 12 ++--- .../translator/ObjectDetectionTranslator.java | 1 + examples/pom.xml | 5 ++ .../djl/examples/inference/MaskDetection.java | 2 + .../examples/inference/MaskDetectionTest.java | 47 +++++++++++++++++++ 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java diff --git a/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java b/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java index 79f7248572d..40735ddeca7 100644 --- a/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java +++ b/api/src/main/java/ai/djl/modality/cv/BufferedImageFactory.java @@ -304,8 +304,8 @@ public void drawBoundingBoxes(DetectedObjects detections) { g.setStroke(new BasicStroke(stroke)); g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); - float widthScale = (float) (image.getWidth() / 640.0); - float heightScale = (float) (image.getHeight() / 640.0); + int imageWidth = image.getWidth(); + int imageHeight = image.getHeight(); List list = detections.items(); int k = 10; @@ -322,13 +322,13 @@ public void drawBoundingBoxes(DetectedObjects detections) { } Rectangle rectangle = box.getBounds(); - int x = (int) (rectangle.getX() * widthScale); - int y = (int) (rectangle.getY() * heightScale); + int x = (int) (rectangle.getX() * imageWidth); + int y = (int) (rectangle.getY() * imageHeight); g.drawRect( x, y, - (int) (rectangle.getWidth() * widthScale), - (int) (rectangle.getHeight() * heightScale)); + (int) (rectangle.getWidth() * imageWidth), + (int) (rectangle.getHeight() * imageHeight)); drawText(g, className, x, y, stroke, 4); // If we have a mask instead of a plain rectangle, draw tha mask if (box instanceof Mask) { diff --git a/api/src/main/java/ai/djl/modality/cv/translator/ObjectDetectionTranslator.java b/api/src/main/java/ai/djl/modality/cv/translator/ObjectDetectionTranslator.java index b0f21bb69f2..1031cb182a5 100644 --- a/api/src/main/java/ai/djl/modality/cv/translator/ObjectDetectionTranslator.java +++ b/api/src/main/java/ai/djl/modality/cv/translator/ObjectDetectionTranslator.java @@ -133,6 +133,7 @@ protected void configPostProcess(Map arguments) { if (ArgumentsUtil.booleanValue(arguments, "rescale")) { optRescaleSize(width, height); } + optApplyRatio(ArgumentsUtil.booleanValue(arguments, "optApplyRatio")); threshold = ArgumentsUtil.floatValue(arguments, "threshold", 0.2f); } } diff --git a/examples/pom.xml b/examples/pom.xml index e9786acba75..f27dbb2facb 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -78,6 +78,11 @@ ai.djl.tensorflow tensorflow-model-zoo + + + ai.djl.onnxruntime + onnxruntime-engine + org.testng testng diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java index 2c2a239815b..0a9ef186aad 100644 --- a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java @@ -61,6 +61,8 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran .optEngine("OnnxRuntime") .optTranslatorFactory(new YoloV5TranslatorFactory()) .optProgress(new ProgressBar()) + .optArgument("optApplyRatio", true) // post process + .optArgument("rescale", true) // post process .build(); try (ZooModel model = criteria.loadModel()) { diff --git a/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java b/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java new file mode 100644 index 00000000000..4d37587d04b --- /dev/null +++ b/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.examples.inference; + +import ai.djl.ModelException; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.testing.TestRequirements; +import ai.djl.translate.TranslateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class MaskDetectionTest { + + private static final Logger logger = LoggerFactory.getLogger(MaskDetectionTest.class); + + @Test + public void testMaskDetection() throws ModelException, TranslateException, IOException { + TestRequirements.engine("OnnxRuntime"); + + DetectedObjects result = MaskDetection.predict(); + logger.info("{}", result); + + Assert.assertTrue(result.getNumberOfObjects() >= 8); + Classifications.Classification obj = result.best(); + String className = obj.getClassName(); + List objects = Arrays.asList("w/o mask", "w/ mask"); + Assert.assertTrue(objects.contains(className)); + Assert.assertTrue(obj.getProbability() > 0.8); + } +} From cdb3269dee62b5d15a27a965f4ae7e14f63e4296 Mon Sep 17 00:00:00 2001 From: KexinFeng Date: Wed, 15 Mar 2023 18:15:53 +0800 Subject: [PATCH 12/12] doc format --- examples/docs/object_detection.md | 23 +++++++------------ .../djl/examples/inference/MaskDetection.java | 6 +++-- .../examples/inference/MaskDetectionTest.java | 1 + 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/examples/docs/object_detection.md b/examples/docs/object_detection.md index a5f3ed43c20..f73cea26871 100644 --- a/examples/docs/object_detection.md +++ b/examples/docs/object_detection.md @@ -48,23 +48,16 @@ An output image with bounding box will be saved as build/output/detected-dog_bik For objection detection application, other than the default model zoo with the default engine, we can also run it with other engines and model zoo. Here, we demonstrate with a pre-trained *YOLOV5s ONNX* model. -First, the following line needs to be added into the dependencies in `build.gradle`. Otherwise, -the ONNX model zoo won't be found. - -```gradle - runtimeOnly project(":engines:onnxruntime:onnxruntime-engine") -``` - -Then use the following criteria +The model can be easily loaded with the following criteria ```java - Criteria criteria = - Criteria.builder() - .optApplication(Application.CV.OBJECT_DETECTION) - .setTypes(Image.class, DetectedObjects.class) - .optEngine("OnnxRuntime") - .optProgress(new ProgressBar()) - .build(); +Criteria criteria = + Criteria.builder() + .optApplication(Application.CV.OBJECT_DETECTION) + .setTypes(Image.class, DetectedObjects.class) + .optEngine("OnnxRuntime") + .optProgress(new ProgressBar()) + .build(); ``` where the `optFilter` is removed and `optEngine` is specified. The rest would be the same. diff --git a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java index 0a9ef186aad..ab06b29ff31 100644 --- a/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java +++ b/examples/src/main/java/ai/djl/examples/inference/MaskDetection.java @@ -50,9 +50,11 @@ public static void main(String[] args) throws IOException, ModelException, Trans } public static DetectedObjects predict() throws IOException, ModelException, TranslateException { + // To feed in local image, use ImageFactory.getInstance().fromFile(...) String imageUrl = "https://resources.djl.ai/images/face_mask_detection/face_mask.png"; Image img = ImageFactory.getInstance().fromUrl(imageUrl); + // modelUrl can be replaced to local onnx model file String modelUrl = "https://resources.djl.ai/demo/onnxruntime/face_mask_detection.zip"; Criteria criteria = Criteria.builder() @@ -61,8 +63,8 @@ public static DetectedObjects predict() throws IOException, ModelException, Tran .optEngine("OnnxRuntime") .optTranslatorFactory(new YoloV5TranslatorFactory()) .optProgress(new ProgressBar()) - .optArgument("optApplyRatio", true) // post process - .optArgument("rescale", true) // post process + .optArgument("optApplyRatio", true) // post process + .optArgument("rescale", true) // post process .build(); try (ZooModel model = criteria.loadModel()) { diff --git a/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java b/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java index 4d37587d04b..db8f4fb74ba 100644 --- a/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java +++ b/examples/src/test/java/ai/djl/examples/inference/MaskDetectionTest.java @@ -17,6 +17,7 @@ import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.testing.TestRequirements; import ai.djl.translate.TranslateException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert;