Merge pull request #58 from Yoonit-Labs/FEATURE/image-quality

Feature/image quality
Yoonit-Labs · Apr 6, 2021 · 4d79e75 · 4d79e75
2 parents ec28300 + 708aaff
commit 4d79e75
Show file tree

Hide file tree

Showing 10 changed files with 306 additions and 43 deletions.
diff --git a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt
@@ -321,7 +321,10 @@ class MainActivity : AppCompatActivity() {
             count: Int,
             total: Int,
             imagePath: String,
-            inferences: ArrayList<android.util.Pair<String, FloatArray>>
+            inferences: ArrayList<android.util.Pair<String, FloatArray>>,
+            darkness: Double,
+            lightness: Double,
+            sharpness: Double
         ) {
             Log.d(TAG, "onImageCaptured . . . . . . . . . . . . . . . . . . . . . . . . .")
 
@@ -346,6 +349,13 @@ class MainActivity : AppCompatActivity() {
                 maskProbabilityTextView.text = probability.toString()
             }
 
+            darknessTextView.text = if (darkness > 0.7) "Too Dark" else "Normal"
+            darknessProbabilityTextView.text = darkness.toString()
+            lightnessTextView.text = if (lightness > 0.65) "Too Light" else "Normal"
+            lightnessProbabilityTextView.text = lightness.toString()
+            sharpnessTextView.text = if (sharpness < 0.1591) "Blurred" else "Normal"
+            sharpnessProbabilityTextView.text = sharpness.toString()
+
             Log.d(TAG, " . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .")
 
             info_textview.text = "$count/$total"

diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml
@@ -348,7 +348,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Left Eye:" />
                 <TextView
                     android:id="@+id/leftEyeTextView"
@@ -370,7 +370,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Right Eye:" />
                 <TextView
                     android:id="@+id/rightEyeTextView"
@@ -392,7 +392,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Smiling:" />
                 <TextView
                     android:id="@+id/smlingTextView"
@@ -414,7 +414,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Vertical:" />
                 <TextView
                     android:id="@+id/headVerticalTextView"
@@ -436,7 +436,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Horizontal:" />
                 <TextView
                     android:id="@+id/headHorizontalTextView"
@@ -458,7 +458,7 @@
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Tilt:" />
                 <TextView
                     android:id="@+id/headTiltTextView"
@@ -475,11 +475,12 @@
             <LinearLayout
                 android:layout_width="match_parent"
                 android:layout_height="wrap_content"
-                android:orientation="horizontal">
+                android:orientation="horizontal"
+                android:layout_marginBottom="16dp">
                 <TextView
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:layout_marginRight="10px"
+                    android:layout_marginRight="10dp"
                     android:text="Using Mask:" />
                 <TextView
                     android:id="@+id/maskTextView"
@@ -493,6 +494,71 @@
                     android:layout_height="wrap_content"
                     android:text="-" />
             </LinearLayout>
+            <LinearLayout
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:orientation="horizontal"
+                android:layout_marginBottom="16dp">
+                <TextView
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_marginRight="10dp"
+                    android:text="Darkness:" />
+                <TextView
+                    android:id="@+id/darknessTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_weight="1"
+                    android:text="-" />
+                <TextView
+                    android:id="@+id/darknessProbabilityTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="-" />
+            </LinearLayout>
+            <LinearLayout
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:orientation="horizontal"
+                android:layout_marginBottom="16dp">
+                <TextView
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_marginRight="10dp"
+                    android:text="Lighness:" />
+                <TextView
+                    android:id="@+id/lightnessTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_weight="1"
+                    android:text="-" />
+                <TextView
+                    android:id="@+id/lightnessProbabilityTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="-" />
+            </LinearLayout>
+            <LinearLayout
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:orientation="horizontal">
+                <TextView
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_marginRight="10dp"
+                    android:text="Sharpness:" />
+                <TextView
+                    android:id="@+id/sharpnessTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:layout_weight="1"
+                    android:text="-" />
+                <TextView
+                    android:id="@+id/sharpnessProbabilityTextView"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="-" />
+            </LinearLayout>
         </LinearLayout>
     </LinearLayout>
 </androidx.constraintlayout.widget.ConstraintLayout>
diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt
@@ -19,6 +19,7 @@ import ai.cyberlabs.yoonit.facefy.model.FaceDetected
 import android.graphics.PointF
 import android.graphics.Rect
 import android.graphics.RectF
+import androidx.camera.core.CameraSelector
 import com.google.mlkit.vision.common.InputImage
 
 /**
@@ -76,7 +77,11 @@ class CoordinatesController(
                     ((this.graphicView.height.toFloat() * imageAspectRatio) - this.graphicView.width.toFloat()) / 2
             }
 
-            val x = this.scale(detectionBox.centerX().toFloat(), scaleFactor) - postScaleWidthOffset
+            var x = this.scale(detectionBox.centerX().toFloat(), scaleFactor) - postScaleWidthOffset
+            if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_BACK) {
+                x = this.graphicView.width - x
+            }
+
             val y = this.scale(detectionBox.centerY().toFloat(), scaleFactor) - postScaleHeightOffset
 
             val left = x - this.scale(detectionBox.width() / 2.0f, scaleFactor)
@@ -186,7 +191,10 @@ class CoordinatesController(
         val faceContours = mutableListOf<PointF>()
 
         contours.forEach { point ->
-            val x = this.scale(point.x, scaleFactor) - postScaleWidthOffset
+            var x = this.scale(point.x, scaleFactor) - postScaleWidthOffset
+            if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_BACK) {
+                x = this.graphicView.width - x
+            }
             val y = this.scale(point.y, scaleFactor) - postScaleHeightOffset
             faceContours.add(PointF(x, y))
         }

diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt
@@ -14,6 +14,7 @@ package ai.cyberlabs.yoonit.camera.analyzers.face
 import ai.cyberlabs.yoonit.camera.CameraGraphicView
 import ai.cyberlabs.yoonit.camera.analyzers.CoordinatesController
 import ai.cyberlabs.yoonit.camera.controllers.ComputerVisionController
+import ai.cyberlabs.yoonit.camera.controllers.ImageQualityController
 import ai.cyberlabs.yoonit.camera.interfaces.CameraCallback
 import ai.cyberlabs.yoonit.camera.interfaces.CameraEventListener
 import ai.cyberlabs.yoonit.camera.models.CaptureOptions
@@ -25,6 +26,7 @@ import android.graphics.Bitmap
 import android.graphics.Rect
 import android.graphics.RectF
 import android.media.Image
+import androidx.camera.core.CameraSelector
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageProxy
 import java.io.File
@@ -55,10 +57,10 @@ class FaceAnalyzer(
 
         val mediaImage = imageProxy.image ?: return
 
-        val bitmap = mediaImage
+        var bitmap = mediaImage
             .toRGBBitmap(context)
             .rotate(imageProxy.imageInfo.rotationDegrees.toFloat())
-            .mirror(imageProxy.imageInfo.rotationDegrees.toFloat())
+            .mirror()
 
         this.facefy.detect(
             bitmap,
@@ -107,16 +109,16 @@ class FaceAnalyzer(
 
                     // Emit face analysis.
                     this.cameraEventListener.onFaceDetected(
-                        detectionBox.left.pxToDPI(this.context),
-                        detectionBox.top.pxToDPI(this.context),
-                        detectionBox.width().pxToDPI(this.context),
-                        detectionBox.height().pxToDPI(this.context),
-                        faceDetected.leftEyeOpenProbability,
-                        faceDetected.rightEyeOpenProbability,
-                        faceDetected.smilingProbability,
-                        faceDetected.headEulerAngleX,
-                        faceDetected.headEulerAngleY,
-                        faceDetected.headEulerAngleZ
+                            detectionBox.left.pxToDPI(this.context),
+                            detectionBox.top.pxToDPI(this.context),
+                            detectionBox.width().pxToDPI(this.context),
+                            detectionBox.height().pxToDPI(this.context),
+                            faceDetected.leftEyeOpenProbability,
+                            faceDetected.rightEyeOpenProbability,
+                            faceDetected.smilingProbability,
+                            faceDetected.headEulerAngleX,
+                            faceDetected.headEulerAngleY,
+                            faceDetected.headEulerAngleZ
                     )
 
                     // Continue only if current time stamp is within the interval.
@@ -140,8 +142,11 @@ class FaceAnalyzer(
                         if (CaptureOptions.saveImageCaptured) this.handleSaveImage(faceBitmap)
                         else ""
 
+                    val imageQuality: Triple<Double, Double, Double> =
+                            ImageQualityController.processImage(faceBitmap, true)
+
                     // Handle to emit image path and the inferences.
-                    this.handleEmitImageCaptured(imagePath, inferences)
+                    this.handleEmitImageCaptured(imagePath, inferences, imageQuality)
                 }
             },
             { errorMessage ->
@@ -210,7 +215,7 @@ class FaceAnalyzer(
 
         val faceBitmap: Bitmap = colorEncodedBitmap
             .rotate(cameraRotation)
-            .mirror(cameraRotation)
+            .mirror()
             .crop(boundingBox)
 
         return Bitmap.createScaledBitmap(
@@ -229,7 +234,8 @@ class FaceAnalyzer(
      */
     private fun handleEmitImageCaptured(
         imagePath: String,
-        inferences: ArrayList<android.util.Pair<String, FloatArray>>
+        inferences: ArrayList<android.util.Pair<String, FloatArray>>,
+        imageQuality: Triple<Double, Double, Double>
     ) {
         if (imagePath == "") return
 
@@ -242,7 +248,10 @@ class FaceAnalyzer(
                     this.numberOfImages,
                     CaptureOptions.numberOfImages,
                     imagePath,
-                    inferences
+                    inferences,
+                        imageQuality.first,
+                        imageQuality.second,
+                        imageQuality.third
                 )
                 return
             }
@@ -259,7 +268,10 @@ class FaceAnalyzer(
             this.numberOfImages,
             CaptureOptions.numberOfImages,
             imagePath,
-            inferences
+            inferences,
+            imageQuality.first,
+            imageQuality.second,
+            imageQuality.third
         )
     }
 

diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt
@@ -13,6 +13,7 @@ package ai.cyberlabs.yoonit.camera.analyzers.frame
 
 import ai.cyberlabs.yoonit.camera.CameraGraphicView
 import ai.cyberlabs.yoonit.camera.controllers.ComputerVisionController
+import ai.cyberlabs.yoonit.camera.controllers.ImageQualityController
 import ai.cyberlabs.yoonit.camera.interfaces.CameraCallback
 import ai.cyberlabs.yoonit.camera.interfaces.CameraEventListener
 import ai.cyberlabs.yoonit.camera.models.CaptureOptions
@@ -80,9 +81,12 @@ class FrameAnalyzer(
                     )
                 }
 
+                val imageQuality: Triple<Double, Double, Double> =
+                        ImageQualityController.processImage(frameBitmap, false)
+
                 // Handle to emit image path and the inference.
                 Handler(Looper.getMainLooper()).post {
-                    this.handleEmitImageCaptured(imagePath, inferences)
+                    this.handleEmitImageCaptured(imagePath, inferences, imageQuality)
                 }
             }
         }
@@ -131,7 +135,8 @@ class FrameAnalyzer(
      */
     private fun handleEmitImageCaptured(
         imagePath: String,
-        inferences: ArrayList<android.util.Pair<String, FloatArray>>
+        inferences: ArrayList<android.util.Pair<String, FloatArray>>,
+        imageQuality: Triple<Double, Double, Double>
     ) {
 
         // process face number of images.
@@ -143,7 +148,10 @@ class FrameAnalyzer(
                     this.numberOfImages,
                     CaptureOptions.numberOfImages,
                     imagePath,
-                    inferences
+                    inferences,
+                    imageQuality.first,
+                    imageQuality.second,
+                    imageQuality.third
                 )
                 return
             }
@@ -160,7 +168,10 @@ class FrameAnalyzer(
             this.numberOfImages,
             CaptureOptions.numberOfImages,
             imagePath,
-            inferences
+            inferences,
+            imageQuality.first,
+            imageQuality.second,
+            imageQuality.third
         )
     }
 
@@ -182,7 +193,7 @@ class FrameAnalyzer(
 
         mediaBitmap
             .rotate(rotationDegrees)
-            .mirror(rotationDegrees)
+            .mirror()
             .compress(
                 Bitmap.CompressFormat.JPEG,
                 100,