From 0d415a7988a10912929f70c1f8c041950589d3e7 Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Thu, 1 Apr 2021 17:02:31 -0300 Subject: [PATCH 1/6] feat: create ImageQualityController to analyze image quality feat: Return image quality at onImageCreated --- .../yoonit/camerademo/MainActivity.kt | 3 +- .../camera/analyzers/face/FaceAnalyzer.kt | 24 +-- .../controllers/ImageQualityController.kt | 150 ++++++++++++++++++ .../camera/interfaces/CameraEventListener.kt | 3 +- .../yoonit/camera/utils/Extensions.kt | 8 + 5 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt diff --git a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt index 26a75f5..0d64524 100644 --- a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt +++ b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt @@ -341,7 +341,8 @@ class MainActivity : AppCompatActivity() { smilingProbability: Float?, headEulerAngleX: Float, headEulerAngleY: Float, - headEulerAngleZ: Float + headEulerAngleZ: Float, + quality: Triple ) { Log.d( TAG, diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt index ade3c22..1156adf 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt @@ -14,6 +14,7 @@ package ai.cyberlabs.yoonit.camera.analyzers.face import ai.cyberlabs.yoonit.camera.CameraGraphicView import ai.cyberlabs.yoonit.camera.analyzers.CoordinatesController import ai.cyberlabs.yoonit.camera.controllers.ComputerVisionController +import ai.cyberlabs.yoonit.camera.controllers.ImageQualityController import ai.cyberlabs.yoonit.camera.interfaces.CameraCallback import ai.cyberlabs.yoonit.camera.interfaces.CameraEventListener import ai.cyberlabs.yoonit.camera.models.CaptureOptions @@ -94,6 +95,8 @@ class FaceAnalyzer( imageProxy.imageInfo.rotationDegrees.toFloat() ) + val quality: Triple = ImageQualityController.faceQuality(faceBitmap) + // Draw or clean the face detection box, face blur and face contours. this.graphicView.handleDraw( detectionBox, @@ -108,16 +111,17 @@ class FaceAnalyzer( // Emit face analysis. this.cameraEventListener.onFaceDetected( - detectionBox.left.pxToDPI(this.context), - detectionBox.top.pxToDPI(this.context), - detectionBox.width().pxToDPI(this.context), - detectionBox.height().pxToDPI(this.context), - faceDetected.leftEyeOpenProbability, - faceDetected.rightEyeOpenProbability, - faceDetected.smilingProbability, - faceDetected.headEulerAngleX, - faceDetected.headEulerAngleY, - faceDetected.headEulerAngleZ + detectionBox.left.pxToDPI(this.context), + detectionBox.top.pxToDPI(this.context), + detectionBox.width().pxToDPI(this.context), + detectionBox.height().pxToDPI(this.context), + faceDetected.leftEyeOpenProbability, + faceDetected.rightEyeOpenProbability, + faceDetected.smilingProbability, + faceDetected.headEulerAngleX, + faceDetected.headEulerAngleY, + faceDetected.headEulerAngleZ, + quality ) // Continue only if current time stamp is within the interval. diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt new file mode 100644 index 0000000..9a888e8 --- /dev/null +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt @@ -0,0 +1,150 @@ +package ai.cyberlabs.yoonit.camera.controllers + +import ai.cyberlabs.yoonit.camera.utils.coerce +import android.graphics.Bitmap +import android.graphics.Color +import android.graphics.Rect +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sqrt + +class ImageQualityController { + companion object { + fun faceQuality(bitmap: Bitmap, boundingBox: Rect = Rect()) : Triple { + + // upper limit for dimensions + val maxWidth = 200 + val maxHeight = 200 + + // crop the face + val faceBitmap: Bitmap = if (boundingBox.isEmpty) { + Bitmap.createBitmap(bitmap) + } else { + boundingBox.coerce(bitmap.width, bitmap.height) + Bitmap.createBitmap(bitmap, boundingBox.left, boundingBox.right, + boundingBox.width(), boundingBox.height()) + } + + // scale the dimensions to at most maxWidth x maxHeight + val scaledFaceBitmap = Bitmap.createScaledBitmap( + faceBitmap, + faceBitmap.width.coerceIn(0, maxWidth), + faceBitmap.height.coerceIn(0, maxHeight), + true + ) + + // define parameters and ellipsoidal mask + val width = scaledFaceBitmap.width + val height = scaledFaceBitmap.height + val ellipseCenterX = width / 2 + val ellipseCenterY = height / 2 + val ellipseRadiusX = width * 35 / 100 + val ellipseRadiusY = height * 50 / 100 + val mask = Ellipse(ellipseCenterX, ellipseCenterY, ellipseRadiusX, ellipseRadiusY) + + // laplacian of gaussian (LoG) kernel + val kernel = arrayOf( + arrayOf(1, 1, 1), + arrayOf(1, -8, 1), + arrayOf(1, 1, 1) + ) + + // create flat array with grayscale image + val pixels = IntArray(width * height) + scaledFaceBitmap.getPixels(pixels, 0, width, 0, 0, width, height) + for (i in pixels.indices) + pixels[i] = min(max((0.299 * Color.red(pixels[i]) + + 0.587 * Color.green(pixels[i]) + + 0.114 * Color.blue(pixels[i])).toInt(), 0), 255) + + // reflect inner pixels on border for convolution -> gfedcb|abcdefgh|gfedcba + // val reflectedBitmap = Bitmap.createBitmap(width, height, scaledFaceBitmap.config) + // reflectedBitmap.setPixels(pixels, 0, width, 1, 1, width+2, height+2) + + // calculate histogram of pixel values inside ellipsoidal mask + val hist = IntArray(256) {0} + for (y in 0 until height) { + for (x in 0 until width) { + if (mask.contains(x, y)) { + val pixel = pixels[y * width + x] + hist[pixel] += 1 + } + } + } + + // calculate percentage of bright and dark pixels based on histogram "tails" + // one measure of image quality is if the image has more pixels in the intermediate + // regions instead of the tails of the histogram + val darkTail = hist.slice(IntRange(0, 63)).sum().toDouble() + val dark = darkTail / hist.sum().toDouble() + val lightTail = hist.slice(IntRange(192, 255)).sum().toDouble() + val light = lightTail / hist.sum().toDouble() + + // determine edges (high frequency signals) via convolution with 3x3 LoG kernel + // conv is the resulting flattened image, the same size as the original + val conv = IntArray(width * height) {0} + + // we iterate on every pixel of the image... + for (y in 0 until height) { + for (x in 0 until width) { + + // ...and on every coefficient of the 3x3 kernel... + for (j in -1 until 2) { + for (i in -1 until 2) { + + // ...and we compute an element-wise multiplication + // of the kernel with the current region of the image + // it is passing through, and store the result on the + // corresponding pixel of the convoluted image + + // if the needed pixel index is pointing to a pixel + // that is outside the image, inner pixels will be mirrored + // otherwise, the pair (x+i, y+j) is computed + val xi = when (x + i) { + -1 -> 1 + width -> width - 2 + else -> x + i + } + val yj = when (y + j) { + -1 -> 1 + height -> height - 2 + else -> y + j + } + + // finally, the needed pixel is fetched and one of the + // element-wise products is computed and the result accumulated + conv[y * width + x] += kernel[i+1][j+1] * pixels[xi * width + yj] + } + } + } + } + + // compute the variance of the pixels. it results in + // a measure of the high frequency signals on the image + val mean = conv.average() + val accVar = conv.fold(0.0, { accVariance, pixel -> + accVariance + (pixel - mean).pow(2) }) + + // get the standard deviation and normalize it + val sharp = sqrt(accVar / (width * height)) / 128 + + return Triple(dark, light, sharp) + } + } + + private class Ellipse(val centerX: Int, val centerY: Int, val radiusX: Int, val radiusY: Int) { + fun contains(x0: Int, y0: Int) : Boolean { + // the ellipse equation is + // + // (x - cx) ^ 2 (y - cy) ^ 2 + // ------------ + ------------ = 1 + // rx ^ 2 ry ^ 2 + // + // if an (x0, y0) point inserted in the equation gives < 1, + // then the point (x0, y0) is inside the ellipse + return (((x0 - centerX).toDouble()/radiusX).pow(2) + + ((y0 - centerY).toDouble()/radiusY).pow(2)) < 1.0 + } + } +} \ No newline at end of file diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt index 19a0995..6a11eff 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt @@ -31,7 +31,8 @@ interface CameraEventListener { smilingProbability: Float?, headEulerAngleX: Float, headEulerAngleY: Float, - headEulerAngleZ: Float + headEulerAngleZ: Float, + quality: Triple ) fun onFaceUndetected() diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt index 3d0de65..12ff23c 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt @@ -130,3 +130,11 @@ fun Rect.scaledBy(percent: Float): Rect { (bottom - deltaY).toInt() ) } + +fun Rect.coerce(width: Int, height: Int) { + // confines Rect to the bitmap's dimensions + this.left = this.left.coerceIn(0, width) + this.top = this.top.coerceIn(0, height) + this.right = this.right.coerceIn(0, width) + this.bottom = this.bottom.coerceIn(0, height) +} From 84552737c1dcd36abf93b27524fb2a978e211ea8 Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Tue, 6 Apr 2021 12:03:49 -0300 Subject: [PATCH 2/6] refactor: upgrade imageQuality algorithm feat: retrieve imageQuality Darkness, Lightness and Sharpness at Demo Analasys tab --- .../yoonit/camerademo/MainActivity.kt | 11 +- app/src/main/res/layout/activity_main.xml | 82 ++++++++- .../camera/analyzers/face/FaceAnalyzer.kt | 23 ++- .../camera/analyzers/frame/FrameAnalyzer.kt | 19 +- .../controllers/ImageQualityController.kt | 172 +++++++++--------- .../camera/interfaces/CameraEventListener.kt | 8 +- 6 files changed, 202 insertions(+), 113 deletions(-) diff --git a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt index e332f06..029e512 100644 --- a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt +++ b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt @@ -321,7 +321,10 @@ class MainActivity : AppCompatActivity() { count: Int, total: Int, imagePath: String, - inferences: ArrayList> + inferences: ArrayList>, + darkness: Double, + lightness: Double, + sharpness: Double ) { Log.d(TAG, "onImageCaptured . . . . . . . . . . . . . . . . . . . . . . . . .") @@ -350,6 +353,9 @@ class MainActivity : AppCompatActivity() { info_textview.text = "$count/$total" image_preview.visibility = View.VISIBLE + darknessProbabilityTextView.text = darkness.toString() + lightnessProbabilityTextView.text = lightness.toString() + sharpnessProbabilityTextView.text = sharpness.toString() } override fun onFaceDetected( @@ -362,8 +368,7 @@ class MainActivity : AppCompatActivity() { smilingProbability: Float?, headEulerAngleX: Float, headEulerAngleY: Float, - headEulerAngleZ: Float, - quality: Triple + headEulerAngleZ: Float ) { Log.d( TAG, diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 9f7c786..774a067 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -348,7 +348,7 @@ + android:orientation="horizontal" + android:layout_marginBottom="16dp"> + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt index 083766a..420d071 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt @@ -94,8 +94,6 @@ class FaceAnalyzer( imageProxy.imageInfo.rotationDegrees.toFloat() ) - val quality: Triple = ImageQualityController.faceQuality(faceBitmap) - // Draw or clean the face detection box, face blur and face contours. this.graphicView.handleDraw( detectionBox, @@ -119,8 +117,7 @@ class FaceAnalyzer( faceDetected.smilingProbability, faceDetected.headEulerAngleX, faceDetected.headEulerAngleY, - faceDetected.headEulerAngleZ, - quality + faceDetected.headEulerAngleZ ) // Continue only if current time stamp is within the interval. @@ -144,8 +141,11 @@ class FaceAnalyzer( if (CaptureOptions.saveImageCaptured) this.handleSaveImage(faceBitmap) else "" + val imageQuality: Triple = + ImageQualityController.processImage(faceBitmap, true) + // Handle to emit image path and the inferences. - this.handleEmitImageCaptured(imagePath, inferences) + this.handleEmitImageCaptured(imagePath, inferences, imageQuality) } }, { errorMessage -> @@ -233,7 +233,8 @@ class FaceAnalyzer( */ private fun handleEmitImageCaptured( imagePath: String, - inferences: ArrayList> + inferences: ArrayList>, + imageQuality: Triple ) { if (imagePath == "") return @@ -246,7 +247,10 @@ class FaceAnalyzer( this.numberOfImages, CaptureOptions.numberOfImages, imagePath, - inferences + inferences, + imageQuality.first, + imageQuality.second, + imageQuality.third ) return } @@ -263,7 +267,10 @@ class FaceAnalyzer( this.numberOfImages, CaptureOptions.numberOfImages, imagePath, - inferences + inferences, + imageQuality.first, + imageQuality.second, + imageQuality.third ) } diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt index 4aa1513..54949bc 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt @@ -13,6 +13,7 @@ package ai.cyberlabs.yoonit.camera.analyzers.frame import ai.cyberlabs.yoonit.camera.CameraGraphicView import ai.cyberlabs.yoonit.camera.controllers.ComputerVisionController +import ai.cyberlabs.yoonit.camera.controllers.ImageQualityController import ai.cyberlabs.yoonit.camera.interfaces.CameraCallback import ai.cyberlabs.yoonit.camera.interfaces.CameraEventListener import ai.cyberlabs.yoonit.camera.models.CaptureOptions @@ -80,9 +81,12 @@ class FrameAnalyzer( ) } + val imageQuality: Triple = + ImageQualityController.processImage(frameBitmap, false) + // Handle to emit image path and the inference. Handler(Looper.getMainLooper()).post { - this.handleEmitImageCaptured(imagePath, inferences) + this.handleEmitImageCaptured(imagePath, inferences, imageQuality) } } } @@ -131,7 +135,8 @@ class FrameAnalyzer( */ private fun handleEmitImageCaptured( imagePath: String, - inferences: ArrayList> + inferences: ArrayList>, + imageQuality: Triple ) { // process face number of images. @@ -143,7 +148,10 @@ class FrameAnalyzer( this.numberOfImages, CaptureOptions.numberOfImages, imagePath, - inferences + inferences, + imageQuality.first, + imageQuality.second, + imageQuality.third ) return } @@ -160,7 +168,10 @@ class FrameAnalyzer( this.numberOfImages, CaptureOptions.numberOfImages, imagePath, - inferences + inferences, + imageQuality.first, + imageQuality.second, + imageQuality.third ) } diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt index 9a888e8..0f5d157 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt @@ -11,128 +11,126 @@ import kotlin.math.sqrt class ImageQualityController { companion object { - fun faceQuality(bitmap: Bitmap, boundingBox: Rect = Rect()) : Triple { - - // upper limit for dimensions - val maxWidth = 200 - val maxHeight = 200 - - // crop the face - val faceBitmap: Bitmap = if (boundingBox.isEmpty) { - Bitmap.createBitmap(bitmap) - } else { - boundingBox.coerce(bitmap.width, bitmap.height) - Bitmap.createBitmap(bitmap, boundingBox.left, boundingBox.right, - boundingBox.width(), boundingBox.height()) + // ellipsoidal mask parameters + private const val fracEllipseCenterX: Double = 0.50 + private const val fracEllipseCenterY: Double = 0.50 + private const val fracEllipseRadiusX: Double = 0.35 + private const val fracEllipseRadiusY: Double = 0.50 + + // kernel for the convolution (3x3 laplacian of gaussian) + private val kernel: IntArray = intArrayOf( + 1, 1, 1, + 1, -8, 1, + 1, 1, 1 + ) + + fun processImage(scaledFaceBitmap: Bitmap, withMask: Boolean) : Triple { + + val pixels = convertToGrayscale(scaledFaceBitmap) + + var mask: Ellipse? = null + + if (withMask) { + mask = Ellipse( + (scaledFaceBitmap.width * fracEllipseCenterX).toInt(), + (scaledFaceBitmap.height * fracEllipseCenterY).toInt(), + (scaledFaceBitmap.width * fracEllipseRadiusX).toInt(), + (scaledFaceBitmap.height * fracEllipseRadiusY).toInt() + ) } - // scale the dimensions to at most maxWidth x maxHeight - val scaledFaceBitmap = Bitmap.createScaledBitmap( - faceBitmap, - faceBitmap.width.coerceIn(0, maxWidth), - faceBitmap.height.coerceIn(0, maxHeight), - true - ) - - // define parameters and ellipsoidal mask - val width = scaledFaceBitmap.width - val height = scaledFaceBitmap.height - val ellipseCenterX = width / 2 - val ellipseCenterY = height / 2 - val ellipseRadiusX = width * 35 / 100 - val ellipseRadiusY = height * 50 / 100 - val mask = Ellipse(ellipseCenterX, ellipseCenterY, ellipseRadiusX, ellipseRadiusY) - - // laplacian of gaussian (LoG) kernel - val kernel = arrayOf( - arrayOf(1, 1, 1), - arrayOf(1, -8, 1), - arrayOf(1, 1, 1) - ) + val histPair = calcHistogramMetrics(scaledFaceBitmap, pixels, mask) + val dark = histPair.first + val light = histPair.second + + val sharpness = calcConvolutionMetrics(scaledFaceBitmap, pixels) + + return Triple(dark, light, sharpness) + } + + private fun convertToGrayscale(bitmap: Bitmap) : IntArray { // create flat array with grayscale image - val pixels = IntArray(width * height) - scaledFaceBitmap.getPixels(pixels, 0, width, 0, 0, width, height) - for (i in pixels.indices) - pixels[i] = min(max((0.299 * Color.red(pixels[i]) - + 0.587 * Color.green(pixels[i]) - + 0.114 * Color.blue(pixels[i])).toInt(), 0), 255) - - // reflect inner pixels on border for convolution -> gfedcb|abcdefgh|gfedcba - // val reflectedBitmap = Bitmap.createBitmap(width, height, scaledFaceBitmap.config) - // reflectedBitmap.setPixels(pixels, 0, width, 1, 1, width+2, height+2) - - // calculate histogram of pixel values inside ellipsoidal mask + val pixelsRGB = IntArray(bitmap.width * bitmap.height) + bitmap.getPixels(pixelsRGB, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + return pixelsRGB.map { pixel -> + (0.299 * Color.red(pixel) + 0.587 * Color.green(pixel) + 0.114 * Color.blue(pixel)) + .toInt().coerceIn(0, 255) + }.toIntArray() + } + + private fun calcHistogramMetrics(bitmap: Bitmap, pixels: IntArray, mask: Ellipse?) : Pair { + + // calculate histogram of pixels inside bit mask val hist = IntArray(256) {0} - for (y in 0 until height) { - for (x in 0 until width) { - if (mask.contains(x, y)) { - val pixel = pixels[y * width + x] + for (y in 0 until bitmap.height) { + for (x in 0 until bitmap.width) { + if ((mask != null && mask.contains(x, y)) + || mask == null) { + val pixel = pixels[y * bitmap.width + x] hist[pixel] += 1 } } } // calculate percentage of bright and dark pixels based on histogram "tails" - // one measure of image quality is if the image has more pixels in the intermediate - // regions instead of the tails of the histogram + // one measure of image quality (or image balance) is to quantify how many pixels + // lie in the tails of the histogram, indicating the image is unbalanced val darkTail = hist.slice(IntRange(0, 63)).sum().toDouble() val dark = darkTail / hist.sum().toDouble() val lightTail = hist.slice(IntRange(192, 255)).sum().toDouble() val light = lightTail / hist.sum().toDouble() + return Pair(dark, light) + } + + private fun calcConvolutionMetrics(bitmap: Bitmap, pixels: IntArray) : Double { + // determine edges (high frequency signals) via convolution with 3x3 LoG kernel // conv is the resulting flattened image, the same size as the original - val conv = IntArray(width * height) {0} + val conv = IntArray(bitmap.width * bitmap.height) {0} // we iterate on every pixel of the image... - for (y in 0 until height) { - for (x in 0 until width) { + for (y in 0 until bitmap.height) { + for (x in 0 until bitmap.width) { // ...and on every coefficient of the 3x3 kernel... + var convPixel = 0 for (j in -1 until 2) { for (i in -1 until 2) { - // ...and we compute an element-wise multiplication - // of the kernel with the current region of the image - // it is passing through, and store the result on the - // corresponding pixel of the convoluted image - - // if the needed pixel index is pointing to a pixel - // that is outside the image, inner pixels will be mirrored - // otherwise, the pair (x+i, y+j) is computed - val xi = when (x + i) { - -1 -> 1 - width -> width - 2 - else -> x + i - } - val yj = when (y + j) { - -1 -> 1 - height -> height - 2 - else -> y + j - } - - // finally, the needed pixel is fetched and one of the - // element-wise products is computed and the result accumulated - conv[y * width + x] += kernel[i+1][j+1] * pixels[xi * width + yj] + // ...and we compute the dot product (the sum of an element-wise multiplication) + // of the kernel (sliding window) with the current region of the image it is + // passing through, and store the result on the corresponding pixel of the convoluted image + + // if the image pixel required is "outside" the image, the border pixels will be + // replicated. otherwise, the sum of indices will point to a valid pixel + val pixelY = (y + j).coerceIn(0, bitmap.height - 1) + val pixelX = (x + i).coerceIn(0, bitmap.width - 1) + val pixelIndex = pixelY * bitmap.width + pixelX + val kernelIndex = (j + 1) * 3 + (i + 1) + + // then, one of the products is computed and accumulated + convPixel += (pixels[pixelIndex] * kernel[kernelIndex]) } } + + // finally, the sum of the products is stored as a pixel + conv[y * bitmap.width + x] = convPixel.coerceIn(0, 255) } } - // compute the variance of the pixels. it results in - // a measure of the high frequency signals on the image + // compute the standard deviation of the pixels. it results in a measure of the amount + // of high frequency signals on the image val mean = conv.average() - val accVar = conv.fold(0.0, { accVariance, pixel -> - accVariance + (pixel - mean).pow(2) }) - - // get the standard deviation and normalize it - val sharp = sqrt(accVar / (width * height)) / 128 + val accVar = conv.fold(0.0, { acc, pixel -> acc + (pixel - mean).pow(2) }) - return Triple(dark, light, sharp) + return sqrt(accVar / conv.size) / 128 } } + + private class Ellipse(val centerX: Int, val centerY: Int, val radiusX: Int, val radiusY: Int) { fun contains(x0: Int, y0: Int) : Boolean { // the ellipse equation is diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt index 6a11eff..02a0f96 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/interfaces/CameraEventListener.kt @@ -18,7 +18,10 @@ interface CameraEventListener { count: Int, total: Int, imagePath: String, - inferences: ArrayList> + inferences: ArrayList>, + darkness: Double, + lightness: Double, + sharpness: Double ) fun onFaceDetected( @@ -31,8 +34,7 @@ interface CameraEventListener { smilingProbability: Float?, headEulerAngleX: Float, headEulerAngleY: Float, - headEulerAngleZ: Float, - quality: Triple + headEulerAngleZ: Float ) fun onFaceUndetected() From 76e3f5e57e280b4cf4820eb2a8a280450bfe6aae Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Tue, 6 Apr 2021 12:19:59 -0300 Subject: [PATCH 3/6] feat: retrieve imageQuality Darkness, Lightness and Sharpness tresholds at Demo Analysis tab --- .../ai/cyberlabs/yoonit/camerademo/MainActivity.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt index 029e512..a787797 100644 --- a/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt +++ b/app/src/main/java/ai/cyberlabs/yoonit/camerademo/MainActivity.kt @@ -349,13 +349,17 @@ class MainActivity : AppCompatActivity() { maskProbabilityTextView.text = probability.toString() } + darknessTextView.text = if (darkness > 0.7) "Too Dark" else "Normal" + darknessProbabilityTextView.text = darkness.toString() + lightnessTextView.text = if (lightness > 0.65) "Too Light" else "Normal" + lightnessProbabilityTextView.text = lightness.toString() + sharpnessTextView.text = if (sharpness < 0.1591) "Blurred" else "Normal" + sharpnessProbabilityTextView.text = sharpness.toString() + Log.d(TAG, " . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .") info_textview.text = "$count/$total" image_preview.visibility = View.VISIBLE - darknessProbabilityTextView.text = darkness.toString() - lightnessProbabilityTextView.text = lightness.toString() - sharpnessProbabilityTextView.text = sharpness.toString() } override fun onFaceDetected( From 434484993fadaa3eaf8f95f9472e27d717374f79 Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Tue, 6 Apr 2021 17:39:56 -0300 Subject: [PATCH 4/6] fix: fix extensions bitmap.mirror bug fix: getBitmap and getContours to work with back camera fix: change qrCode scanner input to RGB Bitmap to work with xiaomi cellphones --- .../yoonit/camera/analyzers/CoordinatesController.kt | 12 ++++++++++-- .../yoonit/camera/analyzers/face/FaceAnalyzer.kt | 11 ++++++++--- .../yoonit/camera/analyzers/frame/FrameAnalyzer.kt | 2 +- .../camera/controllers/ImageQualityController.kt | 4 ---- .../ai/cyberlabs/yoonit/camera/utils/Extensions.kt | 6 ++---- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt index 37096b4..05276ea 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/CoordinatesController.kt @@ -19,6 +19,7 @@ import ai.cyberlabs.yoonit.facefy.model.FaceDetected import android.graphics.PointF import android.graphics.Rect import android.graphics.RectF +import androidx.camera.core.CameraSelector import com.google.mlkit.vision.common.InputImage /** @@ -76,7 +77,11 @@ class CoordinatesController( ((this.graphicView.height.toFloat() * imageAspectRatio) - this.graphicView.width.toFloat()) / 2 } - val x = this.scale(detectionBox.centerX().toFloat(), scaleFactor) - postScaleWidthOffset + var x = this.scale(detectionBox.centerX().toFloat(), scaleFactor) - postScaleWidthOffset + if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_BACK) { + x = this.graphicView.width - x + } + val y = this.scale(detectionBox.centerY().toFloat(), scaleFactor) - postScaleHeightOffset val left = x - this.scale(detectionBox.width() / 2.0f, scaleFactor) @@ -186,7 +191,10 @@ class CoordinatesController( val faceContours = mutableListOf() contours.forEach { point -> - val x = this.scale(point.x, scaleFactor) - postScaleWidthOffset + var x = this.scale(point.x, scaleFactor) - postScaleWidthOffset + if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_BACK) { + x = this.graphicView.width - x + } val y = this.scale(point.y, scaleFactor) - postScaleHeightOffset faceContours.add(PointF(x, y)) } diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt index 420d071..7eaa860 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt @@ -26,6 +26,7 @@ import android.graphics.Bitmap import android.graphics.Rect import android.graphics.RectF import android.media.Image +import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageProxy import java.io.File @@ -56,10 +57,14 @@ class FaceAnalyzer( val mediaImage = imageProxy.image ?: return - val bitmap = mediaImage + var bitmap = mediaImage .toRGBBitmap(context) .rotate(imageProxy.imageInfo.rotationDegrees.toFloat()) - .mirror(imageProxy.imageInfo.rotationDegrees.toFloat()) + .mirror() + + if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_FRONT) { + bitmap = bitmap + } this.facefy.detect( bitmap, @@ -214,7 +219,7 @@ class FaceAnalyzer( val faceBitmap: Bitmap = colorEncodedBitmap .rotate(cameraRotation) - .mirror(cameraRotation) + .mirror() .crop(boundingBox) return Bitmap.createScaledBitmap( diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt index 54949bc..a2ebb15 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/frame/FrameAnalyzer.kt @@ -193,7 +193,7 @@ class FrameAnalyzer( mediaBitmap .rotate(rotationDegrees) - .mirror(rotationDegrees) + .mirror() .compress( Bitmap.CompressFormat.JPEG, 100, diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt index 0f5d157..2e478b0 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/ImageQualityController.kt @@ -1,11 +1,7 @@ package ai.cyberlabs.yoonit.camera.controllers -import ai.cyberlabs.yoonit.camera.utils.coerce import android.graphics.Bitmap import android.graphics.Color -import android.graphics.Rect -import kotlin.math.max -import kotlin.math.min import kotlin.math.pow import kotlin.math.sqrt diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt index 12ff23c..0916805 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/utils/Extensions.kt @@ -43,11 +43,9 @@ fun Bitmap.rotate(rotationDegrees: Float): Bitmap { ) } -fun Bitmap.mirror(rotationDegrees: Float): Bitmap { +fun Bitmap.mirror(): Bitmap { val matrix = Matrix() - if (rotationDegrees == 270f) { - matrix.preScale(-1.0f, 1.0f) - } + matrix.preScale(-1.0f, 1.0f) return Bitmap.createBitmap( this, From 8a0d7b95b4aebbf37bacbe0d95051e54d47ece9f Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Tue, 6 Apr 2021 17:59:47 -0300 Subject: [PATCH 5/6] fix: change qrCode scanner input to RGB Bitmap to work with xiaomi cellphones --- .../yoonit/camera/analyzers/qrcode/QRCodeAnalyzer.kt | 8 ++++++-- .../yoonit/camera/controllers/CameraController.kt | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/qrcode/QRCodeAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/qrcode/QRCodeAnalyzer.kt index eb01c68..7ae4f33 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/qrcode/QRCodeAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/qrcode/QRCodeAnalyzer.kt @@ -14,7 +14,9 @@ package ai.cyberlabs.yoonit.camera.analyzers.qrcode import ai.cyberlabs.yoonit.camera.CameraGraphicView import ai.cyberlabs.yoonit.camera.analyzers.CoordinatesController import ai.cyberlabs.yoonit.camera.interfaces.CameraEventListener +import ai.cyberlabs.yoonit.camera.utils.toRGBBitmap import android.annotation.SuppressLint +import android.content.Context import android.graphics.Rect import android.graphics.RectF import androidx.camera.core.ImageAnalysis @@ -29,6 +31,7 @@ import com.google.mlkit.vision.common.InputImage * Custom camera image analyzer based on barcode detection bounded on [CameraController]. */ class QRCodeAnalyzer( + private val context: Context, private val cameraEventListener: CameraEventListener?, private val graphicView: CameraGraphicView ) : ImageAnalysis.Analyzer { @@ -78,9 +81,10 @@ class QRCodeAnalyzer( onComplete() return } + val imageBitmap = imageProxy.image?.toRGBBitmap(context) - val image: InputImage = InputImage.fromMediaImage( - imageProxy.image, + val image: InputImage = InputImage.fromBitmap( + imageBitmap, imageProxy.imageInfo.rotationDegrees ) diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/CameraController.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/CameraController.kt index 4dc98e7..88f901b 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/CameraController.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/controllers/CameraController.kt @@ -140,6 +140,7 @@ class CameraController( CaptureType.QRCODE -> this.imageAnalyzerController.start( QRCodeAnalyzer( + context, this.cameraEventListener, this.graphicView ) @@ -181,9 +182,7 @@ class CameraController( * Set to enable/disable the device torch. Available only to camera lens "back". */ fun setTorch(enable: Boolean) { - this.camera?.let { - it.cameraControl.enableTorch(enable) - } + this.camera?.cameraControl?.enableTorch(enable) } private fun buildCameraPreview() { From 708aaff33729c51e7d28d54940f78deab5bc4400 Mon Sep 17 00:00:00 2001 From: Goulartvic Date: Tue, 6 Apr 2021 18:06:50 -0300 Subject: [PATCH 6/6] refactor: removed useless if --- .../ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt index 7eaa860..73ce72f 100644 --- a/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt +++ b/yoonit-camera/src/main/java/ai/cyberlabs/yoonit/camera/analyzers/face/FaceAnalyzer.kt @@ -62,10 +62,6 @@ class FaceAnalyzer( .rotate(imageProxy.imageInfo.rotationDegrees.toFloat()) .mirror() - if (CaptureOptions.cameraLens == CameraSelector.LENS_FACING_FRONT) { - bitmap = bitmap - } - this.facefy.detect( bitmap, { faceDetected ->