feat: #1177 - new camera image getter with cropping feature

packages/smooth_app/lib/data_models/user_preferences.dart

@@ -105,4 +105,9 @@ class UserPreferences extends ChangeNotifier {
       _sharedPreferences.setInt(_TAG_DEV_MODE, value);
 
   int get devMode => _sharedPreferences.getInt(_TAG_DEV_MODE) ?? 0;
+
+  Future<void> setDevModeIndex(final String tag, final int index) async =>
+      _sharedPreferences.setInt(tag, index);
+
+  int? getDevModeIndex(final String tag) => _sharedPreferences.getInt(tag);
 }

packages/smooth_app/lib/pages/scan/abstract_camera_image_getter.dart

@@ -0,0 +1,49 @@
+import 'dart:typed_data';
+import 'package:camera/camera.dart';
+import 'package:flutter/material.dart';
+import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart';
+
+/// Abstract getter of Camera Image, for barcode scan.
+///
+/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420
+abstract class AbstractCameraImageGetter {
+  AbstractCameraImageGetter(this.cameraImage, this.cameraDescription);
+
+  final CameraImage cameraImage;
+  final CameraDescription cameraDescription;
+
+  InputImage getInputImage() {
+    final InputImageRotation imageRotation =
+        InputImageRotationMethods.fromRawValue(
+                cameraDescription.sensorOrientation) ??
+            InputImageRotation.Rotation_0deg;
+
+    final InputImageFormat inputImageFormat =
+        InputImageFormatMethods.fromRawValue(
+      int.parse(cameraImage.format.raw.toString()),
+    )!;
+
+    final List<InputImagePlaneMetadata> planeData = getPlaneMetaData();
+
+    final InputImageData inputImageData = InputImageData(
+      size: getSize(),
+      imageRotation: imageRotation,
+      inputImageFormat: inputImageFormat,
+      planeData: planeData,
+    );
+
+    return InputImage.fromBytes(
+      bytes: getBytes(),
+      inputImageData: inputImageData,
+    );
+  }
+
+  @protected
+  Size getSize();
+
+  @protected
+  Uint8List getBytes();
+
+  @protected
+  List<InputImagePlaneMetadata> getPlaneMetaData();
+}

packages/smooth_app/lib/pages/scan/camera_image_cropper.dart

@@ -0,0 +1,127 @@
+import 'dart:typed_data';
+import 'package:camera/camera.dart';
+import 'package:flutter/material.dart';
+import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart';
+import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart';
+import 'package:typed_data/typed_buffers.dart';
+
+/// Camera Image Cropper, in order to limit the barcode scan computations.
+///
+/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420
+/// [left01], [top01], [width01] and [height01] are values between 0 and 1
+/// that delimit the cropping area.
+/// For instance:
+/// * left01: 0, top01: 0, width01: 1, height01: .2 delimit the top 20% banner
+/// * left01: .5, top01: .5, width01: .5, height01: ..5 the bottom right rect
+class CameraImageCropper extends AbstractCameraImageGetter {
+  CameraImageCropper(
+    final CameraImage cameraImage,
+    final CameraDescription cameraDescription, {
+    required this.left01,
+    required this.top01,
+    required this.width01,
+    required this.height01,
+  }) : super(cameraImage, cameraDescription) {
+    _computeCropParameters();
+  }
+
+  final double left01;
+  final double top01;
+  final double width01;
+  final double height01;
+  late int _left;
+  late int _top;
+  late int _width;
+  late int _height;
+
+  void _computeCropParameters() {
+    assert(width01 > 0 && width01 <= 1);
+    assert(height01 > 0 && height01 <= 1);
+    assert(left01 >= 0 && left01 < 1);
+    assert(top01 >= 0 && top01 < 1);
+    assert(left01 + width01 <= 1);
+    assert(top01 + height01 <= 1);
+
+    final int fullWidth = cameraImage.width;
+    final int fullHeight = cameraImage.height;
+    final int orientation = cameraDescription.sensorOrientation;
+
+    int _getEven(final double value) => 2 * (value ~/ 2);
+
+    if (orientation == 0) {
+      _width = _getEven(fullWidth * width01);
+      _height = _getEven(fullHeight * height01);
+      _left = _getEven(fullWidth * left01);
+      _top = _getEven(fullHeight * top01);
+      return;
+    }
+    if (orientation == 90) {
+      _width = _getEven(fullWidth * height01);
+      _height = _getEven(fullHeight * width01);
+      _left = _getEven(fullWidth * top01);
+      _top = _getEven(fullHeight * left01);
+      return;
+    }
+    throw Exception('Orientation $orientation not dealt with for the moment');
+  }
+
+  // cf. https://en.wikipedia.org/wiki/YUV#Y′UV420p_(and_Y′V12_or_YV12)_to_RGB888_conversion
+  static const Map<int, int> _planeDividers = <int, int>{
+    0: 1, // Y
+    1: 2, // U
+    2: 2, // V
+  };
+
+  @override
+  Size getSize() => Size(_width.toDouble(), _height.toDouble());
+
+  @override
+  Uint8List getBytes() {
+    int size = 0;
+    for (final int divider in _planeDividers.values) {
+      size += (_width ~/ divider) * (_height ~/ divider);
+    }
+    final Uint8Buffer buffer = Uint8Buffer(size);
+    final int imageWidth = cameraImage.width;
+    int planeIndex = 0;
+    int bufferOffset = 0;
+    for (final Plane plane in cameraImage.planes) {
+      final int divider = _planeDividers[planeIndex]!;
+      final int fullWidth = imageWidth ~/ divider;
+      final int cropLeft = _left ~/ divider;
+      final int cropTop = _top ~/ divider;
+      final int cropWidth = _width ~/ divider;
+      final int cropHeight = _height ~/ divider;
+
+      for (int i = 0; i < cropHeight; i++) {
+        //buffer.replaceRange(bufferOffset, bufferOffset + cropWidth, plane.bytes.getRange(15, 16));
+        for (int j = 0; j < cropWidth; j++) {
+          buffer[bufferOffset++] =
+              plane.bytes[fullWidth * (cropTop + i) + cropLeft + j];
+        }
+      }
+      planeIndex++;
+    }
+
+    return buffer.buffer.asUint8List();
+  }
+
+  @override
+  List<InputImagePlaneMetadata> getPlaneMetaData() {
+    final List<InputImagePlaneMetadata> planeData = <InputImagePlaneMetadata>[];
+    for (final Plane plane in cameraImage.planes) {
+      planeData.add(
+        InputImagePlaneMetadata(
+          bytesPerRow: (plane.bytesPerRow * _width) ~/ cameraImage.width,
+          height: plane.height == null
+              ? null
+              : (plane.height! * _height) ~/ cameraImage.height,
+          width: plane.width == null
+              ? null
+              : (plane.width! * _width) ~/ cameraImage.width,
+        ),
+      );
+    }
+    return planeData;
+  }
+}

packages/smooth_app/lib/pages/scan/camera_image_full_getter.dart

@@ -0,0 +1,46 @@
+import 'dart:typed_data';
+import 'package:camera/camera.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
+import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart';
+import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart';
+
+/// Camera Image helper where we get the full image.
+///
+/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420
+class CameraImageFullGetter extends AbstractCameraImageGetter {
+  CameraImageFullGetter(
+    final CameraImage cameraImage,
+    final CameraDescription cameraDescription,
+  ) : super(cameraImage, cameraDescription);
+
+  @override
+  Size getSize() => Size(
+        cameraImage.width.toDouble(),
+        cameraImage.height.toDouble(),
+      );
+
+  @override
+  Uint8List getBytes() {
+    final WriteBuffer allBytes = WriteBuffer();
+    for (final Plane plane in cameraImage.planes) {
+      allBytes.putUint8List(plane.bytes);
+    }
+    return allBytes.done().buffer.asUint8List();
+  }
+
+  @override
+  List<InputImagePlaneMetadata> getPlaneMetaData() {
+    final List<InputImagePlaneMetadata> planeData = <InputImagePlaneMetadata>[];
+    for (final Plane plane in cameraImage.planes) {
+      planeData.add(
+        InputImagePlaneMetadata(
+          bytesPerRow: plane.bytesPerRow,
+          height: plane.height,
+          width: plane.width,
+        ),
+      );
+    }
+    return planeData;
+  }
+}

packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart

@@ -1,14 +1,17 @@
-import 'dart:typed_data';
-
 import 'package:camera/camera.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart';
 import 'package:provider/provider.dart';
 import 'package:smooth_app/data_models/continuous_scan_model.dart';
+import 'package:smooth_app/data_models/user_preferences.dart';
 import 'package:smooth_app/main.dart';
+import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart';
+import 'package:smooth_app/pages/scan/camera_image_cropper.dart';
+import 'package:smooth_app/pages/scan/camera_image_full_getter.dart';
 import 'package:smooth_app/pages/scan/lifecycle_manager.dart';
+import 'package:smooth_app/pages/user_preferences_dev_mode.dart';
 
 class MLKitScannerPage extends StatefulWidget {
   const MLKitScannerPage({Key? key}) : super(key: key);
@@ -22,6 +25,7 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {
   BarcodeScanner? barcodeScanner = GoogleMlKit.vision.barcodeScanner();
   CameraLensDirection cameraLensDirection = CameraLensDirection.back;
   late ContinuousScanModel _model;
+  late UserPreferences _userPreferences;
   CameraController? _controller;
   int _cameraIndex = 0;
   bool isBusy = false;
@@ -71,6 +75,7 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {
   @override
   Widget build(BuildContext context) {
     _model = context.watch<ContinuousScanModel>();
+    _userPreferences = context.watch<UserPreferences>();
     return LifeCycleManager(
       onResume: _startLiveFeed,
       onPause: _stopImageStream,
@@ -125,6 +130,7 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {
       camera,
       ResolutionPreset.high,
       enableAudio: false,
+      imageFormatGroup: ImageFormatGroup.yuv420,
     );
 
     // If the controller is initialized update the UI.
@@ -182,53 +188,54 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {
     isBusy = true;
     frameCounter = 0;
 
-    final WriteBuffer allBytes = WriteBuffer();
-    for (final Plane plane in image.planes) {
-      allBytes.putUint8List(plane.bytes);
-    }
-    final Uint8List bytes = allBytes.done().buffer.asUint8List();
+    await _scan(image);
 
-    final Size imageSize =
-        Size(image.width.toDouble(), image.height.toDouble());
+    isBusy = false;
+  }
 
-    final CameraDescription camera = cameras[_cameraIndex];
-    final InputImageRotation imageRotation =
-        InputImageRotationMethods.fromRawValue(camera.sensorOrientation) ??
-            InputImageRotation.Rotation_0deg;
-
-    final InputImageFormat inputImageFormat =
-        InputImageFormatMethods.fromRawValue(
-              int.parse(image.format.raw.toString()),
-            ) ??
-            InputImageFormat.NV21;
-
-    final List<InputImagePlaneMetadata> planeData = image.planes.map(
-      (Plane plane) {
-        return InputImagePlaneMetadata(
-          bytesPerRow: plane.bytesPerRow,
-          height: plane.height,
-          width: plane.width,
-        );
-      },
-    ).toList();
-
-    final InputImageData inputImageData = InputImageData(
-      size: imageSize,
-      imageRotation: imageRotation,
-      inputImageFormat: inputImageFormat,
-      planeData: planeData,
+  Future<void> _scan(final CameraImage image) async {
+    final DevModeScanMode scanMode = DevModeScanModeExtension.fromIndex(
+      _userPreferences
+          .getDevModeIndex(UserPreferencesDevMode.userPreferencesEnumScanMode),
     );
 
-    final InputImage inputImage =
-        InputImage.fromBytes(bytes: bytes, inputImageData: inputImageData);
-
+    final AbstractCameraImageGetter getter;
+    switch (scanMode) {
+      case DevModeScanMode.CAMERA_ONLY:
+        return;
+      case DevModeScanMode.PREPROCESS_FULL_IMAGE:
+      case DevModeScanMode.SCAN_FULL_IMAGE:
+        getter = CameraImageFullGetter(image, cameras[_cameraIndex]);
+        break;
+      case DevModeScanMode.PREPROCESS_HALF_IMAGE:
+      case DevModeScanMode.SCAN_HALF_IMAGE:
+        getter = CameraImageCropper(
+          image,
+          cameras[_cameraIndex],
+          left01: 0,
+          top01: 0,
+          width01: 1,
+          height01: .5,
+        );
+        break;
+    }
+    final InputImage inputImage = getter.getInputImage();
+
+    switch (scanMode) {
+      case DevModeScanMode.CAMERA_ONLY:
+      case DevModeScanMode.PREPROCESS_FULL_IMAGE:
+      case DevModeScanMode.PREPROCESS_HALF_IMAGE:
+        return;
+      case DevModeScanMode.SCAN_FULL_IMAGE:
+      case DevModeScanMode.SCAN_HALF_IMAGE:
+        break;
+    }
     final List<Barcode> barcodes =
         await barcodeScanner!.processImage(inputImage);
 
     for (final Barcode barcode in barcodes) {
-      _model.onScan(barcode.value.rawValue);
+      _model
+          .onScan(barcode.value.rawValue); // TODO(monsieurtanuki): add "await"?
     }
-
-    isBusy = false;
   }
 }