From c068c34d60dc363bc22839efa981b6bab4210f38 Mon Sep 17 00:00:00 2001 From: monsieurtanuki Date: Sat, 5 Mar 2022 19:24:08 +0100 Subject: [PATCH 1/2] feat: #1177 - new camera image getter with cropping feature New files: * `abstract_camera_image_getter.dart`: Abstract getter of Camera Image, for barcode scan. * `camera_image_cropper.dart`: Camera Image Cropper, in order to limit the barcode scan computations. * `camera_image_full_getter.dart`: Camera Image helper where we get the full image. Impacted file: * `ml_kit_scan_page.dart`: now using new `CameraImageFullGetter` class - same full screen camera image capture as before for the moment. --- .../scan/abstract_camera_image_getter.dart | 49 +++++++ .../lib/pages/scan/camera_image_cropper.dart | 127 ++++++++++++++++++ .../pages/scan/camera_image_full_getter.dart | 46 +++++++ .../lib/pages/scan/ml_kit_scan_page.dart | 46 +------ 4 files changed, 229 insertions(+), 39 deletions(-) create mode 100644 packages/smooth_app/lib/pages/scan/abstract_camera_image_getter.dart create mode 100644 packages/smooth_app/lib/pages/scan/camera_image_cropper.dart create mode 100644 packages/smooth_app/lib/pages/scan/camera_image_full_getter.dart diff --git a/packages/smooth_app/lib/pages/scan/abstract_camera_image_getter.dart b/packages/smooth_app/lib/pages/scan/abstract_camera_image_getter.dart new file mode 100644 index 00000000000..a22f24fcd79 --- /dev/null +++ b/packages/smooth_app/lib/pages/scan/abstract_camera_image_getter.dart @@ -0,0 +1,49 @@ +import 'dart:typed_data'; +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart'; + +/// Abstract getter of Camera Image, for barcode scan. +/// +/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420 +abstract class AbstractCameraImageGetter { + AbstractCameraImageGetter(this.cameraImage, this.cameraDescription); + + final CameraImage cameraImage; + final CameraDescription cameraDescription; + + InputImage getInputImage() { + final InputImageRotation imageRotation = + InputImageRotationMethods.fromRawValue( + cameraDescription.sensorOrientation) ?? + InputImageRotation.Rotation_0deg; + + final InputImageFormat inputImageFormat = + InputImageFormatMethods.fromRawValue( + int.parse(cameraImage.format.raw.toString()), + )!; + + final List planeData = getPlaneMetaData(); + + final InputImageData inputImageData = InputImageData( + size: getSize(), + imageRotation: imageRotation, + inputImageFormat: inputImageFormat, + planeData: planeData, + ); + + return InputImage.fromBytes( + bytes: getBytes(), + inputImageData: inputImageData, + ); + } + + @protected + Size getSize(); + + @protected + Uint8List getBytes(); + + @protected + List getPlaneMetaData(); +} diff --git a/packages/smooth_app/lib/pages/scan/camera_image_cropper.dart b/packages/smooth_app/lib/pages/scan/camera_image_cropper.dart new file mode 100644 index 00000000000..55e90044db7 --- /dev/null +++ b/packages/smooth_app/lib/pages/scan/camera_image_cropper.dart @@ -0,0 +1,127 @@ +import 'dart:typed_data'; +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart'; +import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart'; +import 'package:typed_data/typed_buffers.dart'; + +/// Camera Image Cropper, in order to limit the barcode scan computations. +/// +/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420 +/// [left01], [top01], [width01] and [height01] are values between 0 and 1 +/// that delimit the cropping area. +/// For instance: +/// * left01: 0, top01: 0, width01: 1, height01: .2 delimit the top 20% banner +/// * left01: .5, top01: .5, width01: .5, height01: ..5 the bottom right rect +class CameraImageCropper extends AbstractCameraImageGetter { + CameraImageCropper( + final CameraImage cameraImage, + final CameraDescription cameraDescription, { + required this.left01, + required this.top01, + required this.width01, + required this.height01, + }) : super(cameraImage, cameraDescription) { + _computeCropParameters(); + } + + final double left01; + final double top01; + final double width01; + final double height01; + late int _left; + late int _top; + late int _width; + late int _height; + + void _computeCropParameters() { + assert(width01 > 0 && width01 <= 1); + assert(height01 > 0 && height01 <= 1); + assert(left01 >= 0 && left01 < 1); + assert(top01 >= 0 && top01 < 1); + assert(left01 + width01 <= 1); + assert(top01 + height01 <= 1); + + final int fullWidth = cameraImage.width; + final int fullHeight = cameraImage.height; + final int orientation = cameraDescription.sensorOrientation; + + int _getEven(final double value) => 2 * (value ~/ 2); + + if (orientation == 0) { + _width = _getEven(fullWidth * width01); + _height = _getEven(fullHeight * height01); + _left = _getEven(fullWidth * left01); + _top = _getEven(fullHeight * top01); + return; + } + if (orientation == 90) { + _width = _getEven(fullWidth * height01); + _height = _getEven(fullHeight * width01); + _left = _getEven(fullWidth * top01); + _top = _getEven(fullHeight * left01); + return; + } + throw Exception('Orientation $orientation not dealt with for the moment'); + } + + // cf. https://en.wikipedia.org/wiki/YUV#Y′UV420p_(and_Y′V12_or_YV12)_to_RGB888_conversion + static const Map _planeDividers = { + 0: 1, // Y + 1: 2, // U + 2: 2, // V + }; + + @override + Size getSize() => Size(_width.toDouble(), _height.toDouble()); + + @override + Uint8List getBytes() { + int size = 0; + for (final int divider in _planeDividers.values) { + size += (_width ~/ divider) * (_height ~/ divider); + } + final Uint8Buffer buffer = Uint8Buffer(size); + final int imageWidth = cameraImage.width; + int planeIndex = 0; + int bufferOffset = 0; + for (final Plane plane in cameraImage.planes) { + final int divider = _planeDividers[planeIndex]!; + final int fullWidth = imageWidth ~/ divider; + final int cropLeft = _left ~/ divider; + final int cropTop = _top ~/ divider; + final int cropWidth = _width ~/ divider; + final int cropHeight = _height ~/ divider; + + for (int i = 0; i < cropHeight; i++) { + //buffer.replaceRange(bufferOffset, bufferOffset + cropWidth, plane.bytes.getRange(15, 16)); + for (int j = 0; j < cropWidth; j++) { + buffer[bufferOffset++] = + plane.bytes[fullWidth * (cropTop + i) + cropLeft + j]; + } + } + planeIndex++; + } + + return buffer.buffer.asUint8List(); + } + + @override + List getPlaneMetaData() { + final List planeData = []; + for (final Plane plane in cameraImage.planes) { + planeData.add( + InputImagePlaneMetadata( + bytesPerRow: (plane.bytesPerRow * _width) ~/ cameraImage.width, + height: plane.height == null + ? null + : (plane.height! * _height) ~/ cameraImage.height, + width: plane.width == null + ? null + : (plane.width! * _width) ~/ cameraImage.width, + ), + ); + } + return planeData; + } +} diff --git a/packages/smooth_app/lib/pages/scan/camera_image_full_getter.dart b/packages/smooth_app/lib/pages/scan/camera_image_full_getter.dart new file mode 100644 index 00000000000..98517b6d6df --- /dev/null +++ b/packages/smooth_app/lib/pages/scan/camera_image_full_getter.dart @@ -0,0 +1,46 @@ +import 'dart:typed_data'; +import 'package:camera/camera.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart'; +import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart'; + +/// Camera Image helper where we get the full image. +/// +/// Use CameraController with imageFormatGroup: ImageFormatGroup.yuv420 +class CameraImageFullGetter extends AbstractCameraImageGetter { + CameraImageFullGetter( + final CameraImage cameraImage, + final CameraDescription cameraDescription, + ) : super(cameraImage, cameraDescription); + + @override + Size getSize() => Size( + cameraImage.width.toDouble(), + cameraImage.height.toDouble(), + ); + + @override + Uint8List getBytes() { + final WriteBuffer allBytes = WriteBuffer(); + for (final Plane plane in cameraImage.planes) { + allBytes.putUint8List(plane.bytes); + } + return allBytes.done().buffer.asUint8List(); + } + + @override + List getPlaneMetaData() { + final List planeData = []; + for (final Plane plane in cameraImage.planes) { + planeData.add( + InputImagePlaneMetadata( + bytesPerRow: plane.bytesPerRow, + height: plane.height, + width: plane.width, + ), + ); + } + return planeData; + } +} diff --git a/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart b/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart index da5c74014e0..96538171f74 100644 --- a/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart +++ b/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart @@ -1,5 +1,3 @@ -import 'dart:typed_data'; - import 'package:camera/camera.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -8,6 +6,8 @@ import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart'; import 'package:provider/provider.dart'; import 'package:smooth_app/data_models/continuous_scan_model.dart'; import 'package:smooth_app/main.dart'; +import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart'; +import 'package:smooth_app/pages/scan/camera_image_full_getter.dart'; import 'package:smooth_app/pages/scan/lifecycle_manager.dart'; class MLKitScannerPage extends StatefulWidget { @@ -125,6 +125,7 @@ class MLKitScannerPageState extends State { camera, ResolutionPreset.high, enableAudio: false, + imageFormatGroup: ImageFormatGroup.yuv420, ); // If the controller is initialized update the UI. @@ -182,45 +183,12 @@ class MLKitScannerPageState extends State { isBusy = true; frameCounter = 0; - final WriteBuffer allBytes = WriteBuffer(); - for (final Plane plane in image.planes) { - allBytes.putUint8List(plane.bytes); - } - final Uint8List bytes = allBytes.done().buffer.asUint8List(); - - final Size imageSize = - Size(image.width.toDouble(), image.height.toDouble()); - - final CameraDescription camera = cameras[_cameraIndex]; - final InputImageRotation imageRotation = - InputImageRotationMethods.fromRawValue(camera.sensorOrientation) ?? - InputImageRotation.Rotation_0deg; - - final InputImageFormat inputImageFormat = - InputImageFormatMethods.fromRawValue( - int.parse(image.format.raw.toString()), - ) ?? - InputImageFormat.NV21; - - final List planeData = image.planes.map( - (Plane plane) { - return InputImagePlaneMetadata( - bytesPerRow: plane.bytesPerRow, - height: plane.height, - width: plane.width, - ); - }, - ).toList(); - - final InputImageData inputImageData = InputImageData( - size: imageSize, - imageRotation: imageRotation, - inputImageFormat: inputImageFormat, - planeData: planeData, + final AbstractCameraImageGetter getter = CameraImageFullGetter( + image, + cameras[_cameraIndex], ); - final InputImage inputImage = - InputImage.fromBytes(bytes: bytes, inputImageData: inputImageData); + final InputImage inputImage = getter.getInputImage(); final List barcodes = await barcodeScanner!.processImage(inputImage); From 0be9bb35c77550868e851da9cb6593472a217b91 Mon Sep 17 00:00:00 2001 From: monsieurtanuki Date: Sun, 6 Mar 2022 12:32:13 +0100 Subject: [PATCH 2/2] feat: #1177 - added a "scan mode" option on dev mode Impacted files: * `ml_kit_scan_page.dart`: added optional scan / no scan options according to dev mode * `user_preferences.dart`: added methods for dev mode int values * `user_preferences_dev_mode.dart`: added a "scan mode" option on dev mode --- .../lib/data_models/user_preferences.dart | 5 + .../lib/pages/scan/ml_kit_scan_page.dart | 51 ++++++++-- .../lib/pages/user_preferences_dev_mode.dart | 92 +++++++++++++++++++ 3 files changed, 142 insertions(+), 6 deletions(-) diff --git a/packages/smooth_app/lib/data_models/user_preferences.dart b/packages/smooth_app/lib/data_models/user_preferences.dart index 4a9a6630f10..909e5c0c96c 100644 --- a/packages/smooth_app/lib/data_models/user_preferences.dart +++ b/packages/smooth_app/lib/data_models/user_preferences.dart @@ -105,4 +105,9 @@ class UserPreferences extends ChangeNotifier { _sharedPreferences.setInt(_TAG_DEV_MODE, value); int get devMode => _sharedPreferences.getInt(_TAG_DEV_MODE) ?? 0; + + Future setDevModeIndex(final String tag, final int index) async => + _sharedPreferences.setInt(tag, index); + + int? getDevModeIndex(final String tag) => _sharedPreferences.getInt(tag); } diff --git a/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart b/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart index 96538171f74..853f1317af7 100644 --- a/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart +++ b/packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart @@ -5,10 +5,13 @@ import 'package:flutter/services.dart'; import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart'; import 'package:provider/provider.dart'; import 'package:smooth_app/data_models/continuous_scan_model.dart'; +import 'package:smooth_app/data_models/user_preferences.dart'; import 'package:smooth_app/main.dart'; import 'package:smooth_app/pages/scan/abstract_camera_image_getter.dart'; +import 'package:smooth_app/pages/scan/camera_image_cropper.dart'; import 'package:smooth_app/pages/scan/camera_image_full_getter.dart'; import 'package:smooth_app/pages/scan/lifecycle_manager.dart'; +import 'package:smooth_app/pages/user_preferences_dev_mode.dart'; class MLKitScannerPage extends StatefulWidget { const MLKitScannerPage({Key? key}) : super(key: key); @@ -22,6 +25,7 @@ class MLKitScannerPageState extends State { BarcodeScanner? barcodeScanner = GoogleMlKit.vision.barcodeScanner(); CameraLensDirection cameraLensDirection = CameraLensDirection.back; late ContinuousScanModel _model; + late UserPreferences _userPreferences; CameraController? _controller; int _cameraIndex = 0; bool isBusy = false; @@ -71,6 +75,7 @@ class MLKitScannerPageState extends State { @override Widget build(BuildContext context) { _model = context.watch(); + _userPreferences = context.watch(); return LifeCycleManager( onResume: _startLiveFeed, onPause: _stopImageStream, @@ -183,20 +188,54 @@ class MLKitScannerPageState extends State { isBusy = true; frameCounter = 0; - final AbstractCameraImageGetter getter = CameraImageFullGetter( - image, - cameras[_cameraIndex], + await _scan(image); + + isBusy = false; + } + + Future _scan(final CameraImage image) async { + final DevModeScanMode scanMode = DevModeScanModeExtension.fromIndex( + _userPreferences + .getDevModeIndex(UserPreferencesDevMode.userPreferencesEnumScanMode), ); + final AbstractCameraImageGetter getter; + switch (scanMode) { + case DevModeScanMode.CAMERA_ONLY: + return; + case DevModeScanMode.PREPROCESS_FULL_IMAGE: + case DevModeScanMode.SCAN_FULL_IMAGE: + getter = CameraImageFullGetter(image, cameras[_cameraIndex]); + break; + case DevModeScanMode.PREPROCESS_HALF_IMAGE: + case DevModeScanMode.SCAN_HALF_IMAGE: + getter = CameraImageCropper( + image, + cameras[_cameraIndex], + left01: 0, + top01: 0, + width01: 1, + height01: .5, + ); + break; + } final InputImage inputImage = getter.getInputImage(); + switch (scanMode) { + case DevModeScanMode.CAMERA_ONLY: + case DevModeScanMode.PREPROCESS_FULL_IMAGE: + case DevModeScanMode.PREPROCESS_HALF_IMAGE: + return; + case DevModeScanMode.SCAN_FULL_IMAGE: + case DevModeScanMode.SCAN_HALF_IMAGE: + break; + } final List barcodes = await barcodeScanner!.processImage(inputImage); for (final Barcode barcode in barcodes) { - _model.onScan(barcode.value.rawValue); + _model + .onScan(barcode.value.rawValue); // TODO(monsieurtanuki): add "await"? } - - isBusy = false; } } diff --git a/packages/smooth_app/lib/pages/user_preferences_dev_mode.dart b/packages/smooth_app/lib/pages/user_preferences_dev_mode.dart index 56ad56370f0..7db5b987e53 100644 --- a/packages/smooth_app/lib/pages/user_preferences_dev_mode.dart +++ b/packages/smooth_app/lib/pages/user_preferences_dev_mode.dart @@ -39,6 +39,7 @@ class UserPreferencesDevMode extends AbstractUserPreferences { static const String userPreferencesFlagLenientMatching = '__lenientMatching'; static const String userPreferencesFlagAdditionalButton = '__additionalButtonOnProductPage'; + static const String userPreferencesEnumScanMode = '__scanMode'; @override bool isCollapsedByDefault() => true; @@ -175,5 +176,96 @@ class UserPreferencesDevMode extends AbstractUserPreferences { setState(() {}); }, ), + ListTile( + title: const Text('Scan Mode'), + subtitle: Text( + 'Current scan mode is :"' + '${DevModeScanModeExtension.fromIndex(userPreferences.getDevModeIndex(userPreferencesEnumScanMode)).label}' + '"', + ), + onTap: () async { + final DevModeScanMode? scanMode = await showDialog( + context: context, + builder: (BuildContext context) => AlertDialog( + title: const Text('Scan Mode'), + content: SizedBox( + height: 400, + width: 300, + child: ListView.builder( + itemCount: DevModeScanMode.values.length, + itemBuilder: (final BuildContext context, final int index) { + final DevModeScanMode scanMode = + DevModeScanMode.values[index]; + return ListTile( + title: Text(scanMode.label), + onTap: () => Navigator.pop(context, scanMode), + ); + }, + ), + ), + actions: [ + ElevatedButton( + child: const Text('cancel'), + onPressed: () => Navigator.pop(context), + ), + ], + ), + ); + if (scanMode != null) { + await userPreferences.setDevModeIndex( + userPreferencesEnumScanMode, + scanMode.index, + ); + setState(() {}); + } + }, + ), ]; } + +enum DevModeScanMode { + CAMERA_ONLY, + PREPROCESS_FULL_IMAGE, + PREPROCESS_HALF_IMAGE, + SCAN_FULL_IMAGE, + SCAN_HALF_IMAGE, +} + +extension DevModeScanModeExtension on DevModeScanMode { + static const DevModeScanMode defaultScanMode = + DevModeScanMode.SCAN_FULL_IMAGE; + + static const Map _labels = { + DevModeScanMode.CAMERA_ONLY: 'Only camera stream, no scanning', + DevModeScanMode.PREPROCESS_FULL_IMAGE: + 'Camera stream and full image preprocessing, no scanning', + DevModeScanMode.PREPROCESS_HALF_IMAGE: + 'Camera stream and half image preprocessing, no scanning', + DevModeScanMode.SCAN_FULL_IMAGE: 'Full image scanning', + DevModeScanMode.SCAN_HALF_IMAGE: 'Half image scanning', + }; + + static const Map _indices = { + DevModeScanMode.CAMERA_ONLY: 4, + DevModeScanMode.PREPROCESS_FULL_IMAGE: 3, + DevModeScanMode.PREPROCESS_HALF_IMAGE: 2, + DevModeScanMode.SCAN_FULL_IMAGE: 0, + DevModeScanMode.SCAN_HALF_IMAGE: 1, + }; + + String get label => _labels[this]!; + + int get index => _indices[this]!; + + static DevModeScanMode fromIndex(final int? index) { + if (index == null) { + return defaultScanMode; + } + for (final DevModeScanMode scanMode in DevModeScanMode.values) { + if (scanMode.index == index) { + return scanMode; + } + } + throw Exception('Unknown index $index'); + } +}