Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Camera state management #990

Merged
merged 4 commits into from
Jan 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import 'dart:io';

import 'package:device_info_plus/device_info_plus.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/app_localizations.dart';
import 'package:openfoodfacts/openfoodfacts.dart';
Expand Down
27 changes: 12 additions & 15 deletions packages/smooth_app/lib/pages/scan/continuous_scan_page.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import 'package:qr_code_scanner/qr_code_scanner.dart';
import 'package:smooth_app/data_models/continuous_scan_model.dart';
import 'package:smooth_app/pages/scan/scanner_overlay.dart';
import 'package:smooth_app/pages/scan/scanner_state_manager.dart';

class ContinuousScanPage extends StatefulWidget {
const ContinuousScanPage();
Expand All @@ -25,21 +25,18 @@ class _ContinuousScanPageState extends State<ContinuousScanPage> {
constraints.maxHeight / 1.81; // roughly 55% of the available height
final double viewFinderBottomOffset = carouselHeight / 2.0;

return Scaffold(
body: ScannerOverlay(
model: _model,
restartCamera: _resumeLiveFeed,
stopCamera: _stopLiveFeed,
scannerWidget: QRView(
overlay: QrScannerOverlayShape(
// We use [SmoothViewFinder] instead of the overlay.
overlayColor: Colors.transparent,
// This offset adjusts the scanning area on iOS.
cutOutBottomOffset: viewFinderBottomOffset,
),
key: _scannerViewKey,
onQRViewCreated: setupScanner,
return LifeCycleManager(
onResume: _resumeLiveFeed,
onStop: _stopLiveFeed,
child: QRView(
overlay: QrScannerOverlayShape(
// We use [SmoothViewFinder] instead of the overlay.
overlayColor: Colors.transparent,
// This offset adjusts the scanning area on iOS.
cutOutBottomOffset: viewFinderBottomOffset,
),
key: _scannerViewKey,
onQRViewCreated: setupScanner,
),
);
},
Expand Down
101 changes: 61 additions & 40 deletions packages/smooth_app/lib/pages/scan/ml_kit_scan_page.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import 'package:google_ml_barcode_scanner/google_ml_barcode_scanner.dart';
import 'package:provider/provider.dart';
import 'package:smooth_app/data_models/continuous_scan_model.dart';
import 'package:smooth_app/main.dart';
import 'package:smooth_app/pages/scan/scanner_overlay.dart';
import 'package:smooth_app/pages/scan/scanner_state_manager.dart';

class MLKitScannerPage extends StatefulWidget {
const MLKitScannerPage({Key? key}) : super(key: key);
Expand All @@ -19,20 +19,23 @@ class MLKitScannerPage extends StatefulWidget {

class MLKitScannerPageState extends State<MLKitScannerPage> {
BarcodeScanner? barcodeScanner = GoogleMlKit.vision.barcodeScanner();
CameraLensDirection cameraLensDirection = CameraLensDirection.back;
late ContinuousScanModel _model;
CameraController? _controller;
int _cameraIndex = 0;
CameraLensDirection cameraLensDirection = CameraLensDirection.back;
bool isBusy = false;
bool imageStreamActive = false;
//Used when rebuilding to stop the camera
bool stoppingCamera = false;

@override
void initState() {
super.initState();

//Find the most relevant camera to use if none of these criteria are met,
//the default value of [_cameraIndex] will be used to select the first
//camera in the global cameras list.
// Find the most relevant camera to use if none of these criteria are met,
// the default value of [_cameraIndex] will be used to select the first
// camera in the global cameras list.
// if non matching is found we fall back to the first in the list
// initValue of [_cameraIndex]
if (cameras.any(
(CameraDescription element) =>
element.lensDirection == cameraLensDirection &&
Expand All @@ -58,17 +61,28 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {

@override
void dispose() {
_stopImageStream().then(
(_) => _controller?.dispose(),
);
_controller?.dispose();
super.dispose();
}

@override
Widget build(BuildContext context) {
_model = context.watch<ContinuousScanModel>();
if (_controller == null || _controller!.value.isInitialized == false) {
return const Center(child: CircularProgressIndicator());

return LifeCycleManager(
onResume: _startLiveFeed,
onStop: _stopImageStream,
child: _buildScannerWidget(),
);
}

Widget _buildScannerWidget() {
// Showing the black scanner background + the icon when the scanner is
// loading or stopped
if (_controller == null ||
_controller!.value.isInitialized == false ||
stoppingCamera) {
return Container();
}

final Size size = MediaQuery.of(context).size;
Expand All @@ -84,55 +98,62 @@ class MLKitScannerPageState extends State<MLKitScannerPage> {
scale = 1 / scale;
}

return Scaffold(
body: ScannerOverlay(
restartCamera: _resumeImageStream,
stopCamera: _stopImageStream,
model: _model,
scannerWidget: Transform.scale(
scale: scale,
child: Center(
child: CameraPreview(
_controller!,
),
),
return Transform.scale(
scale: scale,
child: Center(
key: ValueKey<bool>(stoppingCamera),
child: CameraPreview(
_controller!,
),
),
);
}

Future<void> _startLiveFeed() async {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of curiosity when is _startLiveFeed called?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In initState after checking which camera to use.

stoppingCamera = false;
final CameraDescription camera = cameras[_cameraIndex];
_controller = CameraController(

final CameraController cameraController = CameraController(
camera,
ResolutionPreset.high,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

have we tested other resolutions ? does it influence the scanning a lot ? it could alleviate some of the resource concerns

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No not tested in any way, just took high that we have a "high" resolution, we fill nearly the full screen with the preview but not max so that we don't spend too much performance. After all it's just the preview not how much we get from the camera which very likely needs more performance.

enableAudio: false,
);
_controller!.setFocusMode(FocusMode.auto);
_controller!.lockCaptureOrientation(DeviceOrientation.portraitUp);
cameraController.setFocusMode(FocusMode.auto);
cameraController.lockCaptureOrientation(DeviceOrientation.portraitUp);

_controller!.initialize().then((_) {
if (!mounted) {
return;
_controller = cameraController;

// If the controller is updated then update the UI.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of curiousity, when does the listener fire, in other words, what does "controller is updated" mean?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I am right this is used to update the page from showing a loading indicator to display the camera preview.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok we should confirm that, because it's rerendering the widget.

cameraController.addListener(() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we keep adding this listener every time _startLiveFeed is called, shouldn't we do it just once?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only call _startLiveFeed once (and again but then the controller is already disposed) so it shouldn't matter.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we also call _startLiveFeed from LifeCycleManager.onResume, so in the two cases you described below.

if (mounted) {
setState(() {});
}
if (cameraController.value.hasError) {
debugPrint(cameraController.value.errorDescription);
}
_controller!.startImageStream(_processCameraImage);
imageStreamActive = true;
setState(() {});
});
}

void _resumeImageStream() {
if (_controller != null && !imageStreamActive) {
_controller!.startImageStream(_processCameraImage);
imageStreamActive = true;
try {
await cameraController.initialize();
_controller?.startImageStream(_processCameraImage);
} on CameraException catch (e) {
if (kDebugMode) {
// TODO(M123): Show error message
debugPrint(e.toString());
}
}

if (mounted) {
setState(() {});
}
}

Future<void> _stopImageStream() async {
if (_controller != null) {
await _controller!.stopImageStream();
imageStreamActive = false;
stoppingCamera = true;
if (mounted) {
setState(() {});
}
await _controller?.dispose();
}

//Convert the [CameraImage] to a [InputImage]
Expand Down
14 changes: 11 additions & 3 deletions packages/smooth_app/lib/pages/scan/scan_page.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'package:smooth_app/data_models/user_preferences.dart';
import 'package:smooth_app/database/local_database.dart';
import 'package:smooth_app/pages/scan/continuous_scan_page.dart';
import 'package:smooth_app/pages/scan/ml_kit_scan_page.dart';
import 'package:smooth_app/pages/scan/scanner_overlay.dart';
import 'package:smooth_app/pages/user_preferences_dev_mode.dart';

class ScanPage extends StatefulWidget {
Expand Down Expand Up @@ -35,10 +36,12 @@ class _ScanPageState extends State<ScanPage> {
}

Future<PermissionStatus> _permissionCheck(
UserPreferences userPreferences) async {
UserPreferences userPreferences,
) async {
final PermissionStatus status = await Permission.camera.status;

//If is denied, is not restricted by for example parental control and is not already declined once
// If is denied, is not restricted by for example parental control and is
// not already declined once
if (status.isDenied &&
!status.isRestricted &&
!userPreferences.cameraDeclinedOnce) {
Expand Down Expand Up @@ -93,7 +96,12 @@ class _ScanPageState extends State<ScanPage> {

return ChangeNotifierProvider<ContinuousScanModel>(
create: (BuildContext context) => _model!,
child: child,
child: Scaffold(
body: ScannerOverlay(
child: child,
model: _model!,
),
),
);
},
);
Expand Down
81 changes: 10 additions & 71 deletions packages/smooth_app/lib/pages/scan/scanner_overlay.dart
Original file line number Diff line number Diff line change
@@ -1,67 +1,27 @@
import 'package:flutter/material.dart';
import 'package:flutter_svg/svg.dart';
import 'package:smooth_app/data_models/continuous_scan_model.dart';
import 'package:smooth_app/pages/scan/scan_page_helper.dart';
import 'package:smooth_app/widgets/smooth_product_carousel.dart';
import 'package:smooth_ui_library/animations/smooth_reveal_animation.dart';
import 'package:smooth_ui_library/widgets/smooth_view_finder.dart';
import 'package:visibility_detector/visibility_detector.dart';

/// This builds all the essential widgets which are displayed above the camera
/// preview, like the [SmoothProductCarousel], the [SmoothViewFinder] and the
/// clear and compare buttons row. It takes the camera preview widget to display
/// and functions to stop and restart the camera, to only activate the camera
/// when the screen is currently visible.
class ScannerOverlay extends StatefulWidget {
/// clear and compare buttons row.
class ScannerOverlay extends StatelessWidget {
const ScannerOverlay({
required this.scannerWidget,
required this.child,
required this.model,
required this.restartCamera,
required this.stopCamera,
});

final Widget scannerWidget;
final Widget child;
final ContinuousScanModel model;
final Function() restartCamera;
final Function() stopCamera;

static const double carouselHeightPct = 0.55;
static const double scannerWidthPct = 0.6;
static const double scannerHeightPct = 0.33;
static const double buttonRowHeightPx = 48;

@override
State<ScannerOverlay> createState() => _ScannerOverlayState();
}

class _ScannerOverlayState extends State<ScannerOverlay>
with WidgetsBindingObserver {
@override
void initState() {
super.initState();
WidgetsBinding.instance!.addObserver(this);
}

@override
void dispose() {
WidgetsBinding.instance!.removeObserver(this);
super.dispose();
}

// Lifecycle changes are not handled by either of the used plugin. This means
// we are responsible to control camera resources when the lifecycle state is
// updated. Failure to do so might lead to unexpected behavior
// didChangeAppLifecycleState is called when the system puts the app in the
// background or returns the app to the foreground.
@override
void didChangeAppLifecycleState(AppLifecycleState state) {
if (state == AppLifecycleState.inactive) {
widget.stopCamera.call();
} else if (state == AppLifecycleState.resumed) {
widget.restartCamera.call();
}
}

@override
Widget build(BuildContext context) {
return LayoutBuilder(
Expand All @@ -76,9 +36,8 @@ class _ScannerOverlayState extends State<ScannerOverlay>
);
final double carouselHeight =
constraints.maxHeight * ScannerOverlay.carouselHeightPct;
final double buttonRowHeight = areButtonsRendered(widget.model)
? ScannerOverlay.buttonRowHeightPx
: 0;
final double buttonRowHeight =
areButtonsRendered(model) ? ScannerOverlay.buttonRowHeightPx : 0;
final double availableScanHeight =
constraints.maxHeight - carouselHeight - buttonRowHeight;

Expand All @@ -87,35 +46,15 @@ class _ScannerOverlayState extends State<ScannerOverlay>
top: (availableScanHeight - scannerSize.height) / 2 +
buttonRowHeight);

return VisibilityDetector(
key: const ValueKey<String>('VisibilityDetector'),
onVisibilityChanged: (VisibilityInfo info) {
if (info.visibleFraction == 0.0) {
widget.stopCamera.call();
} else {
widget.restartCamera.call();
}
},
return Container(
color: Colors.black,
child: Stack(
children: <Widget>[
Container(
alignment: Alignment.center,
color: Colors.black,
child: Padding(
padding: qrScannerPadding,
child: SvgPicture.asset(
'assets/actions/scanner_alt_2.svg',
width: scannerSize.width * 0.8,
height: scannerSize.height * 0.8,
color: Colors.white,
),
),
),
SmoothRevealAnimation(
delay: 400,
startOffset: Offset.zero,
animationCurve: Curves.easeInOutBack,
child: widget.scannerWidget,
child: child,
),
SmoothRevealAnimation(
delay: 400,
Expand All @@ -141,7 +80,7 @@ class _ScannerOverlayState extends State<ScannerOverlay>
child: Column(
mainAxisAlignment: MainAxisAlignment.start,
children: <Widget>[
buildButtonsRow(context, widget.model),
buildButtonsRow(context, model),
const Spacer(),
SmoothProductCarousel(
showSearchCard: true,
Expand Down
Loading