AVCapture session freezes/stuck when locking/unlocking the iPhone - ios

Question

I'm implementing scanner functionality in my xamarin.forms app and for that I'm using iOS native AVCaptureSession. But my problem is while scanning or while capture session is active and the device is being locked, then after unlocking the device freezes the capture session which is quite odd.
I've tried handling it using UIApplication.DidEnterBackgroundNotification|UIApplication.WillEnterForegroundNotification where I stops and starts the capture session again. But the freeze is still occurring.
using System;
using AVFoundation;
using CoreFoundation;
using CoreGraphics;
using Foundation;
using UIKit;
using Xamarin.Forms;
using Xamarin.Forms.Platform.iOS;
namespace MarginPointApp.iOS
{
public class ScannerViewController : UIViewController
{
AVCaptureDevice captureDevice;
AVCaptureVideoPreviewLayer videoPreviewLayer;
AVCaptureSession captureSession;
UIView viewfinderView;
MetadataObjectsDelegate metadataObjectsDelegate;
public event EventHandler<String> OnScanSuccess;
UIView redLineCenter;
UIView overlayView;
UIView overlay;
UIView bottomBarView;
UIButton flashLightButton;
UIButton cancelButton;
UIButton cameraButton;
UILabel bottomTextLabel;
bool isCameraDismissed;
NSObject interuptStartNoti, interuptEndNoti;
AVMetadataObjectType metaTypes = AVMetadataObjectType.Code128Code |
AVMetadataObjectType.Code39Code | AVMetadataObjectType.Code39Mod43Code |
AVMetadataObjectType.DataMatrixCode | AVMetadataObjectType.EAN13Code | AVMetadataObjectType.EAN8Code |
AVMetadataObjectType.Interleaved2of5Code | AVMetadataObjectType.PDF417Code |
AVMetadataObjectType.QRCode | AVMetadataObjectType.UPCECode;
public override void ViewDidLoad()
{
base.ViewDidLoad();
UIView statusBar = UIApplication.SharedApplication.ValueForKey(new NSString("statusBar")) as UIView;
if (statusBar.RespondsToSelector(new ObjCRuntime.Selector("setBackgroundColor:")))
{
statusBar.BackgroundColor = UIColor.Black;
}
NavigationItem.Title = "Scanner";
this.View.BackgroundColor = UIColor.White;
if (marginpoint.im.iOS.AppDelegate.camPosition)
{
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Front);
}
else
{
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Back);
}
CameraSetup();
}
public void CameraSetup()
{
NSError error = null;
if (captureDevice != null)
{
try
{
var input = new AVCaptureDeviceInput(captureDevice, out error);
captureSession = new AVCaptureSession();
if (captureSession == null) { return; }
if (captureSession.CanAddInput(input))
captureSession.AddInput(input);
var captureMetadataOutput = new AVCaptureMetadataOutput();
if (captureSession.CanAddOutput(captureMetadataOutput))
{
captureSession.AddOutput(captureMetadataOutput);
// captureMetadataOutput.MetadataObjectTypes = captureMetadataOutput.AvailableMetadataObjectTypes;
captureMetadataOutput.MetadataObjectTypes = metaTypes;
}
var metadataQueue = new DispatchQueue("com.AVCam.metadata");
metadataObjectsDelegate = new MetadataObjectsDelegate
{
DidOutputMetadataObjectsAction = DidOutputMetadataObjects
};
captureMetadataOutput.SetDelegate(metadataObjectsDelegate, metadataQueue);
videoPreviewLayer = new AVCaptureVideoPreviewLayer(session: captureSession);
videoPreviewLayer.VideoGravity = AVLayerVideoGravity.ResizeAspectFill;
videoPreviewLayer.Frame = View.Layer.Bounds;
View.Layer.AddSublayer(videoPreviewLayer);
}
catch (Exception e)
{
//Console.WriteLine("error device input" + e.ToString());
}
}
// Prepare device for configuration
captureDevice.LockForConfiguration(out error);
if (error != null)
{
// There has been an issue, abort
//Console.WriteLine("Error: {0}", error.LocalizedDescription);
captureDevice.UnlockForConfiguration();
return;
}
addOverlayOnScreen();
/*
string reason = string.Empty;
if (interuptStartNoti == null)
{
interuptStartNoti = AVCaptureSession.Notifications.ObserveWasInterrupted((sender, e) =>
{
reason = e.Notification.UserInfo?.ValueForKey(new NSString("AVCaptureSessionInterruptionReasonKey"))?.ToString();
if (captureSession != null && !reason.Equals(string.Empty) && !reason.Equals("3"))
{
captureSession.StopRunning();
//captureSession.Dispose();
//captureSession = null;
}
});
}
if (interuptEndNoti == null)
{
interuptEndNoti = AVCaptureSession.Notifications.ObserveInterruptionEnded((sender, e) =>
{
Device.BeginInvokeOnMainThread(() =>
{
if (marginpoint.im.iOS.AppDelegate.camPosition)
{
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Front);
}
else
{
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Back);
}
if (!reason.Equals(string.Empty) && !reason.Equals("3"))
{
//CameraSetup();
captureSession.StartRunning();
}
});
});
}
*/
}
NSObject didEnterBackgroundNoti, willEnterForegroundNoti;
public override void ViewWillAppear(bool animated)
{
base.ViewWillAppear(animated);
if (didEnterBackgroundNoti == null)
{
didEnterBackgroundNoti = NSNotificationCenter.DefaultCenter.AddObserver(UIApplication.DidEnterBackgroundNotification, (obj) =>
{
Device.BeginInvokeOnMainThread(() =>
{
AddBlurEffect();
captureSession?.StopRunning();
});
});
}
willEnterForegroundNoti = NSNotificationCenter.DefaultCenter.AddObserver(UIApplication.WillEnterForegroundNotification, (obj) =>
{
Device.BeginInvokeOnMainThread(() =>
{
RemoveBlurEffect();
CameraSetup();
});
});
}
public override void ViewWillDisappear(bool animated)
{
base.ViewWillDisappear(animated);
NSNotificationCenter.DefaultCenter.RemoveObserver(didEnterBackgroundNoti);
NSNotificationCenter.DefaultCenter.RemoveObserver(willEnterForegroundNoti);
}
private AVCaptureDevice GetCameraDevice(AVCaptureDevicePosition position)
{
AVCaptureDevice captureDevice = null;
if (UIDevice.CurrentDevice.CheckSystemVersion(10, 0))
{
captureDevice = AVCaptureDevice.GetDefaultDevice(AVCaptureDeviceType.BuiltInWideAngleCamera, AVMediaType.Video, position);//AVCaptureDevice.GetDefaultDevice(AVMediaTypes.Video);
}
else
{
var devices = AVCaptureDevice.DevicesWithMediaType(AVMediaType.Video);
foreach (var device in devices)
{
if (device.Position == position)
{
captureDevice = device;
}
}
}
return captureDevice;
}
UIVisualEffectView blurView;
/// <summary>
/// Adds the blur effect to camera preview.
/// </summary>
void AddBlurEffect()
{
if (blurView == null)
{
var blur = UIBlurEffect.FromStyle(UIBlurEffectStyle.Light);
blurView = new UIVisualEffectView(blur);
blurView.Frame = View.Frame;
blurView.AutoresizingMask = UIViewAutoresizing.FlexibleHeight | UIViewAutoresizing.FlexibleWidth;
View.AddSubview(blurView);
}
}
void RemoveBlurEffect()
{
if (blurView != null)
{
blurView.RemoveFromSuperview();
blurView = null;
}
}
void addOverlayOnScreen()
{
overlayView = new UIView();
overlayView.Frame = new CGRect(x: 0, y: 0, width: View.Frame.Width, height: View.Frame.Height);
View.AddSubview(overlayView);
View.BringSubviewToFront(overlayView);
var overlayWidth = Application.Current.MainPage.Width * 0.7;
overlay = new UIView();
overlay.Layer.BorderColor = UIColor.Green.CGColor;
overlay.Layer.BorderWidth = 4;
overlay.Frame = new CGRect(x: View.Center.X - overlayWidth / 2, y: View.Center.Y - overlayWidth / 2, width: overlayWidth, height: overlayWidth);
overlayView.AddSubview(overlay);
overlayView.BringSubviewToFront(overlay);
redLineCenter = new UIView();
redLineCenter.BackgroundColor = UIColor.Red;
redLineCenter.Alpha = 0.5f;
redLineCenter.Frame = new CGRect(x: overlay.Frame.X + 4, y: overlay.Center.Y - 2, width: overlay.Frame.Width - 9, height: 4);
overlayView.AddSubview(redLineCenter);
overlayView.BringSubviewToFront(redLineCenter);
// to find Qr code
viewfinderView = new UIView();
viewfinderView.Frame = new CGRect(x: overlay.Frame.X, y: overlay.Center.Y - 50, width: overlay.Frame.Width, height: 100);
overlayView.AddSubview(viewfinderView);
overlayView.BringSubviewToFront(viewfinderView);
bottomBarView = new UIView();
bottomBarView.BackgroundColor = UIColor.White;
bottomBarView.Frame = new CGRect(x: 0, y: View.Frame.Height - 50, width: View.Frame.Width, height: 50);
overlayView.AddSubview(bottomBarView);
overlayView.BringSubviewToFront(bottomBarView);
var centerPoint = (bottomBarView.Frame.Top - overlay.Frame.Bottom) / 2 - 15;
bottomTextLabel = new UILabel
{
Frame = new CGRect(x: View.Frame.X, y: overlay.Frame.Bottom + centerPoint, width: View.Frame.Width, height: 30),
Text = AppResources.ScanAutomatically,
TextColor = UIColor.White,
Font = UIFont.FromName("TitilliumWeb-Regular", 17),
TextAlignment = UITextAlignment.Center
};
View.AddSubview(bottomTextLabel);
View.BringSubviewToFront(bottomTextLabel);
if (captureDevice.Position == AVCaptureDevicePosition.Back)
{
flashLightButton = new UIButton();
flashLightButton.SetImage(new UIImage(filename: "flash_white_light.png"), UIControlState.Normal);
flashLightButton.Frame = new CGRect(x: 0, y: 0, width: 50, height: 50);
bottomBarView.AddSubview(flashLightButton);
flashLightButton.TouchUpInside += async (object sender, EventArgs e) =>
{
NSError error = null;
if (captureDevice == null) return;
captureDevice.LockForConfiguration(out error);
if (error != null)
{
captureDevice.UnlockForConfiguration();
return;
}
else
{
if (!captureDevice.TorchAvailable)
{
var alert = new UIAlertView
{
Title = AppResources.MarginPoint,
Message = AppResources.CameraFlash
};
alert.AddButton(AppResources.OkButtonTitle);
alert.Show();
return;
}
if (captureDevice.TorchMode != AVCaptureTorchMode.On)
{
captureDevice.TorchMode = AVCaptureTorchMode.On;
}
else
{
captureDevice.TorchMode = AVCaptureTorchMode.Off;
}
captureDevice.UnlockForConfiguration();
}
};
}
string blueColor = "#1273B6";
cancelButton = new UIButton();
cancelButton.SetTitleColor(Color.FromHex(blueColor).ToUIColor(), UIControlState.Normal);
cancelButton.SetTitle(AppResources.PickerCancelLabel, UIControlState.Normal);
cancelButton.Font = UIFont.FromName("TitilliumWeb-Regular", 18);
cancelButton.Frame = new CGRect(x: bottomBarView.Center.X - 50, y: 0, width: 100, height: 50);
bottomBarView.AddSubview(cancelButton);
cancelButton.TouchUpInside += (object sender, EventArgs e) =>
{
if (captureSession != null)
captureSession.StopRunning();
DismissViewController(true, null);
};
Device.BeginInvokeOnMainThread(() =>
{
if (captureSession != null)
captureSession.StartRunning();
});
cameraButton = new UIButton();
cameraButton.SetImage(new UIImage(filename: "camera.png"), UIControlState.Normal);
cameraButton.Frame = new CGRect(x: bottomBarView.Frame.Width - 50, y: 0, width: 50, height: 50);
bottomBarView.AddSubview(cameraButton);
cameraButton.TouchUpInside += (object sender, EventArgs e) =>
{
if (captureDevice.Position == AVCaptureDevicePosition.Back)
{
if (captureDevice.TorchAvailable)
captureDevice.TorchMode = AVCaptureTorchMode.Off;
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Front);
marginpoint.im.iOS.AppDelegate.camPosition = true;
}
else
{
if (captureDevice.TorchAvailable)
captureDevice.TorchMode = AVCaptureTorchMode.Off;
captureDevice = GetCameraDevice(AVCaptureDevicePosition.Back);
marginpoint.im.iOS.AppDelegate.camPosition = false;
}
if (captureDevice != null)
{
try
{
NSError error;
var input = new AVCaptureDeviceInput(captureDevice, out error);
captureSession = new AVCaptureSession();
if (captureSession == null) { return; }
if (captureSession.CanAddInput(input))
captureSession.AddInput(input);
var captureMetadataOutput = new AVCaptureMetadataOutput();
if (captureSession.CanAddOutput(captureMetadataOutput))
{
captureSession.AddOutput(captureMetadataOutput);
// captureMetadataOutput.MetadataObjectTypes = captureMetadataOutput.AvailableMetadataObjectTypes;
captureMetadataOutput.MetadataObjectTypes = metaTypes;
}
var metadataQueue = new DispatchQueue("com.AVCam.metadata");
metadataObjectsDelegate = new MetadataObjectsDelegate
{
DidOutputMetadataObjectsAction = DidOutputMetadataObjects
};
captureMetadataOutput.SetDelegate(metadataObjectsDelegate, metadataQueue);
captureSession.StartRunning();
videoPreviewLayer = new AVCaptureVideoPreviewLayer(session: captureSession);
videoPreviewLayer.VideoGravity = AVLayerVideoGravity.ResizeAspectFill;
videoPreviewLayer.Frame = View.Layer.Bounds;
View.Layer.AddSublayer(videoPreviewLayer);
}
catch (Exception ex)
{
// Console.WriteLine("error device input" + ex.ToString());
}
addOverlayOnScreen();
}
};
}
public void DidOutputMetadataObjects(AVCaptureOutput captureOutput,
AVMetadataObject[] metadataObjects,
AVCaptureConnection connection)
{
Device.BeginInvokeOnMainThread(() =>
{
if (metadataObjects != null && metadataObjects.Length == 0)
{
//codeLabel.Text = "No Data";
//displayScanResult(string.Empty);
return;
}
var metadataObject = metadataObjects[0] as AVMetadataMachineReadableCodeObject;
if (metadataObject == null) { return; }
var visualCodeObject = videoPreviewLayer.GetTransformedMetadataObject(metadataObject);
if (metadataObject.Type == AVMetadataObjectType.QRCode)
{
if (viewfinderView.Frame.Contains(visualCodeObject.Bounds))
{
captureSession.StopRunning();
displayScanResult(metadataObject.StringValue);
}
}
else
{
captureSession.StopRunning();
displayScanResult(metadataObject.StringValue);
}
});
}
private async void displayScanResult(string metadataObjectVal)
{
OnScanSuccess?.Invoke(this, string.IsNullOrWhiteSpace(metadataObjectVal) ? string.Empty : metadataObjectVal as String);
captureSession.StopRunning();
DismissViewController(true, null);
//codeLabel.Text = metadataObject.StringValue;
}
}
class MetadataObjectsDelegate : AVCaptureMetadataOutputObjectsDelegate
{
public Action<AVCaptureMetadataOutput, AVMetadataObject[], AVCaptureConnection> DidOutputMetadataObjectsAction;
public override void DidOutputMetadataObjects(AVCaptureMetadataOutput captureOutput, AVMetadataObject[] metadataObjects, AVCaptureConnection connection)
{
if (DidOutputMetadataObjectsAction != null)
DidOutputMetadataObjectsAction(captureOutput, metadataObjects, connection);
}
}
}
Has anyone faced this issue and found a fix for it?

Answer 1

Instead of creating a new captureSession everytime when when user enters foreground, try just starting the existing captureSession.
willEnterForegroundNoti =
NSNotificationCenter.DefaultCenter.AddObserver(UIApplication.WillEnterForegroundNotification,
(obj) => {
Device.BeginInvokeOnMainThread(() => {
RemoveBlurEffect();
captureSession?.startRunning();
});
});