I have integrated the Firebase MLKit SDK as per the document, texts are detected correctly with the back camera photo. When I am using a captured photo from the front camera, texts are detecting wrongly and bad results are coming out.
#IBAction func findTextDidTouch(_ sender: UIButton) {
runTextRecognition(with:imageView.image!)
}
func runCloudTextRecognition(with image: UIImage) {
let visionImage = VisionImage(image: image)
cloudTextDetector.detect(in: visionImage) { features, error in
if let error = error {
print("Received error: \(error)")
return
}
self.processCloudResult(from: features, error: error)
}
}
for text in features {
if let block = text as? VisionTextBlock {
print(block.text)
}
}
I am not able to figure out. Do I need change with the camera or firebase ml kit?
You need to provide image orientation as well. Adding code snippet may help you.
let visionImage = VisionImage(image: image)
let metadata = VisionImageMetadata()
metadata.orientation = self.detectorOrientation(in: image)
visionImage.metadata = metadata
textDetector.process(visionImage) { (features, error) in
if features != nil
{
self.processResult(from: [features!], error: error)
}
}
//Detects orientation of the selected or captured image
func detectorOrientation(in image: UIImage) -> VisionDetectorImageOrientation {
switch image.imageOrientation {
case .up:
return .topLeft
case .down:
return .bottomRight
case .left:
return .leftBottom
case .right:
return .rightTop
case .upMirrored:
return .topRight
case .downMirrored:
return .bottomLeft
case .leftMirrored:
return .leftTop
case .rightMirrored:
return .rightBottom
}
}
Related
Is NSKeyedArchiver appropriate to convert UIImage to Data?
do {
let data = try NSKeyedArchiver.archivedData(withRootObject: UIImage(named: somePath), requiringSecureCoding: true)
...
} catch {
print(error)
}
Or is it overkill and using pngData() is more appropriate?
let image = UIImage(named: somePath)
let data = image?.pngData()
and how can I convert from UIImage to HEIF / HEIC Data ?
The goal is to save the image to the device's file system.
No. Never use NSKeyedArchiver to convert your image to Data. Choose an image format (HEIC, PNG, JPEG, etc) and get its data representation. You should only use PNG when saving images to use in your UI. Most of the time jpeg is the preferred choice. If the device supports HEIC it is an option considering the image quality and reduced data size.
If you need to check if the user device supports HEIC type you can do it as follow:
var isHeicSupported: Bool {
(CGImageDestinationCopyTypeIdentifiers() as! [String]).contains("public.heic")
}
If you need to convert your image to HEIC you need to get a CGImage from your UIImage and convert your UIImage's imageOrientation to CGImagePropertyOrientation to preserve the orientation when creating its data representation:
extension UIImage {
var heic: Data? { heic() }
func heic(compressionQuality: CGFloat = 1) -> Data? {
guard
let mutableData = CFDataCreateMutable(nil, 0),
let destination = CGImageDestinationCreateWithData(mutableData, "public.heic" as CFString, 1, nil),
let cgImage = cgImage
else { return nil }
CGImageDestinationAddImage(destination, cgImage, [kCGImageDestinationLossyCompressionQuality: compressionQuality, kCGImagePropertyOrientation: cgImageOrientation.rawValue] as CFDictionary)
guard CGImageDestinationFinalize(destination) else { return nil }
return mutableData as Data
}
}
extension CGImagePropertyOrientation {
init(_ uiOrientation: UIImage.Orientation) {
switch uiOrientation {
case .up: self = .up
case .upMirrored: self = .upMirrored
case .down: self = .down
case .downMirrored: self = .downMirrored
case .left: self = .left
case .leftMirrored: self = .leftMirrored
case .right: self = .right
case .rightMirrored: self = .rightMirrored
#unknown default:
fatalError()
}
}
}
extension UIImage {
var cgImageOrientation: CGImagePropertyOrientation { .init(imageOrientation) }
}
Usage for lossless compression:
if isHeicSupported, let heicData = image.heic {
// write your heic image data to disk
}
or adding compression to your image:
if isHeicSupported, let heicData = image.heic(compressionQuality: 0.75) {
// write your compressed heic image data to disk
}
I am able to get the rectangles of text detected in vision framework video feed in iOS 11, but I am trying to get the image part of video that was recognized as a text or character. Someone can help in that?
func detectTextHandler(request: VNRequest, error: Error?) {
guard let observations = request.results else {
print("no result")
return
}
let result = observations.map({$0 as? VNTextObservation})
DispatchQueue.main.async() {
self.imageView.layer.sublayers?.removeSubrange(1...)
for region in result {
guard let rg = region else {
continue
}
self.highlightWord(box: rg)
if let boxes = region?.characterBoxes {
for characterBox in boxes {
self.highlightLetters(box: characterBox)
}
}
}
}
}
So how can I get the image part of region?.characterBoxes
This is code which i used in my custom camera for capture photo but it capturing photo correctly but if suppose user rotate device(but camera will still in portrait like iphone camera) and take picture i want that picture in portrait.how can i do that?
let stillImageOutput = self._getStillImageOutput()
if let videoConnection = stillImageOutput.connection(withMediaType: AVMediaTypeVideo) {
// videoConnection.videoOrientation = AVCaptureVideoOrientation.portrait
stillImageOutput.captureStillImageAsynchronously(from: videoConnection , completionHandler: { [weak self] sample, error in
if let error = error {
DispatchQueue.main.async(execute: {
self?._show(NSLocalizedString("Error", comment:""), message: error.localizedDescription)
})
imageCompletion(nil,error as NSError?)
return
}
let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sample)
var capturedImage = UIImage(data: imageData!)[![This how photo showing if i capture by rotating Device][1]][1]
add below code after the initialization of videoConnection and you good to go
switch UIDevice.current.orientation {
case .landscapeLeft:
videoConnection.videoOrientation = AVCaptureVideoOrientation.landscapeRight
break
case .landscapeRight:
videoConnection.videoOrientation = AVCaptureVideoOrientation.landscapeLeft
break
case .portrait:
videoConnection.videoOrientation = AVCaptureVideoOrientation.portrait
break
case .portraitUpsideDown:
videoConnection.videoOrientation = AVCaptureVideoOrientation.portraitUpsideDown
break
default:
videoConnection.videoOrientation = AVCaptureVideoOrientation.portrait
break
}
Set the orientation before capturing the image
// set the image orientation in output
if let photoOutputConnection = self.photoOutput.connection(with: .video) {
photoOutputConnection.videoOrientation = videoPreviewLayerOrientation! // set the orientation you want
}
self.photoOutput.capturePhoto(with: photoSettings, delegate: photoCaptureProcessor) // capture image
How can you set a camera into landscape mode? Everytime I take a photo, the image gets saved as a portrait image. When the device is in landscape mode the photo looks fine but if I see it in the camera roll it's still portrait mode.
This is my take photo function:
// take a photo
#IBAction func takePhoto(sender: AnyObject) {
self.fullScreenView.hidden = false
self.recordButton.enabled = false
self.takephoto.enabled = false
self.recordButton.hidden = true
self.takephoto.hidden = true
session.startRunning()
// customize the quality level or bitrate of the output photo
session.sessionPreset = AVCaptureSessionPresetPhoto
// add the AVCaptureVideoPreviewLayer to the view and set the view in fullscreen
fullScreenView.frame = view.bounds
videoPreviewLayer.frame = fullScreenView.bounds
fullScreenView.layer.addSublayer(videoPreviewLayer)
// add action to fullScreenView
gestureFullScreenView = UITapGestureRecognizer(target: self, action: #selector(ViewController.takePhoto(_:)))
self.fullScreenView.addGestureRecognizer(gestureFullScreenView)
// add action to myView
gestureView = UITapGestureRecognizer(target: self, action: #selector(ViewController.setFrontpage(_:)))
self.view.addGestureRecognizer(gestureView)
if (preview == true) {
if let videoConnection = stillImageOutput!.connectionWithMediaType(AVMediaTypeVideo) {
// code for photo capture goes here...
stillImageOutput?.captureStillImageAsynchronouslyFromConnection(videoConnection, completionHandler: { (sampleBuffer, error) -> Void in
// process the image data (sampleBuffer) here to get an image file we can put in our view
if (sampleBuffer != nil) {
let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer)
let image = UIImage(data: imageData, scale: 1.0)
self.fullScreenView.hidden = true
self.fullScreenView.gestureRecognizers?.forEach(self.fullScreenView.removeGestureRecognizer)
self.session.stopRunning()
// save image to the library
UIImageWriteToSavedPhotosAlbum(image!, nil, nil, nil)
self.imageViewBackground = UIImageView(frame: self.view.bounds)
self.imageViewBackground.image = image
self.imageViewBackground.tag = self.key
self.view.addSubview(self.imageViewBackground)
}
})
}
}
else {
preview = true
}
}
My preview looks like that and that's ok:
http://img5.fotos-hochladen.net/uploads/bildschirmfotom4s7diaehy.png
but in the end it looks like that:
http://img5.fotos-hochladen.net/uploads/bildschirmfoto3c2rlwtevf.png
Thanks in advance!
Because your videoConnection's orientation is always in portrait no matters your device is in portrait or landscape. So you should adjust the videoConnection's orientation to the correct one before taking the still photo
Add the following method to get videoOrientation for the current deviceOrientation
func videoOrientation(for deviceOrientation: UIDeviceOrientation) -> AVCaptureVideoOrientation {
switch deviceOrientation {
case UIDeviceOrientation.portrait:
return AVCaptureVideoOrientation.portrait
case UIDeviceOrientation.landscapeLeft:
return AVCaptureVideoOrientation.landscapeRight
case UIDeviceOrientation.landscapeRight:
return AVCaptureVideoOrientation.landscapeLeft
case UIDeviceOrientation.portraitUpsideDown:
return AVCaptureVideoOrientation.portraitUpsideDown
default:
return AVCaptureVideoOrientation.portrait
}
}
Right after the following line
if let videoConnection = stillImageOutput.connectionWithMediaType(AVMediaTypeVideo) {
Add the following line
videoConnection.videoOrientation = videoOrientation(for: UIDevice.current.orientation)
Note: If your app supports only portrait or landscape mode, this issue still happens because UIDevice.current.orientation will always return the supported orientation. To overcome this, you can use CoreMotion to detect device orientation, then pass it to videoOrientation(for:) method.
Hope this helps :)
I got the answer from #Yodagama but he didn't add how to detect the orientation
if let photoOutputConnection = self.photoOutput.connection(with: .video) {
// USE the below function HERE
photoOutputConnection.videoOrientation = videoOrientation()
}
photoOutput.capturePhoto(with: settings, delegate: self)
func to detect device orientation:
func videoOrientation() -> AVCaptureVideoOrientation {
var videoOrientation: AVCaptureVideoOrientation!
let orientation: UIDeviceOrientation = UIDevice.current.orientation
switch orientation {
case .faceUp, .faceDown, .unknown:
// let interfaceOrientation = UIApplication.shared.statusBarOrientation
if let interfaceOrientation = UIApplication.shared.windows.first(where: { $0.isKeyWindow })?.windowScene?.interfaceOrientation {
switch interfaceOrientation {
case .portrait, .portraitUpsideDown, .unknown:
videoOrientation = .portrait
case .landscapeLeft:
videoOrientation = .landscapeRight
case .landscapeRight:
videoOrientation = .landscapeLeft
#unknown default:
videoOrientation = .portrait
}
}
case .portrait, .portraitUpsideDown:
videoOrientation = .portrait
case .landscapeLeft:
videoOrientation = .landscapeRight
case .landscapeRight:
videoOrientation = .landscapeLeft
#unknown default:
videoOrientation = .portrait
}
return videoOrientation
}
Possible Solution: Saving Image as JPEG instead of PNG
This occurs because PNGs do not store orientation information. Save the photo as a JPG instead and it will be oriented correctly.
Use this code to convert your image to JPG immediately after taking the image (the second line is the operative one here):
let image = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer)
let imageData:NSData = UIImageJPEGRepresentation(image, 0.9)! // 0.9 is compression value: 0.0 is most compressed/lowest quality and 1.0 is least compressed/highest quality
Source + more info: https://stackoverflow.com/a/34796890/5700898
If that doesn't work
I've edited your code in a couple places, see if the below code now works as you hope:
// initialize saving photo capabilities
func image(image: UIImage, didFinishSavingWithError error: NSError?, contextInfo:UnsafePointer<Void>) {
if error == nil {
print("image saved")
} else {
print("save error: \(error?.localizedDescription)")
}
}
// take a photo
#IBAction func takePhoto(sender: AnyObject) {
self.fullScreenView.hidden = false
self.recordButton.enabled = false
self.takephoto.enabled = false
self.recordButton.hidden = true
self.takephoto.hidden = true
session.startRunning()
// customize the quality level or bitrate of the output photo
session.sessionPreset = AVCaptureSessionPresetPhoto
// add the AVCaptureVideoPreviewLayer to the view and set the view in fullscreen
fullScreenView.frame = view.bounds
videoPreviewLayer.frame = fullScreenView.bounds
fullScreenView.layer.addSublayer(videoPreviewLayer)
// add action to fullScreenView
gestureFullScreenView = UITapGestureRecognizer(target: self, action: #selector(ViewController.takePhoto(_:)))
self.fullScreenView.addGestureRecognizer(gestureFullScreenView)
// add action to myView
gestureView = UITapGestureRecognizer(target: self, action: #selector(ViewController.setFrontpage(_:)))
self.view.addGestureRecognizer(gestureView)
if (preview == true) {
if let videoConnection = stillImageOutput!.connectionWithMediaType(AVMediaTypeVideo) {
// code for photo capture goes here...
stillImageOutput?.captureStillImageAsynchronouslyFromConnection(videoConnection, completionHandler: { (sampleBuffer, error) -> Void in
// process the image data (sampleBuffer) here to get an image file we can put in our view
if (sampleBuffer != nil) {
self.stillImageOutput!.outputSettings = [AVVideoCodecKey: AVVideoCodecJPEG]
if let videoConnection = self.stillImageOutput!.connectionWithMediaType(AVMediaTypeVideo){
videoConnection.videoOrientation = self.interfaceToVideoOrientation()
self.stillImageOutput!.captureStillImageAsynchronouslyFromConnection(videoConnection, completionHandler: {
(sampleBuffer, error) in
let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer)
let image = UIImage(data: imageData)
self.fullScreenView.hidden = true
self.fullScreenView.gestureRecognizers?.forEach(self.fullScreenView.removeGestureRecognizer)
self.session.stopRunning()
// save image to the library
UIImageWriteToSavedPhotosAlbum(image, self, #selector(ViewController.image(_:didFinishSavingWithError:contextInfo:)), nil)
self.imageViewBackground = UIImageView(frame: self.view.bounds)
self.imageViewBackground.image = image
self.imageViewBackground.tag = self.key
self.view.addSubview(self.imageViewBackground)
}
})
}
}
else {
preview = true
}
}
I am having a hard time for something I think shouldn’t be so difficult, so I presume I must be looking at the problem from the wrong angle.
In order to understand how AVCaptureStillImageOutput and the camera work I made a tiny app.
This app is able to take a picture and save it as a PNG file (I do not want JPEG). The next time the app is launched, it checks if a file is present and if it is, the image stored inside the file is used as the background view of the app. The idea is rather simple.
The problem is that it does not work. If someone can tell me what I am doing wrong that will be very helpful.
I would like the picture to appear as a background the same way it was on the display when it was taken, but it is rotated or has the wrong scale ..etc..
Here is the relevant code (I can provide more information if ever needed).
The viewDidLoad method:
override func viewDidLoad() {
super.viewDidLoad()
// For the photo capture:
captureSession.sessionPreset = AVCaptureSessionPresetHigh
// Select the appropriate capture devices:
for device in AVCaptureDevice.devices() {
// Make sure this particular device supports video.
if (device.hasMediaType(AVMediaTypeVideo)) {
// Finally check the position and confirm we've got the back camera.
if(device.position == AVCaptureDevicePosition.Back) {
captureDevice = device as? AVCaptureDevice
}
}
}
tapGesture = UITapGestureRecognizer(target: self, action: Selector("takePhoto"))
self.view.addGestureRecognizer(tapGesture)
let filePath = self.toolBox.getDocumentsDirectory().stringByAppendingPathComponent("BackGroundImage#2x.png")
if !NSFileManager.defaultManager().fileExistsAtPath(filePath) {return}
let bgImage = UIImage(contentsOfFile: filePath),
bgView = UIImageView(image: bgImage)
self.view.addSubview(bgView)
}
The method to handle the picture taking:
func takePhoto() {
if !captureSession.running {
beginPhotoCaptureSession()
return
}
if let videoConnection = stillImageOutput.connectionWithMediaType(AVMediaTypeVideo) {
stillImageOutput.captureStillImageAsynchronouslyFromConnection(videoConnection) {
(imageDataSampleBuffer, error) -> Void in
if error == nil {
var localImage = UIImage(fromSampleBuffer: imageDataSampleBuffer)
UIGraphicsBeginImageContext(localImage!.size)
CGContextRotateCTM (UIGraphicsGetCurrentContext(), CGFloat(M_PI_2))
//localImage!.drawAtPoint(CGPointZero)
localImage!.drawAtPoint(CGPoint(x: -localImage!.size.height, y: -localImage!.size.width))
//localImage!.drawAtPoint(CGPoint(x: -localImage!.size.width, y: -localImage!.size.height))
localImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
localImage = resizeImage(localImage!, toSize: self.view.frame.size)
if let data = UIImagePNGRepresentation(localImage!) {
let bitMapName = "BackGroundImage#2x"
let filename = self.toolBox.getDocumentsDirectory().stringByAppendingPathComponent("\(bitMapName).png")
data.writeToFile(filename, atomically: true)
print("Picture saved: \(bitMapName)\n\(filename)")
}
} else {print("Error on taking a picture:\n\(error)")}
}
}
captureSession.stopRunning()
previewLayer!.removeFromSuperlayer()
}
The method to start the AVCaptureSession:
func beginPhotoCaptureSession() {
do {let input = try AVCaptureDeviceInput(device: captureDevice)
captureSession.addInput(input)
} catch let error as NSError {
// Handle any errors:
print(error)
}
previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer?.frame = self.view.layer.frame
self.view.layer.addSublayer(previewLayer!)
captureSession.startRunning()
stillImageOutput.outputSettings = [kCVPixelBufferPixelFormatTypeKey:Int(kCVPixelFormatType_32BGRA)]
if captureSession.canAddOutput(stillImageOutput) {
captureSession.addOutput(stillImageOutput)
}
}
As an example here is an image of a picture taken with the app:
Now here is what I get as the background of the app when it is relaunched:
If it was working correctly the 2 pictures would be similar.
I cant see rotation in screenshot.. But the scale is a problem which is related to your code when you are drawing it in function takePhoto. You can try
func takePhoto() {
if !captureSession.running {
beginPhotoCaptureSession()
return
}
if let videoConnection = stillImageOutput.connectionWithMediaType(AVMediaTypeVideo) {
stillImageOutput.captureStillImageAsynchronouslyFromConnection(videoConnection) {
(imageDataSampleBuffer, error) -> Void in
if error == nil {
if let data = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(imageDataSampleBuffer) {
let bitMapName = "BackGroundImage#2x"
let filename = self.toolBox.getDocumentsDirectory().stringByAppendingPathComponent("\(bitMapName).png")
data.writeToFile(filename, atomically: true)
print("Picture saved: \(bitMapName)\n\(filename)")
}
} else {print("Error on taking a picture:\n\(error)")}
}
}
captureSession.stopRunning()
previewLayer!.removeFromSuperlayer()
}
For those who might be hitting the same issue at one point, I post below what I fixed in the code to make it work. There may still be some improvement needed to support all possible orientation, but it's OK for a start.
if let videoConnection = stillImageOutput.connectionWithMediaType(AVMediaTypeVideo) {
stillImageOutput.captureStillImageAsynchronouslyFromConnection(videoConnection) {
(imageDataSampleBuffer, error) -> Void in
if error == nil {
var localImage = UIImage(fromSampleBuffer: imageDataSampleBuffer)
var imageSize = CGSize(width: UIScreen.mainScreen().bounds.height * UIScreen.mainScreen().scale,
height: UIScreen.mainScreen().bounds.width * UIScreen.mainScreen().scale)
localImage = resizeImage(localImage!, toSize: imageSize)
imageSize = CGSize(width: imageSize.height, height: imageSize.width)
UIGraphicsBeginImageContext(imageSize)
CGContextRotateCTM (UIGraphicsGetCurrentContext(), CGFloat(M_PI_2))
localImage!.drawAtPoint(CGPoint(x: 0.0, y: -imageSize.width))
localImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
if let data = UIImagePNGRepresentation(localImage!) {
data.writeToFile(self.bmpFilePath, atomically: true)
}
} else {print("Error on taking a picture:\n\(error)")}
}
}