I'm making iOS video recording app using AVFoundation. Every frame of a video should have timestamp showing time when the video frame was recorded. I looked internet but could not find any reasonable articles.
Here is my current code around video record.
private var session = AVCaptureSession()
private let videoOutput = AVCaptureVideoDataOutput()
private let audioOutput = AVCaptureAudioDataOutput()
private func setupCaptureSession() {
session.sessionPreset = .vga640x480
guard
let videoDevice = AVCaptureDevice.default(for: .video),
let audioDevice = AVCaptureDevice.default(for: .audio),
let videoInput = try? AVCaptureDeviceInput(device: videoDevice),
let audioInput = try? AVCaptureDeviceInput(device: audioDevice) else {
fatalError()
}
session.beginConfiguration()
session.addInput(videoInput)
session.addInput(audioInput)
session.addOutput(videoOutput)
session.addOutput(audioOutput)
session.commitConfiguration()
DispatchQueue.main.async {
let previewView = PreviewView()
previewView.videoPreviewLayer.session = self.session
previewView.videoPreviewLayer.connection?.videoOrientation = .landscapeRight
previewView.frame = self.view.bounds
previewView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
self.compositeImageView.insertSubview(previewView, at: 0)
self.session.startRunning()
}
}
private func startRecording() {
// AVAssetWriter
assetWriter = try! AVAssetWriter(outputURL: self.exportURL!, fileType: .mov)
// video
let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: [
AVVideoCodecKey : AVVideoCodecType.h264,
AVVideoWidthKey : 640,
AVVideoHeightKey : 480
])
videoInput.expectsMediaDataInRealTime = true
assetWriter?.add(videoInput)
// audio
let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
audioInput.expectsMediaDataInRealTime = true
assetWriter?.add(audioInput)
assetWriter?.startWriting()
let queue = DispatchQueue.global()
videoOutput.setSampleBufferDelegate(self, queue: queue)
audioOutput.setSampleBufferDelegate(self, queue: queue)
}
private func finishRecording() {
videoOutput.setSampleBufferDelegate(nil, queue: nil)
audioOutput.setSampleBufferDelegate(nil, queue: nil)
startTime = nil
assetWriter?.finishWriting { [weak self] in
guard let self = self else { return }
guard self.assetWriter!.status == .completed else { fatalError("failed recording") }
self.saveToPhotoLibrary { isSaveToPhotoLibrarySucceed in
print("vide saved to photo library")
guard isSaveToPhotoLibrarySucceed else {
print("Save to photo library failed")
return
}
self.saveToRealmFromTempVideo {
self.uploadVideoToServer()
}
}
}
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard isRecording else { fatalError() }
guard CMSampleBufferDataIsReady(sampleBuffer) else {
print("not ready")
return
}
if startTime == nil {
startTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
assetWriter?.startSession(atSourceTime: startTime!)
}
// Append video or audio
let mediaType: AVMediaType = output is AVCaptureVideoDataOutput ? .video : .audio
if mediaType == .video {
appendVideo(from: sampleBuffer)
} else if mediaType == .audio {
appendAudio(from: sampleBuffer)
} else {
fatalError("should not reach here")
}
}
Related
I want to save image at a high frame rate.
IO is expensive, not 240fps exactly.But maybe more than 120fps.
What I'm I do is set captureDivece in viewDidLoad,
and log timestamp in captureOutput and view the rate.
But I notice that captureOutput is always 30fps.
Could you please tell me which I get wrong?
Thanks for your time and answer!
Here is my code and result:
//
// ViewController.swift
// CustomCamera
//
// Created by chunibyo on 2021/3/8.
//
import UIKit
import AVFoundation
import Vision
import VideoToolbox
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
#IBOutlet weak var captureButton: UIButton!
let sessionQueue = DispatchQueue(label: "Session Queue")
var status = false
private var MyCaptureDevice: AVCaptureDevice?
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view.
captureButton.layer.cornerRadius = captureButton.frame.width / 2
captureButton.layer.masksToBounds = true
captureButton.layer.zPosition = 10
guard let captureDevice = AVCaptureDevice.default(.builtInDualCamera, for: .video, position: .back) else {return}
guard let input = try? AVCaptureDeviceInput(device: captureDevice) else {return}
// 1
for vFormat in captureDevice.formats {
// 2
let ranges = vFormat.videoSupportedFrameRateRanges as [AVFrameRateRange]
let frameRates = ranges[0]
// 3
if frameRates.maxFrameRate == 240 {
// 4
try? captureDevice.lockForConfiguration()
captureDevice.activeFormat = vFormat as AVCaptureDevice.Format
captureDevice.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
captureDevice.activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
captureDevice.videoZoomFactor = captureDevice.minAvailableVideoZoomFactor
captureDevice.unlockForConfiguration()
}
}
let captureSession = AVCaptureSession();
// captureSession.sessionPreset = .photo
captureSession.addInput(input)
captureSession.startRunning();
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
view.layer.addSublayer(previewLayer)
previewLayer.frame = view.frame
let dataOutput = AVCaptureVideoDataOutput()
dataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
dataOutput.alwaysDiscardsLateVideoFrames = true;
captureSession.addOutput(dataOutput)
print(captureDevice.minAvailableVideoZoomFactor)
print(captureDevice.maxAvailableVideoZoomFactor)
MyCaptureDevice = captureDevice
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
print(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)))
// if !status { return }
// guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// guard let uiImage = UIImage(pixelBuffer: pixelBuffer) else { return }
//
// sessionQueue.async {
// guard let captureDevice = self.MyCaptureDevice else { return }
// if captureDevice.videoZoomFactor >= (captureDevice.maxAvailableVideoZoomFactor - 0.2) { return }
// UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
// try? captureDevice.lockForConfiguration()
// captureDevice.videoZoomFactor += 0.1
// captureDevice.unlockForConfiguration()
// }
}
#IBAction func captureControl(_ sender: UIButton) {
DispatchQueue.main.async {
if self.status {
self.captureButton.backgroundColor = .white
print("stop")
self.status = !self.status
}
else {
self.captureButton.backgroundColor = .red
print("recording...")
self.status = !self.status
}
}
}
}
extension UIImage {
public convenience init?(pixelBuffer: CVPixelBuffer) {
var cgImage: CGImage?
VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
guard let _cgImage = cgImage else { return nil }
self.init(cgImage: _cgImage)
}
}
console log (stackoverflow not allow me to post image, sorry)
captureOutput only logs is 240fps.
captureOutput with take photos to album is about 70~100fps.
This code can get 240fps logs.
//
// ViewController.swift
// CustomCamera
//
// Created by chunibyo on 2021/3/8.
//
import UIKit
import AVFoundation
import Vision
import VideoToolbox
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
#IBOutlet weak var captureButton: UIButton!
let sessionQueue = DispatchQueue(label: "Session Queue")
var status = false
var zoomStatus = 1
private var MyCaptureDevice: AVCaptureDevice?
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view.
captureButton.layer.cornerRadius = captureButton.frame.width / 2
captureButton.layer.masksToBounds = true
captureButton.layer.zPosition = 10
guard let captureDevice = AVCaptureDevice.default(for: AVMediaType.video) else {return}
guard let input = try? AVCaptureDeviceInput(device: captureDevice) else {return}
let captureSession = AVCaptureSession();
// captureSession.sessionPreset = .photo
captureSession.addInput(input)
// 1
for vFormat in captureDevice.formats {
// 2
let ranges = vFormat.videoSupportedFrameRateRanges as [AVFrameRateRange]
let frameRates = ranges[0]
// 3
if frameRates.maxFrameRate == 240 {
// 4
try? captureDevice.lockForConfiguration()
captureDevice.activeFormat = vFormat as AVCaptureDevice.Format
captureDevice.activeVideoMinFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
captureDevice.activeVideoMaxFrameDuration = CMTimeMake(value: 1, timescale: Int32(240))
captureDevice.videoZoomFactor = captureDevice.minAvailableVideoZoomFactor
captureDevice.unlockForConfiguration()
}
}
captureSession.startRunning();
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
view.layer.addSublayer(previewLayer)
previewLayer.frame = view.frame
let dataOutput = AVCaptureVideoDataOutput()
dataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
dataOutput.alwaysDiscardsLateVideoFrames = true;
captureSession.addOutput(dataOutput)
print(captureDevice.minAvailableVideoZoomFactor)
print(captureDevice.maxAvailableVideoZoomFactor)
MyCaptureDevice = captureDevice
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
print(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)))
// if !status { return }
// guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// guard let uiImage = UIImage(pixelBuffer: pixelBuffer) else { return }
// UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
//
// guard let captureDevice = self.MyCaptureDevice else { return }
// if self.zoomStatus == 1 && captureDevice.videoZoomFactor >= CGFloat(Int32(captureDevice.maxAvailableVideoZoomFactor * 0.6)) { self.zoomStatus = -1
// }
// else if self.zoomStatus == -1 && captureDevice.videoZoomFactor <= (captureDevice.minAvailableVideoZoomFactor + 1.0) {
// self.zoomStatus = 1
// }
// UIImageWriteToSavedPhotosAlbum(uiImage, nil, nil, nil)
// try? captureDevice.lockForConfiguration()
// captureDevice.videoZoomFactor += (0.1 * CGFloat(self.zoomStatus))
// captureDevice.unlockForConfiguration()
}
#IBAction func captureControl(_ sender: UIButton) {
DispatchQueue.main.async {
if self.status {
self.captureButton.backgroundColor = .white
print("stop")
self.status = !self.status
}
else {
self.captureButton.backgroundColor = .red
print("recording...")
self.status = !self.status
}
}
}
}
extension UIImage {
public convenience init?(pixelBuffer: CVPixelBuffer) {
var cgImage: CGImage?
VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
guard let _cgImage = cgImage else { return nil }
self.init(cgImage: _cgImage)
}
}
I am trying to record depth data from the TrueDepth camera along with a photo. But when calling
AVCapturePhotoOutput capturePhoto(withSettings,delegate)
I get an exception stating:
No active and enabled video connection
I configure the camera and outputs like so (basically following the guide from Apple about photo capturing and capturing depth):
func configurePhotoOutput() throws {
self.captureSession = AVCaptureSession()
guard self.captureSession != nil else {
return
}
// Select a depth-capable capture device.
guard let videoDevice = AVCaptureDevice.default(.builtInTrueDepthCamera,
for: .video, position: .unspecified)
else { fatalError("No dual camera.") }
// Select a depth (not disparity) format that works with the active color format.
let availableFormats = videoDevice.activeFormat.supportedDepthDataFormats
let depthFormat = availableFormats.first(where: { format in
let pixelFormatType = CMFormatDescriptionGetMediaSubType(format.formatDescription)
return (pixelFormatType == kCVPixelFormatType_DepthFloat16 ||
pixelFormatType == kCVPixelFormatType_DepthFloat32)
})
do {
try videoDevice.lockForConfiguration()
videoDevice.activeDepthDataFormat = depthFormat
videoDevice.unlockForConfiguration()
} catch {
print("Could not lock device for configuration: \(error)")
return
}
self.captureSession!.beginConfiguration()
// add video input
guard let videoDeviceInput = try? AVCaptureDeviceInput(device: videoDevice),
self.captureSession!.canAddInput(videoDeviceInput)
else { fatalError("Can't add video input.") }
self.captureSession!.addInput(videoDeviceInput)
// add video output
if self.captureSession!.canAddOutput(videoOutput) {
self.captureSession!.addOutput(videoOutput)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]
} else { fatalError("Can't add video output.") }
// Set up photo output for depth data capture.
let photoOutput = AVCapturePhotoOutput()
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
guard self.captureSession!.canAddOutput(photoOutput)
else { fatalError("Can't add photo output.") }
self.captureSession!.addOutput(photoOutput)
self.captureSession!.sessionPreset = .photo
self.captureSession!.commitConfiguration()
self.captureSession!.startRunning()
}
And the code responsible for capturing the photo:
func captureImage(delegate: AVCapturePhotoCaptureDelegate,completion: #escaping (UIImage?, Error?) -> Void) {
let photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
photoSettings.isDepthDataDeliveryEnabled =
self.photoOutput.isDepthDataDeliverySupported
photoSettings.isDepthDataFiltered = false
self.photoOutput.capturePhoto(with: photoSettings, delegate: delegate) // <---- error is being thrown on this call
self.photoCaptureCompletionBlock = completion
}
What I am I doing wrong in this configuration?
solved it with the following implementation:
Any comments / remarks are highly appreciated!
import AVFoundation
import UIKit
class CameraController: NSObject {
var captureSession: AVCaptureSession?
var videoDevice: AVCaptureDevice?
var previewLayer: AVCaptureVideoPreviewLayer?
var videoOutput = AVCaptureVideoDataOutput()
var photoOutput = AVCapturePhotoOutput()
func prepare(completionHandler: #escaping (Error?) -> Void) {
func createCaptureSession() {
captureSession = AVCaptureSession()
}
func configureCaptureDevices() throws {
// Select a depth-capable capture device.
guard let vd = AVCaptureDevice.default(.builtInTrueDepthCamera,
for: .video, position: .unspecified)
else { fatalError("No dual camera.") }
videoDevice = vd
// Select a depth (not disparity) format that works with the active color format.
let availableFormats = videoDevice!.activeFormat.supportedDepthDataFormats
let depthFormat = availableFormats.first(where: { format in
let pixelFormatType = CMFormatDescriptionGetMediaSubType(format.formatDescription)
return (pixelFormatType == kCVPixelFormatType_DepthFloat16 ||
pixelFormatType == kCVPixelFormatType_DepthFloat32)
})
do {
try videoDevice!.lockForConfiguration()
videoDevice!.activeDepthDataFormat = depthFormat
videoDevice!.unlockForConfiguration()
} catch {
print("Could not lock device for configuration: \(error)")
return
}
}
func configureDeviceInputs() throws {
if( captureSession == nil) {
throw CameraControllerError.captureSessionIsMissing
}
captureSession?.beginConfiguration()
// add video input
guard let videoDeviceInput = try? AVCaptureDeviceInput(device: self.videoDevice!),
captureSession!.canAddInput(videoDeviceInput)
else { fatalError("Can't add video input.") }
captureSession!.addInput(videoDeviceInput)
captureSession?.commitConfiguration()
}
func configurePhotoOutput() throws {
guard let captureSession = self.captureSession else { throw CameraControllerError.captureSessionIsMissing }
captureSession.beginConfiguration()
// Set up photo output for depth data capture.
photoOutput = AVCapturePhotoOutput()
photoOutput.setPreparedPhotoSettingsArray([AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])], completionHandler: nil)
guard captureSession.canAddOutput(photoOutput)
else { fatalError("Can't add photo output.") }
captureSession.addOutput(photoOutput)
// must be set after photoOutput is added to captureSession. Why???
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
captureSession.sessionPreset = .photo
captureSession.commitConfiguration()
captureSession.startRunning()
}
DispatchQueue(label: "prepare").async {
do {
createCaptureSession()
try configureCaptureDevices()
try configureDeviceInputs()
try configurePhotoOutput()
}
catch {
DispatchQueue.main.async {
completionHandler(error)
}
return
}
DispatchQueue.main.async {
completionHandler(nil)
}
}
}
func displayPreview(on view: UIView) throws {
guard let captureSession = self.captureSession, captureSession.isRunning else { throw CameraControllerError.captureSessionIsMissing }
self.previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
self.previewLayer?.videoGravity = AVLayerVideoGravity.resizeAspectFill
self.previewLayer?.connection?.videoOrientation = .portrait
view.layer.insertSublayer(self.previewLayer!, at: 0)
self.previewLayer?.frame = view.frame
}
func captureImage(delegate: AVCapturePhotoCaptureDelegate,completion: #escaping (UIImage?, Error?) -> Void) {
let photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
photoSettings.isDepthDataDeliveryEnabled = true
photoSettings.isDepthDataFiltered = false
self.photoOutput.capturePhoto(with: photoSettings, delegate: delegate)
self.photoCaptureCompletionBlock = completion
}
var photoCaptureCompletionBlock: ((UIImage?, Error?) -> Void)?
}
extension CameraController {
public enum CameraPosition {
case front
case rear
}
enum CameraControllerError: Swift.Error {
case captureSessionAlreadyRunning
case captureSessionIsMissing
case inputsAreInvalid
case invalidOperation
case noCamerasAvailable
case unknown
}
}
I don't understand why torch is not continuously on.
It blinks at the start and immediately goes off.
I did lock and unlock as well.
I added part of my code which I think is relevent with the problem here.
override func viewDidLoad() {
super.viewDidLoad()
initCamera()
}
func initCamera(){
captureSession.sessionPreset = .medium
guard let device = AVCaptureDevice.default(for: .video) else {
print("no camera")
return
}
try? device.lockForConfiguration()
device.activeVideoMaxFrameDuration = CMTimeMake(1, 30)
if device.hasTorch && device.isTorchAvailable && !device.isTorchActive{
do {
try device.setTorchModeOn(level: 1.0)
device.torchMode = .on
} catch {
print("Torch could not be used")
}
} else {
print("Torch is not available")
}
if device.isFocusModeSupported(.continuousAutoFocus){
device.focusMode = .continuousAutoFocus
}
device.unlockForConfiguration()
guard let captureInput = try? AVCaptureDeviceInput(device: device) else {return;}
if captureSession.canAddInput(captureInput) {
captureSession.addInput(captureInput)
}
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
view.layer.addSublayer(previewLayer)
let videoOutput = AVCaptureVideoDataOutput()
let queue = DispatchQueue(label: "queue image delegate", attributes: .concurrent)
videoOutput.setSampleBufferDelegate(self, queue: queue)
if captureSession.canAddOutput(videoOutput)
{
captureSession.addOutput(videoOutput)
}
captureSession.startRunning()
}
I am new to swift and trying to build a camera app which can apply real-time filters, and save with the applied filters.
So far i can preview real-time with the applied filters, but when i save the video its all black.
import UIKit
import AVFoundation
import AssetsLibrary
import CoreMedia
import Photos
class ViewController: UIViewController , AVCaptureVideoDataOutputSampleBufferDelegate {
var captureSession: AVCaptureSession!
#IBOutlet weak var previewView: UIView!
#IBOutlet weak var recordButtton: UIButton!
#IBOutlet weak var imageView: UIImageView!
var assetWriter: AVAssetWriter?
var assetWriterPixelBufferInput: AVAssetWriterInputPixelBufferAdaptor?
var isWriting = false
var currentSampleTime: CMTime?
var currentVideoDimensions: CMVideoDimensions?
override func viewDidLoad() {
super.viewDidLoad()
FilterVendor.register()
setupCaptureSession()
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
func setupCaptureSession() {
let captureSession = AVCaptureSession()
captureSession.sessionPreset = AVCaptureSessionPresetPhoto
guard let captureDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo), let input = try? AVCaptureDeviceInput(device: captureDevice) else {
print("Can't access the camera")
return
}
if captureSession.canAddInput(input) {
captureSession.addInput(input)
}
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue.main)
if captureSession.canAddOutput(videoOutput) {
captureSession.addOutput(videoOutput)
}
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
if((previewLayer) != nil) {
view.layer.addSublayer(previewLayer!)
}
captureSession.startRunning()
}
#IBAction func record(_ sender: Any) {
if isWriting {
print("stop record")
self.isWriting = false
assetWriterPixelBufferInput = nil
assetWriter?.finishWriting(completionHandler: {[unowned self] () -> Void in
self.saveMovieToCameraRoll()
})
} else {
print("start record")
createWriter()
assetWriter?.startWriting()
assetWriter?.startSession(atSourceTime: currentSampleTime!)
isWriting = true
}
}
func saveMovieToCameraRoll() {
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: self.movieURL() as URL)
}) { saved, error in
if saved {
print("saved")
}
}
}
func movieURL() -> NSURL {
let tempDir = NSTemporaryDirectory()
let url = NSURL(fileURLWithPath: tempDir).appendingPathComponent("tmpMov.mov")
return url! as NSURL
}
func checkForAndDeleteFile() {
let fm = FileManager.default
let url = movieURL()
let exist = fm.fileExists(atPath: url.path!)
if exist {
do {
try fm.removeItem(at: url as URL)
} catch let error as NSError {
print(error.localizedDescription)
}
}
}
func createWriter() {
self.checkForAndDeleteFile()
do {
assetWriter = try AVAssetWriter(outputURL: movieURL() as URL, fileType: AVFileTypeQuickTimeMovie)
} catch let error as NSError {
print(error.localizedDescription)
return
}
let outputSettings = [
AVVideoCodecKey : AVVideoCodecH264,
AVVideoWidthKey : Int(currentVideoDimensions!.width),
AVVideoHeightKey : Int(currentVideoDimensions!.height)
] as [String : Any]
let assetWriterVideoInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: outputSettings as? [String : AnyObject])
assetWriterVideoInput.expectsMediaDataInRealTime = true
assetWriterVideoInput.transform = CGAffineTransform(rotationAngle: CGFloat(M_PI / 2.0))
let sourcePixelBufferAttributesDictionary = [
String(kCVPixelBufferPixelFormatTypeKey) : Int(kCVPixelFormatType_32BGRA),
String(kCVPixelBufferWidthKey) : Int(currentVideoDimensions!.width),
String(kCVPixelBufferHeightKey) : Int(currentVideoDimensions!.height),
String(kCVPixelFormatOpenGLESCompatibility) : kCFBooleanTrue
] as [String : Any]
assetWriterPixelBufferInput = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: assetWriterVideoInput,
sourcePixelBufferAttributes: sourcePixelBufferAttributesDictionary)
if assetWriter!.canAdd(assetWriterVideoInput) {
assetWriter!.add(assetWriterVideoInput)
} else {
print("no way\(assetWriterVideoInput)")
}
}
func captureOutput(_ captureOutput: AVCaptureOutput, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection) {
autoreleasepool {
connection.videoOrientation = AVCaptureVideoOrientation.landscapeLeft;
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let cameraImage = CIImage(cvPixelBuffer: pixelBuffer)
let filter = CIFilter(name: "Fİlter")!
filter.setValue(cameraImage, forKey: kCIInputImageKey)
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer)!
self.currentVideoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription)
self.currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer)
if self.isWriting {
if self.assetWriterPixelBufferInput?.assetWriterInput.isReadyForMoreMediaData == true {
var newPixelBuffer: CVPixelBuffer? = nil
CVPixelBufferPoolCreatePixelBuffer(nil, self.assetWriterPixelBufferInput!.pixelBufferPool!, &newPixelBuffer)
let success = self.assetWriterPixelBufferInput?.append(newPixelBuffer!, withPresentationTime: self.currentSampleTime!)
if success == false {
print("Pixel Buffer failed")
}
}
}
DispatchQueue.main.async {
if let outputValue = filter.value(forKey: kCIOutputImageKey) as? CIImage {
let filteredImage = UIImage(ciImage: outputValue)
self.imageView.image = filteredImage
}
}
}
}
}
I've added some comments to the critical part below:
func captureOutput(_ captureOutput: AVCaptureOutput, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection) {
autoreleasepool {
connection.videoOrientation = AVCaptureVideoOrientation.landscapeLeft;
// COMMENT: This line makes sense - this is your pixelbuffer from the camera.
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// COMMENT: OK, so you turn pixelBuffer into a CIImage...
let cameraImage = CIImage(cvPixelBuffer: pixelBuffer)
// COMMENT: And now you've create a CIImage with a Filter instruction...
let filter = CIFilter(name: "Fİlter")!
filter.setValue(cameraImage, forKey: kCIInputImageKey)
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer)!
self.currentVideoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription)
self.currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer)
if self.isWriting {
if self.assetWriterPixelBufferInput?.assetWriterInput.isReadyForMoreMediaData == true {
// COMMENT: Here's where it gets weird. You've declared a new, empty pixelBuffer... but you already have one (pixelBuffer) that contains the image you want to write...
var newPixelBuffer: CVPixelBuffer? = nil
// COMMENT: And you grabbed memory from the pool.
CVPixelBufferPoolCreatePixelBuffer(nil, self.assetWriterPixelBufferInput!.pixelBufferPool!, &newPixelBuffer)
// COMMENT: And now you wrote an empty pixelBuffer back <-- this is what's causing the black frame.
let success = self.assetWriterPixelBufferInput?.append(newPixelBuffer!, withPresentationTime: self.currentSampleTime!)
if success == false {
print("Pixel Buffer failed")
}
}
}
// COMMENT: And now you're sending the filtered image back to the screen.
DispatchQueue.main.async {
if let outputValue = filter.value(forKey: kCIOutputImageKey) as? CIImage {
let filteredImage = UIImage(ciImage: outputValue)
self.imageView.image = filteredImage
}
}
}
}
It looks to me like you're basically getting the screen image, creating a filtered copy, then making a NEW pixel buffer which is empty and writing that out.
If you write the pixelBuffer you grabbed instead of the new one you're creating, you should successfully write the image.
What you need to successfully write out the filtered video is to create a new CVPixelBuffer from a CIImage - that solution exists here on StackOverflow already, I know because I needed that step myself!
I am making a sample application that utilizes AVFoundation to record video. The whole point is so I can have more control over how the video is recorded. In my sample project I have the video capturing but am struggling with handling orientation correctly.
I have done a lot of searching around the web and found that others are suggesting that I should NOT allow my capture view or capture session to rotate based on orientation, but rather set a transformation to rotate the video during playback. I have this working fine on iOS and Mac devices, but am wondering if I will have issues on other platforms such as Windows or Android.
Also, when I view the recorded video's metadata I see that the width and height are not set properly for the orientation. This makes sense as I am only transforming the presentation of the video and not it's actual resolution.
My question here is how do I correctly support portrait and landscape orientations and have it reflected correctly in the video file output? I need these videos to play on all platforms correctly so I am thinking the resolution is going to matter a great deal.
Below is the full source I have written thus far. I appreciate any advice you all can provide.
import UIKit
import AVFoundation
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
//MARK: - Outlet
#IBOutlet weak var previewView: UIView!
#IBOutlet var playStopButton: UIButton!
//MARK: - Private Variables
let videoQueue = DispatchQueue(label: "VideoQueue", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
private let captureSession = AVCaptureSession()
var outputUrl: URL {
get {
if let url = _outputUrl {
return url
}
_outputUrl = outputDirectory.appendingPathComponent("video.mp4")
return _outputUrl!
}
}
private var _outputUrl: URL?
var outputDirectory: URL {
get {
if let url = _outputDirectory {
return url
}
_outputDirectory = getDocumentsDirectory().appendingPathComponent("recording")
return _outputDirectory!
}
}
private var _outputDirectory: URL?
private var assetWriter: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var videoOutput: AVCaptureVideoDataOutput?
private var audioOutput: AVCaptureAudioDataOutput?
private var isRecording = false
private var isWriting = false
private var videoSize = CGSize(width: 640, height: 480)
//MARK: - View Life-cycle
override func viewDidLoad() {
super.viewDidLoad()
videoQueue.async {
do {
try self.configureCaptureSession()
try self.configureAssetWriter()
DispatchQueue.main.async {
self.configurePreview()
}
} catch {
DispatchQueue.main.async {
self.showAlert("Unable to configure video output")
}
}
}
}
override var preferredInterfaceOrientationForPresentation: UIInterfaceOrientation {
return .portrait
}
override var supportedInterfaceOrientations: UIInterfaceOrientationMask {
return .portrait
}
//MARK: - Capture Session
private func configureCaptureSession() throws {
do {
// configure the session
if captureSession.canSetSessionPreset(AVCaptureSessionPreset640x480) {
captureSession.sessionPreset = AVCaptureSessionPreset640x480
}
// configure capture devices
let camDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo)
let micDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeAudio)
let camInput = try AVCaptureDeviceInput(device: camDevice)
let micInput = try AVCaptureDeviceInput(device: micDevice)
if captureSession.canAddInput(camInput) {
captureSession.addInput(camInput)
}
if captureSession.canAddInput(micInput) {
captureSession.addInput(micInput)
}
// configure audio/video output
videoOutput = AVCaptureVideoDataOutput()
videoOutput?.alwaysDiscardsLateVideoFrames = false // TODO: is this necessary?
videoOutput?.setSampleBufferDelegate(self, queue: videoQueue)
if let v = videoOutput {
captureSession.addOutput(v)
}
audioOutput = AVCaptureAudioDataOutput()
audioOutput?.setSampleBufferDelegate(self, queue: videoQueue)
if let a = audioOutput {
captureSession.addOutput(a)
}
// configure audio session
let audioSession = AVAudioSession.sharedInstance()
try audioSession.setCategory(AVAudioSessionCategoryPlayAndRecord)
try audioSession.setActive(true)
var micPort: AVAudioSessionPortDescription?
if let inputs = audioSession.availableInputs {
for port in inputs {
if port.portType == AVAudioSessionPortBuiltInMic {
micPort = port
break;
}
}
}
if let port = micPort, let dataSources = port.dataSources {
for source in dataSources {
if source.orientation == AVAudioSessionOrientationFront {
try audioSession.setPreferredInput(port)
break
}
}
}
} catch {
print("Failed to configure audio/video capture session")
throw error
}
}
private func configureAssetWriter() throws {
prepareVideoFile()
do {
assetWriter = try AVAssetWriter(url: outputUrl, fileType: AVFileTypeMPEG4)
guard let writer = assetWriter else {
print("Asset writer not created")
return
}
let vidSize = videoSize
let videoSettings: [String: Any] = [AVVideoCodecKey: AVVideoCodecH264,
AVVideoWidthKey: NSNumber(value: Float(vidSize.width)),
AVVideoHeightKey: NSNumber(value: Float(vidSize.height))]
videoInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: videoSettings)
videoInput?.expectsMediaDataInRealTime = true
videoInput?.transform = getVideoTransform()
var channelLayout = AudioChannelLayout()
memset(&channelLayout, 0, MemoryLayout<AudioChannelLayout>.size);
channelLayout.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo;
let audioSettings: [String: Any] = [AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2]
audioInput = AVAssetWriterInput(mediaType: AVMediaTypeAudio, outputSettings: audioSettings)
audioInput?.expectsMediaDataInRealTime = true
guard let vi = videoInput else {
print("Video input not configured")
return
}
guard let ai = audioInput else {
print("Audio input not configured")
return
}
if writer.canAdd(vi) {
writer.add(vi)
}
if writer.canAdd(ai) {
writer.add(ai)
}
} catch {
print("Failed to configure asset writer")
throw error
}
}
private func prepareVideoFile() {
if FileManager.default.fileExists(atPath: outputUrl.path) {
do {
try FileManager.default.removeItem(at: outputUrl)
} catch {
print("Unable to remove file at URL \(outputUrl)")
}
}
if !FileManager.default.fileExists(atPath: outputDirectory.path) {
do {
try FileManager.default.createDirectory(at: outputDirectory, withIntermediateDirectories: true, attributes: nil)
} catch {
print("Unable to create directory at URL \(outputDirectory)")
}
}
}
private func configurePreview() {
if let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession) {
previewLayer.frame = previewView.bounds
previewView.layer.addSublayer(previewLayer)
}
}
private func getVideoSize() -> CGSize {
if UIDevice.current.orientation == .landscapeLeft || UIDevice.current.orientation == .landscapeRight {
if videoSize.width > videoSize.height {
return videoSize
} else {
return CGSize(width: videoSize.height, height: videoSize.width)
}
} else {
if videoSize.width < videoSize.height {
return videoSize
} else {
return CGSize(width: videoSize.height, height: videoSize.width)
}
}
}
private func getVideoTransform() -> CGAffineTransform {
switch UIDevice.current.orientation {
case .portraitUpsideDown:
return CGAffineTransform(rotationAngle: CGFloat((M_PI * -90.0)) / 180.0)
case .landscapeLeft:
return CGAffineTransform(rotationAngle: CGFloat((M_PI * -180.0)) / 180.0) // TODO: Add support for front facing camera
// return CGAffineTransform(rotationAngle: CGFloat((M_PI * 0.0)) / 180.0) // TODO: For front facing camera
case .landscapeRight:
return CGAffineTransform(rotationAngle: CGFloat((M_PI * 0.0)) / 180.0) // TODO: Add support for front facing camera
// return CGAffineTransform(rotationAngle: CGFloat((M_PI * -180.0)) / 180.0) // TODO: For front facing camera
default:
return CGAffineTransform(rotationAngle: CGFloat((M_PI * 90.0)) / 180.0)
}
}
//MARK: - Controls
private func startRecording() {
videoQueue.async {
self.captureSession.startRunning()
}
isRecording = true
playStopButton.setTitle("Stop Recording", for: .normal)
print("Recording did start")
}
private func stopRecording() {
if !isRecording {
return
}
videoQueue.async {
self.assetWriter?.finishWriting {
print("Asset writer did finish writing")
self.isWriting = false
}
self.captureSession.stopRunning()
}
isRecording = false
playStopButton.setTitle("Start Recording", for: .normal)
print("Recording did stop")
}
//MARK: - AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {
guard let w = assetWriter else {
print("Asset writer not configured")
return
}
guard let vo = videoOutput else {
print("Video output not configured")
return
}
guard let ao = audioOutput else {
print("Audio output not configured")
return
}
guard let vi = videoInput else {
print("Video input not configured")
return
}
guard let ai = audioInput else {
print("Audio input not configured")
return
}
let st = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
print("Writer status \(w.status.rawValue)")
if let e = w.error {
print("Writer error \(e)")
stopRecording()
return
}
switch w.status {
case .unknown:
if !isWriting {
isWriting = true
w.startWriting()
w.startSession(atSourceTime: st)
}
return
case .completed:
print("Video writing completed")
return
case .cancelled:
print("Video writing cancelled")
return
case .failed:
print("Video writing failed")
return
default:
print("Video is writing")
}
if vo == captureOutput {
if !vi.append(sampleBuffer) {
print("Unable to write to video buffer")
}
} else if ao == captureOutput {
if !ai.append(sampleBuffer) {
print("Unable to write to audio buffer")
}
}
}
//MARK: Helpers
private func getDocumentsDirectory() -> URL {
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let documentsDirectory = paths[0]
return documentsDirectory
}
//MARK: Actions
#IBAction func startStopTapped(sender: AnyObject) {
if isRecording {
stopRecording()
} else {
startRecording()
}
}
}
Video orientation is handled by the AVAssetWriterInput.transform, looks like the getVideoTransform() implementation is not correct - CGAffineTransform expects the rotation angle to be in radians, so need to change to something like this:
private func getVideoTransform() -> CGAffineTransform {
switch UIDevice.current.orientation {
case .portrait:
return .identity
case .portraitUpsideDown:
return CGAffineTransform(rotationAngle: .pi)
case .landscapeLeft:
return CGAffineTransform(rotationAngle: .pi/2)
case .landscapeRight:
return CGAffineTransform(rotationAngle: -.pi/2)
default:
return .identity
}
}
From Apple Technical Q&A:
https://developer.apple.com/library/archive/qa/qa1744/_index.html
If you are using an AVAssetWriter object to write a movie file, you
can use the transform property of the associated AVAssetWriterInput to
specify the output file orientation. This will write a display
transform property into the output file as the preferred
transformation of the visual media data for display purposes. See the
AVAssetWriterInput.h interface file for the details.
I have found a solution to my problem. The solution is to export the video using AVAssetExportSession to handle setting the video size and then handling the rotation at the time of export and not during recording. I still have an issue were I need to fix the scale factor to go from my original video size to a smaller 640x480 resolution, but at least I solved my rotation issues. Please see updated code below.
import UIKit
import AVFoundation
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
//MARK: - Outlet
#IBOutlet weak var previewView: UIView!
#IBOutlet var playStopButton: UIButton!
//MARK: - Private Variables
let videoQueue = DispatchQueue(label: "VideoQueue", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
private let captureSession = AVCaptureSession()
var outputUrl: URL {
get {
if let url = _outputUrl {
return url
}
_outputUrl = outputDirectory.appendingPathComponent("video.mp4")
return _outputUrl!
}
}
private var _outputUrl: URL?
var exportUrl: URL {
get {
if let url = _exportUrl {
return url
}
_exportUrl = outputDirectory.appendingPathComponent("video_encoded.mp4")
return _exportUrl!
}
}
private var _exportUrl: URL?
var outputDirectory: URL {
get {
if let url = _outputDirectory {
return url
}
_outputDirectory = getDocumentsDirectory().appendingPathComponent("recording")
return _outputDirectory!
}
}
private var _outputDirectory: URL?
private var assetWriter: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var videoOutput: AVCaptureVideoDataOutput?
private var audioOutput: AVCaptureAudioDataOutput?
private var isRecording = false
private var isWriting = false
private var videoSize = CGSize(width: 640, height: 480)
private var exportPreset = AVAssetExportPreset640x480
//MARK: - View Life-cycle
override func viewDidLoad() {
super.viewDidLoad()
videoQueue.async {
do {
try self.configureCaptureSession()
DispatchQueue.main.sync {
self.configurePreview()
}
} catch {
DispatchQueue.main.async {
self.showAlert("Unable to configure capture session")
}
}
}
}
override var preferredInterfaceOrientationForPresentation: UIInterfaceOrientation {
return .portrait
}
override var supportedInterfaceOrientations: UIInterfaceOrientationMask {
return .portrait
}
//MARK: - Capture Session
private func configureCaptureSession() throws {
do {
// configure capture devices
let camDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo)
let micDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeAudio)
let camInput = try AVCaptureDeviceInput(device: camDevice)
let micInput = try AVCaptureDeviceInput(device: micDevice)
if captureSession.canAddInput(camInput) {
captureSession.addInput(camInput)
}
if captureSession.canAddInput(micInput) {
captureSession.addInput(micInput)
}
// configure audio/video output
videoOutput = AVCaptureVideoDataOutput()
videoOutput?.alwaysDiscardsLateVideoFrames = false // TODO: is this necessary?
videoOutput?.setSampleBufferDelegate(self, queue: videoQueue)
if let v = videoOutput {
captureSession.addOutput(v)
}
audioOutput = AVCaptureAudioDataOutput()
audioOutput?.setSampleBufferDelegate(self, queue: videoQueue)
if let a = audioOutput {
captureSession.addOutput(a)
}
// configure audio session
let audioSession = AVAudioSession.sharedInstance()
try audioSession.setCategory(AVAudioSessionCategoryPlayAndRecord)
try audioSession.setActive(true)
var micPort: AVAudioSessionPortDescription?
if let inputs = audioSession.availableInputs {
for port in inputs {
if port.portType == AVAudioSessionPortBuiltInMic {
micPort = port
break;
}
}
}
if let port = micPort, let dataSources = port.dataSources {
for source in dataSources {
if source.orientation == AVAudioSessionOrientationFront {
try audioSession.setPreferredInput(port)
break
}
}
}
} catch {
print("Failed to configure audio/video capture session")
throw error
}
}
private func configureAssetWriter() throws {
prepareVideoFile()
do {
if assetWriter != nil {
assetWriter = nil
videoInput = nil
audioInput = nil
}
assetWriter = try AVAssetWriter(url: outputUrl, fileType: AVFileTypeMPEG4)
guard let writer = assetWriter else {
print("Asset writer not created")
return
}
let videoSettings: [String: Any] = [AVVideoCodecKey: AVVideoCodecH264,
AVVideoWidthKey: NSNumber(value: Float(videoSize.width)),
AVVideoHeightKey: NSNumber(value: Float(videoSize.height))]
videoInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: videoSettings)
videoInput?.expectsMediaDataInRealTime = true
var channelLayout = AudioChannelLayout()
memset(&channelLayout, 0, MemoryLayout<AudioChannelLayout>.size);
channelLayout.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo;
let audioSettings: [String: Any] = [AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2]
audioInput = AVAssetWriterInput(mediaType: AVMediaTypeAudio, outputSettings: audioSettings)
audioInput?.expectsMediaDataInRealTime = true
guard let vi = videoInput else {
print("Video input not configured")
return
}
guard let ai = audioInput else {
print("Audio input not configured")
return
}
if writer.canAdd(vi) {
writer.add(vi)
}
if writer.canAdd(ai) {
writer.add(ai)
}
} catch {
print("Failed to configure asset writer")
throw error
}
}
private func prepareVideoFile() {
if FileManager.default.fileExists(atPath: outputUrl.path) {
do {
try FileManager.default.removeItem(at: outputUrl)
} catch {
print("Unable to remove file at URL \(outputUrl)")
}
}
if !FileManager.default.fileExists(atPath: outputDirectory.path) {
do {
try FileManager.default.createDirectory(at: outputDirectory, withIntermediateDirectories: true, attributes: nil)
} catch {
print("Unable to create directory at URL \(outputDirectory)")
}
}
}
private func configurePreview() {
if let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession) {
previewLayer.frame = previewView.bounds
previewView.layer.addSublayer(previewLayer)
}
}
private func getVideoSize() -> CGSize {
if UIDevice.current.orientation == .landscapeLeft || UIDevice.current.orientation == .landscapeRight {
if videoSize.width > videoSize.height {
return videoSize
} else {
return CGSize(width: videoSize.height, height: videoSize.width)
}
} else {
if videoSize.width < videoSize.height {
return videoSize
} else {
return CGSize(width: videoSize.height, height: videoSize.width)
}
}
}
//MARK: - Controls
private func startRecording() {
videoQueue.async {
do {
try self.configureAssetWriter()
self.captureSession.startRunning()
} catch {
print("Unable to start recording")
DispatchQueue.main.async { self.showAlert("Unable to start recording") }
}
}
isRecording = true
playStopButton.setTitle("Stop Recording", for: .normal)
print("Recording did start")
}
private func stopRecording() {
if !isRecording {
return
}
videoQueue.async {
self.assetWriter?.finishWriting {
print("Asset writer did finish writing")
self.isWriting = false
}
self.captureSession.stopRunning()
do {
try self.export()
} catch {
print("Export failed")
DispatchQueue.main.async { self.showAlert("Unable to export video") }
}
}
isRecording = false
playStopButton.setTitle("Start Recording", for: .normal)
print("Recording did stop")
}
//MARK: - AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {
guard let w = assetWriter else {
print("Asset writer not configured")
return
}
guard let vo = videoOutput else {
print("Video output not configured")
return
}
guard let ao = audioOutput else {
print("Audio output not configured")
return
}
guard let vi = videoInput else {
print("Video input not configured")
return
}
guard let ai = audioInput else {
print("Audio input not configured")
return
}
let st = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
print("Writer status \(w.status.rawValue)")
if let e = w.error {
print("Writer error \(e)")
stopRecording()
return
}
switch w.status {
case .unknown:
if !isWriting {
isWriting = true
w.startWriting()
w.startSession(atSourceTime: st)
}
return
case .completed:
print("Video writing completed")
return
case .cancelled:
print("Video writing cancelled")
return
case .failed:
print("Video writing failed")
return
default:
print("Video is writing")
}
if vo == captureOutput {
if !vi.append(sampleBuffer) {
print("Unable to write to video buffer")
}
} else if ao == captureOutput {
if !ai.append(sampleBuffer) {
print("Unable to write to audio buffer")
}
}
}
//MARK: - Export
private func getVideoComposition(asset: AVAsset, videoSize: CGSize) -> AVMutableVideoComposition? {
guard let videoTrack = asset.tracks(withMediaType: AVMediaTypeVideo).first else {
print("Unable to get video tracks")
return nil
}
let videoComposition = AVMutableVideoComposition()
videoComposition.renderSize = videoSize
let seconds: Float64 = Float64(1.0 / videoTrack.nominalFrameRate)
videoComposition.frameDuration = CMTimeMakeWithSeconds(seconds, 600);
let layerInst = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack)
var transforms = asset.preferredTransform
var isPortrait = true;
if (transforms.a == 0.0 && transforms.b == 1.0 && transforms.c == -1.0 && transforms.d == 0)
|| (transforms.a == 0.0 && transforms.b == -1.0 && transforms.c == 1.0 && transforms.d == 0) {
isPortrait = false;
}
if isPortrait {
transforms = transforms.concatenating(CGAffineTransform(rotationAngle: CGFloat(90.0.degreesToRadians)))
transforms = transforms.concatenating(CGAffineTransform(translationX: videoSize.width, y: 0))
}
layerInst.setTransform(transforms, at: kCMTimeZero)
let inst = AVMutableVideoCompositionInstruction()
inst.backgroundColor = UIColor.black.cgColor
inst.layerInstructions = [layerInst]
inst.timeRange = CMTimeRange(start: kCMTimeZero, duration: asset.duration)
videoComposition.instructions = [inst]
return videoComposition
}
private func export() throws {
let videoAsset = AVURLAsset(url: outputUrl)
if FileManager.default.fileExists(atPath: exportUrl.path) {
try FileManager.default.removeItem(at: exportUrl)
}
let videoSize = getVideoSize()
guard let encoder = AVAssetExportSession(asset: videoAsset, presetName: exportPreset) else {
print("Unable to create encoder")
return
}
guard let vidcomp = getVideoComposition(asset: videoAsset, videoSize: videoSize) else {
print("Unable to create video composition")
return
}
encoder.videoComposition = vidcomp
encoder.outputFileType = AVFileTypeMPEG4 // MP4 format
encoder.outputURL = exportUrl
encoder.shouldOptimizeForNetworkUse = true
encoder.exportAsynchronously(completionHandler: {
print("Video exported successfully")
})
}
//MARK: Helpers
private func getDocumentsDirectory() -> URL {
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let documentsDirectory = paths[0]
return documentsDirectory
}
//MARK: Actions
#IBAction func startStopTapped(sender: AnyObject) {
if isRecording {
stopRecording()
} else {
startRecording()
}
}
}
I found that the easiest is to set the preferredTransform on the video composition track depending on the required orientation.
Solution
// Devices orientation
var orientation = UIDevice.current.orientation
// The composition
let audioVideoComposition = AVMutableComposition()
// The video track of the composition
let videoCompositionTrack = audioVideoComposition
.addMutableTrack(withMediaType: .video, preferredTrackID: .init())!
// Set preferred transform
videoCompositionTrack.preferredTransform = getVideoTransform()
Helper function and extension
func getVideoTransform() -> CGAffineTransform {
switch orientation {
case .portrait:
return CGAffineTransform(rotationAngle: 90.degreesToRadians)
case .portraitUpsideDown:
return CGAffineTransform(rotationAngle: 180)
case .landscapeLeft:
return CGAffineTransform(rotationAngle: 0.degreesToRadians)
case .landscapeRight:
return CGAffineTransform(rotationAngle: 180.degreesToRadians)
default:
return CGAffineTransform(rotationAngle: 90.degreesToRadians)
}
}
extension BinaryInteger {
var degreesToRadians: CGFloat { CGFloat(self) * .pi / 180 }
}
extension FloatingPoint {
var degreesToRadians: Self { self * .pi / 180 }
var radiansToDegrees: Self { self * 180 / .pi }
}
just swap width and height in writer settings
and don't forget about the HEVC
assetWriter = try AVAssetWriter(url: outputUrl, fileType: AVFileTypeMPEG4)
https://developer.apple.com/videos/play/wwdc2017/503
https://developer.apple.com/videos/play/wwdc2017/511
- (BOOL)
configureWriterInput {
const BOOL isError = YES;
AVFileType
mov = AVFileTypeQuickTimeMovie;
NSDictionary <NSString *, id> *settings;
// HEVC
if (#available(iOS 11.0, *)) {
NSArray <AVVideoCodecType> *available = [self.sessionOutput availableVideoCodecTypesForAssetWriterWithOutputFileType:
mov];
const BOOL isHEVC = [available containsObject:AVVideoCodecTypeHEVC];
if (isHEVC) {
settings = [self.sessionOutput recommendedVideoSettingsForVideoCodecType:
AVVideoCodecTypeHEVC
assetWriterOutputFileType:
mov];
}
else {
settings = [self.sessionOutput recommendedVideoSettingsForAssetWriterWithOutputFileType:
mov];
}
}
else {
settings = [self.sessionOutput recommendedVideoSettingsForAssetWriterWithOutputFileType:
mov];
}
if ([writer
canApplyOutputSettings:settings forMediaType:AVMediaTypeVideo]) {
// swap width and height to fix orientation
NSMutableDictionary <NSString *, id> *rotate =
[settings mutableCopy];
if (YES
&& settings[AVVideoHeightKey]
&& settings[AVVideoWidthKey]
) {
rotate[AVVideoHeightKey] = settings[AVVideoWidthKey];
rotate[AVVideoWidthKey] = settings[AVVideoHeightKey];
if ([writer
canApplyOutputSettings:rotate forMediaType:AVMediaTypeVideo]) {
settings = rotate;
}
else {
}
}
else {
}
}
else {
return isError;
}
writerInput = [AVAssetWriterInput
assetWriterInputWithMediaType:AVMediaTypeVideo
outputSettings:settings];
{
// AVCaptureConnection *con =
// [self.sessionOutput connectionWithMediaType:AVMediaTypeVideo];
// const AVCaptureVideoOrientation o = con.videoOrientation;
// writerInput.transform = [[self class] configureOrientationTransform:o];
}
if ([writer canAddInput:writerInput]) {
[writer addInput:writerInput];
return ! isError;
}
else {
return isError;
}
}