AVAssetExportSession Crop Video Transformation problem - ios

I am trying to crop the video with AVMutableComposition. The exported video seem rotated to an unknown angle. It does not even choose the correct frame from the video. I have debugged the frame, and the provided crop frame is correct. Even if the video's preferredTransformation is Identity, then too it does not crop the correct video.
My code is like below:
public func exportVideo(asset: AVAsset, destinationDirectory: URL? = nil, editingOptions: MediaEditingOptions? = nil,
completion: #escaping VideoExportCompletion) -> ExportOperation? {
let composition = AVMutableComposition()
guard let videoTrack = composition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid),
let audioTrack = composition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) else {
completion(.failure(Error.couldNotAddMutableTracks))
return nil
}
let timeRange = editingOptions?.videoEditingOptions.timeRange ?? CMTimeRange(start: .zero, duration: asset.duration)
if let videoAssetTrack = asset.tracks(withMediaType: .video).first {
do {
try videoTrack.insertTimeRange(timeRange, of: videoAssetTrack, at: .zero)
} catch {
completion(.failure(error))
return nil
}
} else {
completion(.failure(Error.couldNotFindVideoTrack))
return nil
}
if let audioAssetTrack = asset.tracks(withMediaType: .audio).first {
do {
try audioTrack.insertTimeRange(timeRange, of: audioAssetTrack, at: .zero)
} catch {
completion(.failure(error))
return nil
}
}
let preset = editingOptions?.videoEditingOptions.preset ?? VideoEditingOptions.defaultPreset
guard let exportSession = AVAssetExportSession(asset: composition, presetName: preset) else {
completion(.failure(Error.couldNotInitiateExportSession))
return nil
}
guard var exportURL = destinationDirectory ?? cacheDirectoryURL() else {
completion(.failure(Error.couldNotDetermineOutputDirectory))
return nil
}
/// Prepare Instructions
let compositionInstructions = AVMutableVideoCompositionInstruction()
compositionInstructions.timeRange = CMTimeRange(start: .zero, duration: asset.duration)
let layerInstructions = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack)
let cropFrame = editingOptions?.scrollBounds ?? CGRect(x: 0, y: 0, width: videoTrack.naturalSize.width, height: videoTrack.naturalSize.height)
let transform = getTransform(for: videoTrack, cropFrame: cropFrame)
layerInstructions.setTransform(transform, at: .zero)
layerInstructions.setOpacity(1.0, at: CMTime.zero)
compositionInstructions.layerInstructions = [layerInstructions]
/// Add video composition.
let videoComposition = AVMutableVideoComposition()
videoComposition.renderSize = cropFrame.size
videoComposition.instructions = [compositionInstructions]
videoComposition.frameDuration = videoTrack.minFrameDuration
exportURL.appendPathComponent(UUID().uuidString + ".mp4")
exportSession.videoComposition = videoComposition
exportSession.outputURL = exportURL
exportSession.outputFileType = AVFileType.mov
exportSession.exportAsynchronously {
if exportSession.status == .failed {
completion(.failure(exportSession.error ?? Error.unknown))
} else if exportSession.status == .completed {
completion(.success(exportURL))
} else {
completion(.failure(Error.unknown))
}
}
return exportSession
}
func getTransform(for videoTrack: AVAssetTrack, cropFrame: CGRect) -> CGAffineTransform {
let renderSize = cropFrame.size
let renderScale = renderSize.width / cropFrame.width
let offset = CGPoint(x: -cropFrame.origin.x, y: -cropFrame.origin.y)
let rotation = atan2(videoTrack.preferredTransform.b, videoTrack.preferredTransform.a)
var rotationOffset = CGPoint(x: 0, y: 0)
if videoTrack.preferredTransform.b == -1.0 {
rotationOffset.y = videoTrack.naturalSize.width
} else if videoTrack.preferredTransform.c == -1.0 {
rotationOffset.x = videoTrack.naturalSize.height
} else if videoTrack.preferredTransform.a == -1.0 {
rotationOffset.x = videoTrack.naturalSize.width
rotationOffset.y = videoTrack.naturalSize.height
}
var transform = CGAffineTransform.identity
transform = transform.scaledBy(x: renderScale, y: renderScale)
transform = transform.translatedBy(x: offset.x + rotationOffset.x, y: offset.y + rotationOffset.y)
transform = transform.rotated(by: rotation)
print("track size \(videoTrack.naturalSize)")
print("preferred Transform = \(videoTrack.preferredTransform)")
print("rotation angle \(rotation)")
print("rotation offset \(rotationOffset)")
print("actual Transform = \(transform)")
return transform
}
I don't find anything obvious here. Am I missing anything here?

Related

Swift - Add watermark to a video is very slow

Here is my code that adds image & text overlays to a local video. The problem is that it's extremely SLOW. Any ideas how to fix it?
Also I would appreciate if you can suggest 3rd party libraries that can do watermarking.
public func addWatermark(
fromVideoAt videoURL: URL,
watermark: Watermark,
fileName: String,
onSuccess: #escaping (URL) -> Void,
onFailure: #escaping ((Error?) -> Void)
) {
let asset = AVURLAsset(url: videoURL)
let composition = AVMutableComposition()
guard
let compositionTrack = composition.addMutableTrack(
withMediaType: .video,
preferredTrackID: kCMPersistentTrackID_Invalid
),
let assetTrack = asset.tracks(withMediaType: .video).first
else {
onFailure(nil)
return
}
do {
let timeRange = CMTimeRange(start: .zero, duration: assetTrack.timeRange.duration)
try compositionTrack.insertTimeRange(timeRange, of: assetTrack, at: .zero)
if let audioAssetTrack = asset.tracks(withMediaType: .audio).first,
let compositionAudioTrack = composition.addMutableTrack(
withMediaType: .audio,
preferredTrackID: kCMPersistentTrackID_Invalid
) {
try compositionAudioTrack.insertTimeRange(
timeRange,
of: audioAssetTrack,
at: .zero
)
}
} catch {
onFailure(error)
return
}
compositionTrack.preferredTransform = assetTrack.preferredTransform
let videoInfo = orientation(from: assetTrack.preferredTransform)
let videoSize: CGSize
if videoInfo.isPortrait {
videoSize = CGSize(
width: assetTrack.naturalSize.height,
height: assetTrack.naturalSize.width
)
} else {
videoSize = assetTrack.naturalSize
}
let videoLayer = CALayer()
videoLayer.frame = CGRect(origin: .zero, size: videoSize)
let overlayLayer = CALayer()
overlayLayer.frame = CGRect(origin: .zero, size: videoSize)
videoLayer.frame = CGRect(x: 0, y: 0, width: videoSize.width, height: videoSize.height)
let imageFrame = watermark.calculateImageFrame(parentSize: videoSize)
addImage(watermark.image, to: overlayLayer, frame: imageFrame)
let textOrigin = CGPoint(x: imageFrame.minX + 4, y: imageFrame.minY)
if let text = watermark.text {
addText(
text,
to: overlayLayer,
origin: textOrigin,
textAttributes: Watermark.textAttributes(type: watermark.type)
)
}
let outputLayer = CALayer()
outputLayer.frame = CGRect(origin: .zero, size: videoSize)
outputLayer.addSublayer(videoLayer)
outputLayer.addSublayer(overlayLayer)
let videoComposition = AVMutableVideoComposition()
videoComposition.renderSize = videoSize
videoComposition.frameDuration = CMTime(value: 1, timescale: 60)
videoComposition.animationTool = AVVideoCompositionCoreAnimationTool(
postProcessingAsVideoLayer: videoLayer,
in: outputLayer
)
videoComposition.colorPrimaries = AVVideoColorPrimaries_ITU_R_709_2
videoComposition.colorTransferFunction = "sRGB"
videoComposition.colorYCbCrMatrix = nil
let instruction = AVMutableVideoCompositionInstruction()
instruction.timeRange = CMTimeRange(start: .zero, duration: composition.duration)
videoComposition.instructions = [instruction]
let layerInstruction = compositionLayerInstruction(
for: compositionTrack,
assetTrack: assetTrack
)
instruction.layerInstructions = [layerInstruction]
guard let export = AVAssetExportSession(
asset: composition,
presetName: AVAssetExportPresetHighestQuality
)
else {
onFailure(nil)
return
}
let exportURL = URL(fileURLWithPath: NSTemporaryDirectory())
.appendingPathComponent(fileName)
.appendingPathExtension("mov")
export.videoComposition = videoComposition
export.outputFileType = .mov
export.outputURL = exportURL
export.exportAsynchronously {
DispatchQueue.main.async {
switch export.status {
case .completed:
onSuccess(exportURL)
default:
onFailure(export.error)
}
}
}
}
Watermark is the wrapper struct. It contains image/text, text attributes, size and other similar helpful information.
I've tried without any luck:
export.shouldOptimizeForNetworkUse = true. It did not work.
AVAssetExportPresetPassthrough instead of AVAssetExportPresetHighestQuality. It removed overlays.
I have the following code which is relatively fast. It watermarks an 8 second video in about 2.56 seconds. When I ran it under Metal System Trace Instrument it seemed to be balanced and using GPU-acceleration the whole time. You just call exportIt()
As a side matter, this code uses async await wrapping of AVKit functions and migrates off any deprecated interfaces as of iOS 16.
A tidied up and working sample app with resource files is https://github.com/faisalmemon/watermark
The core code is as follows:
//
// WatermarkHelper.swift
// watermark
//
// Created by Faisal Memon on 09/02/2023.
//
import Foundation
import AVKit
struct WatermarkHelper {
enum WatermarkError: Error {
case cannotLoadResources
case cannotAddTrack
case cannotLoadVideoTrack(Error?)
case cannotCopyOriginalAudioVideo(Error?)
case noVideoTrackPresent
case exportSessionCannotBeCreated
}
func compositionAddMediaTrack(_ composition: AVMutableComposition, withMediaType mediaType: AVMediaType) throws -> AVMutableCompositionTrack {
guard let compositionTrack = composition.addMutableTrack(
withMediaType: mediaType,
preferredTrackID: kCMPersistentTrackID_Invalid) else {
throw WatermarkError.cannotAddTrack
}
return compositionTrack
}
func loadTrack(inputVideo: AVAsset, withMediaType mediaType: AVMediaType) async throws -> AVAssetTrack? {
return try await withCheckedThrowingContinuation({
(continuation: CheckedContinuation<AVAssetTrack?, Error>) in
inputVideo.loadTracks(withMediaType: mediaType) { tracks, error in
if let tracks = tracks {
continuation.resume(returning: tracks.first)
} else {
continuation.resume(throwing: WatermarkError.cannotLoadVideoTrack(error))
}
}
})
}
func bringOverVideoAndAudio(inputVideo: AVAsset, assetTrack: AVAssetTrack, compositionTrack: AVMutableCompositionTrack, composition: AVMutableComposition) async throws {
do {
let timeRange = await CMTimeRange(start: .zero, duration: try inputVideo.load(.duration))
try compositionTrack.insertTimeRange(timeRange, of: assetTrack, at: .zero)
if let audioAssetTrack = try await loadTrack(inputVideo: inputVideo, withMediaType: .audio) {
let compositionAudioTrack = try compositionAddMediaTrack(composition, withMediaType: .audio)
try compositionAudioTrack.insertTimeRange(timeRange, of: audioAssetTrack, at: .zero)
}
} catch {
print(error)
throw WatermarkError.cannotCopyOriginalAudioVideo(error)
}
}
private func orientation(from transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
var assetOrientation = UIImage.Orientation.up
var isPortrait = false
if transform.a == 0 && transform.b == 1.0 && transform.c == -1.0 && transform.d == 0 {
assetOrientation = .right
isPortrait = true
} else if transform.a == 0 && transform.b == -1.0 && transform.c == 1.0 && transform.d == 0 {
assetOrientation = .left
isPortrait = true
} else if transform.a == 1.0 && transform.b == 0 && transform.c == 0 && transform.d == 1.0 {
assetOrientation = .up
} else if transform.a == -1.0 && transform.b == 0 && transform.c == 0 && transform.d == -1.0 {
assetOrientation = .down
}
return (assetOrientation, isPortrait)
}
func preferredTransformAndSize(compositionTrack: AVMutableCompositionTrack, assetTrack: AVAssetTrack) async throws -> (preferredTransform: CGAffineTransform, videoSize: CGSize) {
let transform = try await assetTrack.load(.preferredTransform)
let videoInfo = orientation(from: transform)
let videoSize: CGSize
let naturalSize = try await assetTrack.load(.naturalSize)
if videoInfo.isPortrait {
videoSize = CGSize(
width: naturalSize.height,
height: naturalSize.width)
} else {
videoSize = naturalSize
}
return (transform, videoSize)
}
private func compositionLayerInstruction(for track: AVCompositionTrack, assetTrack: AVAssetTrack, preferredTransform: CGAffineTransform) -> AVMutableVideoCompositionLayerInstruction {
let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: track)
instruction.setTransform(preferredTransform, at: .zero)
return instruction
}
private func addImage(to layer: CALayer, watermark: UIImage, videoSize: CGSize) {
let imageLayer = CALayer()
let aspect: CGFloat = watermark.size.width / watermark.size.height
let width = videoSize.width
let height = width / aspect
imageLayer.frame = CGRect(
x: 0,
y: -height * 0.15,
width: width,
height: height)
imageLayer.contents = watermark.cgImage
layer.addSublayer(imageLayer)
}
func composeVideo(composition: AVMutableComposition, videoComposition: AVMutableVideoComposition, compositionTrack: AVMutableCompositionTrack, assetTrack: AVAssetTrack, preferredTransform: CGAffineTransform) {
let instruction = AVMutableVideoCompositionInstruction()
instruction.timeRange = CMTimeRange(
start: .zero,
duration: composition.duration)
videoComposition.instructions = [instruction]
let layerInstruction = compositionLayerInstruction(
for: compositionTrack,
assetTrack: assetTrack, preferredTransform: preferredTransform)
instruction.layerInstructions = [layerInstruction]
}
func exportSession(composition: AVMutableComposition, videoComposition: AVMutableVideoComposition, outputURL: URL) throws -> AVAssetExportSession {
guard let export = AVAssetExportSession(
asset: composition,
presetName: AVAssetExportPresetHighestQuality)
else {
print("Cannot create export session.")
throw WatermarkError.exportSessionCannotBeCreated
}
export.videoComposition = videoComposition
export.outputFileType = .mp4
export.outputURL = outputURL
return export
}
func executeSession(_ session: AVAssetExportSession) async throws -> AVAssetExportSession.Status {
return try await withCheckedThrowingContinuation({
(continuation: CheckedContinuation<AVAssetExportSession.Status, Error>) in
session.exportAsynchronously {
DispatchQueue.main.async {
if let error = session.error {
continuation.resume(throwing: error)
} else {
continuation.resume(returning: session.status)
}
}
}
})
}
func addWatermarkTopDriver(inputVideo: AVAsset, outputURL: URL, watermark: UIImage) async throws -> AVAssetExportSession.Status {
let composition = AVMutableComposition()
let compositionTrack = try compositionAddMediaTrack(composition, withMediaType: .video)
guard let videoAssetTrack = try await loadTrack(inputVideo: inputVideo, withMediaType: .video) else {
throw WatermarkError.noVideoTrackPresent
}
try await bringOverVideoAndAudio(inputVideo: inputVideo, assetTrack: videoAssetTrack, compositionTrack: compositionTrack, composition: composition)
let transformAndSize = try await preferredTransformAndSize(compositionTrack: compositionTrack, assetTrack: videoAssetTrack)
compositionTrack.preferredTransform = transformAndSize.preferredTransform
let videoLayer = CALayer()
videoLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
let overlayLayer = CALayer()
overlayLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
addImage(to: overlayLayer, watermark: watermark, videoSize: transformAndSize.videoSize)
let outputLayer = CALayer()
outputLayer.frame = CGRect(origin: .zero, size: transformAndSize.videoSize)
outputLayer.addSublayer(videoLayer)
outputLayer.addSublayer(overlayLayer)
let videoComposition = AVMutableVideoComposition()
videoComposition.renderSize = transformAndSize.videoSize
videoComposition.frameDuration = CMTime(value: 1, timescale: 30)
videoComposition.animationTool = AVVideoCompositionCoreAnimationTool(
postProcessingAsVideoLayer: videoLayer,
in: outputLayer)
composeVideo(composition: composition, videoComposition: videoComposition, compositionTrack: compositionTrack, assetTrack: videoAssetTrack, preferredTransform: transformAndSize.preferredTransform)
let session = try exportSession(composition: composition, videoComposition: videoComposition, outputURL: outputURL)
return try await executeSession(session)
}
/// Creates a watermarked movie and saves it to the documents directory.
///
/// For an 8 second video (251 frames), this code takes 2.56 seconds on iPhone 11 producing a high quality video at 30 FPS.
/// - Returns: Time interval taken for processing.
public func exportIt() async throws -> TimeInterval {
let timeStart = Date()
guard
let filePath = Bundle.main.path(forResource: "donut-spinning", ofType: "mp4"),
let docUrl = try? FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true),
let watermarkImage = UIImage(systemName: "seal") else {
throw WatermarkError.cannotLoadResources
}
let videoAsset = AVAsset(url: URL(filePath: filePath))
let outputURL = docUrl.appending(component: "watermark-donut-spinning.mp4")
try? FileManager.default.removeItem(at: outputURL)
print(outputURL)
let result = try await addWatermarkTopDriver(inputVideo: videoAsset, outputURL: outputURL, watermark: watermarkImage)
let timeEnd = Date()
let duration = timeEnd.timeIntervalSince(timeStart)
print(result)
return duration
}
}
Use this below method for super fast watermark adding to video
func addWatermark(inputURL: URL, outputURL: URL, handler:#escaping (_ exportSession: AVAssetExportSession?)-> Void) {
let mixComposition = AVMutableComposition()
let asset = AVAsset(url: inputURL)
let videoTrack = asset.tracks(withMediaType: AVMediaType.video)[0]
let timerange = CMTimeRangeMake(start: CMTime.zero, duration: asset.duration)
let compositionVideoTrack:AVMutableCompositionTrack = mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: CMPersistentTrackID(kCMPersistentTrackID_Invalid))!
do {
try compositionVideoTrack.insertTimeRange(timerange, of: videoTrack, at: CMTime.zero)
compositionVideoTrack.preferredTransform = videoTrack.preferredTransform
} catch {
print(error)
}
let watermarkFilter = CIFilter(name: "CISourceOverCompositing")!
let watermarkImage = CIImage(image: UIImage(named: "waterMark")!)
let videoComposition = AVVideoComposition(asset: asset) { (filteringRequest) in
let source = filteringRequest.sourceImage.clampedToExtent()
watermarkFilter.setValue(source, forKey: "inputBackgroundImage")
let transform = CGAffineTransform(translationX: filteringRequest.sourceImage.extent.width - (watermarkImage?.extent.width)! - 2, y: 0)
watermarkFilter.setValue(watermarkImage?.transformed(by: transform), forKey: "inputImage")
filteringRequest.finish(with: watermarkFilter.outputImage!, context: nil)
}
guard let exportSession = AVAssetExportSession(asset: asset, presetName: AVAssetExportPreset640x480) else {
handler(nil)
return
}
exportSession.outputURL = outputURL
exportSession.outputFileType = AVFileType.mp4
exportSession.shouldOptimizeForNetworkUse = true
exportSession.videoComposition = videoComposition
exportSession.exportAsynchronously { () -> Void in
handler(exportSession)
}
}
Call this method when you want to add watermark easily
let outputURL = NSURL.fileURL(withPath: "TempPath")
let inputURL = NSURL.fileURL(withPath: "VideoWithWatermarkPath")
addWatermark(inputURL: inputURL, outputURL: outputURL, handler: { (exportSession) in
guard let session = exportSession else {
// Error
return
}
switch session.status {
case .completed:
guard NSData(contentsOf: outputURL) != nil else {
// Error
return
}
// Now you can find the video with the watermark in the location outputURL
default:
// Error
}
})
Have you checked out Apple's documentation? It adds a title layer (CALayer) on top of an existing AVMutableComposition or an AVAsset? Since it's a legacy doc from iOS 6, you'll need to refactor a bit, but it should be fast on today's tech.

Merging video with original orientation

I have a link I can DM for a minimum working example!
Recording Videos
For recording, the AVCaptureConnection for an AVCaptureSession, I set isVideoMirrored to true when using the front camera and false when using the back camera. All in portrait orientation.
Saving Videos
When I save videos, I perform an AVAssetExportSession. If I used the front camera, I want to maintain the isVideoMirrored = true, so I create an AVMutableComposition to set the AVAsset video track's preferredTransform to CGAffineTransform(scaleX: -1.0, y: 1.0).rotated(by: CGFloat(Double.pi/2)). For the back camera, I export the AVAsset as outputted.
Part of my saving code:
if didCaptureWithFrontCamera {
let composition = AVMutableComposition()
let assetVideoTrack = asset.tracks(withMediaType: .video).last!
let assetAudioTrack = asset.tracks(withMediaType: .audio).last!
let compositionVideoTrack = composition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: CMPersistentTrackID(kCMPersistentTrackID_Invalid))
let compositionAudioTrack = composition.addMutableTrack(withMediaType: AVMediaType.audio, preferredTrackID: CMPersistentTrackID(kCMPersistentTrackID_Invalid))
try? compositionVideoTrack?.insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: asset.duration), of: assetVideoTrack, at: CMTime.zero)
try? compositionAudioTrack?.insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: asset.duration), of: assetAudioTrack, at: CMTime.zero)
compositionVideoTrack?.preferredTransform = CGAffineTransform(scaleX: -1.0, y: 1.0).rotated(by: CGFloat(Double.pi/2))
guard let exportSession = AVAssetExportSession(asset: composition, presetName: AVAssetExportPreset1280x720) else {
handler(nil)
return
}
exportSession.outputURL = outputURL
exportSession.outputFileType = .mp4
exportSession.shouldOptimizeForNetworkUse = true
exportSession.exportAsynchronously { handler(exportSession) }
} else {
guard let exportSession = AVAssetExportSession(asset: asset, presetName: AVAssetExportPreset1280x720) else {
handler(nil)
return
}
exportSession.outputURL = outputURL
exportSession.outputFileType = .mp4
exportSession.shouldOptimizeForNetworkUse = true
exportSession.exportAsynchronously { handler(exportSession) }
}
Merging Videos
Later, to view the saved videos, I want to merge them together as a single video and maintain each by their original orientation via AVMutableComposition.
What partially has worked is setting the video track of AVMutableComposition to the preferredTransform property of the video track of an individual AVAsset video. The only problem is that a single orientation is applied to all the videos (i.e. mirroring isn't applied in a back camera recorded video and the same is applied to the front camera video too).
From solutions I've come across it appears I need to apply AVMutableVideoCompositionInstruction, but in trying to do so, the AVAssetExportSession doesn't seem to factor in the videoComposition instructions at all.
Any guidance would be extremely appreciated as I haven't been able to solve it for the life of me...
My attempted merge code:
func merge(videos: [AVURLAsset], for date: Date, completion: #escaping (_ url: URL, _ asset: AVAssetExportSession)->()) {
let videoComposition = AVMutableComposition()
var lastTime: CMTime = .zero
var count = 0
var instructions = [AVMutableVideoCompositionInstruction]()
let renderSize = CGSize(width: 720, height: 1280)
guard let videoCompositionTrack = videoComposition.addMutableTrack(withMediaType: .video, preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else { return }
guard let audioCompositionTrack = videoComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else { return }
for video in videos {
if let videoTrack = video.tracks(withMediaType: .video)[safe: 0] {
//this is the only thing that seems to work, but work not in the way i'd hope where each video keeps its original orientation
//videoCompositionTrack.preferredTransform = videoTrack.preferredTransform
if let audioTrack = video.tracks(withMediaType: .audio)[safe: 0] {
do {
try videoCompositionTrack.insertTimeRange(CMTimeRangeMake(start: .zero, duration: video.duration), of: videoTrack, at: lastTime)
try audioCompositionTrack.insertTimeRange(CMTimeRangeMake(start: .zero, duration: video.duration), of: audioTrack, at: lastTime)
let layerInstruction = videoCompositionInstruction(videoTrack, asset: video, count: count)
let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
videoCompositionInstruction.timeRange = CMTimeRangeMake(start: lastTime, duration: video.duration)
videoCompositionInstruction.layerInstructions = [layerInstruction]
instructions.append(videoCompositionInstruction)
} catch {
return
}
lastTime = CMTimeAdd(lastTime, video.duration)
count += 1
} else {
do {
try videoCompositionTrack.insertTimeRange(CMTimeRangeMake(start: .zero, duration: video.duration), of: videoTrack, at: lastTime)
let layerInstruction = videoCompositionInstruction(videoTrack, asset: video, count: count)
let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
videoCompositionInstruction.timeRange = CMTimeRangeMake(start: lastTime, duration: video.duration)
videoCompositionInstruction.layerInstructions = [layerInstruction]
instructions.append(videoCompositionInstruction)
} catch {
return
}
lastTime = CMTimeAdd(lastTime, video.duration)
count += 1
}
}
}
let mutableVideoComposition = AVMutableVideoComposition()
mutableVideoComposition.instructions = instructions
mutableVideoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
mutableVideoComposition.renderSize = renderSize
dateFormatter.dateStyle = .long
dateFormatter.timeStyle = .short
let date = dateFormatter.string(from: date)
let mergedURL = NSURL.fileURL(withPath: NSTemporaryDirectory() + "merged-\(date)" + ".mp4")
guard let exporter = AVAssetExportSession(asset: videoComposition, presetName: AVAssetExportPresetHighestQuality) else { return }
exporter.outputURL = mergedURL
exporter.outputFileType = .mp4
exporter.videoComposition = mutableVideoComposition
exporter.shouldOptimizeForNetworkUse = true
completion(mergedURL, exporter)
}
func videoCompositionInstruction(_ firstTrack: AVAssetTrack, asset: AVAsset, count: Int) -> AVMutableVideoCompositionLayerInstruction {
let renderSize = CGSize(width: 720, height: 1280)
let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: firstTrack)
let assetTrack = asset.tracks(withMediaType: .video)[0]
let t = assetTrack.fixedPreferredTransform // new transform fix
let assetInfo = orientationFromTransform(t)
if assetInfo.isPortrait {
let scaleToFitRatio = renderSize.width / assetTrack.naturalSize.height
let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
var finalTransform = assetTrack.fixedPreferredTransform.concatenating(scaleFactor)
if assetInfo.orientation == .rightMirrored || assetInfo.orientation == .leftMirrored {
finalTransform = finalTransform.translatedBy(x: -t.ty, y: 0)
}
instruction.setTransform(t, at: CMTime.zero)
} else {
let renderRect = CGRect(x: 0, y: 0, width: renderSize.width, height: renderSize.height)
let videoRect = CGRect(origin: .zero, size: assetTrack.naturalSize).applying(assetTrack.fixedPreferredTransform)
let scale = renderRect.width / videoRect.width
let transform = CGAffineTransform(scaleX: renderRect.width / videoRect.width, y: (videoRect.height * scale) / assetTrack.naturalSize.height)
let translate = CGAffineTransform(translationX: .zero, y: ((renderSize.height - (videoRect.height * scale))) / 2)
instruction.setTransform(assetTrack.fixedPreferredTransform.concatenating(transform).concatenating(translate), at: .zero)
}
if count == 0 {
instruction.setOpacity(0.0, at: asset.duration)
}
return instruction
}
func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
var assetOrientation = UIImage.Orientation.up
var isPortrait = false
if transform.a == 0 && transform.b == 1.0 && transform.c == -1.0 && transform.d == 0 {
assetOrientation = .right
isPortrait = true
} else if transform.a == 0 && transform.b == 1.0 && transform.c == 1.0 && transform.d == 0 {
assetOrientation = .rightMirrored
isPortrait = true
} else if transform.a == 0 && transform.b == -1.0 && transform.c == 1.0 && transform.d == 0 {
assetOrientation = .left
isPortrait = true
} else if transform.a == 0 && transform.b == -1.0 && transform.c == -1.0 && transform.d == 0 {
assetOrientation = .leftMirrored
isPortrait = true
} else if transform.a == 1.0 && transform.b == 0 && transform.c == 0 && transform.d == 1.0 {
assetOrientation = .up
} else if transform.a == -1.0 && transform.b == 0 && transform.c == 0 && transform.d == -1.0 {
assetOrientation = .down
}
return (assetOrientation, isPortrait)
}
extension AVAssetTrack {
var fixedPreferredTransform: CGAffineTransform {
var t = preferredTransform
switch(t.a, t.b, t.c, t.d) {
case (1, 0, 0, 1):
t.tx = 0
t.ty = 0
case (1, 0, 0, -1):
t.tx = 0
t.ty = naturalSize.height
case (-1, 0, 0, 1):
t.tx = naturalSize.width
t.ty = 0
case (-1, 0, 0, -1):
t.tx = naturalSize.width
t.ty = naturalSize.height
case (0, -1, 1, 0):
t.tx = 0
t.ty = naturalSize.width
case (0, 1, -1, 0):
t.tx = naturalSize.height
t.ty = 0
case (0, 1, 1, 0):
t.tx = 0
t.ty = 0
case (0, -1, -1, 0):
t.tx = naturalSize.height
t.ty = naturalSize.width
default:
break
}
return t
}
}
Assuming your transformations are correct, I updated your merge function.
The main change is using a single AVMutableVideoCompositionInstruction with multiple AVMutableVideoCompositionLayerInstruction, and passing the correct CMTime value to for the layer instruction to be executed at.
func merge(videos: [AVURLAsset],
for date: Date,
completion: #escaping (_ url: URL, _ asset: AVAssetExportSession)->()) {
let videoComposition = AVMutableComposition()
guard let videoCompositionTrack = videoComposition.addMutableTrack(withMediaType: .video,
preferredTrackID: Int32(kCMPersistentTrackID_Invalid)),
let audioCompositionTrack = videoComposition.addMutableTrack(withMediaType: .audio,
preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
else { return }
var lastTime: CMTime = .zero
var layerInstructions = [AVMutableVideoCompositionLayerInstruction]()
for video in videos {
guard let videoTrack = video.tracks(withMediaType: .video)[safe: 0] else { return }
// add audio track if available
if let audioTrack = video.tracks(withMediaType: .audio)[safe: 0] {
do {
try audioCompositionTrack.insertTimeRange(CMTimeRangeMake(start: .zero, duration: video.duration),
of: audioTrack,
at: lastTime)
} catch {
return
}
}
// add video track
do {
try videoCompositionTrack.insertTimeRange(CMTimeRangeMake(start: .zero, duration: video.duration),
of: videoTrack,
at: lastTime)
let layerInstruction = makeVideoCompositionInstruction(videoTrack,
asset: video,
atTime: lastTime)
layerInstructions.append(layerInstruction)
} catch {
return
}
lastTime = CMTimeAdd(lastTime, video.duration)
} // end for..in videos
let renderSize = CGSize(width: 720, height: 1280)
let videoInstruction = AVMutableVideoCompositionInstruction()
videoInstruction.timeRange = CMTimeRangeMake(start: .zero, duration: lastTime)
videoInstruction.layerInstructions = layerInstructions
let mutableVideoComposition = AVMutableVideoComposition()
mutableVideoComposition.instructions = [videoInstruction]
mutableVideoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
mutableVideoComposition.renderSize = renderSize
let dateFormatter = DateFormatter()
dateFormatter.dateStyle = .long
dateFormatter.timeStyle = .short
let date = dateFormatter.string(from: date)
let mergedURL = NSURL.fileURL(withPath: NSTemporaryDirectory() + "merged-\(date)" + ".mp4")
guard let exporter = AVAssetExportSession(asset: videoComposition,
presetName: AVAssetExportPresetHighestQuality) else { return }
exporter.outputURL = mergedURL
exporter.outputFileType = .mp4
exporter.videoComposition = mutableVideoComposition
exporter.shouldOptimizeForNetworkUse = true
completion(mergedURL, exporter)
}
func makeVideoCompositionInstruction(_ videoTrack: AVAssetTrack,
asset: AVAsset,
atTime: CMTime) -> AVMutableVideoCompositionLayerInstruction {
let renderSize = CGSize(width: 720, height: 1280)
let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack)
let assetTrack = asset.tracks(withMediaType: .video)[0]
let t = assetTrack.fixedPreferredTransform // new transform fix
let assetInfo = orientationFromTransform(t)
if assetInfo.isPortrait {
let scaleToFitRatio = renderSize.width / assetTrack.naturalSize.height
let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
var finalTransform = assetTrack.fixedPreferredTransform.concatenating(scaleFactor)
if assetInfo.orientation == .rightMirrored || assetInfo.orientation == .leftMirrored {
finalTransform = finalTransform.translatedBy(x: -t.ty, y: 0)
}
instruction.setTransform(t, at: atTime)
} else {
let renderRect = CGRect(x: 0, y: 0, width: renderSize.width, height: renderSize.height)
let videoRect = CGRect(origin: .zero, size: assetTrack.naturalSize).applying(assetTrack.fixedPreferredTransform)
let scale = renderRect.width / videoRect.width
let transform = CGAffineTransform(scaleX: renderRect.width / videoRect.width,
y: (videoRect.height * scale) / assetTrack.naturalSize.height)
let translate = CGAffineTransform(translationX: .zero,
y: ((renderSize.height - (videoRect.height * scale))) / 2)
instruction.setTransform(assetTrack.fixedPreferredTransform.concatenating(transform).concatenating(translate),
at: atTime)
}
// if atTime = 0, we can assume this is the first track being added
if atTime == .zero {
instruction.setOpacity(0.0,
at: asset.duration)
}
return instruction
}

Overlay two videos swift

I wanted to dig this question back up. I am having an issue overlaying two videos. I believe it has something to do with the transparency of the first AVMutableVideoCompositionLayerInstruction but I have played around with it extensively with no luck. Any suggestions would be greatly appreciated!:
func overlay(video firstAsset: AVURLAsset, withSecondVideo secondAsset: AVURLAsset) {
let mixComposition = AVMutableComposition()
let firstTrack = mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: kCMPersistentTrackID_Invalid)
let secondTrack = mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: kCMPersistentTrackID_Invalid)
guard let firstMediaTrack = firstAsset.tracks(withMediaType: AVMediaType.video).first else { return }
guard let secondMediaTrack = secondAsset.tracks(withMediaType: AVMediaType.video).first else { return }
do {
try firstTrack?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, firstAsset.duration), of: firstMediaTrack, at: kCMTimeZero)
try secondTrack?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, secondAsset.duration), of: secondMediaTrack, at: kCMTimeZero)
} catch (let error) {
print(error)
}
self.width = max(firstMediaTrack.naturalSize.width, secondMediaTrack.naturalSize.width)
self.height = max(firstMediaTrack.naturalSize.height, secondMediaTrack.naturalSize.height)
let videoComposition = AVMutableVideoComposition()
videoComposition.renderSize = CGSize(width: width!, height: height!)
videoComposition.frameDuration = firstMediaTrack.minFrameDuration
let firstLayerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: firstMediaTrack)
let scale = CGAffineTransform(scaleX: 0.3, y: 0.3)
let move = CGAffineTransform(translationX: self.width! - ((self.width! * 0.3) + 10), y: 10)
firstLayerInstruction.setTransform(scale.concatenating(move), at: kCMTimeZero)
firstLayerInstruction.setOpacity(1.0, at: kCMTimeZero)
let secondlayerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: secondMediaTrack)
secondlayerInstruction.setTransform((secondTrack?.preferredTransform)!, at: kCMTimeZero)
secondlayerInstruction.setOpacity(1.0, at: kCMTimeZero)
let combined = AVMutableVideoCompositionInstruction()
combined.timeRange = CMTimeRangeMake(kCMTimeZero, max(firstAsset.duration, secondAsset.duration))
combined.backgroundColor = UIColor.clear.cgColor
combined.layerInstructions = [firstLayerInstruction, secondlayerInstruction]
videoComposition.instructions = [combined]
let outputUrl = self.getPathForTempFileNamed(filename: "output.mov")
self.exportCompositedVideo(compiledVideo: mixComposition, toURL: outputUrl, withVideoComposition: videoComposition)
self.removeTempFileAtPath(path: outputUrl.absoluteString)
}
The expected result is one video with two videos overplayed inside of it. the first layer being a full screen video and the second layer being a smaller video positioned in the upper right hand corner. Oddly enough, when I use on AVMutableVideoCompositionInstruction and put both AVMutableVideoCompositionLayerInstruction's inside of its layer instructions, it works! - but the video used in the FirstMediaTrack is used for both layers? Played around with that for a while and then tried to implement the approach detailed here which has individual instructions for both layers but this approach results in just the first layer showing a full screen video with the second layer completely invisible.
Here is the code that is working for me, i based it off this tutorial. I found the key was setting the backgrounds as clear (found on this thread). It also has a scale in there as I was experimenting with making one video smaller.
import AVFoundation
import AVKit
import Photos
var myurl: URL?
func newoverlay(video firstAsset: AVURLAsset, withSecondVideo secondAsset: AVURLAsset) {
// 1 - Create AVMutableComposition object. This object will hold your AVMutableCompositionTrack instances.
let mixComposition = AVMutableComposition()
// 2 - Create two video tracks
guard let firstTrack = mixComposition.addMutableTrack(withMediaType: .video,
preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else { return }
do {
try firstTrack.insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: firstAsset.duration),
of: firstAsset.tracks(withMediaType: .video)[0],
at: CMTime.zero)
} catch {
print("Failed to load first track")
return
}
guard let secondTrack = mixComposition.addMutableTrack(withMediaType: .video,
preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else { return }
do {
try secondTrack.insertTimeRange(CMTimeRangeMake(start: CMTime.zero, duration: secondAsset.duration),
of: secondAsset.tracks(withMediaType: .video)[0],
at: CMTime.zero)
} catch {
print("Failed to load second track")
return
}
// 2.1
let mainInstruction = AVMutableVideoCompositionInstruction()
mainInstruction.timeRange = CMTimeRangeMake(start: CMTime.zero, duration: CMTimeAdd(firstAsset.duration, secondAsset.duration))
// 2.2
let firstInstruction = ViewController.videoCompositionInstruction(firstTrack, asset: firstAsset)
let scale = CGAffineTransform(scaleX: 0.3, y: 0.3)
let move = CGAffineTransform(translationX: 10, y: 10)
firstInstruction.setTransform(scale.concatenating(move), at: CMTime.zero)
let secondInstruction = ViewController.videoCompositionInstruction(secondTrack, asset: secondAsset)
// 2.3
mainInstruction.layerInstructions = [firstInstruction, secondInstruction]
let mainComposition = AVMutableVideoComposition()
mainComposition.instructions = [mainInstruction]
mainComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
let width = max(firstTrack.naturalSize.width, secondTrack.naturalSize.width)
let height = max(firstTrack.naturalSize.height, secondTrack.naturalSize.height)
mainComposition.renderSize = CGSize(width: width, height: height)
mainInstruction.backgroundColor = UIColor.clear.cgColor
// 4 - Get path
guard let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else { return }
let dateFormatter = DateFormatter()
dateFormatter.dateStyle = .long
dateFormatter.timeStyle = .short
let date = dateFormatter.string(from: Date())
let url = documentDirectory.appendingPathComponent("mergeVideo-\(date).mov")
// Check exists and remove old file
FileManager.default.removeItemIfExisted(url as URL)
// 5 - Create Exporter
guard let exporter = AVAssetExportSession(asset: mixComposition, presetName: AVAssetExportPresetHighestQuality) else { return }
exporter.outputURL = url
exporter.outputFileType = AVFileType.mov
exporter.shouldOptimizeForNetworkUse = true
exporter.videoComposition = mainComposition
// 6 - Perform the Export
exporter.exportAsynchronously() {
DispatchQueue.main.async {
print("Movie complete")
self.myurl = url as URL
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: url as URL)
}) { saved, error in
if saved {
print("Saved")
}
}
self.playVideo()
}
}
}
func playVideo() {
let player = AVPlayer(url: myurl!)
let playerLayer = AVPlayerLayer(player: player)
playerLayer.frame = self.view.bounds
self.view.layer.addSublayer(playerLayer)
player.play()
print("playing...")
}
static func videoCompositionInstruction(_ track: AVCompositionTrack, asset: AVAsset) -> AVMutableVideoCompositionLayerInstruction {
let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: track)
let assetTrack = asset.tracks(withMediaType: AVMediaType.video)[0]
let transform = assetTrack.preferredTransform
let assetInfo = orientationFromTransform(transform)
var scaleToFitRatio = UIScreen.main.bounds.width / assetTrack.naturalSize.width
if assetInfo.isPortrait {
scaleToFitRatio = UIScreen.main.bounds.width / assetTrack.naturalSize.height
let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
instruction.setTransform(assetTrack.preferredTransform.concatenating(scaleFactor), at: CMTime.zero)
} else {
let scaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
var concat = assetTrack.preferredTransform.concatenating(scaleFactor)
.concatenating(CGAffineTransform(translationX: 0, y: UIScreen.main.bounds.width / 2))
if assetInfo.orientation == .down {
let fixUpsideDown = CGAffineTransform(rotationAngle: CGFloat(Double.pi))
let windowBounds = UIScreen.main.bounds
let yFix = assetTrack.naturalSize.height + windowBounds.height
let centerFix = CGAffineTransform(translationX: assetTrack.naturalSize.width, y: yFix)
concat = fixUpsideDown.concatenating(centerFix).concatenating(scaleFactor)
}
instruction.setTransform(concat, at: CMTime.zero)
}
return instruction
}
static func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
var assetOrientation = UIImage.Orientation.up
var isPortrait = false
if transform.a == 0 && transform.b == 1.0 && transform.c == -1.0 && transform.d == 0 {
assetOrientation = .right
isPortrait = true
} else if transform.a == 0 && transform.b == -1.0 && transform.c == 1.0 && transform.d == 0 {
assetOrientation = .left
isPortrait = true
} else if transform.a == 1.0 && transform.b == 0 && transform.c == 0 && transform.d == 1.0 {
assetOrientation = .up
} else if transform.a == -1.0 && transform.b == 0 && transform.c == 0 && transform.d == -1.0 {
assetOrientation = .down
}
return (assetOrientation, isPortrait)
}
}
extension FileManager {
func removeItemIfExisted(_ url:URL) -> Void {
if FileManager.default.fileExists(atPath: url.path) {
do {
try FileManager.default.removeItem(atPath: url.path)
}
catch {
print("Failed to delete file")
}
}
}
}

Merge two videos in Ios

I can merge two videos, but when I see the final result the duration of the video is correct but it only plays the first video and for the duration of the second video remains a static image.
For example:
two videos of 6 seconds each makes a video of 12 seconds, i can see it correctly until 6 seconds, later it blocks the images
func mergeVideos(videoMergedUrl:URL) {
let mainComposition = AVMutableVideoComposition()
var startDuration:CMTime = kCMTimeZero
let mainInstruction = AVMutableVideoCompositionInstruction()
let mixComposition = AVMutableComposition()
var allVideoInstruction = [AVMutableVideoCompositionLayerInstruction]()
for i:Int in 0 ..< listSegment.count {
let currentAsset = listSegment[i]
let currentTrack = mixComposition.addMutableTrack(withMediaType: AVMediaType.video, preferredTrackID: Int32(kCMPersistentTrackID_Invalid))
do {
try currentTrack?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, currentAsset.duration), of: currentAsset.tracks(withMediaType: AVMediaType.video)[0], at: startDuration)
let currentInstruction:AVMutableVideoCompositionLayerInstruction = videoCompositionInstructionForTrack(currentTrack!, asset: currentAsset)
//currentInstruction.setOpacityRamp(fromStartOpacity: 0.0, toEndOpacity: 1.0, timeRange:CMTimeRangeMake(startDuration, CMTimeMake(1, 1)))
/*if i != assets.count - 1 {
//Sets Fade out effect at the end of the video.
currentInstruction.setOpacityRamp(fromStartOpacity: 1.0,
toEndOpacity: 0.0,
timeRange:CMTimeRangeMake(
CMTimeSubtract(
CMTimeAdd(currentAsset.duration, startDuration),
CMTimeMake(1, 1)),
CMTimeMake(2, 1)))
}*/
/*let transform:CGAffineTransform = currentTrack!.preferredTransform
if orientationFromTransform(transform).isPortrait {
let outputSize:CGSize = CGSize(width: 640, height: 480)
let horizontalRatio = CGFloat(outputSize.width) / (currentTrack?.naturalSize.width)!
let verticalRatio = CGFloat(outputSize.height) / (currentTrack?.naturalSize.height)!
let scaleToFitRatio = max(horizontalRatio, verticalRatio) // ScaleAspectFill
let FirstAssetScaleFactor = CGAffineTransform(scaleX: scaleToFitRatio, y: scaleToFitRatio)
if currentAsset.g_orientation == .landscapeLeft {
let rotation = CGAffineTransform(rotationAngle: .pi)
let translateToCenter = CGAffineTransform(translationX: 640, y: 480)
let mixedTransform = rotation.concatenating(translateToCenter)
currentInstruction.setTransform((currentTrack?.preferredTransform.concatenating(FirstAssetScaleFactor).concatenating(mixedTransform))!, at: kCMTimeZero)
} else {
currentInstruction.setTransform((currentTrack?.preferredTransform.concatenating(FirstAssetScaleFactor))!, at: kCMTimeZero)
}
}*/
allVideoInstruction.append(currentInstruction) //Add video instruction in Instructions Array.
startDuration = CMTimeAdd(startDuration, currentAsset.duration)
} catch _ {
print("ERROR_LOADING_VIDEO")
}
}
mainInstruction.timeRange = CMTimeRangeMake(kCMTimeZero, startDuration)
mainInstruction.layerInstructions = allVideoInstruction
mainComposition.instructions = [mainInstruction]
mainComposition.frameDuration = CMTimeMake(1, 30)
mainComposition.renderSize = CGSize(width: 640, height: 480)
let manager = FileManager.default
_ = try? manager.removeItem(at: videoMergedUrl)
guard let exporter = AVAssetExportSession(asset: mixComposition, presetName: AVAssetExportPreset640x480) else { return }
exporter.outputURL = videoMergedUrl
exporter.outputFileType = AVFileType.mp4
exporter.shouldOptimizeForNetworkUse = false
exporter.videoComposition = mainComposition
// Perform the Export
exporter.exportAsynchronously() {
DispatchQueue.main.async {
self.exportDidFinish(exporter)
}
}
}
I had the same problem after following this tutorial. I fixed it by adding clips to the composition using AVMutableComposition.insertTimeRange instead of addMutableTrack.

Swift 3 : How to export video with text using AVVideoComposition

I am trying to useAVVideoComposition to add some text on top of a video and save the video.
This is the code I use:
I Create an AVMutableComposition and AVVideoComposition
var mutableComp = AVMutableComposition()
var mutableVidComp = AVMutableVideoComposition()
var compositionSize : CGSize?
func configureAsset(){
let options = [AVURLAssetPreferPreciseDurationAndTimingKey : "true"]
let videoAsset = AVURLAsset(url: Bundle.main.url(forResource: "Car", withExtension: "mp4")! , options : options)
let videoAssetSourceTrack = videoAsset.tracks(withMediaType: AVMediaTypeVideo).first! as AVAssetTrack
compositionSize = videoAssetSourceTrack.naturalSize
let mutableVidTrack = mutableComp.addMutableTrack(withMediaType: AVMediaTypeVideo, preferredTrackID: kCMPersistentTrackID_Invalid)
let trackRange = CMTimeRangeMake(kCMTimeZero, videoAsset.duration)
do {
try mutableVidTrack.insertTimeRange( trackRange, of: videoAssetSourceTrack, at: kCMTimeZero)
mutableVidTrack.preferredTransform = videoAssetSourceTrack.preferredTransform
}catch { print(error) }
snapshot = mutableComp
mutableVidComp = AVMutableVideoComposition(propertiesOf: videoAsset)
}
II Setup the layers
func applyVideoEffectsToComposition() {
// 1 - Set up the text layer
let subTitle1Text = CATextLayer()
subTitle1Text.font = "Helvetica-Bold" as CFTypeRef
subTitle1Text.frame = CGRect(x: self.view.frame.midX - 60 , y: self.view.frame.midY - 50, width: 120, height: 100)
subTitle1Text.string = "Bench"
subTitle1Text.foregroundColor = UIColor.black.cgColor
subTitle1Text.alignmentMode = kCAAlignmentCenter
// 2 - The usual overlay
let overlayLayer = CALayer()
overlayLayer.addSublayer(subTitle1Text)
overlayLayer.frame = CGRect(x: 0, y: 0, width: compositionSize!.width, height: compositionSize!.height)
overlayLayer.masksToBounds = true
// 3 - set up the parent layer
let parentLayer = CALayer()
let videoLayer = CALayer()
parentLayer.frame = CGRect(x: 0, y: 0, width: compositionSize!.width, height: compositionSize!.height)
videoLayer.frame = CGRect(x: 0, y: 0, width: compositionSize!.width, height: compositionSize!.height)
parentLayer.addSublayer(videoLayer)
parentLayer.addSublayer(overlayLayer)
mutableVidComp.animationTool = AVVideoCompositionCoreAnimationTool(postProcessingAsVideoLayer: videoLayer, in: parentLayer)
}
III . Save video with AVMutbaleVideoComposition
func saveAsset (){
func deleteFile(_ filePath:URL) {
guard FileManager.default.fileExists(atPath: filePath.path) else { return }
do {
try FileManager.default.removeItem(atPath: filePath.path) }
catch {fatalError("Unable to delete file: \(error) : \(#function).")} }
let documentsDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] as URL
let filePath = documentsDirectory.appendingPathComponent("rendered-audio.mp4")
deleteFile(filePath)
if let exportSession = AVAssetExportSession(asset: mutableComp , presetName: AVAssetExportPresetHighestQuality){
exportSession.videoComposition = mutableVidComp
// exportSession.canPerformMultiplePassesOverSourceMediaData = true
exportSession.outputURL = filePath
exportSession.shouldOptimizeForNetworkUse = true
exportSession.timeRange = CMTimeRangeMake(kCMTimeZero, mutableComp.duration)
exportSession.outputFileType = AVFileTypeQuickTimeMovie
exportSession.exportAsynchronously {
print("finished: \(filePath) : \(exportSession.status.rawValue) ")
if exportSession.status.rawValue == 4 {
print("Export failed -> Reason: \(exportSession.error!.localizedDescription))")
print(exportSession.error!)
}
}
}
}
Then I run all three methods in the viewDidLoad method for a quick test. The problem is that when I run the app ,the result of the export is the original video without the title on it.
What am I missing here?
UPDATE
I notice that adding a subTitle1Text.backgroundColor property in
part II of the code makes a colored CGRect corresponding to subTitle1Text.frame appear on top of the video when exported.
(See Image)
When this code is modified for playback using AVSynchronizedLayer the desired layer can be seen on top of the video with text on it.
So perhaps this is a bug in AVFoundation itself.
I suppose I am only left with the option of using a customVideoCompositorClass. The problem with that is that it takes a lot of time to render the video . Here is an example that uses AVVideoCompositing
Here is full working code which I used in my project. It will show CATextLayer at bottom (0,0). And in export session finish it will replace new path in player item. I used one model from Objective C code to get orientation. Please do testing in device. AVPLayer will not show text layer properly in simulator.
let composition = AVMutableComposition.init()
let videoComposition = AVMutableVideoComposition()
videoComposition.frameDuration = CMTimeMake(1, 30)
videoComposition.renderScale = 1.0
let compositionCommentaryTrack: AVMutableCompositionTrack? = composition.addMutableTrack(withMediaType: AVMediaTypeAudio, preferredTrackID: kCMPersistentTrackID_Invalid)
let compositionVideoTrack: AVMutableCompositionTrack? = composition.addMutableTrack(withMediaType: AVMediaTypeVideo, preferredTrackID: kCMPersistentTrackID_Invalid)
let clipVideoTrack:AVAssetTrack = self.currentAsset.tracks(withMediaType: AVMediaTypeVideo)[0]
let audioTrack: AVAssetTrack? = self.currentAsset.tracks(withMediaType: AVMediaTypeAudio)[0]
try? compositionCommentaryTrack?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, self.currentAsset.duration), of: audioTrack!, at: kCMTimeZero)
try? compositionVideoTrack?.insertTimeRange(CMTimeRangeMake(kCMTimeZero, self.currentAsset.duration), of: clipVideoTrack, at: kCMTimeZero)
let orientation = VideoModel.videoOrientation(self.currentAsset)
var isPortrait = false
switch orientation {
case .landscapeRight:
isPortrait = false
case .landscapeLeft:
isPortrait = false
case .portrait:
isPortrait = true
case .portraitUpsideDown:
isPortrait = true
}
var naturalSize = clipVideoTrack.naturalSize
if isPortrait
{
naturalSize = CGSize.init(width: naturalSize.height, height: naturalSize.width)
}
videoComposition.renderSize = naturalSize
let scale = CGFloat(1.0)
var transform = CGAffineTransform.init(scaleX: CGFloat(scale), y: CGFloat(scale))
switch orientation {
case .landscapeRight: break
// isPortrait = false
case .landscapeLeft:
transform = transform.translatedBy(x: naturalSize.width, y: naturalSize.height)
transform = transform.rotated(by: .pi)
case .portrait:
transform = transform.translatedBy(x: naturalSize.width, y: 0)
transform = transform.rotated(by: CGFloat(M_PI_2))
case .portraitUpsideDown:break
}
let frontLayerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack!)
frontLayerInstruction.setTransform(transform, at: kCMTimeZero)
let MainInstruction = AVMutableVideoCompositionInstruction()
MainInstruction.timeRange = CMTimeRangeMake(kCMTimeZero, composition.duration)
MainInstruction.layerInstructions = [frontLayerInstruction]
videoComposition.instructions = [MainInstruction]
let parentLayer = CALayer.init()
parentLayer.frame = CGRect.init(x: 0, y: 0, width: naturalSize.width, height: naturalSize.height)
let videoLayer = CALayer.init()
videoLayer.frame = parentLayer.frame
let layer = CATextLayer()
layer.string = "HELLO ALL"
layer.foregroundColor = UIColor.white.cgColor
layer.backgroundColor = UIColor.orange.cgColor
layer.fontSize = 32
layer.frame = CGRect.init(x: 0, y: 0, width: 300, height: 100)
var rct = layer.frame;
let widthScale = self.playerView.frame.size.width/naturalSize.width
rct.size.width /= widthScale
rct.size.height /= widthScale
rct.origin.x /= widthScale
rct.origin.y /= widthScale
parentLayer.addSublayer(videoLayer)
parentLayer.addSublayer(layer)
videoComposition.animationTool = AVVideoCompositionCoreAnimationTool.init(postProcessingAsVideoLayer: videoLayer, in: parentLayer)
let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
let videoPath = documentsPath+"/cropEditVideo.mov"
let fileManager = FileManager.default
if fileManager.fileExists(atPath: videoPath)
{
try! fileManager.removeItem(atPath: videoPath)
}
print("video path \(videoPath)")
var exportSession = AVAssetExportSession.init(asset: composition, presetName: AVAssetExportPresetHighestQuality)
exportSession?.videoComposition = videoComposition
exportSession?.outputFileType = AVFileTypeQuickTimeMovie
exportSession?.outputURL = URL.init(fileURLWithPath: videoPath)
exportSession?.videoComposition = videoComposition
var exportProgress: Float = 0
let queue = DispatchQueue(label: "Export Progress Queue")
queue.async(execute: {() -> Void in
while exportSession != nil {
// int prevProgress = exportProgress;
exportProgress = (exportSession?.progress)!
print("current progress == \(exportProgress)")
sleep(1)
}
})
exportSession?.exportAsynchronously(completionHandler: {
if exportSession?.status == AVAssetExportSessionStatus.failed
{
print("Failed \(exportSession?.error)")
}else if exportSession?.status == AVAssetExportSessionStatus.completed
{
exportSession = nil
let asset = AVAsset.init(url: URL.init(fileURLWithPath: videoPath))
DispatchQueue.main.async {
let item = AVPlayerItem.init(asset: asset)
self.player.replaceCurrentItem(with: item)
let assetDuration = CMTimeGetSeconds(composition.duration)
self.progressSlider.maximumValue = Float(assetDuration)
self.syncLayer.removeFromSuperlayer()
self.lblIntro.isHidden = true
self.player.play()
// let url = URL.init(fileURLWithPath: videoPath)
// let activityVC = UIActivityViewController(activityItems: [url], applicationActivities: [])
// self.present(activityVC, animated: true, completion: nil)
}
}
})
Below is code of My VideoModel class
-(AVCaptureVideoOrientation)videoOrientation:(AVAsset *)asset
{
AVCaptureVideoOrientation result = 0;
NSArray *tracks = [asset tracksWithMediaType:AVMediaTypeVideo];
if([tracks count] > 0) {
AVAssetTrack *videoTrack = [tracks objectAtIndex:0];
CGAffineTransform t = videoTrack.preferredTransform;
// Portrait
if(t.a == 0 && t.b == 1.0 && t.c == -1.0 && t.d == 0)
{
result = AVCaptureVideoOrientationPortrait;
}
// PortraitUpsideDown
if(t.a == 0 && t.b == -1.0 && t.c == 1.0 && t.d == 0) {
result = AVCaptureVideoOrientationPortraitUpsideDown;
}
// LandscapeRight
if(t.a == 1.0 && t.b == 0 && t.c == 0 && t.d == 1.0)
{
result = AVCaptureVideoOrientationLandscapeRight;
}
// LandscapeLeft
if(t.a == -1.0 && t.b == 0 && t.c == 0 && t.d == -1.0)
{
result = AVCaptureVideoOrientationLandscapeLeft;
}
}
return result;
}
Let me know if you need any more help in this.

Resources