Related
I build a App, where user can make a Video of a few images.
I took the code from this post: Making video from UIImage array with different transition animations
Everything is working fine, but I have one big problem and I don't know how to solve it.
When the video was created, the image is not filling the hole screen and sometimes it turns to another side or the image is cut in half.
First image is the original image and the second image is the created video with the problems I have.
I really need some help.
var outputSize = CGSize(width: 1920 , height: 1280)
var imagesPerSecond: TimeInterval = 0
var fps: Int32 = 0
var selectedPhotosArray = [UIImage]()
var imageArrayToVideoURL = NSURL()
var asset: AVAsset!
var imageCount = 1
My Code
func buildVideoFromImageArray() {
for image in arrayOfImages {
selectedPhotosArray.append(image)
}
imageArrayToVideoURL = NSURL(fileURLWithPath: NSHomeDirectory() + "/Documents/video1.MP4")
removeFileAtURLIfExists(url: imageArrayToVideoURL)
guard let videoWriter = try? AVAssetWriter(outputURL: imageArrayToVideoURL as URL, fileType: AVFileType.mp4) else {
fatalError("AVAssetWriter error")
}
let outputSettings = [AVVideoCodecKey : AVVideoCodecType.h264, AVVideoWidthKey : NSNumber(value: Float(outputSize.width)), AVVideoHeightKey : NSNumber(value: Float(outputSize.height))] as [String : Any]
guard videoWriter.canApply(outputSettings: outputSettings, forMediaType: AVMediaType.video) else {
fatalError("Negative : Can't applay the Output settings...")
}
let videoWriterInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: outputSettings)
let sourcePixelBufferAttributesDictionary = [kCVPixelBufferPixelFormatTypeKey as String : NSNumber(value: kCVPixelFormatType_32ARGB), kCVPixelBufferWidthKey as String: NSNumber(value: Float(outputSize.width)), kCVPixelBufferHeightKey as String: NSNumber(value: Float(outputSize.height))]
let pixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriterInput, sourcePixelBufferAttributes: sourcePixelBufferAttributesDictionary)
if videoWriter.canAdd(videoWriterInput) {
videoWriter.add(videoWriterInput)
}
if videoWriter.startWriting() {
let zeroTime = CMTimeMake(value: Int64(imagesPerSecond),timescale: self.fps)
videoWriter.startSession(atSourceTime: zeroTime)
assert(pixelBufferAdaptor.pixelBufferPool != nil)
let media_queue = DispatchQueue(label: "mediaInputQueue")
videoWriterInput.requestMediaDataWhenReady(on: media_queue, using: { () -> Void in
//let fps: Int32 = 1
let framePerSecond: Int64 = Int64(self.imagesPerSecond)
let frameDuration = CMTimeMake(value: Int64(self.imagesPerSecond), timescale: self.fps)
var frameCount: Int64 = 0
var appendSucceeded = true
while (!self.selectedPhotosArray.isEmpty) { // wird so lange ausgeführt, bis noch etwas im Array steht
if (videoWriterInput.isReadyForMoreMediaData) {
let nextPhoto = self.selectedPhotosArray.remove(at: 0) // foto wird aus dem selectedPhotosArray gelöscht
let lastFrameTime = CMTimeMake(value: frameCount * framePerSecond, timescale: self.fps)
let presentationTime = frameCount == 0 ? lastFrameTime : CMTimeAdd(lastFrameTime, frameDuration)
var pixelBuffer: CVPixelBuffer? = nil
let status: CVReturn = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferAdaptor.pixelBufferPool!, &pixelBuffer)
if let pixelBuffer = pixelBuffer, status == 0 {
let managedPixelBuffer = pixelBuffer
CVPixelBufferLockBaseAddress(managedPixelBuffer, CVPixelBufferLockFlags(rawValue: CVOptionFlags(0)))
let data = CVPixelBufferGetBaseAddress(managedPixelBuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let context = CGContext(data: data, width: Int(self.outputSize.width), height: Int(self.outputSize.height), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(managedPixelBuffer), space: rgbColorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue)
context!.clear(CGRect(x: 0, y: 0, width: CGFloat(self.outputSize.width), height: CGFloat(self.outputSize.height)))
let horizontalRatio = CGFloat(self.outputSize.width) / nextPhoto.size.width
let verticalRatio = CGFloat(self.outputSize.height) / nextPhoto.size.height
//let aspectRatio = max(horizontalRatio, verticalRatio) // ScaleAspectFill
let aspectRatio = min(horizontalRatio, verticalRatio) // ScaleAspectFit
let newSize: CGSize = CGSize(width: nextPhoto.size.width * aspectRatio, height: nextPhoto.size.height * aspectRatio)
let x = newSize.width < self.outputSize.width ? (self.outputSize.width - newSize.width) / 2 : 0
let y = newSize.height < self.outputSize.height ? (self.outputSize.height - newSize.height) / 2 : 0
context?.draw(nextPhoto.cgImage!, in: CGRect(x: x, y: y, width: newSize.width, height: newSize.height))
CVPixelBufferUnlockBaseAddress(managedPixelBuffer, CVPixelBufferLockFlags(rawValue: CVOptionFlags(0)))
appendSucceeded = pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: presentationTime)
} else {
print("Failed to allocate pixel buffer")
appendSucceeded = false
}
}
if !appendSucceeded {
break
}
frameCount += 1
}
videoWriterInput.markAsFinished()
videoWriter.finishWriting { () -> Void in
print("-----video1 url = \(self.imageArrayToVideoURL)")
//self.asset = AVAsset(url: self.imageArrayToVideoURL as URL)
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: self.imageArrayToVideoURL as URL)
}) { saved, error in
if saved {
let fetchOptions = PHFetchOptions()
fetchOptions.sortDescriptors = [NSSortDescriptor(key: "creationDate", ascending: false)]
let fetchResult = PHAsset.fetchAssets(with: .video, options: fetchOptions).firstObject
// fetchResult is your latest video PHAsset
// To fetch latest image replace .video with .image
}
}
}
})
}
}
func removeFileAtURLIfExists(url: NSURL) {
if let filePath = url.path {
let fileManager = FileManager.default
if fileManager.fileExists(atPath: filePath) {
do{
try fileManager.removeItem(atPath: filePath)
} catch let error as NSError {
print("Couldn't remove existing destination file: \(error)")
}
}
}
}`
After captured photo from camera, I was doing image compression For (400kb and 1 Mb), it look almost 3 seconds in iPhone 6 and less than a second in iPhone 6s.
Is there any way to get thumbnail and original image without doing manual compression?
Code used for image compression
Extension for UIImage
extension UIImage {
// MARK: - UIImage+Resize
func compressTo(_ expectedSizeInMb:Int) -> Data? {
let sizeInBytes = expectedSizeInMb * 1024 * 1024
var needCompress:Bool = true
var imgData:Data?
var compressingValue:CGFloat = 1.0
while (needCompress && compressingValue > 0.0) {
if let data:Data = jpegData(compressionQuality: compressingValue) {
if data.count < sizeInBytes {
needCompress = false
imgData = data
} else {
compressingValue -= 0.1
}
}
}
if let data = imgData {
if (data.count < sizeInBytes) {
return data
}
}
return nil
}
}
usage:
if let imageData = image.compressTo(1) {
print(imageData)
}
For images saved in Photos Library :
Try :
let phAsset = info[UIImagePickerController.InfoKey.phAsset] as! PHAsset
let options = PHImageRequestOptions()
options.deliveryMode = .fastFormat
options.isSynchronous = false
// you can change your target size to CGSize(width: Int , height: Int) any number you want.
PHImageManager.default().requestImage(for: phAsset, targetSize: PHImageManagerMaximumSize, contentMode: .default, options: options, resultHandler: { image , _ in
let thumbnail = image
// use your thumbnail
})
For Captured images from Camera, you can get image pixels without recalculating data count :
let image = info[UIImagePickerController.InfoKey.originalImage] as! UIImage
// pixels are the same on each device’s camera
let widthPixels = image.size.width * image.scale
let heightPixels = image.size.height * image.scale
let sizeInBytes = 1024 * 1024
var thumbnail : UIImage! = nil
if Int(widthPixels * heightPixels) > sizeInBytes {
// assign custom width and height you need
let rect = CGRect(x: 0.0, y: 0.0, width: 100, height: 100)
UIGraphicsBeginImageContextWithOptions(rect.size, false, 1)
let context = UIGraphicsGetCurrentContext()
context?.interpolationQuality = .low
image.draw(in: rect)
let resizedImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
thumbnail = resizedImage
} else {
thumbnail = image
}
I have a project where users can take a video and later add filters to them or change basic settings like brightness and contrast. To accomplish this, I use BBMetalImage, which basically returns the video in a MTKView (named a BBMetalView in the project).
Everything works great - I can play the video, add filters and the desired effects, but there is no audio. I asked the author about this, who recommended using an AVPlayer (or AVAudioPlayer) for this. So I did. However, the video and audio are out of sync. Possibly because of different bitrates in the first place, and the author of the library also mentioned the frame rate can differ because of the filter process (the time this consumes is variable):
The render view FPS is not exactly the same to the actual rate.
Because the video source output frame is processed by filters and the
filter process time is variable.
First, I crop my video to the desired aspect ratio (4:5). I save this file (480x600) locally, using AVVideoProfileLevelH264HighAutoLevel as AVVideoProfileLevelKey. My audio configuration, using NextLevelSessionExporter, has the following setup: AVEncoderBitRateKey: 128000, AVNumberOfChannelsKey: 2, AVSampleRateKey: 44100.
Then, the BBMetalImage library takes this saved audio file and provides a MTKView (BBMetalView) to display the video, allowing me to add filters and effects in real time. The setup kind of looks like this:
self.metalView = BBMetalView(frame: CGRect(x: 0, y: self.view.center.y - ((UIScreen.main.bounds.width * 1.25) / 2), width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.width * 1.25))
self.view.addSubview(self.metalView)
self.videoSource = BBMetalVideoSource(url: outputURL)
self.videoSource.playWithVideoRate = true
self.videoSource.audioConsumer = self.metalAudio
self.videoSource.add(consumer: self.metalView)
self.videoSource.add(consumer: self.videoWriter)
self.audioItem = AVPlayerItem(url: outputURL)
self.audioPlayer = AVPlayer(playerItem: self.audioItem)
self.playerLayer = AVPlayerLayer(player: self.audioPlayer)
self.videoPreview.layer.addSublayer(self.playerLayer!)
self.playerLayer?.frame = CGRect(x: 0, y: 0, width: 0, height: 0)
self.playerLayer?.backgroundColor = UIColor.black.cgColor
self.startVideo()
And startVideo() goes like this:
audioPlayer.seek(to: .zero)
audioPlayer.play()
videoSource.start(progress: { (frameTime) in
print(frameTime)
}) { [weak self] (finish) in
guard let self = self else { return }
self.startVideo()
}
This is all probably pretty vague because of the external library/libraries. However, my question is pretty simple: is there any way I can sync the MTKView with my AVPlayer? It would help me a lot and I'm sure Silence-GitHub would also implement this feature into the library to help a lot of other users. Any ideas on how to approach this are welcome!
I custom the BBMetalVideoSource as follow then it worked:
Create a delegate in BBMetalVideoSource to get the current time of the audio player with which we want to sync
In func private func processAsset(progress:, completion:), I replace this block of code if useVideoRate { //... } by:
if useVideoRate {
if let playerTime = delegate.getAudioPlayerCurrentTime() {
let diff = CMTimeGetSeconds(sampleFrameTime) - playerTime
if diff > 0.0 {
sleepTime = diff
if sleepTime > 1.0 {
sleepTime = 0.0
}
usleep(UInt32(1000000 * sleepTime))
} else {
sleepTime = 0
}
}
}
This code help us resolve both problems: 1. No audio when preview video effect, and 2. Sync audio with video.
Due to your circumstances, you seem to need to try 1 of 2 things:
1) Try and apply some sort of overlay that has the desired effect for your video. I could attempt something like this, but I have personally not done this.
2) This takes a little more time beforehand - in the sense that the program would have to take a few moments (depending on your filtering, time varies), to recreate a new video with the desired effects. You can try this out and see if it works for you.
I have made my own VideoCreator using some sourcecode from SO from somewhere.
//Recreates a new video with applied filter
public static func createFilteredVideo(asset: AVAsset, completionHandler: #escaping (_ asset: AVAsset) -> Void) {
let url = (asset as? AVURLAsset)!.url
let snapshot = url.videoSnapshot()
guard let image = snapshot else { return }
let fps = Int32(asset.tracks(withMediaType: .video)[0].nominalFrameRate)
let writer = VideoCreator(fps: Int32(fps), width: image.size.width, height: image.size.height, audioSettings: nil)
let timeScale = asset.duration.timescale
let timeValue = asset.duration.value
let frameTime = 1/Double(fps) * Double(timeScale)
let numberOfImages = Int(Double(timeValue)/Double(frameTime))
let queue = DispatchQueue(label: "com.queue.queue", qos: .utility)
let composition = AVVideoComposition(asset: asset) { (request) in
let source = request.sourceImage.clampedToExtent()
//This is where you create your filter and get your filtered result.
//Here is an example
let filter = CIFilter(name: "CIBlendWithMask")
filter!.setValue(maskImage, forKey: "inputMaskImage")
filter!.setValue(regCIImage, forKey: "inputImage")
let filteredImage = filter!.outputImage.clamped(to: source.extent)
request.finish(with: filteredImage, context: nil)
}
var i = 0
getAudioFromURL(url: url) { (buffer) in
writer.addAudio(audio: buffer, time: .zero)
i == 0 ? writer.startCreatingVideo(initialBuffer: buffer, completion: {}) : nil
i += 1
}
let group = DispatchGroup()
for i in 0..<numberOfImages {
group.enter()
autoreleasepool {
let time = CMTime(seconds: Double(Double(i) * frameTime / Double(timeScale)), preferredTimescale: timeScale)
let image = url.videoSnapshot(time: time, composition: composition)
queue.async {
writer.addImageAndAudio(image: image!, audio: nil, time: time.seconds)
group.leave()
}
}
}
group.notify(queue: queue) {
writer.finishWriting()
let url = writer.getURL()
//Now create exporter to add audio then do completion handler
completionHandler(AVAsset(url: url))
}
}
static func getAudioFromURL(url: URL, completionHandlerPerBuffer: #escaping ((_ buffer:CMSampleBuffer) -> Void)) {
let asset = AVURLAsset(url: url, options: [AVURLAssetPreferPreciseDurationAndTimingKey: NSNumber(value: true as Bool)])
guard let assetTrack = asset.tracks(withMediaType: AVMediaType.audio).first else {
fatalError("Couldn't load AVAssetTrack")
}
guard let reader = try? AVAssetReader(asset: asset)
else {
fatalError("Couldn't initialize the AVAssetReader")
}
reader.timeRange = CMTimeRange(start: .zero, duration: asset.duration)
let outputSettingsDict: [String : Any] = [
AVFormatIDKey: Int(kAudioFormatLinearPCM),
AVLinearPCMBitDepthKey: 16,
AVLinearPCMIsBigEndianKey: false,
AVLinearPCMIsFloatKey: false,
AVLinearPCMIsNonInterleaved: false
]
let readerOutput = AVAssetReaderTrackOutput(track: assetTrack,
outputSettings: outputSettingsDict)
readerOutput.alwaysCopiesSampleData = false
reader.add(readerOutput)
while reader.status == .reading {
guard let readSampleBuffer = readerOutput.copyNextSampleBuffer() else { break }
completionHandlerPerBuffer(readSampleBuffer)
}
}
extension URL {
func videoSnapshot(time:CMTime? = nil, composition:AVVideoComposition? = nil) -> UIImage? {
let asset = AVURLAsset(url: self)
let generator = AVAssetImageGenerator(asset: asset)
generator.appliesPreferredTrackTransform = true
generator.requestedTimeToleranceBefore = .zero
generator.requestedTimeToleranceAfter = .zero
generator.videoComposition = composition
let timestamp = time == nil ? CMTime(seconds: 1, preferredTimescale: 60) : time
do {
let imageRef = try generator.copyCGImage(at: timestamp!, actualTime: nil)
return UIImage(cgImage: imageRef)
}
catch let error as NSError
{
print("Image generation failed with error \(error)")
return nil
}
}
}
Below is the VideoCreator
//
// VideoCreator.swift
// AKPickerView-Swift
//
// Created by Impression7vx on 7/16/19.
//
import UIKit
import AVFoundation
import UIKit
import Photos
#available(iOS 11.0, *)
public class VideoCreator: NSObject {
private var settings:RenderSettings!
private var imageAnimator:ImageAnimator!
public override init() {
self.settings = RenderSettings()
self.imageAnimator = ImageAnimator(renderSettings: self.settings)
}
public convenience init(fps: Int32, width: CGFloat, height: CGFloat, audioSettings: [String:Any]?) {
self.init()
self.settings = RenderSettings(fps: fps, width: width, height: height)
self.imageAnimator = ImageAnimator(renderSettings: self.settings, audioSettings: audioSettings)
}
public convenience init(width: CGFloat, height: CGFloat) {
self.init()
self.settings = RenderSettings(width: width, height: height)
self.imageAnimator = ImageAnimator(renderSettings: self.settings)
}
func startCreatingVideo(initialBuffer: CMSampleBuffer?, completion: #escaping (() -> Void)) {
self.imageAnimator.render(initialBuffer: initialBuffer) {
completion()
}
}
func finishWriting() {
self.imageAnimator.isDone = true
}
func addImageAndAudio(image:UIImage, audio:CMSampleBuffer?, time:CFAbsoluteTime) {
self.imageAnimator.addImageAndAudio(image: image, audio: audio, time: time)
}
func getURL() -> URL {
return settings!.outputURL
}
func addAudio(audio: CMSampleBuffer, time: CMTime) {
self.imageAnimator.videoWriter.addAudio(buffer: audio, time: time)
}
}
#available(iOS 11.0, *)
public struct RenderSettings {
var width: CGFloat = 1280
var height: CGFloat = 720
var fps: Int32 = 2 // 2 frames per second
var avCodecKey = AVVideoCodecType.h264
var videoFilename = "video"
var videoFilenameExt = "mov"
init() { }
init(width: CGFloat, height: CGFloat) {
self.width = width
self.height = height
}
init(fps: Int32) {
self.fps = fps
}
init(fps: Int32, width: CGFloat, height: CGFloat) {
self.fps = fps
self.width = width
self.height = height
}
var size: CGSize {
return CGSize(width: width, height: height)
}
var outputURL: URL {
// Use the CachesDirectory so the rendered video file sticks around as long as we need it to.
// Using the CachesDirectory ensures the file won't be included in a backup of the app.
let fileManager = FileManager.default
if let tmpDirURL = try? fileManager.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true) {
return tmpDirURL.appendingPathComponent(videoFilename).appendingPathExtension(videoFilenameExt)
}
fatalError("URLForDirectory() failed")
}
}
#available(iOS 11.0, *)
public class ImageAnimator {
// Apple suggests a timescale of 600 because it's a multiple of standard video rates 24, 25, 30, 60 fps etc.
static let kTimescale: Int32 = 600
let settings: RenderSettings
let videoWriter: VideoWriter
var imagesAndAudio:SynchronizedArray<(UIImage, CMSampleBuffer?, CFAbsoluteTime)> = SynchronizedArray<(UIImage, CMSampleBuffer?, CFAbsoluteTime)>()
var isDone:Bool = false
let semaphore = DispatchSemaphore(value: 1)
var frameNum = 0
class func removeFileAtURL(fileURL: URL) {
do {
try FileManager.default.removeItem(atPath: fileURL.path)
}
catch _ as NSError {
// Assume file doesn't exist.
}
}
init(renderSettings: RenderSettings, audioSettings:[String:Any]? = nil) {
settings = renderSettings
videoWriter = VideoWriter(renderSettings: settings, audioSettings: audioSettings)
}
func addImageAndAudio(image: UIImage, audio: CMSampleBuffer?, time:CFAbsoluteTime) {
self.imagesAndAudio.append((image, audio, time))
// print("Adding to array -- \(self.imagesAndAudio.count)")
}
func render(initialBuffer: CMSampleBuffer?, completion: #escaping ()->Void) {
// The VideoWriter will fail if a file exists at the URL, so clear it out first.
ImageAnimator.removeFileAtURL(fileURL: settings.outputURL)
videoWriter.start(initialBuffer: initialBuffer)
videoWriter.render(appendPixelBuffers: appendPixelBuffers) {
//ImageAnimator.saveToLibrary(self.settings.outputURL)
completion()
}
}
// This is the callback function for VideoWriter.render()
func appendPixelBuffers(writer: VideoWriter) -> Bool {
//Don't stop while images are NOT empty
while !imagesAndAudio.isEmpty || !isDone {
if(!imagesAndAudio.isEmpty) {
let date = Date()
if writer.isReadyForVideoData == false {
// Inform writer we have more buffers to write.
// print("Writer is not ready for more data")
return false
}
autoreleasepool {
//This should help but truly doesn't suffice - still need a mutex/lock
if(!imagesAndAudio.isEmpty) {
semaphore.wait() // requesting resource
let imageAndAudio = imagesAndAudio.first()!
let image = imageAndAudio.0
// let audio = imageAndAudio.1
let time = imageAndAudio.2
self.imagesAndAudio.removeAtIndex(index: 0)
semaphore.signal() // releasing resource
let presentationTime = CMTime(seconds: time, preferredTimescale: 600)
// if(audio != nil) { videoWriter.addAudio(buffer: audio!) }
let success = videoWriter.addImage(image: image, withPresentationTime: presentationTime)
if success == false {
fatalError("addImage() failed")
}
else {
// print("Added image # frame \(frameNum) with presTime: \(presentationTime)")
}
frameNum += 1
let final = Date()
let timeDiff = final.timeIntervalSince(date)
// print("Time: \(timeDiff)")
}
else {
// print("Images was empty")
}
}
}
}
print("Done writing")
// Inform writer all buffers have been written.
return true
}
}
#available(iOS 11.0, *)
public class VideoWriter {
let renderSettings: RenderSettings
var audioSettings: [String:Any]?
var videoWriter: AVAssetWriter!
var videoWriterInput: AVAssetWriterInput!
var pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor!
var audioWriterInput: AVAssetWriterInput!
static var ci:Int = 0
var initialTime:CMTime!
var isReadyForVideoData: Bool {
return (videoWriterInput == nil ? false : videoWriterInput!.isReadyForMoreMediaData )
}
var isReadyForAudioData: Bool {
return (audioWriterInput == nil ? false : audioWriterInput!.isReadyForMoreMediaData)
}
class func pixelBufferFromImage(image: UIImage, pixelBufferPool: CVPixelBufferPool, size: CGSize, alpha:CGImageAlphaInfo) -> CVPixelBuffer? {
var pixelBufferOut: CVPixelBuffer?
let status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &pixelBufferOut)
if status != kCVReturnSuccess {
fatalError("CVPixelBufferPoolCreatePixelBuffer() failed")
}
let pixelBuffer = pixelBufferOut!
CVPixelBufferLockBaseAddress(pixelBuffer, [])
let data = CVPixelBufferGetBaseAddress(pixelBuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let context = CGContext(data: data, width: Int(size.width), height: Int(size.height),
bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), space: rgbColorSpace, bitmapInfo: alpha.rawValue)
context!.clear(CGRect(x: 0, y: 0, width: size.width, height: size.height))
let horizontalRatio = size.width / image.size.width
let verticalRatio = size.height / image.size.height
//aspectRatio = max(horizontalRatio, verticalRatio) // ScaleAspectFill
let aspectRatio = min(horizontalRatio, verticalRatio) // ScaleAspectFit
let newSize = CGSize(width: image.size.width * aspectRatio, height: image.size.height * aspectRatio)
let x = newSize.width < size.width ? (size.width - newSize.width) / 2 : 0
let y = newSize.height < size.height ? (size.height - newSize.height) / 2 : 0
let cgImage = image.cgImage != nil ? image.cgImage! : image.ciImage!.convertCIImageToCGImage()
context!.draw(cgImage!, in: CGRect(x: x, y: y, width: newSize.width, height: newSize.height))
CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
return pixelBuffer
}
#available(iOS 11.0, *)
init(renderSettings: RenderSettings, audioSettings:[String:Any]? = nil) {
self.renderSettings = renderSettings
self.audioSettings = audioSettings
}
func start(initialBuffer: CMSampleBuffer?) {
let avOutputSettings: [String: AnyObject] = [
AVVideoCodecKey: renderSettings.avCodecKey as AnyObject,
AVVideoWidthKey: NSNumber(value: Float(renderSettings.width)),
AVVideoHeightKey: NSNumber(value: Float(renderSettings.height))
]
let avAudioSettings = audioSettings
func createPixelBufferAdaptor() {
let sourcePixelBufferAttributesDictionary = [
kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: kCVPixelFormatType_32ARGB),
kCVPixelBufferWidthKey as String: NSNumber(value: Float(renderSettings.width)),
kCVPixelBufferHeightKey as String: NSNumber(value: Float(renderSettings.height))
]
pixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriterInput,
sourcePixelBufferAttributes: sourcePixelBufferAttributesDictionary)
}
func createAssetWriter(outputURL: URL) -> AVAssetWriter {
guard let assetWriter = try? AVAssetWriter(outputURL: outputURL, fileType: AVFileType.mov) else {
fatalError("AVAssetWriter() failed")
}
guard assetWriter.canApply(outputSettings: avOutputSettings, forMediaType: AVMediaType.video) else {
fatalError("canApplyOutputSettings() failed")
}
return assetWriter
}
videoWriter = createAssetWriter(outputURL: renderSettings.outputURL)
videoWriterInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: avOutputSettings)
// if(audioSettings != nil) {
audioWriterInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
audioWriterInput.expectsMediaDataInRealTime = true
// }
if videoWriter.canAdd(videoWriterInput) {
videoWriter.add(videoWriterInput)
}
else {
fatalError("canAddInput() returned false")
}
// if(audioSettings != nil) {
if videoWriter.canAdd(audioWriterInput) {
videoWriter.add(audioWriterInput)
}
else {
fatalError("canAddInput() returned false")
}
// }
// The pixel buffer adaptor must be created before we start writing.
createPixelBufferAdaptor()
if videoWriter.startWriting() == false {
fatalError("startWriting() failed")
}
self.initialTime = initialBuffer != nil ? CMSampleBufferGetPresentationTimeStamp(initialBuffer!) : CMTime.zero
videoWriter.startSession(atSourceTime: self.initialTime)
precondition(pixelBufferAdaptor.pixelBufferPool != nil, "nil pixelBufferPool")
}
func render(appendPixelBuffers: #escaping (VideoWriter)->Bool, completion: #escaping ()->Void) {
precondition(videoWriter != nil, "Call start() to initialze the writer")
let queue = DispatchQueue(__label: "mediaInputQueue", attr: nil)
videoWriterInput.requestMediaDataWhenReady(on: queue) {
let isFinished = appendPixelBuffers(self)
if isFinished {
self.videoWriterInput.markAsFinished()
self.videoWriter.finishWriting() {
DispatchQueue.main.async {
print("Done Creating Video")
completion()
}
}
}
else {
// Fall through. The closure will be called again when the writer is ready.
}
}
}
func addAudio(buffer: CMSampleBuffer, time: CMTime) {
if(isReadyForAudioData) {
print("Writing audio \(VideoWriter.ci) of a time of \(CMSampleBufferGetPresentationTimeStamp(buffer))")
let duration = CMSampleBufferGetDuration(buffer)
let offsetBuffer = CMSampleBuffer.createSampleBuffer(fromSampleBuffer: buffer, withTimeOffset: time, duration: duration)
if(offsetBuffer != nil) {
print("Added audio")
self.audioWriterInput.append(offsetBuffer!)
}
else {
print("Not adding audio")
}
}
VideoWriter.ci += 1
}
func addImage(image: UIImage, withPresentationTime presentationTime: CMTime) -> Bool {
precondition(pixelBufferAdaptor != nil, "Call start() to initialze the writer")
//1
let pixelBuffer = VideoWriter.pixelBufferFromImage(image: image, pixelBufferPool: pixelBufferAdaptor.pixelBufferPool!, size: renderSettings.size, alpha: CGImageAlphaInfo.premultipliedFirst)!
return pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: presentationTime + self.initialTime)
}
}
I was looking a little further into this - and while I could have updated my answer, I'd rather open this tangent in a new area to separate these ideas. Apple states that we can use an AVVideoComposition to "To use the created video composition for playback, create an AVPlayerItem object from the same asset used as the composition’s source, then assign the composition to the player item’s videoComposition property. To export the composition to a new movie file, create an AVAssetExportSession object from the same source asset, then assign the composition to the export session’s videoComposition property.".
https://developer.apple.com/documentation/avfoundation/avasynchronousciimagefilteringrequest
So, what you COULD try is using the AVPlayer for the ORIGINAL URL. Then try applying your filter.
let filter = CIFilter(name: "CIGaussianBlur")!
let composition = AVVideoComposition(asset: asset, applyingCIFiltersWithHandler: { request in
// Clamp to avoid blurring transparent pixels at the image edges
let source = request.sourceImage.imageByClampingToExtent()
filter.setValue(source, forKey: kCIInputImageKey)
// Vary filter parameters based on video timing
let seconds = CMTimeGetSeconds(request.compositionTime)
filter.setValue(seconds * 10.0, forKey: kCIInputRadiusKey)
// Crop the blurred output to the bounds of the original image
let output = filter.outputImage!.imageByCroppingToRect(request.sourceImage.extent)
// Provide the filter output to the composition
request.finishWithImage(output, context: nil)
})
let asset = AVAsset(url: originalURL)
let item = AVPlayerItem(asset: asset)
item.videoComposition = composition
let player = AVPlayer(playerItem: item)
I'm sure you know what to do from here. This may allow you to do a "Real-time" of your filtering. What I could see as a potential issue is that this runs into the same issues as your original thing, whereas it still takes a set time to run each frame and leading to a delay between audio and video. However, this may not happen. If you do get this working, once the user selects their filter, you can use AVAssetExportSession to export the specific videoComposition.
More here if you need help!
I am trying to convert a PDF file and all its pages to png images.
I have put together the code below filling the example on this thread
How to convert PDF to PNG efficiently?
When I run the code, it crashes on the pdf file source (sourceURL) there is definitely a file there. and when I print sourceURl it prints the URL to the file.
The crash says it found nil - My understanding is that means it could not find the file? even though I can physically see and open the file and also print the URL to the file.
Can someone point out what I'm doing wrong?
Code:
func convertPDFtoPNG() {
let sourceURL = pptURL
print("pptURL:", pptURL!)
let destinationURL = pngURL
let urls = try? convertPDF(at: sourceURL!, to: destinationURL!, fileType: .png, dpi: 200)
}
func convertPDF(at sourceURL: URL, to destinationURL: URL, fileType: ImageFileType, dpi: CGFloat = 200) throws -> [URL] {
let pdfDocument: CGPDFDocument! = CGPDFDocument(sourceURL as CFURL)! //Thread 1: Fatal error: Unexpectedly found nil while unwrapping an Optional value
let colorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGImageAlphaInfo.noneSkipLast.rawValue
var urls = [URL](repeating: URL(fileURLWithPath : "/"), count: pdfDocument.numberOfPages)
DispatchQueue.concurrentPerform(iterations: pdfDocument.numberOfPages) { i in
let pdfPage = pdfDocument.page(at: i + 1)!
let mediaBoxRect = pdfPage.getBoxRect(.mediaBox)
let scale = dpi / 72.0
let width = Int(mediaBoxRect.width * scale)
let height = Int(mediaBoxRect.height * scale)
let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace, bitmapInfo: bitmapInfo)!
context.interpolationQuality = .high
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
context.scaleBy(x: scale, y: scale)
context.drawPDFPage(pdfPage)
let image = context.makeImage()!
let imageName = sourceURL.deletingPathExtension().lastPathComponent
let imageURL = destinationURL.appendingPathComponent("\(imageName)-Page\(i+1).\(fileType.fileExtention)")
let imageDestination = CGImageDestinationCreateWithURL(imageURL as CFURL, fileType.uti, 1, nil)!
CGImageDestinationAddImage(imageDestination, image, nil)
CGImageDestinationFinalize(imageDestination)
urls[i] = imageURL
}
return urls
}
import Foundation
import Photos
// 1: 目前主要用来操作pdf转为图片
// 2: 图片保存到自定义相册中
struct HBPhotosAlbumHelperUtil {
static let shared = HBPhotosAlbumHelperUtil()
// url链接的pdf转为image
// pageNumber :表示pdf的对应的页面,默认为第一页
func drawToImagePDFFromURL(pdfurl url: String?, pageNumber index: Int = 1, scaleX scalex: CGFloat = 1.0, scaleY scaley: CGFloat = -1.0) -> UIImage? {
guard let pdfUrl = url, pdfUrl.count > 0, let formatterUrl = pdfUrl.urlValue else {
return nil
}
guard let document = CGPDFDocument(formatterUrl as CFURL) else {
return nil
}
guard let page = document.page(at: index) else {
return nil
}
let pageRect = page.getBoxRect(.mediaBox)
if #available(iOS 10.0, *) {
let renderGraph = UIGraphicsImageRenderer(size: pageRect.size)
let drawImage = renderGraph.image { context in
UIColor.white.set()
context.fill(pageRect)
context.cgContext.translateBy(x: 0.0, y: pageRect.size.height)
context.cgContext.scaleBy(x: scalex, y: scaley)
context.cgContext.drawPDFPage(page)
}
return drawImage
} else {
UIGraphicsBeginImageContextWithOptions(pageRect.size, false, 1.0)
let context = UIGraphicsGetCurrentContext()
context?.setFillColor(UIColor.white.cgColor)
context?.fill(pageRect)
context?.translateBy(x: 0.0, y: pageRect.size.height)
context?.scaleBy(x: scalex, y: scaley)
context?.drawPDFPage(page)
let pdfImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return pdfImage
}
}
}
// 用来表示保存图片到自定义相册或者系统相册的操作结果
enum HBPhotosAlbumUtilResult {
case success, error, denied
}
extension HBPhotosAlbumHelperUtil {
// 请求获取操作系统相册权限
// 返回true说明已经得到授权
static var photoAlbumAuthorized: Bool {
return PHPhotoLibrary.authorizationStatus() == .authorized || PHPhotoLibrary.authorizationStatus() == .notDetermined
}
// 保存图片到自定义相册中
func saveImageToCustomAlbum(saveImage markImage: UIImage, customAlbumName albumName: String = "丰巢管家电子发票", completion: ((_ result: HBPhotosAlbumUtilResult) -> Void)?) {
guard HBPhotosAlbumHelperUtil.photoAlbumAuthorized else {
completion?(.denied)
return
}
var assetAlbum: PHAssetCollection?
// 如果相册名称为空,则图片默认保存到系统相册里面
if albumName.isEmpty {
let assetCollection = PHAssetCollection.fetchAssetCollections(with: .smartAlbum, subtype: .smartAlbumUserLibrary,
options: nil)
assetAlbum = assetCollection.firstObject
} else {
// 获取指定的相册是否存在
let assetList = PHAssetCollection.fetchAssetCollections(with: .album, subtype: .any, options: nil)
assetList.enumerateObjects { albumOption, _, stop in
let assetCollection = albumOption
if albumName == assetCollection.localizedTitle {
assetAlbum = assetCollection
stop.initialize(to: true)
}
}
// 自定义相册不存在就创建
if assetAlbum == nil {
PHPhotoLibrary.shared().performChanges({
PHAssetCollectionChangeRequest.creationRequestForAssetCollection(withTitle: albumName)
}) { _, _ in
self.saveImageToCustomAlbum(saveImage: markImage, customAlbumName: albumName, completion: completion)
}
}
}
// 保存图片
PHPhotoLibrary.shared().performChanges({
let result = PHAssetChangeRequest.creationRequestForAsset(from: markImage)
if !albumName.isEmpty {
if let assetPlaceHolder = result.placeholderForCreatedAsset,
let lastAssetAlbum = assetAlbum,
let albumChangeRequset = PHAssetCollectionChangeRequest(for:
lastAssetAlbum) {
albumChangeRequset.addAssets([assetPlaceHolder] as NSArray)
}
}
}) { isSuccess, _ in
guard isSuccess else {
completion?(.error)
return
}
completion?(.success)
}
}
}
extension String {
/// URL legalization
public var urlValue: URL? {
if let url = URL(string: self) {
return url
}
var set = CharacterSet()
set.formUnion(.urlHostAllowed)
set.formUnion(.urlPathAllowed)
set.formUnion(.urlQueryAllowed)
set.formUnion(.urlFragmentAllowed)
return self.addingPercentEncoding(withAllowedCharacters: set).flatMap { URL(string: $0) }
}
}
You can use the api like this:
// Use this way to achieve pdf to image
HBPhotosAlbumHelperUtil.shared.drawToImagePDFFromURL(pdfurl: "link to pdf file")
// In this way, you can save pictures to the system custom album.
HBPhotosAlbumHelperUtil.shared.saveImageToCustomAlbum(saveImage: UIImage()) { (result) in
}
Make sure that your pptURL is file url.
URL(string: "path/to/pdf") and URL(fileURLWithPath: "path/to/pdf") are different things and you must use the last one while initiating your url.
The output should start with "file:///" prefix, f.e.
file:///Users/dev/Library/Developer/CoreSimulator/Devices/4FF18699-D82F-4308-88D6-44E3C11C955A/data/Containers/Bundle/Application/8F230041-AC15-45D9-863F-5778B565B12F/myApp.app/example.pdf
I have create pdf of tableview all cells. but i have create the pdf it does not show all data of tableview & also when pdf page height set according to cells data. I dont know how to resolve this issue. I am using this code for create pdf.please help me.
let _: NSData!
do {
let priorBounds = table_view.bounds
let fittedSize = table_view.sizeThatFits(CGSizeMake(priorBounds.size.width, .infinity))
table_view.bounds = CGRectMake(0, 0, fittedSize.width, fittedSize.height)
let pdfPageBounds = CGRectMake(0, 0, 612, 800)
let pdfData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, pdfPageBounds, nil)
do {
var pageOriginY = 0
while pageOriginY < Int(fittedSize.height) {
UIGraphicsBeginPDFPageWithInfo(pdfPageBounds, nil)
CGContextSaveGState(UIGraphicsGetCurrentContext()!)
do {
CGContextTranslateCTM(UIGraphicsGetCurrentContext()!, 0, -CGFloat(pageOriginY))
table_view.layer.renderInContext(UIGraphicsGetCurrentContext()!)
}
CGContextRestoreGState(UIGraphicsGetCurrentContext()!)
pageOriginY += Int(pdfPageBounds.size.height)
}
}
UIGraphicsEndPDFContext()
table_view.bounds = priorBounds
if let documentDirectories = NSSearchPathForDirectoriesInDomains(.DocumentDirectory, .UserDomainMask, true).first {
let documentsFileName = documentDirectories + "/" + (filename as String)
debugPrint(documentsFileName)
pdfData.writeToFile(documentsFileName, atomically: true)
}
}
Try this.
func createPdfFromTableView()
{
let priorBounds: CGRect = self.tblView.bounds
let fittedSize: CGSize = self.tblView.sizeThatFits(CGSize(width: priorBounds.size.width, height: self.tblView.contentSize.height))
self.tblView.bounds = CGRect(x: 0, y: 0, width: fittedSize.width, height: fittedSize.height)
self.tblView.reloadData()
let pdfPageBounds: CGRect = CGRect(x: 0, y: 0, width: fittedSize.width, height: (fittedSize.height))
let pdfData: NSMutableData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, pdfPageBounds, nil)
UIGraphicsBeginPDFPageWithInfo(pdfPageBounds, nil)
self.tblView.layer.render(in: UIGraphicsGetCurrentContext()!)
UIGraphicsEndPDFContext()
let documentDirectories = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true).first
let documentsFileName = documentDirectories! + "/" + "pdfName"
pdfData.write(toFile: documentsFileName, atomically: true)
print(documentsFileName)
}
One addition to Rajeskumar's answer below is as follows:
If your UITableView does not occupy the entire device screen with its original bounds, you might want to insert the following line:
self.tblview.bounds = priorBounds
after the render(in:) step. Otherwise, the table will be drawn on the device in a manner in which it will cover more space that originally intended.
Above answers did not work for creating PDF after tableview is scrolled at the bottom.
While reloading tableview(reloadData()) after scrolling, correct PDF is not generated and removing, it won't generate PDF for unloaded items.
So I have added extra condition and refactored as below:
extension UITableView {
func convertToPDF() -> Data? {
let priorBounds = self.bounds
setBoundsForAllItems()
self.layoutIfNeeded()
let pdfData = createPDF()
self.bounds = priorBounds
return pdfData.copy() as? Data
}
private func getContentFrame() -> CGRect {
return CGRect(x: 0, y: 0, width: self.contentSize.width, height: self.contentSize.height)
}
private func createPDF() -> NSMutableData {
let pdfPageBounds: CGRect = getContentFrame()
let pdfData: NSMutableData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, pdfPageBounds, nil)
UIGraphicsBeginPDFPageWithInfo(pdfPageBounds, nil)
self.layer.render(in: UIGraphicsGetCurrentContext()!)
UIGraphicsEndPDFContext()
return pdfData
}
private func setBoundsForAllItems() {
if self.isEndOfTheScroll() {
self.bounds = getContentFrame()
} else {
self.bounds = getContentFrame()
self.reloadData()
}
}
private func isEndOfTheScroll() -> Bool {
let contentYoffset = contentOffset.y
let distanceFromBottom = contentSize.height - contentYoffset
return distanceFromBottom < frame.size.height
}
}
Create PDF of tableview using this way
#IBAction func onClickBtnGeneratePDF(_ sender: Any) {
if enterPDFNAME.text?.isEmpty() == true{
Utility.showToast(message: "Please Enter PDF name")
}else{
self.viewOverlaySavePDF.isHidden = true
Utility.showLoading()
let pdfFilePath = tableview.exportAsPdfFromTable(pdfName: enterPDFNAME.text!)
if pdfFilePath != nil {
self.enterPDFNAME.text = ""
Utility.showToast(message: "Your PDF file saved")
Utility.hideLoading()
}
print(pdfFilePath)
}
}
}
extension UITableView {
// Export pdf from UITableView and save pdf in drectory and return pdf file path
func exportAsPdfFromTable(pdfName:String) -> String {
/* let html = "<b>Hello <i>World!</i></b> <p>Generate PDF file from HTML in Swift</p>"
let fmt = UIMarkupTextPrintFormatter(markupText: html)
// 2. Assign print formatter to UIPrintPageRenderer
let render = UIPrintPageRenderer()
render.addPrintFormatter(fmt, startingAtPageAt: 0)
// 3. Assign paperRect and printableRect
let page = CGRect(x: 0, y: 0, width: 595.2, height: 841.8) // A4, 72 dpi
let printable = page.insetBy(dx: 0, dy: 0)
render.setValue(NSValue(cgRect: page), forKey: "paperRect")
render.setValue(NSValue(cgRect: printable), forKey: "printableRect")
// 4. Create PDF context and draw
let pdfData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, .zero, nil)
for i in 1...render.numberOfPages {
UIGraphicsBeginPDFPage();
let bounds = UIGraphicsGetPDFContextBounds()
render.drawPage(at: i - 1, in: bounds)
}
UIGraphicsEndPDFContext();
*/
let originalBounds = self.bounds
self.bounds = CGRect(x:originalBounds.origin.x, y: originalBounds.origin.y, width: self.contentSize.width, height: self.contentSize.height)
let pdfPageFrame = CGRect(x: 0, y: 0, width: self.bounds.size.width, height: self.contentSize.height)
let pdfData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, pdfPageFrame, nil)
UIGraphicsBeginPDFPageWithInfo(pdfPageFrame, nil)
guard let pdfContext = UIGraphicsGetCurrentContext() else { return "" }
self.layer.render(in: pdfContext)
UIGraphicsEndPDFContext()
self.bounds = originalBounds
// Save pdf data
return self.saveTablePdf(data: pdfData,name:pdfName)
}
// Save pdf file in document directory
func saveTablePdf(data: NSMutableData,name:String) -> String {
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let docDirectoryPath = paths[0]
let pdfPath = docDirectoryPath.appendingPathComponent("\(name).pdf")
if data.write(to: pdfPath, atomically: true) {
return pdfPath.path
} else {
return ""
}
}
}
Only one solution for exporting all tableView items: make snapshot of every part of tableview as an image -> merge them into an imageView -> render imageView to pdf.
NOTE: It's the same way with UIScrollView & UICollectionView
func exportPDF() {
// 1. estimate tableview contentsize
UIGraphicsBeginImageContextWithOptions(tableView.contentSize, false, 0)
tableView.scrollToRow(at: IndexPath(row: 0, section: 0), at: .top, animated: false)
tableView.layer.render(in: UIGraphicsGetCurrentContext()!)
// 2. calculate number of part of tableview, loop and get snapshot
let iterationCount = Int(ceil(tableView.contentSize.height / tableView.bounds.size.height))
for i in 0..<iterationCount {
tableView.setContentOffset(CGPoint(x: 0, y: Int(tableView.bounds.size.height) * i), animated: false)
tableView.layer.render(in: UIGraphicsGetCurrentContext()!)
}
// 3. merge and make UIImageView from snapshot
let image = UIGraphicsGetImageFromCurrentImageContext()
let pdfImageView = UIImageView(image: image)
//4. render imageView to PDF
let pdfData: NSMutableData = NSMutableData()
UIGraphicsBeginPDFContextToData(pdfData, pdfImageView.bounds, nil)
UIGraphicsBeginPDFPageWithInfo(pdfImageView.bounds, nil)
pdfImageView.layer.render(in: UIGraphicsGetCurrentContext()!)
UIGraphicsEndPDFContext()
let documentDirectories = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true).first
let documentsFileName = documentDirectories! + "/" + "MyPDF.pdf"
pdfData.write(toFile: documentsFileName, atomically: true)
// 5. We have pdf file at documentsFileName
}