Improve efficiency on downloading multiple XML files and parsing them - ios

I'm downloading five XML files from the web server using Alamofire and parsing them using SWXMLHash. The last file depends on the first four files in a way that some of its entries refer to the entries contain in the first four. I use cascading style structure for download code to ensure all four files are downloaded before downloading of the last file can start.
let destination: DownloadRequest.DownloadFileDestination = { _, _ in
let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory,
.userDomainMask, true)[0]
let documentsURL = URL(fileURLWithPath: documentsPath, isDirectory: true)
let fileURL = documentsURL.appendingPathComponent("image.png")
return (fileURL, [.removePreviousFile, .createIntermediateDirectories])
}
// Download Category files
Alamofire.download(self.ottawaOpenDataCategoriesURL, to:
destination)
.downloadProgress { progress in
//print("Download Progress for Category: \(progress.fractionCompleted)")
if let pv = self.progressViewController {
pv.updateProgress(progress.fractionCompleted * 100.0) //Alamofire returns 1 for complete process
}
}
.responseData { response in
// Check 304 response to see if new file is available
if response.response?.statusCode == self.HTTP_STATUS_FILE_NOT_CHANGE {
return
}
if let data = response.result.value {
let xml = self.initXMLParser(data: data)
self.storeCategoryXMLStream(xml)
}
// After complete downloading or get error, try download Options
Alamofire.download(self.ottawaOpenDataOptionsURL, to:
destination)
.downloadProgress { progress in
if let pv = self.progressViewController {
pv.updateProgress(progress.fractionCompleted * 100.0) //Alamofire returns 1 for complete process
}
}
.responseData { response in
// Check 304 response to see if new file is available
if response.response?.statusCode == self.HTTP_STATUS_FILE_NOT_CHANGE {
return
}
if let data = response.result.value {
let xml = self.initXMLParser(data: data)
self.storeEventOptionsXMLStream(xml)
}
// After complete downloading or get error, try download Locations
Alamofire.download(self.ottawaOpenDataLocationsURL, to:
destination)
.downloadProgress { progress in
if let pv = self.progressViewController {
pv.updateProgress(progress.fractionCompleted * 100.0) //Alamofire returns 1 for complete process
}
}
.responseData { response in
// Check 304 response to see if new file is available
if response.response?.statusCode == self.HTTP_STATUS_FILE_NOT_CHANGE {
return
}
if let data = response.result.value {
let xml = self.initXMLParser(data: data)
self.storeVenuesXMLStream(xml)
}
// After complete downloading or get error, try download CitrSectors
Alamofire.download(self.ottawaOpenDataCitySectorsURL, to:
destination)
.downloadProgress { progress in
if let pv = self.progressViewController {
pv.updateProgress(progress.fractionCompleted * 100.0) //Alamofire returns 1 for complete process
}
}
.responseData { response in
// Check 304 response to see if new file is available
if response.response?.statusCode == self.HTTP_STATUS_FILE_NOT_CHANGE {
return
}
if let data = response.result.value {
let xml = self.initXMLParser(data: data)
self.storeCitySectorsXMLStream(xml)
}
// After complete downloading or get error, try download events
Alamofire.download(self.ottawaOpenDataEventsURL, to:
destination)
.downloadProgress { progress in
if let pv = self.progressViewController {
pv.updateProgress(progress.fractionCompleted * 100.0) //Alamofire returns 1 for complete process
}
}
.responseData { response in
// Check 304 response to see if new file is available
if response.response?.statusCode == self.HTTP_STATUS_FILE_NOT_CHANGE {
return
}
if let data = response.result.value {
let xml = self.initXMLParser(data: data)
self.storeEventsXMLStream(xml)
}
}
}
}
}
}
As you can see, each download segment waits for the previous one to complete. Also, the progress bar is updated during download process. Most XML files are from small to decent size (80 Lines - 10K lines) and the last one is the largest and contains about 200K lines.
I'm not sure if it's because of cascading style structure, but it takes about 10 seconds to download and parse the first four files. Is it possible to make it faster? I just want to know if I can improve efficiency. Here's a screenshot of cpu usage and mem usage.
P.S I'm running this app on simulator.

Since the first 4 downloads aren't interdependent, you could download them simultaneously, and only when all 4 of those are received start the 5th download. Use a dispatch group to synchronize the execution of the 5th download.
In short, the whole operation looks like:
let group = DispatchGroup()
group.enter()
Alamofire.download(first url).responseData {
process first url
group.leave()
}
group.enter()
Alamofire.download(second url).responseData {
process second url
group.leave()
}
repeat for third and fourth url
group.notify(queue:Dispatch.main) {
Alamofire.download(fifth url).responseData {
process 5th url
}
}

Related

Almofire multiple images Download and save them locally

I have more than 500 image links I want to download those images and store locally in app document directory when app starts. I am using Almofire for download but I am getting error like
"URLSession task was cancelled" and Request timeOut
func downloadAllImages(images:[String: String], retryCount: Int = 0,completion: #escaping((Bool)->Void)){
var success: Bool = true
var failedImages = [String: String]()
for (localName, severPath) in images {
self.dispatchGroup.enter()
let destination: DownloadRequest.Destination = { _, _ in
let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
let fileURL = documentsURL.appendingPathComponent(localName)
return (fileURL, [.removePreviousFile, .createIntermediateDirectories])
}
let path = severPath.replacingOccurrences(of: "\\", with: "/").addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? ""
//AF.sessionConfiguration.httpShouldSetCookies = false
AF.download(path, to: destination).response { response in
switch response.result {
case .success(_):
break
case .failure(let error):
if response.response?.statusCode != 404 {
success = false
failedImages[localName] = path
print("Image Download Error = \(error.localizedDescription)")
}
break
}
debugPrint(response)
self.dispatchGroup.leave()
}
}
dispatchGroup.notify(queue: .main) {
//retry If some Images failed to download
if failedImages.isEmpty || retryCount >= self.maximumRetryCount {
completion(success)
}else {
self.downloadAllImages(images: failedImages, retryCount: retryCount + 1) { (success) in
completion(success)
}
}
}
}
images dictionary contains
localName as key
serverPath as value
AFImageDownloaders have limit of active downloads, and I believe changing maximumActiveDownloads or something similar in your API will fix that. The new downloads just cancel the previous ones. But it's better to download them in chunks.
For example this one is for ImageDownloader
public init(session: Session,
downloadPrioritization: DownloadPrioritization = .fifo,
maximumActiveDownloads: Int = 4,
imageCache: ImageRequestCache? = AutoPurgingImageCache()) {
precondition(!session.startRequestsImmediately, "Session must set `startRequestsImmediately` to `false`.")
self.session = session
self.downloadPrioritization = downloadPrioritization
self.maximumActiveDownloads = maximumActiveDownloads
self.imageCache = imageCache
}
UPD:
The limit is not on AF, but URLSession's. And one AF downloader uses one URLSession. You have to pass custom URLSessionConfigurations to handle more active downloads HTTPMaximumConnectionsPerHost. And pass it AF Session class

iOS, Swift : Download multiple file serially and showing single progress bar for all file as a one progress

I am downloading the zip file using the Alamofire in my ios(swift4) application. I am able to download the file serially using Alamofire.
But I also want to show the one progress bar for the all the downloaded files. Means If I have 4 zip files and when all the file is downloaded then the progress should be 100%.
Till I have tried below code, which gives a progress value for each of the url and progress is shown as one file downloaded showing progress as 100% then again it start from 0 for second url and when second url is downloaded then progress is shown 100% complete.
Please guide me for this. I want to get progress value as 100% when all the files are downloaded using Alamofire.
Can it be possible with Alamofire?
CODE:
func serialZipFileDownload(downloadPath: String){
let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first!
let name = self.offlineDownloadFileName?[urlCount]
let currentVideoURL = documentsURL.appendingPathComponent(name ?? "Default.zip")
let str = downloadPath
let urlString = str.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)
let url = URL(string: urlString ?? "")
if super.isConnectedToNetwork() == true {
let manager = Alamofire.SessionManager.default
let headers = ["Accept-Encoding" : ""]
manager.request(url ?? "", method: .get, parameters: nil, encoding: JSONEncoding.default, headers: headers).downloadProgress { (progress) in
print(progress.fractionCompleted)
DispatchQueue.main.async{
self.progressDownload.setProgress((Float(progress.fractionCompleted)), animated: true)
let per = round((Float(progress.fractionCompleted)) * 100)
self.lblDownloadPercent.text = "\(Int(per))%"
}
}.responseData { (response) in
switch (response.result){
case .success(_) :
print(response)
print(response.result.value!)
print(response.result.description)
if let data = response.result.value {
do {
try data.write(to: currentVideoURL)
self.showToast(message: "File downloaded successfully")
}
catch {
print("Something went wrong!")
}
}
case .failure(let error) :
print(response)
if error._code == NSURLErrorNetworkConnectionLost {
DispatchQueue.main.async {
super.showPopup(title: msgStruct.networkTitle, message: msgStruct.noInternet)
}
}
else if error._code == NSURLErrorTimedOut {
DispatchQueue.main.async {
super.showPopup(title: msgStruct.networkTitle, message: msgStruct.noInternet)
}
}
else if error._code == NSURLErrorDownloadDecodingFailedMidStream {
print("error",error.localizedDescription)
}
break
}
}
}
else{
super.showPopup(title: msgStruct.networkTitle, message: msgStruct.noInternet)
}
You can do it like this:
Maintain a global variable - tatalPercentage some other swift class.
static let tatalPercentage = 0
tatalPercentage = tatalPercentage + Int(per/4)
self.lblDownloadPercent.text = "\(tatalPercentage) %"

Swift: downloading data from url causes semaphore_wait_trap freeze

In my application, button tapping downloads data from an Internet site. The site is a list of links containing binary data. Sometimes, the first link may not contain the proper data. In this case, the application takes the next link in the array and gets data from there. The links are correct.
The problem I have is that frequently (not always though) the application freezes for seconds when I tap on the button. After 5-30 seconds, it unfreezes and downloading implements normally. I understand, something is blocking the main thread. When stopping the process in xCode, I get this (semaphore_wait_trap noted):
This is how I do it:
// Button Action
#IBAction func downloadWindNoaa(_ sender: UIButton)
{
// Starts activity indicator
startActivityIndicator()
// Starts downloading and processing data
// Either use this
DispatchQueue.global(qos: .default).async
{
DispatchQueue.main.async
{
self.downloadWindsAloftData()
}
}
// Or this - no difference.
//downloadWindsAloftData()
}
}
func downloadWindsAloftData()
{
// Creates a list of website addresses to request data: CHECKED.
self.listOfLinks = makeGribWebAddress()
// Extract and save the data
saveGribFile()
}
// This downloads the data and saves it in a required format. I suspect, this is the culprit
func saveGribFile()
{
// Check if the links have been created
if (!self.listOfLinks.isEmpty)
{
/// Instance of OperationQueue
queue = OperationQueue()
// Convert array of Strings to array of URL links
let urls = self.listOfLinks.map { URL(string: $0)! }
guard self.urlIndex != urls.count else
{
NSLog("report failure")
return
}
// Current link
let url = urls[self.urlIndex]
// Increment the url index
self.urlIndex += 1
// Add operation to the queue
queue.addOperation { () -> Void in
// Variables for Request, Queue, and Error
let request = URLRequest(url: url)
let session = URLSession.shared
// Array of bytes that will hold the data
var dataReceived = [UInt8]()
// Read data
let task = session.dataTask(with: request) {(data, response, error) -> Void in
if error != nil
{
print("Request transport error")
}
else
{
let response = response as! HTTPURLResponse
let data = data!
if response.statusCode == 200
{
//Converting data to String
dataReceived = [UInt8](data)
}
else
{
print("Request server-side error")
}
}
// Main thread
OperationQueue.main.addOperation(
{
// If downloaded data is less than 2 KB in size, repeat the operation
if dataReceived.count <= 2000
{
self.saveGribFile()
}
else
{
self.setWindsAloftDataFromGrib(gribData: dataReceived)
// Reset the URL Index back to 0
self.urlIndex = 0
}
}
)
}
task.resume()
}
}
}
// Processing data further
func setWindsAloftDataFromGrib(gribData: [UInt8])
{
// Stops spinning activity indicator
stopActivityIndicator()
// Other code to process data...
}
// Makes Web Address
let GRIB_URL = "http://xxxxxxxxxx"
func makeGribWebAddress() -> [String]
{
var finalResult = [String]()
// Main address site
let address1 = "http://xxxxxxxx"
// Address part with type of data
let address2 = "file=gfs.t";
let address4 = "z.pgrb2.1p00.anl&lev_250_mb=on&lev_450_mb=on&lev_700_mb=on&var_TMP=on&var_UGRD=on&var_VGRD=on"
let leftlon = "0"
let rightlon = "359"
let toplat = "90"
let bottomlat = "-90"
// Address part with coordinates
let address5 = "&leftlon="+leftlon+"&rightlon="+rightlon+"&toplat="+toplat+"&bottomlat="+bottomlat
// Vector that includes all Grib files available for download
let listOfFiles = readWebToString()
if (!listOfFiles.isEmpty)
{
for i in 0..<listOfFiles.count
{
// Part of the link that includes the file
let address6 = "&dir=%2F"+listOfFiles[i]
// Extract time: last 2 characters
let address3 = listOfFiles[i].substring(from:listOfFiles[i].index(listOfFiles[i].endIndex, offsetBy: -2))
// Make the link
let addressFull = (address1 + address2 + address3 + address4 + address5 + address6).trimmingCharacters(in: .whitespacesAndNewlines)
finalResult.append(addressFull)
}
}
return finalResult;
}
func readWebToString() -> [String]
{
// Final array to return
var finalResult = [String]()
guard let dataURL = NSURL(string: self.GRIB_URL)
else
{
print("IGAGribReader error: No URL identified")
return []
}
do
{
// Get contents of the page
let contents = try String(contentsOf: dataURL as URL)
// Regular expression
let expression : String = ">gfs\\.\\d+<"
let range = NSRange(location: 0, length: contents.characters.count)
do
{
// Match the URL content with regex expression
let regex = try NSRegularExpression(pattern: expression, options: NSRegularExpression.Options.caseInsensitive)
let contentsNS = contents as NSString
let matches = regex.matches(in: contents, options: [], range: range)
for match in matches
{
for i in 0..<match.numberOfRanges
{
let resultingNS = contentsNS.substring(with: (match.rangeAt(i))) as String
finalResult.append(resultingNS)
}
}
// Remove "<" and ">" from the strings
if (!finalResult.isEmpty)
{
for i in 0..<finalResult.count
{
finalResult[i].remove(at: finalResult[i].startIndex)
finalResult[i].remove(at: finalResult[i].index(before: finalResult[i].endIndex))
}
}
}
catch
{
print("IGAGribReader error: No regex match")
}
}
catch
{
print("IGAGribReader error: URL content is not read")
}
return finalResult;
}
I have been trying to fix it for the past several weeks but in vain. Any help would be much appreciated!
let contents = try String(contentsOf: dataURL as URL)
You are calling String(contentsOf: url) on the main thread (main queue). This downloads the content of the URL into a string synchronously The main thread is used to drive the UI, running synchronous network code is going to freeze the UI. This is a big no-no.
You should never call readWebToString() in the main queue. Doing DispatchQueue.main.async { self.downloadWindsAloftData() } exactly put the block in the main queue which we should avoid. (async just means "execute this later", it is still executed on Dispatch.main.)
You should just run downloadWindsAloftData in the global queue instead of main queue
DispatchQueue.global(qos: .default).async {
self.downloadWindsAloftData()
}
Only run DispatchQueue.main.async when you want to update the UI.
Your stack trace is telling you that it's stopping at String(contentsOf:), called by readWebToString, called by makeGribWebAddress.
The problem is that String(contentsOf:) performs a synchronous network request. If that request takes any time, it will block that thread. And if you call this from the main thread, your app may freeze.
Theoretically, you could just dispatch that process to a background queue, but that merely hides the deeper problem, that you are doing a network request with an API that is synchronous, non-cancellable, and offers no meaningful error reporting.
You really should doing asynchronous requests with URLSession, like you have elsewhere. Avoid using String(contentsOf:) with remote URL.

Alamofire Pause Request not contine other Requests wating in Queue

I'm using Alamofire library to make a download manager.
I'm configuring the Alamofire Manager with
HTTPMaximumConnectionsPerHost = 4.
And I'm start Downloading 5 items, then 4 items start downloading and
the the 5th item is waiting in queue.
When I Pause one of the first 4th item, I'm expecting the 5th item to
start downloading. This is not happening and this is my issue 💃 👯
This is my Code:
func startDownload(thisItem url: String, folderName: String) {
let startDownloadRequest = AlamofireManager!.download(.GET, url,
destination: { temporaryURL, response in
let fileManager = NSFileManager.defaultManager()
let directoryURL = fileManager.URLsForDirectory(.DocumentDirectory, inDomains: .UserDomainMask)[0]
self.createFolderWithName(folderName)
let pathComponent = "\(folderName)/\(response.suggestedFilename!)"
return directoryURL.URLByAppendingPathComponent(pathComponent)
})
.progress { bytesRead, totalBytesRead, totalBytesExpectedToRead in
dispatch_async(dispatch_get_main_queue())
{
let progress = String(Float(totalBytesRead) / Float(totalBytesExpectedToRead))
}
}
.response { _, _, _, error in
if let error = error {
print("Failed with error: \(error)")
} else {
print("Downloaded file successfully")
}
}
}
And this is the method for Pause Request
func pauseDownload(thisItem item: String) {
var request : Request
for req in DownloadCenter.defaultCenter.currentDownloadsList {
if req.request?.URLString == item {
request = req
request.suspend()
break
}
}
}
This is my case:
Start Downloading 5 items, 4 are Downloading and the 5th item is waiting..
After Pausing...

Swift (iOS), waiting for all images to finish downloading before returning

I am writing a Swift iOS app (my first, so please bear with me) where I use Swifter HTTP server to process various requests. One such request is an HTTP POST with a JSON array specifying images to download from the web (and do some other stuff, not pertinent to the issue at hand).
I use Alamofire to download the images (this works fine), but I am looking for good (preferably simple) way to wait for all the images to finish downloading before returning a response to the POST request above (since the response has to contain JSON indicating the result, including any failed downloads).
What is a good way to accomplish this (preferably w/o blocking the main thread)?
Here are some snippets to illustrate:
public func webServer(publicDir: String?) -> HttpServer {
let server = HttpServer()
server.POST["/images/update"] = { r in
let images = ...(from JSON array in body)
let updateResult = ImageUtil.updateImages(images)
let resultJson: String = Mapper().toJSONString(updateResult, prettyPrint: true)!
if updateResult.success {
return .OK(.Text(resultJson))
}
return HttpResponse.RAW(500, "Error", nil, { $0.write([UInt8](updateResult.errorMessage.utf8)) })
}
}
static func updateImages(images: [ImageInfo]) -> UpdateResult {
let updateResult = UpdateResult()
for image in images {
Alamofire.download(.GET, serverFile.imageUrl) { temporaryURL, response in return destinationPath }
.validate()
.response{_, _, _, error in
if let error = error {
Log.error?.message("Error downloading file \(image.imageUrl) to \(image.fileName): \(error)")
} else {
updateResult.filesDownloaded++
Log.info?.message("Downloaded file \(image.imageUrl) to \(image.fileName)")
}}
}
return updateResult // It obviously returns before any images finish downloading. I need to wait until all images have downloaded before I can return an accurate result.
}
Update 1/23/2016, using dispatcher per bbum
This is an attempt to use the dispatcher mechanism, but the call to updateImages still return right away (even when using dispatch_sync).
How can I await the completion of all downloads before returning my HTTP response to the caller?
public func webServer(publicDir: String?) -> HttpServer {
let server = HttpServer()
server.POST["/images/update"] = { r in
let imageDownloader = ImageDownloader()
imageDownloader.updateimageFiles(adFilesOnServer)
let resultJson: String = Mapper().toJSONString(imageDownloader.updateResult, prettyPrint: true)!
if imageDownloader.updateResult.success {
return .OK(.Text(resultJson))
}
return HttpResponse.RAW(500, "Error", nil, { $0.write([UInt8](imageDownloader.updateResult.errorMessage.utf8)) })
}
}
class ImageDownloader {
var updateResult = AdUpdateResult()
private var imageFilesOnServer = [ImageFile]()
private let fileManager = NSFileManager.defaultManager()
private let imageDirectoryURL = NSURL(fileURLWithPath: Settings.imageDirectory, isDirectory: true)
private let semaphore = dispatch_semaphore_create(4)
private let downloadQueue = dispatch_queue_create("com.acme.downloader", DISPATCH_QUEUE_SERIAL)
func updateimageFiles(imageFilesOnServer: [ImageFile]) {
self.imageFilesOnServer = imageFilesOnServer
dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER)
for serverFile in imageFilesOnServer {
downloadImageFileFromServer(serverFile)
}
dispatch_sync(downloadQueue) {
dispatch_sync(dispatch_get_main_queue()) {
print("done") // It gets here before images have downloaded.
}
}
}
private func downloadImageFileFromServer(serverFile: ImageFile) {
let destinationPath = imageDirectoryURL.URLByAppendingPathComponent(serverFile.fileName)
Alamofire.download(.GET, serverFile.imageUrl) { temporaryURL, response in return destinationPath }
.validate()
.response { _, _, _, error in
if let error = error {
Log.error?.message("Error downloading file \(serverFile.imageUrl) to \(serverFile.fileName): \(error)")
} else {
self.updateResult.filesDownloaded++
Log.info?.message("Downloaded file \(serverFile.imageUrl) to \(serverFile.fileName)")
}
dispatch_semaphore_signal(self.semaphore)
}
}
}
First, you really don't want to be firing off a request-per-image without some kind of a throttle. Semaphores work well for that sort of thing.
Secondly, you need to basically count the number of operations outstanding and then fire a completion handler when they are all done. Or, if new operations can be started at any time, you'll probably want to group operations.
So, pseudo code:
sema = dispatch_semaphore_create(4) // 4 being # of concurrent operations allowed
serialQ = dispatch_queue_create(.., SERIAL)
dispatch_async(serialQ) {
dispatch_semaphore_wait(sema, FOREVER) // will block if there are 4 in flight already
for image in images {
downloader.downloadAsync(image, ...) { // completion
dispatch_semaphore_signal(sema) // signal that we are done with one
... handle downloaded image or error ...
... add downloaded images to downloadedImages ...
}
}
}
dispatch_async(serialQ) {
// since serialQ is serial, this will be executed after the downloads are done
dispatch_async(main_queue()) {
yo_main_queue_here_be_yer_images(... downloadedImages ...)
}
}

Resources