Loading data taking too much time CoreData - ios

Question

I am facing problems when i tries to save 40,000 records into CoreData Entity.
I am getting 40,000 records by consuming the webservice using AFNetworking, the response is in JSON. Than i divide the data into 4 , 10000 record chunks and then assign these 4 chunks to separate NSOperation objects (i have created subclass of NSOperation) and add these NSOperation Objects to NSOperationQueue.
The problem is that this way it is taking too much time to save the data into CoreData. And i want to find a solution where i can load the data very quickly.
This is the code in which i am creating NSOperation objects and adding them to NSOperationQueue.
- (void)casesResponseReceived:(NSArray*)array
{
id responseObject = [array objectAtIndex:0];
NSManagedObjectContext *moc = [array objectAtIndex:1];
NSString *responseString = [[NSString alloc] initWithData:responseObject encoding:NSUTF8StringEncoding];
NSArray *response = [responseString JSONValue];
NSString *responseStr = [response JSONRepresentation];
NSRange range = [responseStr rangeOfString:#"["];
int index = 0;
int objectsCount = 5000;
if (range.location == 0) {
NSInteger count = objectsCount;
totalOperationsCount = 0;
completedOperationsCount = 0;
self.myQueue = [[NSOperationQueue alloc] init];
while (count == objectsCount) {
if ((index+count) > [response count]) {
count = [response count] - index;
}
NSArray *subArray = [response subarrayWithRange:NSMakeRange(index, count)];
index += objectsCount;
CaseParseOperation *operation = [[CaseParseOperation alloc] initWithData:subArray MOC:moc];
operation.delegate = self;
totalOperationsCount++;
[self.myQueue addOperation:operation];
}
/*
if (self.delegate && [self.delegate respondsToSelector:#selector(serviceHelperDidCasesReceivedSuccessful:)]) {
[self.delegate serviceHelperDidCasesReceivedSuccessful:self];
}*/
}
else {
if (self.delegate && [self.delegate respondsToSelector:#selector(serviceHelperDidCasesReceivedFailed:)]) {
[self.delegate serviceHelperDidCasesReceivedFailed:self];
}
}}
CaseOperation.h
#class CaseParseOperation;
#protocol CaseParseOperationProtocol <NSObject>
-(void)caseParseOperationDidOperationComplete: (CaseParseOperation*)caseParseOperation;
#end
#interface CaseParseOperation : NSOperation
#property (nonatomic, weak) id<CaseParseOperationProtocol> delegate;
-(id)initWithData:(NSArray*)parseData MOC:(NSManagedObjectContext*)moc;
#end
CaseOperation.m
#interface CaseParseOperation()
#property (nonatomic, copy) NSArray *casesData;
#property (nonatomic, strong) NSManagedObjectContext *mainMOC;
#property (nonatomic, strong) NSManagedObjectContext *localMOC;
#end
#implementation CaseParseOperation
- (id)initWithData:(NSArray*)parseData MOC:(NSManagedObjectContext*)moc
{
self = [super init];
if (self) {
self.casesData = [parseData copy];
self.mainMOC = moc;
}
return self;
}
- (void)main
{
#autoreleasepool {
self.localMOC = [[NSManagedObjectContext alloc] init];
self.localMOC.persistentStoreCoordinator = self.mainMOC.persistentStoreCoordinator;
[[NSNotificationCenter defaultCenter] addObserver: self
selector: #selector(mergeChanges:)
name: NSManagedObjectContextDidSaveNotification
object: self.localMOC];
[self parseData];
}
}
-(void) mergeChanges: (NSNotification*) saveNotification {
dispatch_async(dispatch_get_main_queue(), ^{
[self.mainMOC mergeChangesFromContextDidSaveNotification:saveNotification];
});
if (self.delegate && [self.delegate respondsToSelector:#selector(caseParseOperationDidOperationComplete:)]) {
[self.delegate caseParseOperationDidOperationComplete:self];
}
}
- (void)parseData
{
NSFetchRequest *fetchRequest = [[NSFetchRequest alloc] init];
NSEntityDescription *ent = [NSEntityDescription entityForName:#"Case" inManagedObjectContext:self.localMOC];
fetchRequest.entity = ent;
NSString *predicateString = [NSString stringWithFormat:#"caseNumber == $caseNumber"];
NSPredicate *predicate = [NSPredicate predicateWithFormat:predicateString];
//NSMutableArray *insertedObjects = [[NSMutableArray alloc] init];
for (NSMutableDictionary *dic in self.casesData) {
if (self.isCancelled) {
break;
}
NSString *desc = [dic valueForKey:#"description"];
BOOL enabled = [[dic valueForKey:#"enabled"] boolValue];
NSString *billToCustomerNo = [dic valueForKey:#"billToCustomerNo"];
NSString *caseNo = [dic valueForKey:#"caseNo"];
NSString *billToName = [dic valueForKey:#"billToName"];
NSString *personResponsible = [dic valueForKey:#"personResponsible"];
NSDictionary *variables = #{ #"caseNumber" : caseNo };
fetchRequest.predicate = [predicate predicateWithSubstitutionVariables:variables];
NSArray *matchedObj = [self.localMOC executeFetchRequest:fetchRequest error:nil];
if ([matchedObj count] > 0) {
Case *caseObj = [matchedObj objectAtIndex:0];
caseObj.isEnabled = [NSNumber numberWithBool:enabled];
caseObj.caseDescription = desc;
caseObj.customerNumber = billToCustomerNo;
caseObj.customerName = billToName;
caseObj.personResponsible = personResponsible;
}
else {
/*
Case *caseObj = [[Case alloc] initWithEntity:[NSEntityDescription entityForName:#"Case"
inManagedObjectContext:self.localMOC] insertIntoManagedObjectContext:nil];
caseObj.caseNumber = caseNo;
caseObj.customerName = billToName;
caseObj.customerAddress = #"";
caseObj.customerPhone = #"";
caseObj.caseDescription = desc;
caseObj.customerNumber = billToCustomerNo;
caseObj.isEnabled = [NSNumber numberWithBool:enabled];
caseObj.personResponsible = personResponsible;
[insertedObjects addObject:caseObj];
*/
[Case createObjectWithCaseNumber:caseNo customerName:billToName customerAddress:#"" customerPhone:#"" caseDescription:desc customerNumber:billToCustomerNo isEnabled:enabled personResponsible:personResponsible MOC:self.localMOC];
}
}
/*
if ([insertedObjects count] > 0) {
NSError *error = nil;
BOOL isInserted = [self.localMOC obtainPermanentIDsForObjects:insertedObjects error:&error];
if (error || !isInserted) {
NSLog(#"Error occured");
}
}
*/
if ([self.localMOC hasChanges]) {
[self.localMOC save:nil];
}
}
#end

Answer 1

The first thing to do is run Instruments and find the bottlenecks, as #jrturton recommends.
But there's one huge glaring bottleneck that's apparent from reading the code. To avoid duplicates, you're doing a fetch-- for every incoming instance. With 40k records you'll have to do 40k fetches during the import process, and that's going to be slow no matter what.
You can improve that by processing the data in batches:
Get a bunch of caseNumber values into an array
Do a fetch with a predicate of caseNumber IN %#, with the array as the argument.
Use that array to check for duplicates.
You'll need to experiment a little to see how many "a bunch" is in step 1. Higher numbers mean fewer fetches, which is good for speed. But higher numbers also mean more memory use.
For a more detailed discussion, see Apple's Efficiently Importing Data guide, especially the section named "Implementing Find-or-Create Efficiently".

Answer 2

Thanks guys valuable suggestions. But i have solved that issue by just altering some technique in the parseData function.
-(void)parseData
{
NSString *predicateString = [NSString stringWithFormat:#"caseNumber == $caseNumber"];
NSPredicate *predicate = [NSPredicate predicateWithFormat:predicateString];
NSArray *allCases = [Case getAllCaseObjectsWithMOC:self.localMOC];
for (NSMutableDictionary *dic in self.casesData) {
if (self.isCancelled) {
break;
}
NSString *caseNo = [dic valueForKey:#"caseNo"];
NSDictionary *variables = #{ #"caseNumber" : caseNo };
predicate = [predicate predicateWithSubstitutionVariables:variables];
NSArray *matchedObj = [allCases filteredArrayUsingPredicate:predicate];
if ([matchedObj count] == 0) {
NSString *desc = [dic valueForKey:#"description"];
BOOL enabled = [[dic valueForKey:#"enabled"] boolValue];
NSString *billToCustomerNo = [dic valueForKey:#"billToCustomerNo"];
NSString *billToName = [dic valueForKey:#"billToName"];
NSString *personResponsible = [dic valueForKey:#"personResponsible"];
[Case createObjectWithCaseNumber:caseNo customerName:billToName customerAddress:#"" customerPhone:#"" caseDescription:desc customerNumber:billToCustomerNo isEnabled:enabled personResponsible:personResponsible MOC:self.localMOC];
}
}
if ([self.localMOC hasChanges]) {
[self.localMOC save:nil];
}
}