Failing Parse background job when using beforesave with thousands of objects - ios

Question

I am using a background job to query a json with thousands of objects to initially populate my database. I have also implemented the beforesave function to prevent any duplicate entries. However, once I implemented this, it seems my background job called response.error and does not save all objects. It looks like I might be exceeding the requests/sec? I would really appreciate if someone could take a look at my code and tell me why it is not saving all entries successfully.
Here is my background job:
Parse.Cloud.job("testing", function(request, response) {
var json;
Parse.Cloud.httpRequest({
url: stringURL + pageNumber.toString(),
success: function(httpResponse) {
json = httpResponse.data;
console.log("total is: " + json["meta"].total);
console.log("object 1 is: " + json["events"][1].title);
return json;
}
//after getting the json, save all 1000
}).then(function() {
//helper function called
saveObjects(json).then(function() {
response.success("success");
},
function(error) {
response.error("nooooo");
});
});
});
function saveObjects(json) {
var promises = [];
for(var i = 0; i < 1000; i++) {
var newEvent = new Event();
promises.push(newEvent.save(new Event(json["events"][i])));
}
return Parse.Promise.when(promises);
}
Here is my beforesave code:
Parse.Cloud.beforeSave("Event", function(request, response) {
var newEvent = request.object;
var Event = Parse.Object.extend("Event");
var query = new Parse.Query("Event");
query.equalTo("title", newEvent.get("title"));
query.equalTo("datetime_utc", newEvent.get("datetime_utc"));
query.equalTo("url", newEvent.get("url"));
query.first({
success: function(temp) {
response.error({errorCode:123,errorMsg:"Event already exist!"});
},
error: function(error) {
response.success();
}
});
});
Thanks I really appreciate any help... I've been stuck for a while.

Answer 1

If it's a request rate issue, then you could probably use something like node-function-rate-limit but it's fairly simple to write your own rate limiting batcher. See doInBatches() below.
Also, when using promise-returning methods that also offer a "success:..." callback, it's better not to mix the two styles. It may behave as expected but you are denied the opportunity to pass results from the "success:..." callback to the rest of the promise chain. As you can see below, the "success:..." code has simply been shuffled into the .then() callback.
Parse.Cloud.job("testing", function(request, response) {
Parse.Cloud.httpRequest({
url: stringURL + pageNumber.toString()
}).then(function(httpResponse) {
var json = httpResponse.data;
// console.log("total is: " + json.meta.total);
// console.log("object 1 is: " + json.events[1].title);
/* helper function called */
doInBatches(json.events, 30, 1000, function(evt, i) {
var newEvent = new Event();
return newEvent.save(new Event(evt));
}).then(function() {
response.success('success');
}, function(error) {
response.error('nooooo');
});
});
});
// Async batcher.
function doInBatches(arr, batchSize, delay, fn) {
function delayAsync() {
var p = new Parse.Promise();
setTimeout(p.resolve, delay);
return p;
}
function saveBatch(start) {
if(start < arr.length) {
return Parse.Promise.when(arr.slice(start, start+batchSize).map(fn))
.then(delayAsync) // delay between batches
.then(function() {
return saveBatch(start + batchSize);
});
} else {
return Parse.Promise.as();
}
}
return saveBatch(0);
}
I can't see how or why the beforesave code might affect things.