We are trying web crawl and get contents from multiple pages. I am taking the advantage of async API with Promise ALL which can execute requests in parallel.
Is there a limitation on the number of contexts which can be opened parallel?
const fs = require('fs');
let browser;
const batch_size = 4; // control the number of async parallel calls
(async () => { // main function
let urls = [];
urls = fs.readFileSync('./resources/input_selenium_urls.csv').toString().split("\n");
browser = await chromium.launch();
let context_size = 0;
let processUrls = [];
let total_length = 0;
for (let i=0;i<urls.length;i++,total_length++) {
if ((context_size==batch_size)||(i==urls.length-1)){
await Promise.all(processUrls.map(x => getHTMLPageSource(x)));
context_size = 0;
processUrls = [];
} else {
processUrls.push(urls[i]);
context_size++;
}
}
await browser.close();
})();
async function getHTMLPageSource(url) {
const context = await browser.newContext();
const page = await context.newPage();
let response = {}
try {
await page.goto(url, { waitUntil: 'networkidle' });
response = {
url : url,
content: await page.title(),
error : null
}
console.log(response);
}
catch {
response = {
error : "Timeout error"
}
}
context.close;
return response;
}
Browser contexts are cheap to create, but it's not clear whether there is a hard-coded limit on them from the docs perhaps the limit might depend on the browser you chose and your OS resources. I think you might only be able to find out by creating a lot of contexts.
Related
Is there some one who can really help here?
I can't get anything from sso.getGraphData in the office addin generated from Yeoman generator.
Here is my code in ssoauthhelper.js:
export async function getGraphData() {
try {
let bootstrapToken = await OfficeRuntime.auth.getAccessToken({ allowSignInPrompt: true });
let exchangeResponse = await sso.getGraphToken(bootstrapToken);
if (exchangeResponse.claims) {
// Microsoft Graph requires an additional form of authentication. Have the Office host
// get a new token using the Claims string, which tells AAD to prompt the user for all
// required forms of authentication.
let mfaBootstrapToken = await OfficeRuntime.auth.getAccessToken({ authChallenge: exchangeResponse.claims });
exchangeResponse = sso.getGraphToken(mfaBootstrapToken);
}
if (exchangeResponse.error) {
// AAD errors are returned to the client with HTTP code 200, so they do not trigger
// the catch block below.
documentHelper.writeDataToOfficeDocument("response");
handleAADErrors(exchangeResponse);
} else {
/*const response = await sso.makeGraphApiCall(exchangeResponse.access_token);
documentHelper.writeDataToOfficeDocument(response);*/
const _EndPoint = "/me/messages";
const _UrlParams = "?$select=receivedDateTime,subject,isRead&$orderby=receivedDateTime&$top=10";
const response = await sso.getGraphData(exchangeResponse.access_token, _EndPoint, _UrlParams);
documentHelper.writeDataToOfficeDocument(response);
sso.showMessage("Your data has been added to the document.");
}
} catch (exception) {
if (exception.code) {
if (sso.handleClientSideErrors(exception)) {
fallbackAuthHelper.dialogFallback();
}
} else {
sso.showMessage("EXCEPTION: " + JSON.stringify(exception));
}
}
}
the code in documentHelper.js:
function writeDataToExcel(result) {
/*return Excel.run(function (context) {
const sheet = context.workbook.worksheets.getActiveWorksheet();
let data = [];
let userProfileInfo = filterUserProfileInfo(result);
for (let i = 0; i < userProfileInfo.length; i++) {
if (userProfileInfo[i] !== null) {
let innerArray = [];
innerArray.push(userProfileInfo[i]);
data.push(innerArray);
}
}
const rangeAddress = `B5:B${5 + (data.length - 1)}`;
const range = sheet.getRange(rangeAddress);
range.values = data;
range.format.autofitColumns();
return context.sync();
});*/
return Excel.run(function (context) {
const sheet = context.workbook.worksheets.getActiveWorksheet();
const rangeHeadings = sheet.getRange("A1:D1");
rangeHeadings.valueTypes = [["ReceivedDateTime", "subject", "Read?", "ID"]];
const _Contents = result.value;
for (let i = 0; i < _Contents.length; i++) {
if (_Contents !== null) {
let _TempArr = [];
_TempArr.push(_Contents[i].receivedDateTime);
_TempArr.push(_Contents[i].subject);
_TempArr.push(_Contents[i].isRead);
_TempArr.push(_Contents[i].id);
let _Data = [];
_Data.push(_TempArr);
const _rangeaddress = `A${2 + i}:D${2 + i}`;
const _rangedata = sheet.getRange(_rangeaddress);
_rangedata.values = _Data;
}
}
rangeHeadings.format.autofitColumns();
return context.sync();
});
}
When i run the add-in, it does not proceed after GET/ auth? as shown in below image.
Here are my permission in the Azure APP Registration:
I can't understand what is really happening. When i click on the button in the sideloaded task pane, nothing is sent from the Graph API as shown in the command prompt image. I am really grateful, if someone could point out where the error is.
I have defined the scopes in my manifiest file as well.
Please help.
I'm having an issue getting a Drive ID when making a GET request.
First I'm creating the Team using "https://graph.microsoft.com/v1.0/teams". The request returns no response but a Status 202 accepted, the new Team is listed on Teams. I then extract the Team ID from Response Headers "Location".
Here is the failure, I make a request to https://graph.microsoft.com/v1.0/groups/{{TeamsID}}/drive
Here is my response back
{"error":{"code":"ResourceNotFound","message":"Resource is not found.","innerError":{"date":"2021-05-03T19:17:03","request-id":"75b4888d-de91-41aa-89ef-d983fcb5238d","client-request-id":"75b4888d-de91-41aa-89ef-d983fcb5238d"}}}
I make the same request on via postman and it works fine.
Here is my code
try {
let crt = await createTeam(msalData, data);
crt.headers.forEach( async (val, key) => {
if (key === "location") {
let id = val.split("/teams('")[1].split("')/operations")[0];
let gdd = await getDocumentDrive(msalData, id);
console.log(gdd)
}
});
} catch (err) {
console.log(err);
}
I can't figure out what I'm doing wrong.
Apparently the error (ResourceNotFound) occurs since the moment when Get Drive request is submitted the Group is not yet completely provisioned, documentations says the following in this regard:
If the group was created less than 15 minutes ago, it's possible for
the Create team call to fail with a 404 error code due to replication
delays. The recommended pattern is to retry the Create team call three
times, with a 10 second delay between calls.
Since you haven't revealed how exactly createTeam function is implemented, retry pattern come to the rescue here, for example:
async function withRetry(fn, maxRetry = 5, ms = 1000) {
try {
const val = await fn();
return val;
} catch (error) {
if (maxRetry) {
await new Promise((r) => setTimeout(r, ms));
return withRetry(fn, maxRetry - 1, ms);
} else throw new Error(`Max retries reached for function ${fn.name}`);
}
}
and now Teams could be created like this (note, in retry logic Get Group endpoint request is submitted to ensure it has been provisioned):
async function ensureTeam(client, teamName, defaultOwnerId) {
const teamResp = await createTeam(client, teamName, defaultOwnerId);
const teamloc = teamResp.headers.get("Content-Location");
return await withRetry(
async () => {
const groupUrl = teamloc.replace("teams", "groups");
return await client.api(groupUrl).get();
},
5,
10000
);
}
where createTeam could look like this:
async function createTeam(client, teamName, defaultOwnerId) {
const team = {
"template#odata.bind":
"https://graph.microsoft.com/v1.0/teamsTemplates('standard')",
displayName: teamName,
description: teamName,
members: [
{
"#odata.type": "#microsoft.graph.aadUserConversationMember",
roles: ["owner"],
"user#odata.bind": `https://graph.microsoft.com/v1.0/users('${defaultOwnerId}')`,
},
],
};
return await client.api("/teams").responseType(ResponseType.RAW).post(team);
}
Usage
const graphClient = Client.initWithMiddleware({ authProvider });
const group = await ensureTeam(graphClient,"Demo team",ownerId);
const driveUrl = `/groups/${group.id}/drive`;
const drive = await graphClient.api(driveUrl).get();
I tried to share data between Safari browser and standalone PWA on iPhone12 with iOS 14.3.
The information, that this should work are here: https://firt.dev/ios-14/
I#ve tried this: https://www.netguru.com/codestories/how-to-share-session-cookie-or-state-between-pwa-in-standalone-mode-and-safari-on-ios
Without success.
Are there any suggestions to running this? Or is it not possible ...
This is the code
const CACHE_NAME = "auth";
const TOKEN_KEY = "token";
const FAKE_TOKEN = "sRKWQu6hCJgR25lslcf5s12FFVau0ugi";
// Cache Storage was designed for caching
// network requests with service workers,
// mainly to make PWAs work offline.
// You can give it any value you want in this case.
const FAKE_ENDPOINT = "/fake-endpoint";
const saveToken = async (token: string) => {
try {
const cache = await caches.open(CACHE_NAME);
const responseBody = JSON.stringify({
[TOKEN_KEY]: token
});
const response = new Response(responseBody);
await cache.put(FAKE_ENDPOINT, response);
console.log("Token saved! 🎉");
} catch (error) {
// It's up to you how you resolve the error
console.log("saveToken error:", { error });
}
};
const getToken = async () => {
try {
const cache = await caches.open(CACHE_NAME);
const response = await cache.match(FAKE_ENDPOINT);
if (!response) {
return null;
}
const responseBody = await response.json();
return responseBody[TOKEN_KEY];
} catch (error) {
// Gotta catch 'em all
console.log("getToken error:", { error });
}
};
const displayCachedToken = async () => {
const cachedToken = await getToken();
console.log({ cachedToken });
};
// Uncomment the line below to save the fake token
// saveToken(FAKE_TOKEN);
displayCachedToken();
Without success means no result, i've tried to set data in safari and get them in standalone pwa
Im looking to invoke a twilio function and obtain a count of available workers.
I have a feeling it might be related to TaskQueues and the Matching Workers who are available?
Ive come up with the following. However, users are still listed as available when they are interacting with a task, which means this wont work necessarily.
exports.handler = function (context, event, callback) {
const client = require('twilio')(context.ACCOUNT_SID, context.AUTH_TOKEN);
client.taskrouter
.workspaces('eee')
.workers.list()
.then(workers => {
data = {
availWorkersCount: Object.keys(workers.filter(x=> x.available === true && x.attributes.includes("sales"))).length
};
const response = new Twilio.Response();
response.appendHeader('Access-Control-Allow-Origin', '*');
response.appendHeader('Access-Control-Allow-Methods', 'OPTIONS POST GET');
response.appendHeader('Access-Control-Allow-Headers', 'Content-Type');
response.appendHeader('Content-Type', 'application/json');
response.setBody(data);
callback(null, response);
});
};
I know this is a bit late, but this is the chunk of code I use in a twilio function to do just this. We call the function from the studio flow and use the results to decide if I am going to route the call into voicemail or play some IVR options for picking what queue they should be put in. To use it you can pass in an optional skill to filter down the agents even more. We grab all available workers then filter down to only the ones that have the needed skill.
You could then take this and check if any of the agents that are available also have a task assigned to them then remove them from the count.
const fetch = require("node-fetch");
exports.handler = function(context, event, callback) {
let response = new Twilio.Response();
// Set the status code to 200 OK
response.setStatusCode(200);
// Set the Content-Type Header
response.appendHeader('Access-Control-Allow-Origin', '*');
response.appendHeader('Access-Control-Allow-Methods', 'OPTIONS, POST, GET');
response.appendHeader('Access-Control-Allow-Headers', 'Content-Type');
response.appendHeader('Content-Type', 'application/json');
let body = {
TotalAvailable: 0
};
let client = context.getTwilioClient();
client.taskrouter.workspaces(context.WorkspaceSid)
.workers
.list({
available: 'true',
limit: 50
})
.then((workers) => {
let agents = [];
let i = 0;
if(workers){
for(i = 0; i < workers.length; i++){
let worker = workers[i];
let item = {};
let attributes = JSON.parse(worker.attributes);
if(attributes && attributes.routing && attributes.routing.skills && attributes.routing.skills.length > 0){
item.skills = attributes.routing.skills;
item.nid = attributes.nid;
item.first_name = attributes.first_name;
item.last_name = attributes.last_name;
if(event.skill){
if(item.skills.includes(event.skill)){
// TODO: filter here
agents.push(item);
}
}else{
agents.push(item);
}
}
}
}
body.TotalAvailable = agents.length;
body.Agents = agents;
response.setBody(body);
callback(null, response);
})
.catch((ex) => {
body.error = true;
body.message = ex;
response.setBody(body);
callback(null, response);
});
};
I am calling an api. I am getting a streamed response after sending the request. But i cannot parse the response and convert it to String/JSON. This is where I am calling the api.
static Future<String> callDeviceListFetchApi() async {
Completer completer = new Completer();
String jsonResponse;
String url = Constants.BASE_URL + Constants.DEVICE_REGISTER_URL;
var client = new http.Client();
var request = new http.Request('GET', Uri.parse(url));
request.headers[HttpHeaders.CONTENT_TYPE] = 'application/json';
request.headers[HttpHeaders.AUTHORIZATION] = '<auth code>';
await client.send(request).then((response) {
response.stream.bytesToString().then((value) {
print(value.toString());
jsonResponse = value.toString();
completer.complete(jsonResponse);
});
}).catchError((error) {
print(error.toString());
});
return completer.future;
}
I am getting the error,
Bad state: Stream has already been listened to Flutter error. Any idea why this is happening?
There's a couple of things wrong with your code. I think you have a slight misunderstanding about how Async and Futures work in dart - you should re-read the docs and this tutorial (part 1 and part 2).
Basically, the problem is that you were returning a 'Future' from an async function. If you return a future from an async function, it has issues (I don't know why the analyzer doesn't catch that).
Future<String> callDeviceListFetchApi() async {
Completer completer = new Completer();
String url = "<url>";
var client = new http.Client();
var request = new http.Request('GET', Uri.parse(url));
request.headers[HttpHeaders.CONTENT_TYPE] = 'application/json';
request.headers[HttpHeaders.AUTHORIZATION] =
'<auth string>';
var response = await client.send(request);
String jsonResponse;
try {
var value = await response.stream.bytesToString();
print(value.toString());
jsonResponse = value.toString();
} catch (error) {
print(error.toString());
}
return completer.complete(jsonResponse);
}
Or not async:
Future<String> callDeviceListFetchApiNotAsync() {
String url = "<url>";
var client = new http.Client();
var request = new http.Request('GET', Uri.parse(url));
request.headers[HttpHeaders.CONTENT_TYPE] = 'application/json';
request.headers[HttpHeaders.AUTHORIZATION] =
'<auth string>';
Completer completer = new Completer();
return client.send(request).then((response) {
return response.stream.bytesToString();
}).then((value) {
print(value.toString());
return value.toString();
}).catchError((error) {
print(error.toString());
// if you use catchError, whatever you return from it
// is the value you'll get wherever you resolve the future.
return null;
});
}
But unless you're trying to do something I'm not seeing, there's a way easier way to do this (assuming all you want to do is get a string from a server):
Future<String> getList() async {
var response = await http.get("<url>", headers: {
HttpHeaders.CONTENT_TYPE: 'application/json',
HttpHeaders.AUTHORIZATION: '<auth string>',
});
if (response.statusCode == 200) {
return response.body;
} else {
throw Error();
}
}