We are trying web crawl and get contents from multiple pages. I am taking the advantage of async API with Promise ALL which can execute requests in parallel.
Is there a limitation on the number of contexts which can be opened parallel?
const fs = require('fs');
let browser;
const batch_size = 4; // control the number of async parallel calls
(async () => { // main function
let urls = [];
urls = fs.readFileSync('./resources/input_selenium_urls.csv').toString().split("\n");
browser = await chromium.launch();
let context_size = 0;
let processUrls = [];
let total_length = 0;
for (let i=0;i<urls.length;i++,total_length++) {
if ((context_size==batch_size)||(i==urls.length-1)){
await Promise.all(processUrls.map(x => getHTMLPageSource(x)));
context_size = 0;
processUrls = [];
} else {
processUrls.push(urls[i]);
context_size++;
}
}
await browser.close();
})();
async function getHTMLPageSource(url) {
const context = await browser.newContext();
const page = await context.newPage();
let response = {}
try {
await page.goto(url, { waitUntil: 'networkidle' });
response = {
url : url,
content: await page.title(),
error : null
}
console.log(response);
}
catch {
response = {
error : "Timeout error"
}
}
context.close;
return response;
}
Browser contexts are cheap to create, but it's not clear whether there is a hard-coded limit on them from the docs perhaps the limit might depend on the browser you chose and your OS resources. I think you might only be able to find out by creating a lot of contexts.
I would like to run a Lighthouse audit programatically. I have found multiple examples on how to accomplish this with Puppeteer. However, is there a way to run a Lighthouse audit using the Chrome DevTools Protocol?
You can run Lighthouse programatically by using the PageSpeed Insights API: https://developers.google.com/speed/docs/insights/v5/about
Javascript example:
function run() {
const url = setUpQuery();
fetch(url)
.then(response => response.json())
.then(json => {
// See https://developers.google.com/speed/docs/insights/v5/reference/pagespeedapi/runpagespeed#response
// to learn more about each of the properties in the response object.
showInitialContent(json.id);
const cruxMetrics = {
"First Contentful Paint": json.loadingExperience.metrics.FIRST_CONTENTFUL_PAINT_MS.category,
"First Input Delay": json.loadingExperience.metrics.FIRST_INPUT_DELAY_MS.category
};
showCruxContent(cruxMetrics);
const lighthouse = json.lighthouseResult;
const lighthouseMetrics = {
'First Contentful Paint': lighthouse.audits['first-contentful-paint'].displayValue,
'Speed Index': lighthouse.audits['speed-index'].displayValue,
'Time To Interactive': lighthouse.audits['interactive'].displayValue,
'First Meaningful Paint': lighthouse.audits['first-meaningful-paint'].displayValue,
'First CPU Idle': lighthouse.audits['first-cpu-idle'].displayValue,
'Estimated Input Latency': lighthouse.audits['estimated-input-latency'].displayValue
};
showLighthouseContent(lighthouseMetrics);
});
}
function setUpQuery() {
const api = 'https://www.googleapis.com/pagespeedonline/v5/runPagespeed';
const parameters = {
url: encodeURIComponent('https://developers.google.com')
};
let query = `${api}?`;
for (key in parameters) {
query += `${key}=${parameters[key]}`;
}
return query;
}
function showInitialContent(id) {
document.body.innerHTML = '';
const title = document.createElement('h1');
title.textContent = 'PageSpeed Insights API Demo';
document.body.appendChild(title);
const page = document.createElement('p');
page.textContent = `Page tested: ${id}`;
document.body.appendChild(page);
}
function showCruxContent(cruxMetrics) {
const cruxHeader = document.createElement('h2');
cruxHeader.textContent = "Chrome User Experience Report Results";
document.body.appendChild(cruxHeader);
for (key in cruxMetrics) {
const p = document.createElement('p');
p.textContent = `${key}: ${cruxMetrics[key]}`;
document.body.appendChild(p);
}
}
function showLighthouseContent(lighthouseMetrics) {
const lighthouseHeader = document.createElement('h2');
lighthouseHeader.textContent = "Lighthouse Results";
document.body.appendChild(lighthouseHeader);
for (key in lighthouseMetrics) {
const p = document.createElement('p');
p.textContent = `${key}: ${lighthouseMetrics[key]}`;
document.body.appendChild(p);
}
}
run();
Source: https://developers.google.com/speed/docs/insights/v5/get-started
For example, I have a car1 that was first owner by the manufacturer and then it was transferred over to the retailer and then to the user
In the fabcar example, I can know who is its current owner by i don't know who is the previous owner.
Is there a way to do it?
Here is the http://hyperledger-fabric.readthedocs.io/en/latest/write_first_app.html example I was following
Is not implemented in the chaincode. To do this you could implement a new method which returns the history of the asset.
Here the link to the official documentation for nodeJS (you can find it also for GoLang):
https://fabric-shim.github.io/ChaincodeStub.html#getHistoryForKey__anchor
Here an example:
async queryValueHistory(stub,args){
if (args.length != 1) {
throw new Error('Incorrect number of arguments. Expecting identifier ex: CAR01');
}
let carId = args[0];
let iterator = await stub.getHistoryForKey(carId);
let allResults = [];
while (true) {
let res = await iterator.next();
if (res.value && res.value.value.toString()) {
let jsonRes = {};
console.log(res.value.value.toString('utf8'));
jsonRes.TxId = res.value.tx_id;
jsonRes.Timestamp = res.value.timestamp;
jsonRes.IsDelete = res.value.is_delete.toString();
try {
jsonRes.Value = JSON.parse(res.value.value.toString('utf8'));
} catch (err) {
console.log(err);
jsonRes.Value = res.value.value.toString('utf8');
}
console.info(jsonRes);
allResults.push(jsonRes);
}
if (res.done) {
console.log('end of data');
try{
await iterator.close();
}catch(err){
console.log(err);
}
console.info(allResults);
return Buffer.from(JSON.stringify(allResults));
}
}
}
I have got following script in Google docs spreadsheet which is fetching data from Fitbit. Script worked fine so far but recently on 6th July Google stopped using OAuthConfig so script is not working since:-(
I am not programmer, I am just advanced user. So I would like to kindly ask some programmer to help tune script below in order to make it work again.
// Key of ScriptProperty for Firtbit consumer key.
var CONSUMER_KEY_PROPERTY_NAME = "fitbitConsumerKey";
// Key of ScriptProperty for Fitbit consumer secret.
var CONSUMER_SECRET_PROPERTY_NAME = "fitbitConsumerSecret";
// Default loggable resources (from Fitbit API docs).
var LOGGABLES = ["activities/log/steps", "activities/log/distance",
"activities/log/activeScore", "activities/log/activityCalories",
"activities/log/calories", "foods/log/caloriesIn",
"activities/log/minutesSedentary",
"activities/log/minutesLightlyActive",
"activities/log/minutesFairlyActive",
"activities/log/minutesVeryActive", "sleep/timeInBed",
"sleep/minutesAsleep", "sleep/minutesAwake", "sleep/awakeningsCount",
"body/weight", "body/bmi", "body/fat",];
// function authorize() makes a call to the Fitbit API to fetch the user profile
function authorize() {
var oAuthConfig = UrlFetchApp.addOAuthService("fitbit");
oAuthConfig.setAccessTokenUrl("https://api.fitbit.com/oauth/access_token");
oAuthConfig.setRequestTokenUrl("https://api.fitbit.com/oauth/request_token");
oAuthConfig.setAuthorizationUrl("https://api.fitbit.com/oauth/authorize");
oAuthConfig.setConsumerKey(getConsumerKey());
oAuthConfig.setConsumerSecret(getConsumerSecret());
var options = {
"oAuthServiceName": "fitbit",
"oAuthUseToken": "always",
};
// get the profile to force authentication
Logger.log("Function authorize() is attempting a fetch...");
try {
var result = UrlFetchApp.fetch("https://api.fitbit.com/1/user/-/profile.json", options);
var o = Utilities.jsonParse(result.getContentText());
return o.user;
}
catch (exception) {
Logger.log(exception);
Browser.msgBox("Error attempting authorization");
return null;
}
}
// function setup accepts and stores the Consumer Key, Consumer Secret, firstDate, and list of Data Elements
function setup() {
var doc = SpreadsheetApp.getActiveSpreadsheet();
var app = UiApp.createApplication().setTitle("Setup Fitbit Download");
app.setStyleAttribute("padding", "10px");
var consumerKeyLabel = app.createLabel("Fitbit OAuth Consumer Key:*");
var consumerKey = app.createTextBox();
consumerKey.setName("consumerKey");
consumerKey.setWidth("100%");
consumerKey.setText(getConsumerKey());
var consumerSecretLabel = app.createLabel("Fitbit OAuth Consumer Secret:*");
var consumerSecret = app.createTextBox();
consumerSecret.setName("consumerSecret");
consumerSecret.setWidth("100%");
consumerSecret.setText(getConsumerSecret());
var firstDate = app.createTextBox().setId("firstDate").setName("firstDate");
firstDate.setName("firstDate");
firstDate.setWidth("100%");
firstDate.setText(getFirstDate());
// add listbox to select data elements
var loggables = app.createListBox(true).setId("loggables").setName(
"loggables");
loggables.setVisibleItemCount(4);
// add all possible elements (in array LOGGABLES)
var logIndex = 0;
for (var resource in LOGGABLES) {
loggables.addItem(LOGGABLES[resource]);
// check if this resource is in the getLoggables list
if (getLoggables().indexOf(LOGGABLES[resource]) > -1) {
// if so, pre-select it
loggables.setItemSelected(logIndex, true);
}
logIndex++;
}
// create the save handler and button
var saveHandler = app.createServerClickHandler("saveSetup");
var saveButton = app.createButton("Save Setup", saveHandler);
// put the controls in a grid
var listPanel = app.createGrid(6, 3);
listPanel.setWidget(1, 0, consumerKeyLabel);
listPanel.setWidget(1, 1, consumerKey);
listPanel.setWidget(2, 0, consumerSecretLabel);
listPanel.setWidget(2, 1, consumerSecret);
listPanel.setWidget(3, 0, app.createLabel(" * (obtain these at dev.fitbit.com)"));
listPanel.setWidget(4, 0, app.createLabel("Start Date for download (yyyy-mm-dd)"));
listPanel.setWidget(4, 1, firstDate);
listPanel.setWidget(5, 0, app.createLabel("Data Elements to download:"));
listPanel.setWidget(5, 1, loggables);
// Ensure that all controls in the grid are handled
saveHandler.addCallbackElement(listPanel);
// Build a FlowPanel, adding the grid and the save button
var dialogPanel = app.createFlowPanel();
dialogPanel.add(listPanel);
dialogPanel.add(saveButton);
app.add(dialogPanel);
doc.show(app);
}
// function sync() is called to download all desired data from Fitbit API to the spreadsheet
function sync() {
// if the user has never performed setup, do it now
if (!isConfigured()) {
setup();
return;
}
var user = authorize();
// Spatny kod, oprava nize - var doc = SpreadsheetApp.getActiveSpreadsheet();
var doc = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Pavel');
doc.setFrozenRows(1);
var options = {
"oAuthServiceName": "fitbit",
"oAuthUseToken": "always",
"method": "GET"
};
// prepare and format today's date, and a list of desired data elements
var dateString = formatToday();
var activities = getLoggables();
// for each data element, fetch a list beginning from the firstDate, ending with today
for (var activity in activities) {
var currentActivity = activities[activity];
try {
var result = UrlFetchApp.fetch("https://api.fitbit.com/1/user/-/"
+ currentActivity + "/date/" + getFirstDate() + "/"
+ dateString + ".json", options);
} catch (exception) {
Logger.log(exception);
Browser.msgBox("Error downloading " + currentActivity);
}
var o = Utilities.jsonParse(result.getContentText());
// set title
var titleCell = doc.getRange("a1");
titleCell.setValue("date");
var cell = doc.getRange('a2');
// fill the spreadsheet with the data
var index = 0;
for (var i in o) {
// set title for this column
var title = i.substring(i.lastIndexOf('-') + 1);
titleCell.offset(0, 1 + activity * 1.0).setValue(title);
var row = o[i];
for (var j in row) {
var val = row[j];
cell.offset(index, 0).setValue(val["dateTime"]);
// set the date index
cell.offset(index, 1 + activity * 1.0).setValue(val["value"]);
// set the value index index
index++;
}
}
}
}
function isConfigured() {
return getConsumerKey() != "" && getConsumerSecret() != "";
}
function setConsumerKey(key) {
ScriptProperties.setProperty(CONSUMER_KEY_PROPERTY_NAME, key);
}
function getConsumerKey() {
var key = ScriptProperties.getProperty(CONSUMER_KEY_PROPERTY_NAME);
if (key == null) {
key = "";
}
return key;
}
function setLoggables(loggable) {
ScriptProperties.setProperty("loggables", loggable);
}
function getLoggables() {
var loggable = ScriptProperties.getProperty("loggables");
if (loggable == null) {
loggable = LOGGABLES;
} else {
loggable = loggable.split(',');
}
return loggable;
}
function setFirstDate(firstDate) {
ScriptProperties.setProperty("firstDate", firstDate);
}
function getFirstDate() {
var firstDate = ScriptProperties.getProperty("firstDate");
if (firstDate == null) {
firstDate = "2012-01-01";
}
return firstDate;
}
function formatToday() {
var todayDate = new Date;
return todayDate.getFullYear()
+ '-'
+ ("00" + (todayDate.getMonth() + 1)).slice(-2)
+ '-'
+ ("00" + todayDate.getDate()).slice(-2);
}
function setConsumerSecret(secret) {
ScriptProperties.setProperty(CONSUMER_SECRET_PROPERTY_NAME, secret);
}
function getConsumerSecret() {
var secret = ScriptProperties.getProperty(CONSUMER_SECRET_PROPERTY_NAME);
if (secret == null) {
secret = "";
}
return secret;
}
// function saveSetup saves the setup params from the UI
function saveSetup(e) {
setConsumerKey(e.parameter.consumerKey);
setConsumerSecret(e.parameter.consumerSecret);
setLoggables(e.parameter.loggables);
setFirstDate(e.parameter.firstDate);
var app = UiApp.getActiveApplication();
app.close();
return app;
}
// function onOpen is called when the spreadsheet is opened; adds the Fitbit menu
function onOpen() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var menuEntries = [{
name: "Sync",
functionName: "sync"
}, {
name: "Setup",
functionName: "setup"
}, {
name: "Authorize",
functionName: "authorize"
}];
ss.addMenu("Fitbit", menuEntries);
}
// function onInstall is called when the script is installed (obsolete?)
function onInstall() {
onOpen();
}
Problem solved with updated code at https://github.com/loghound/Fitbit-for-Google-App-Script
I have already try to understand the API doc, the articles about them, and this post: How do you create a Stream in Dart
I'm making a simple web app using WebSocket. Actually, it's working well, but I want add a feature (enjoy learn).
This is my class (can be optimized I guess)
library Ask;
import 'dart:html';
import 'dart:async';
import 'dart:convert';
class Ask {
final String addr;
String _protocol;
String _port;
WebSocket _ws;
bool openned;
Map<int, Completer> _completer_list = {};
int _counter = 0;
static final Map<String, Ask> _cache = <String, Ask>{};
factory Ask(String addr) {
if (_cache.containsKey(addr)) {
return _cache[addr];
} else {
final ask_server = new Ask._internal(addr);
_cache[addr] = ask_server;
return ask_server;
}
}
Ask._internal(this.addr);
Future<bool> open() {
if (openned)
return true;
_completer_list[0] = new Completer();
if (window.location.protocol == 'http:') {
_port = ':8080/ws';
_protocol = 'ws://';
} else {
_port = ':8443/ws';
_protocol = 'wss://';
}
_ws = new WebSocket(_protocol + addr + _port);
_ws.onOpen.listen((e) {
_get_data();
_get_close();
openned = true;
_completer_list[0].complete(true);
});
return _completer_list[0].future;
}
Future<String> send(Map data) {
bool check = false;
int id;
_completer_list.forEach((k, v) {
if (v.isCompleted) {
id = data['ws_id'] = k;
_completer_list[k] = new Completer();
_ws.send(JSON.encode(data));
check = true;
}
});
if (!check) {
_counter++;
id = data['ws_id'] = _counter;
_completer_list[id] = new Completer();
_ws.send(JSON.encode(data));
}
return _completer_list[id].future;
}
void _get_data() {
_ws.onMessage.listen((MessageEvent data) {
var response = JSON.decode(data.data);
_completer_list[response['ws_id']].complete(response);
});
}
void _get_close() {
_ws.onClose.listen((_) {
print('Server have been lost. Try to reconnect in 3 seconds.');
new Timer(new Duration(seconds: 3), () {
_ws = new WebSocket(_protocol + addr + _port);
_get_data();
_get_close();
_ws.onOpen.listen((e) => print('Server is alive again.'));
});
});
}
}
Example of use:
void showIndex() {
Element main = querySelector('main');
Ask connect = new Ask('127.0.0.1');
Map request = {};
request['index'] = true;
connect.open().then((_) {
connect.send(request).then((data) {
main.setInnerHtml(data['response']);
});
});
}
I would replace the then by a listen who will be canceled when the message will completed. By this way, I can add a progress bar, I think...
So my question, my send function can be a stream and keep my concept of one websocket for all ? (yes, if my function is used when a request is in progress, it's sent and if she's finish before the first, I recovered her properly. Thank you ws_id).
Thank you.
I think what you need is a StreamController
https://api.dartlang.org/apidocs/channels/stable/dartdoc-viewer/dart-async.StreamController