I want to send parallel POST requests in puppeteer. I have to change the payload with every request (URL remains the same).
I tried using puppeteer cluster, but how do I change payload with every request when I queue the same request?
Using normal puppeteer
(async() => {
const browser = await puppeteer.launch({
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-web-security",
],
executablePath: 'C:/Program Files/..',
headless: false,
});
for(const id of Ids) {
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(60000);
await page.evaluateOnNewDocument(() => {
// Some code
})
await page.setRequestInterception(true);
// Request intercept handler... will be triggered with
// each page.goto() statement
page.on('request', interceptedRequest => {
// Here, is where you change the request method and
// add your post data
var data = {
'method': 'POST',
'postData': JSON.stringify({
....
"similarMaterialId": `${id}`,
}),
'headers': {
.....
},
};
// Request modified... finish sending!
interceptedRequest.continue(data);
});
const response = await page.goto('https://.../getProductInfo');
const responseBody = await response.json();
try {
let title = responseBody.description;
let price = responseBody.price;
fs.appendFile('temp.tsv', `${title}\t${price}\n`, function (err) {
if (err) throw err;
})
}
catch {
console.log(id)
}
await page.close();
}
console.log("Code ended!!")
await browser.close();
})();
I want to create many pages in parallel on a single browser.
I am using Capacitor v3, NextJS static export, and a Django backend to build out an iOS app based on a production website.
The current backend authentication scheme uses Django sessions via cookies as well as setting the CSRF token via cookies. The CSRF token can be bypassed pretty easily for the app and not worried about disabling that but forking our authentication scheme would be somewhat of a hassle. The capacitor-community/http claims to allow Cookies but I haven't been able to configure that correctly.
Capacitor Config:
import { CapacitorConfig } from '#capacitor/cli';
const config: CapacitorConfig = {
appId: 'com.nextwebapp.app',
appName: 'nextwebapp',
webDir: 'out',
bundledWebRuntime: false
};
export default config;
Note that I have tried setting server.hostname to myapp.com as well.
Based on the comments at the bottom of the capacitor http readme I set the following Info.plist values.
App/Info.plist
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
....
<key>WKAppBoundDomains</key>
<array>
<string>staging.myapp.com</string>
<string>myapp.com</string>
</array>
</dict>
</plist>
The web app uses a react hooks wrapper package for axios so in order to keep changes minimal I made a hook that mimics the state returned from that package.
hooks/useNativeRequest.ts
import { useEffect, useState } from "react";
import { Http } from "#capacitor-community/http";
import {
BASE_URL,
DEFAULT_HEADERS,
HOST_NAME,
ERROR_MESSAGE,
Refetch,
RequestOptions,
ResponseValues,
RequestConfig,
} from "#utils/http";
import { handleResponseToast } from "#utils/toast";
const makeUrl = (url): string => `${BASE_URL}${url}`;
const getCSRFToken = async () =>
await Http.getCookie({ key: "csrftoken", url: HOST_NAME });
const combineHeaders = async (headers: any) => {
const newHeaders = Object.assign(DEFAULT_HEADERS, headers);
const csrfHeader = await getCSRFToken();
if (csrfHeader.value) {
newHeaders["X-CSRFToken"] = csrfHeader.value;
}
return newHeaders;
};
function useNativeRequest<T>(
config?: RequestConfig,
options?: RequestOptions
): [ResponseValues<T>, Refetch<T>] {
const [responseState, setResponseState] = useState({
data: null,
error: null,
loading: false,
});
let method = "get";
let url = config;
let headers = {};
let params = undefined;
let data = undefined;
if (config && typeof config !== "string") {
url = config.url;
method = config.method?.toLowerCase() ?? method;
headers = config.headers;
params = config.params;
data = config.data;
}
const requestMethod = Http[method];
const makeRequest = async () => {
setResponseState({ error: null, data: null, loading: true });
try {
const reqHeaders = await combineHeaders(headers);
console.log({
url,
reqHeaders,
params,
data
})
const response = await requestMethod({
url: makeUrl(url),
headers: reqHeaders,
params,
data,
});
if (response?.status === 200) {
setResponseState({ error: null, data: response.data, loading: false });
handleResponseToast(response?.data?.detail);
} else {
const errorMessage = response?.data?.detail || ERROR_MESSAGE;
handleResponseToast(errorMessage);
setResponseState({
error: errorMessage,
data: response.data,
loading: false,
});
}
return response;
} catch {
setResponseState({
error: ERROR_MESSAGE,
data: null,
loading: false,
});
return Promise.reject(ERROR_MESSAGE);
}
};
useEffect(() => {
if (!options?.manual) {
makeRequest();
}
}, [options?.manual]);
return [responseState, makeRequest];
}
export { useNativeRequest };
The console.log above never includes the additional csrf cookie and in the getter logs it doesn't contain a value.
Backend Django
MIDDLEWARE = [
...
'myapp_webapp.middle.CustomCSRFMiddleWare',
]
CORS_ALLOWED_ORIGINS = [
...
"capacitor://localhost",
]
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.BasicAuthentication',
'rest_framework.authentication.TokenAuthentication',
'rest_framework.authentication.SessionAuthentication',
),
}
middleware
class CustomCSRFMiddleWare(CsrfViewMiddleware):
def process_request(self, request):
# Special Processing for API Requests
if "/api/v1" in request.path:
try:
requested_with = request.headers['X-Requested-With']
myapp_request = request.headers['X-Myapp-Request']
# Check Custom Headers
if not (requested_with == 'XMLHttpRequest' and myapp_request == '1'):
raise PermissionDenied()
return None
except KeyError:
# All API Requests should include the above headers
raise PermissionDenied()
# Call original CSRF Middleware
return super(CustomCSRFMiddleWare, self).process_request(request)
Occasionally the backend will also show that X-Requested-With is not being sent but it is included in the DEFAULT_HEADERS constant I have in the UI and appears in the console.log.
Is anything above preventing me from being able to read and send cookies from Capacitor on iOS? Does Cookie based authentication even work with capacitor?
Here is my updated react hook that combine's my above question and thread mentioned in the comments as well as some manual cookie setting.
The below client side code worked without changes to existing Django Session authentication.
The changes from my code above
Added credentials: "include" to webFetchExtra
Added "Content-Type": "application/json" to headers
Handle override of the initial config for manual request & refetch
Set Session Cookie After Response
Based on the docs this shouldn't be necessary but I am keeping in my code for now.
import { useCallback, useEffect, useState } from "react";
import { AxiosRequestConfig } from "axios";
import { Http } from "#capacitor-community/http";
const DEFAULT_HEADERS = {
"X-Requested-With": "XMLHttpRequest",
"X-MyApp-Request": "1",
"Content-Type": "application/json",
};
const makeUrl = (url): string => `${BASE_URL}${url}`;
const getCSRFToken = async () =>
await Http.getCookie({ key: "csrftoken", url: HOST_NAME });
const setSessionCookie = async () => {
const sessionId = await Http.getCookie({ key: "sessionid", url: HOST_NAME });
if (sessionId.value) {
await Http.setCookie({
key: "sessionid",
value: sessionId.value,
url: HOST_NAME,
});
}
};
const combineHeaders = async (headers: any) => {
const newHeaders = Object.assign(DEFAULT_HEADERS, headers);
const csrfHeader = await getCSRFToken();
if (csrfHeader.value) {
newHeaders["X-CSRFToken"] = csrfHeader.value;
}
return newHeaders;
};
const parseConfig = (config: RequestConfig, configOverride?: RequestConfig) => {
let method = "get";
let url = config;
let headers = {};
let params = undefined;
let data = undefined;
if (config && typeof config !== "string") {
url = config.url;
method = config.method ?? method;
headers = config.headers;
params = config.params;
data = config.data;
}
return {
url,
method,
headers,
params,
data,
...(configOverride as AxiosRequestConfig),
};
};
function useNativeRequest<T>(
config?: RequestConfig,
options?: RequestOptions
): [ResponseValues<T>, Refetch<T>] {
const [responseState, setResponseState] = useState({
data: null,
error: null,
loading: false,
});
const makeRequest = useCallback(
async (configOverride) => {
setResponseState({ error: null, data: null, loading: true });
const { url, method, headers, params, data } = parseConfig(
config,
configOverride
);
try {
const reqHeaders = await combineHeaders(headers);
const response = await Http.request({
url: makeUrl(url),
headers: reqHeaders,
method,
params,
data,
webFetchExtra: {
credentials: "include",
},
});
if (response?.status === 200) {
setResponseState({
error: null,
data: response.data,
loading: false,
});
await setSessionCookie();
} else {
setResponseState({
error: errorMessage,
data: response.data,
loading: false,
});
}
return response;
} catch {
setResponseState({
error: ERROR_MESSAGE,
data: null,
loading: false,
});
return Promise.reject(ERROR_MESSAGE);
}
},
[config]
);
useEffect(() => {
if (!options?.manual) {
makeRequest(config);
}
}, [options?.manual]);
return [responseState, makeRequest];
}
export { useNativeRequest };
I'm using docker network and try to use apollo-client, apollo-upload(createUploadLink) and I try to sent Barear token in headers too. the error show up Request with GET/HEAD method cannot have body
But if I change my url into real url [ not dockerNetwork everything work fine]
export const client = (req) => {
const uri = http://dockerNetwork:3000
return new ApolloClient({
link: authLink(req).concat(createUploadLink({
uri: uri ',
});),
cache: new InMemoryCache(),
});
};
const authLink = req => {
return setContext(_ => {
return {
headers: {
...req.headers,
authorization: `Bearer ${req.cookies.token)}`,
},
};
});
};
How can I fix this error by using docker network
Finally I found solution, first I use
"#apollo/client": "3.4.20"
"apollo-upload-client": "^16.0.0",
and I downgrade apollo/client to 3.3.20
"#apollo/client": "3.3.20",
I want to do what the MS Graph sample node app is doing in its integrationTests.js, but that test doesn't work. Here's what I've tried:
Followed the quick start for creating a node.js app.
Ran the app. Ensured it worked by sending an e-mail.
Modified the test Checking that the sample can send an email to use my account parameters.
Tried to run the test. It fails with 403: insufficient scope. The call to get the token returned scopes, but lacked Mail.Send.
In the post data for the call to login.microsoftonline.com, I added "scope: 'Mail.Send'"
I still receive a valid token, and the return scope includes Mail.Send, but when I try to post with that token, I get 400: cannot POST /beta/me/sendMail
I tried adding scope (Mail.Send) in the query string and as a header (thought I saw that somewhere), but it made no difference.
I added the Mail.Send permission (under "Application Permissions") for the app in the application registration portal.
I compared the token (using https://jwt.ms) from my test call to the call from the app when it works. I see no real difference. They both contain the Mail.Send scope.
Here is the code (which is only slightly different from what's in the sample):
// in graphHelper.js
function postSendMail(accessToken, message, callback) {
request
.post('https://graph.microsoft.com/beta/me/sendMail')
//.post('https://graph.microsoft.com/beta/me/sendMail?scope=Mail.Send') // nope
.send(message)
.set('Authorization', 'Bearer ' + accessToken)
.set('Content-Type', 'application/json')
.set('Content-Length', message.length)
.set('scope', 'Mail.Send') // nope
.end((err, res) => {
callback(err, res);
});
}
describe('Integration', function () { // mocha
var accessToken;
var scope;
const config = getConfig();
// My account variables in testConfig.json file
function getConfig() {
var configFilePath = path.join(__dirname, 'testConfig.json');
return JSON.parse(fs.readFileSync(configFilePath, { encoding: 'utf8' }));
}
function getAccessToken(done) {
var postData = querystring.stringify(
{
grant_type: 'password',
//grant_type: 'client_id', // not supported
//grant_type: 'authorization_code', // This assumes you've requested an auth code.
resource: 'https://graph.microsoft.com/',
scope: 'Mail.Send',
client_id: config.test_client_id_v2,
client_secret: config.test_client_secret_v2,
username: config.test_username,
password: config.test_password
}
);
var postOptions = {
host: 'login.microsoftonline.com',
port: 443,
path: '/common/oauth2/token',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
var postRequest = https.request(postOptions, function (res) {
var data = '';
res.setEncoding('utf8');
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
const response = JSON.parse(data);
accessToken = response.access_token;
scope = response.scope;
done();
});
});
postRequest.on('error', function (e) {
console.log('Error: ' + e.message);
done(e);
});
postRequest.write(postData);
postRequest.end();
}
before( // eslint-disable-line no-undef
function (done) {
getAccessToken(done);
}
);
it('Checking that the sample can send an email',
function (done) {
var postBody = emailer.generateMailBody(config.test_name, config.test_username);
graphHelper.postSendMail(
accessToken, scope,
JSON.stringify(postBody),
function (error) {
assert(error === null, `The sample failed to send an email: ${error}`);
done();
});
}
);
});
I can't get the offline.html page to display. I keep getting the The FetchEvent for "https://my-domain.com" resulted in a network error response: a redirected response was used for a request whose redirect mode is not "follow".
Here's the snippet of my service-worker.js which should return the offline.html when the network is unavailable.
self.addEventListener('fetch', function(event) {
if (event.request.mode === 'navigate' || (event.request.method === 'GET' && event.request.headers.get('accept').includes('text/html'))) {
if(event.request.url.includes("my-domain.com")){
console.log(event.request);
event.respondWith(
caches.match(event.request).then(function(resp) {
return resp || fetch(event.request).then(function(response) {
let responseClone = response.clone();
caches.open(CACHE_NAME).then(function(cache) {
cache.put(event.request, responseClone);
});
return response;
});
}).catch(function() {
return caches.match("/offline.html");
})
);
}
}
});
Below is the console.log of my network request (page refresh when offline)
Request {method: "GET", url: "https://my-domain.com", headers: Headers, destination: "unknown", referrer: "", …}
bodyUsed:false
cache:"no-cache"
credentials:"include"
destination:"unknown"
headers:Headers {}
integrity:""
keepalive:false
method:"GET"
mode:"navigate"
redirect:"manual"
referrer:""
referrerPolicy:"no-referrer-when-downgrade"
signal:AbortSignal {aborted: false, onabort: null}
url:"https://my-domain.com"
__proto__:Request
I got this working / found the fix. It was related to a redirected response security issue in the browser. From the Chromium Bugs Blog, Response.redirected and a new security restriction.
Solution: To avoid this failure, you have 2 options.
You can either change the install event handler to store the response generated from res.body:
self.oninstall = evt => {
evt.waitUntil(
caches.open('cache_name')
.then(cache => {
return fetch('/')
.then(response => cache.put('/', new Response(response.body));
}));
};
Or change both handlers to store the non-redirected response by setting redirect mode to ‘manual’:
self.oninstall = function (evt) {
evt.waitUntil(caches.open('cache_name').then(function (cache) {
return Promise.all(['/', '/index.html'].map(function (url) {
return fetch(new Request(url, { redirect: 'manual' })).then(function (res) {
return cache.put(url, res);
});
}));
}));
};
self.onfetch = function (evt) {
var url = new URL(evt.request.url);
if (url.pathname != '/' && url.pathname != '/index.html') return;
evt.respondWith(caches.match(evt.request, { cacheName: 'cache_name' }));
};