498 lines
16 KiB
JavaScript
498 lines
16 KiB
JavaScript
const fs = require('fs');
|
|
const URL = require('url').URL;
|
|
const URLParse = require('url').parse;
|
|
|
|
if (typeof global.ReadableStream === 'undefined') {
|
|
const {ReadableStream} = require("stream/web");
|
|
global.ReadableStream = ReadableStream;
|
|
}
|
|
|
|
const [, , ...args] = process.argv;
|
|
|
|
/**
|
|
* There are two ways for Browsershot to communicate with puppeteer:
|
|
* - By giving a options JSON dump as an argument
|
|
* - Or by providing a temporary file with the options JSON dump,
|
|
* the path to this file is then given as an argument with the flag -f
|
|
*/
|
|
const request = args[0].startsWith('-f ')
|
|
? JSON.parse(fs.readFileSync(new URL(args[0].substring(3))))
|
|
: JSON.parse(args[0]);
|
|
|
|
const requestsList = [];
|
|
|
|
const redirectHistory = [];
|
|
|
|
const consoleMessages = [];
|
|
|
|
const failedRequests = [];
|
|
|
|
const pageErrors = [];
|
|
|
|
const getOutput = async (request, page = null) => {
|
|
let output = {
|
|
requestsList,
|
|
consoleMessages,
|
|
failedRequests,
|
|
redirectHistory,
|
|
pageErrors,
|
|
};
|
|
|
|
if (
|
|
![
|
|
'requestsList',
|
|
'consoleMessages',
|
|
'failedRequests',
|
|
'redirectHistory',
|
|
'pageErrors',
|
|
].includes(request.action) &&
|
|
page
|
|
) {
|
|
if (request.action == 'evaluate') {
|
|
output.result = await page.evaluate(request.options.pageFunction);
|
|
} else {
|
|
const result = await page[request.action](request.options);
|
|
|
|
// Ignore output result when saving to a file
|
|
output.result = request.options.path
|
|
? ''
|
|
: (result instanceof Uint8Array ? Buffer.from(result) : result).toString('base64');
|
|
}
|
|
}
|
|
|
|
if (page) {
|
|
return JSON.stringify(output);
|
|
}
|
|
|
|
// this will allow adding additional error info (only reach this point when there's an exception)
|
|
return output;
|
|
};
|
|
|
|
const callChrome = async pup => {
|
|
let browser;
|
|
let page;
|
|
let remoteInstance;
|
|
const puppet = (pup || require('puppeteer'));
|
|
const options = request.options ?? {};
|
|
|
|
const closeBrowser = async () => {
|
|
if (!browser) return;
|
|
if (remoteInstance && page) {
|
|
await page.close();
|
|
}
|
|
await (remoteInstance ? browser.disconnect() : browser.close());
|
|
};
|
|
|
|
try {
|
|
if (options.remoteInstanceUrl || options.browserWSEndpoint) {
|
|
// default options
|
|
let connectOptions = {
|
|
acceptInsecureCerts: options.acceptInsecureCerts
|
|
};
|
|
|
|
// choose only one method to connect to the browser instance
|
|
if (options.remoteInstanceUrl) {
|
|
connectOptions.browserURL = options.remoteInstanceUrl;
|
|
} else if (options.browserWSEndpoint) {
|
|
connectOptions.browserWSEndpoint = options.browserWSEndpoint;
|
|
}
|
|
|
|
try {
|
|
browser = await puppet.connect(connectOptions);
|
|
|
|
remoteInstance = true;
|
|
} catch (exception) {
|
|
|
|
if (options.throwOnRemoteConnectionError) {
|
|
console.error(exception.toString());
|
|
process.exit(4);
|
|
}
|
|
|
|
/** fallback to launching a chromium instance */
|
|
}
|
|
}
|
|
|
|
if (!browser) {
|
|
browser = await puppet.launch({
|
|
headless: options.newHeadless ? true : 'shell',
|
|
acceptInsecureCerts: options.acceptInsecureCerts,
|
|
executablePath: options.executablePath,
|
|
args: options.args || [],
|
|
pipe: options.pipe || false,
|
|
env: {
|
|
...(options.env || {}),
|
|
...process.env
|
|
},
|
|
protocolTimeout: options.protocolTimeout ?? 30000,
|
|
});
|
|
}
|
|
|
|
page = await browser.newPage();
|
|
|
|
if (options.disableJavascript) {
|
|
await page.setJavaScriptEnabled(false);
|
|
}
|
|
|
|
const contentUrl = options.contentUrl;
|
|
const parsedContentUrl = contentUrl ? contentUrl.replace(/\/$/, "") : undefined;
|
|
let pageContent;
|
|
|
|
if (contentUrl) {
|
|
pageContent = fs.readFileSync(request.url.replace('file://', ''));
|
|
request.url = contentUrl;
|
|
}
|
|
|
|
// The request listener captures URLs unconditionally (backing triggeredRequests()) and,
|
|
// when interception is enabled, walks a set of rules that may abort/respond/continue
|
|
// the request. setRequestInterception(true) is only enabled when needed. Unconditional
|
|
// interception routes every request through the CDP, introducing timing anomalies that
|
|
// anti-bot systems can detect.
|
|
const hasItems = (value) => Array.isArray(value) && value.length > 0;
|
|
const hasKeys = (value) => value && typeof value === 'object' && Object.keys(value).length > 0;
|
|
|
|
const needsInterception = !!(
|
|
options.disableImages ||
|
|
hasItems(options.blockDomains) ||
|
|
hasItems(options.blockUrls) ||
|
|
options.disableRedirects ||
|
|
hasKeys(options.extraNavigationHTTPHeaders) ||
|
|
pageContent ||
|
|
request.postParams
|
|
);
|
|
|
|
const captureUrl = (req) => {
|
|
if (!options.disableCaptureURLS) {
|
|
requestsList.push({ url: req.url() });
|
|
}
|
|
};
|
|
|
|
// Each rule below returns true if it terminated the request (via abort/respond/continue).
|
|
// The listener walks them top-to-bottom and stops at the first match.
|
|
const blockImage = (req) => {
|
|
if (!options.disableImages || req.resourceType() !== 'image') return false;
|
|
req.abort();
|
|
return true;
|
|
};
|
|
|
|
const blockDomain = (req) => {
|
|
if (!hasItems(options.blockDomains)) return false;
|
|
const hostname = URLParse(req.url()).hostname;
|
|
if (!options.blockDomains.includes(hostname)) return false;
|
|
req.abort();
|
|
return true;
|
|
};
|
|
|
|
const blockUrl = (req) => {
|
|
if (!hasItems(options.blockUrls)) return false;
|
|
const matches = options.blockUrls.some(fragment => req.url().indexOf(fragment) >= 0);
|
|
if (!matches) return false;
|
|
req.abort();
|
|
return true;
|
|
};
|
|
|
|
const blockRedirect = (req) => {
|
|
if (!options.disableRedirects) return false;
|
|
if (!req.isNavigationRequest() || !req.redirectChain().length) return false;
|
|
req.abort();
|
|
return true;
|
|
};
|
|
|
|
const buildHeaders = (req) => {
|
|
const headers = req.headers();
|
|
if (hasKeys(options.extraNavigationHTTPHeaders) && req.isNavigationRequest()) {
|
|
return { ...headers, ...options.extraNavigationHTTPHeaders };
|
|
}
|
|
return headers;
|
|
};
|
|
|
|
const respondWithPageContent = (req, headers) => {
|
|
if (!pageContent) return false;
|
|
if (req.url().replace(/\/$/, "") !== parsedContentUrl) return false;
|
|
req.respond({ headers, body: pageContent });
|
|
return true;
|
|
};
|
|
|
|
// Note: postParams intentionally uses raw req.headers() rather than the merged
|
|
// `headers`. extraNavigationHTTPHeaders does not apply to postParams (pre-existing).
|
|
const continueWithPostParams = (req) => {
|
|
if (!request.postParams) return false;
|
|
const queryString = Object.entries(request.postParams)
|
|
.map(([key, value]) => `${key}=${value}`)
|
|
.join('&');
|
|
req.continue({
|
|
method: "POST",
|
|
postData: queryString,
|
|
headers: {
|
|
...req.headers(),
|
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
},
|
|
});
|
|
return true;
|
|
};
|
|
|
|
if (needsInterception) {
|
|
await page.setRequestInterception(true);
|
|
}
|
|
|
|
page.on('request', (req) => {
|
|
captureUrl(req);
|
|
|
|
if (!needsInterception) return;
|
|
|
|
if (blockImage(req)) return;
|
|
if (blockDomain(req)) return;
|
|
if (blockUrl(req)) return;
|
|
if (blockRedirect(req)) return;
|
|
|
|
const headers = buildHeaders(req);
|
|
|
|
if (respondWithPageContent(req, headers)) return;
|
|
if (continueWithPostParams(req)) return;
|
|
|
|
req.continue({ headers });
|
|
});
|
|
|
|
page.on('console', (message) =>
|
|
consoleMessages.push({
|
|
type: message.type(),
|
|
message: message.text(),
|
|
location: message.location(),
|
|
stackTrace: message.stackTrace(),
|
|
})
|
|
);
|
|
|
|
page.on('pageerror', (msg) => {
|
|
pageErrors.push({
|
|
name: msg?.name || 'unknown error',
|
|
message: msg?.message || msg?.toString() || 'null'
|
|
});
|
|
});
|
|
|
|
page.on('response', function (response) {
|
|
const frame = response.request().frame();
|
|
if (response.request().isNavigationRequest() && frame && frame.parentFrame() === null) {
|
|
redirectHistory.push({
|
|
url: response.request().url(),
|
|
status: response.status(),
|
|
reason: response.statusText(),
|
|
headers: response.headers()
|
|
})
|
|
}
|
|
|
|
if (response.status() >= 200 && response.status() <= 399) {
|
|
return;
|
|
}
|
|
|
|
failedRequests.push({
|
|
status: response.status(),
|
|
url: response.url(),
|
|
});
|
|
});
|
|
|
|
if (options.dismissDialogs) {
|
|
page.on('dialog', async dialog => {
|
|
await dialog.dismiss();
|
|
});
|
|
}
|
|
|
|
if (options.userAgent) {
|
|
await page.setUserAgent(options.userAgent);
|
|
}
|
|
|
|
if (options.device) {
|
|
const devices = puppet.KnownDevices;
|
|
const device = devices[options.device];
|
|
await page.emulate(device);
|
|
}
|
|
|
|
if (options.emulateMedia) {
|
|
await page.emulateMediaType(options.emulateMedia);
|
|
}
|
|
|
|
if (options.emulateMediaFeatures) {
|
|
await page.emulateMediaFeatures(JSON.parse(options.emulateMediaFeatures));
|
|
}
|
|
|
|
if (options.viewport) {
|
|
await page.setViewport(options.viewport);
|
|
}
|
|
|
|
if (options.extraHTTPHeaders) {
|
|
await page.setExtraHTTPHeaders(options.extraHTTPHeaders);
|
|
}
|
|
|
|
if (options.authentication) {
|
|
await page.authenticate(options.authentication);
|
|
}
|
|
|
|
if (options.cookies) {
|
|
await page.setCookie(...options.cookies);
|
|
}
|
|
|
|
if (options.timeout) {
|
|
await page.setDefaultNavigationTimeout(options.timeout);
|
|
}
|
|
|
|
const requestOptions = {};
|
|
|
|
if (options.networkIdleTimeout) {
|
|
requestOptions.waitUntil = 'networkidle';
|
|
requestOptions.networkIdleTimeout = options.networkIdleTimeout;
|
|
} else if (options.waitUntil) {
|
|
requestOptions.waitUntil = options.waitUntil;
|
|
}
|
|
|
|
const response = await page.goto(request.url, requestOptions);
|
|
|
|
if (options.preventUnsuccessfulResponse) {
|
|
const status = response.status()
|
|
|
|
if (status >= 400 && status < 600) {
|
|
throw {type: "UnsuccessfulResponse", status};
|
|
}
|
|
}
|
|
|
|
if (options.disableImages) {
|
|
await page.evaluate(() => {
|
|
let images = document.getElementsByTagName('img');
|
|
while (images.length > 0) {
|
|
images[0].parentNode.removeChild(images[0]);
|
|
}
|
|
});
|
|
}
|
|
|
|
if (options.types) {
|
|
for (const typeOptions of options.types) {
|
|
await page.type(typeOptions.selector, typeOptions.text, {
|
|
delay: typeOptions.delay,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (options.selects) {
|
|
for (const selectOptions of options.selects) {
|
|
await page.select(selectOptions.selector, selectOptions.value);
|
|
}
|
|
}
|
|
|
|
if (options.clicks) {
|
|
for (const clickOptions of options.clicks) {
|
|
await page.click(clickOptions.selector, {
|
|
button: clickOptions.button,
|
|
clickCount: clickOptions.clickCount,
|
|
delay: clickOptions.delay,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (options.locatorClicks) {
|
|
for (const clickOptions of options.locatorClicks) {
|
|
try {
|
|
await page.locator(clickOptions.selector).click({
|
|
button: clickOptions.button,
|
|
clickCount: clickOptions.clickCount,
|
|
delay: clickOptions.delay,
|
|
});
|
|
} catch (error) {
|
|
console.error('Timeout error:', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (options.addStyleTag) {
|
|
await page.addStyleTag(JSON.parse(options.addStyleTag));
|
|
}
|
|
|
|
if (options.addScriptTag) {
|
|
await page.addScriptTag(JSON.parse(options.addScriptTag));
|
|
}
|
|
|
|
if (options.delay) {
|
|
await new Promise(r => setTimeout(r, options.delay));
|
|
}
|
|
|
|
if (options.initialPageNumber) {
|
|
await page.evaluate((initialPageNumber) => {
|
|
window.pageStart = initialPageNumber;
|
|
|
|
const style = document.createElement('style');
|
|
style.type = 'text/css';
|
|
style.innerHTML = '.empty-page { page-break-after: always; visibility: hidden; }';
|
|
document.getElementsByTagName('head')[0].appendChild(style);
|
|
|
|
const emptyPages = Array.from({length: window.pageStart}).map(() => {
|
|
const emptyPage = document.createElement('div');
|
|
emptyPage.className = "empty-page";
|
|
emptyPage.textContent = "empty";
|
|
return emptyPage;
|
|
});
|
|
document.body.prepend(...emptyPages);
|
|
}, options.initialPageNumber);
|
|
}
|
|
|
|
if (options.function) {
|
|
const functionOptions = {
|
|
polling: options.functionPolling,
|
|
timeout: options.functionTimeout || options.timeout
|
|
};
|
|
await page.waitForFunction(options.function, functionOptions);
|
|
}
|
|
|
|
if (options.waitForSelector) {
|
|
await page.waitForSelector(options.waitForSelector, options.waitForSelectorOptions ?? undefined);
|
|
}
|
|
|
|
if (options.selector) {
|
|
let element;
|
|
const index = options.selectorIndex || 0;
|
|
if (index) {
|
|
element = await page.$$(options.selector);
|
|
if (!element.length || typeof element[index] === 'undefined') {
|
|
element = null;
|
|
} else {
|
|
element = element[index];
|
|
}
|
|
} else {
|
|
element = await page.$(options.selector);
|
|
}
|
|
if (element === null) {
|
|
throw {type: 'ElementNotFound'};
|
|
}
|
|
|
|
options.clip = await element.boundingBox();
|
|
}
|
|
|
|
console.log(await getOutput(request, page));
|
|
|
|
await closeBrowser();
|
|
} catch (exception) {
|
|
await closeBrowser();
|
|
|
|
const output = await getOutput(request);
|
|
|
|
if (exception.type === 'UnsuccessfulResponse') {
|
|
output.exception = exception.toString();
|
|
console.error(exception.status);
|
|
console.log(JSON.stringify(output));
|
|
process.exit(3);
|
|
}
|
|
|
|
output.exception = exception.toString();
|
|
|
|
console.error(exception);
|
|
console.log(JSON.stringify(output));
|
|
|
|
if (exception.type === 'ElementNotFound') {
|
|
process.exit(2);
|
|
}
|
|
|
|
process.exit(1);
|
|
}
|
|
};
|
|
|
|
if (require.main === module) {
|
|
callChrome();
|
|
}
|
|
|
|
exports.callChrome = callChrome;
|