Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor to make chrome happy #2185

Closed
wants to merge 6 commits into from
Closed

Refactor to make chrome happy #2185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
62b55df
Refactor to make chrome happy
dgtlmoon Feb 14, 2024
b045a72
More JS tweaks
dgtlmoon Feb 14, 2024
71a4613
add handy comment
dgtlmoon Feb 14, 2024
491715b
more tweaks
dgtlmoon Feb 14, 2024
72c4d35
Merge branch 'master' into restock-visualselector-refactor
dgtlmoon Feb 20, 2024
8a2afaa
Merge branch 'master' into restock-visualselector-refactor
dgtlmoon Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
159 changes: 75 additions & 84 deletions changedetectionio/content_fetchers/res/puppeteer_fetch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ module.exports = async ({page, context}) => {

await page.setBypassCSP(true)
await page.setExtraHTTPHeaders(req_headers);
var total_size = 0;

if (user_agent) {
await page.setUserAgent(user_agent);
Expand All @@ -42,102 +43,89 @@ module.exports = async ({page, context}) => {
height: 768,
deviceScaleFactor: 1,
});

await page.setRequestInterception(true);
if (disk_cache_dir) {
console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
await page.setCacheEnabled(false);


await page.evaluateOnNewDocument('navigator.serviceWorker.register = () => { console.warn("Service Worker registration blocked by Playwright")}');

await page.evaluateOnNewDocument(`

const toBlob = HTMLCanvasElement.prototype.toBlob;
const toDataURL = HTMLCanvasElement.prototype.toDataURL;

HTMLCanvasElement.prototype.manipulate = function() {
console.warn("ma");
const {width, height} = this;
const context = this.getContext('2d');
var dt = new Date();

const shift = {
'r': dt.getDay()-3,
'g': dt.getDay()-3,
'b': dt.getDay()-3
};
console.log(shift);
const matt = context.getImageData(0, 0, width, height);
for (let i = 0; i < height; i += Math.max(1, parseInt(height / 10))) {
for (let j = 0; j < width; j += Math.max(1, parseInt(width / 10))) {
const n = ((i * (width * 4)) + (j * 4));
matt.data[n + 0] = matt.data[n + 0] + shift.r;
matt.data[n + 1] = matt.data[n + 1] + shift.g;
matt.data[n + 2] = matt.data[n + 2] + shift.b;
}
}
const fs = require('fs');
const crypto = require('crypto');
context.putImageData(matt, 0, 0);
};

function file_is_expired(file_path) {
if (!fs.existsSync(file_path)) {
return true;
Object.defineProperty(HTMLCanvasElement.prototype, 'toBlob', {
value: function() {
console.warn("toblob");
if (true) {
try {
this.manipulate();
}
var stats = fs.statSync(file_path);
const now_date = new Date();
const expire_seconds = 300;
if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
console.log("CACHE EXPIRED: " + file_path);
return true;
catch(e) {
console.warn('manipulation failed', e);
}
return false;

}
return toBlob.apply(this, arguments);
}

page.on('request', async (request) => {
// General blocking of requests that waste traffic
if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();

if (disk_cache_dir) {
const url = request.url();
const key = crypto.createHash('md5').update(url).digest("hex");
const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';

// https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js

if (fs.existsSync(dir_path + key)) {
console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
const cached_data = fs.readFileSync(dir_path + key);
// @todo headers can come from dir_path+key+".meta" json file
request.respond({
status: 200,
//contentType: 'text/html', //@todo
body: cached_data
});
return;
}
});
Object.defineProperty(HTMLCanvasElement.prototype, 'toDataURL', {
value: function() {
console.warn("todata");
if (true) {
try {
this.manipulate();
}
request.continue();
});
catch(e) {
console.warn('manipulation failed', e);
}
}
return toDataURL.apply(this, arguments);
}
});


if (disk_cache_dir) {
page.on('response', async (response) => {
const url = response.url();
// Basic filtering for sane responses
if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
return;
}
if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
console.log("Skipping (no_cache_list) - " + url);
return;
}
if (url.toLowerCase().includes('data:')) {
console.log("Skipping (embedded-data) - " + url);
return;
}
response.buffer().then(buffer => {
if (buffer.length > 100) {
console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());

const key = crypto.createHash('md5').update(url).digest("hex");
const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';

if (!fs.existsSync(dir_path)) {
fs.mkdirSync(dir_path, {recursive: true})
}

if (fs.existsSync(dir_path + key)) {
if (file_is_expired(dir_path + key)) {
fs.writeFileSync(dir_path + key, buffer);
}
} else {
fs.writeFileSync(dir_path + key, buffer);
}
}
});
});
}
Object.defineProperty(navigator, 'webdriver', {get: () => false});
`)

await page.emulateTimezone('America/Chicago');

const r = await page.goto(url, {
waitUntil: 'load'
var r = await page.goto(url, {
waitUntil: 'load', timeout: 0
});

await page.waitForTimeout(1000);
// https://github.com/puppeteer/puppeteer/issues/2479#issuecomment-408263504
if (r === null) {
r = await page.waitForResponse(() => true);
}

await page.waitForTimeout(4000);
await page.waitForTimeout(extra_wait_ms);


if (execute_js) {
await page.evaluate(execute_js);
await page.waitForTimeout(200);
Expand Down Expand Up @@ -176,15 +164,18 @@ module.exports = async ({page, context}) => {
}

var html = await page.content();
page.close();

return {
data: {
'content': html,
'headers': r.headers(),
'instock_data': instock_data,
'screenshot': b64s,
'status_code': r.status(),
'xpath_data': xpath_data
'xpath_data': xpath_data,
'total_size': total_size
},
type: 'application/json',
};
};
};
2 changes: 2 additions & 0 deletions changedetectionio/content_fetchers/res/stock-not-in-stock.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ function isItemInStock() {
const element = elementsToScan[i];
// outside the 'fold' or some weird text in the heading area
// .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden

// Should be in the "above the fold" plus about 150px
if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
continue
}
Expand Down