-
Notifications
You must be signed in to change notification settings - Fork 38
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Replace PhantomJS with Puppeteer #46
base: master
Are you sure you want to change the base?
Changes from 1 commit
12373a4
6720fd7
08c161e
3cfae93
53c5ee4
0fc17a3
74864f4
f31a09b
3bfddc5
14a0354
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
node_modules | ||
yarn.lock |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
package-lock=false |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,4 +3,3 @@ language: node_js | |
node_js: | ||
- '8' | ||
- '6' | ||
- '4' |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,85 +1,48 @@ | ||
'use strict'; | ||
const fs = require('fs'); | ||
const path = require('path'); | ||
const urlMod = require('url'); | ||
const base64Stream = require('base64-stream'); | ||
const parseCookiePhantomjs = require('parse-cookie-phantomjs'); | ||
const phantomBridge = require('phantom-bridge'); | ||
const byline = require('byline'); | ||
|
||
const handleCookies = (cookies, url) => { | ||
const parsedUrl = urlMod.parse(url); | ||
|
||
return (cookies || []).map(x => { | ||
const ret = typeof x === 'string' ? parseCookiePhantomjs(x) : x; | ||
|
||
if (!ret.domain) { | ||
ret.domain = parsedUrl.hostname; | ||
} | ||
|
||
if (!ret.path) { | ||
ret.path = parsedUrl.path; | ||
} | ||
|
||
return ret; | ||
}); | ||
}; | ||
const parseResolution = require('parse-resolution'); | ||
const puppeteer = require('puppeteer'); | ||
const Screenshot = require('./screenshot'); | ||
|
||
module.exports = (url, size, opts) => { | ||
const {width, height} = parseResolution(size); | ||
|
||
opts = Object.assign({ | ||
delay: 0, | ||
cookies: [], | ||
format: 'png', | ||
fullPage: true, | ||
hide: [], | ||
scale: 1, | ||
format: 'png' | ||
viewport: {} | ||
}, opts); | ||
|
||
const args = Object.assign(opts, { | ||
url, | ||
width: size.split(/x/i)[0] * opts.scale, | ||
height: size.split(/x/i)[1] * opts.scale, | ||
cookies: handleCookies(opts.cookies, url), | ||
format: opts.format === 'jpg' ? 'jpeg' : opts.format, | ||
css: /\.css$/.test(opts.css) ? fs.readFileSync(opts.css, 'utf8') : opts.css, | ||
script: /\.js$/.test(opts.script) ? fs.readFileSync(opts.script, 'utf8') : opts.script | ||
}); | ||
|
||
const cp = phantomBridge(path.join(__dirname, 'stream.js'), [ | ||
'--ignore-ssl-errors=true', | ||
'--local-to-remote-url-access=true', | ||
'--ssl-protocol=any', | ||
JSON.stringify(args) | ||
]); | ||
|
||
const stream = base64Stream.decode(); | ||
|
||
process.stderr.setMaxListeners(0); | ||
|
||
cp.stderr.setEncoding('utf8'); | ||
cp.stdout.pipe(stream); | ||
|
||
byline(cp.stderr).on('data', data => { | ||
data = data.trim(); | ||
|
||
if (/ phantomjs\[/.test(data)) { | ||
return; | ||
} | ||
|
||
if (/http:\/\/requirejs.org\/docs\/errors.html#mismatch/.test(data)) { | ||
return; | ||
} | ||
|
||
if (data.startsWith('WARN: ')) { | ||
stream.emit('warning', data.replace(/^WARN: /, '')); | ||
stream.emit('warn', data.replace(/^WARN: /, '')); // TODO: deprecate this event in v5 | ||
return; | ||
} | ||
opts.type = opts.format === 'jpg' ? 'jpeg' : opts.format; | ||
|
||
if (data.length > 0) { | ||
const err = new Error(data); | ||
err.noStack = true; | ||
cp.stdout.unpipe(stream); | ||
stream.emit('error', err); | ||
} | ||
opts.viewport = Object.assign({}, opts.viewport, { | ||
width, | ||
height | ||
}); | ||
|
||
return stream; | ||
if (opts.crop) { | ||
opts.fullPage = false; | ||
} | ||
|
||
if (opts.scale !== 1) { | ||
opts.viewport.deviceScaleFactor = opts.scale; | ||
} | ||
|
||
if (opts.transparent) { | ||
opts.omitBackground = true; | ||
} | ||
|
||
return puppeteer.launch() | ||
.then(browser => browser.newPage() | ||
.then(page => new Screenshot(browser, page))) | ||
.then(Screenshot => Screenshot.authenticate(opts.username, opts.password) | ||
.then(() => Screenshot.setCookie(opts.cookies)) | ||
.then(() => Screenshot.setHeaders(opts.headers)) | ||
.then(() => Screenshot.setUserAgent(opts.userAgent)) | ||
.then(() => Screenshot.open(url, opts)) | ||
.then(() => Screenshot.hideElements(opts.hide)) | ||
.then(() => Screenshot.getRect(opts.selector)) | ||
.then(clip => Screenshot.screenshot(Object.assign(opts, clip)))); | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,21 +1,9 @@ | ||
The MIT License (MIT) | ||
MIT License | ||
|
||
Copyright (c) Kevin Mårtensson <[email protected]> | ||
Copyright (c) Kevin Mårtensson <[email protected]> (github.com/kevva) | ||
|
||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: | ||
|
||
The above copyright notice and this permission notice shall be included in | ||
all copies or substantial portions of the Software. | ||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. | ||
|
||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
THE SOFTWARE. | ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,67 +1,49 @@ | ||
{ | ||
"name": "screenshot-stream", | ||
"version": "4.2.0", | ||
"description": "Capture screenshot of a website and return it as a stream", | ||
"license": "MIT", | ||
"repository": "kevva/screenshot-stream", | ||
"author": { | ||
"name": "Kevin Mårtensson", | ||
"email": "[email protected]", | ||
"url": "github.com/kevva" | ||
}, | ||
"engines": { | ||
"node": ">=4" | ||
}, | ||
"scripts": { | ||
"test": "xo && ava" | ||
}, | ||
"files": [ | ||
"index.js", | ||
"stream.js" | ||
], | ||
"keywords": [ | ||
"image", | ||
"page", | ||
"phantom", | ||
"phantomjs", | ||
"resolution", | ||
"screen", | ||
"screenshot", | ||
"size", | ||
"stream", | ||
"url" | ||
], | ||
"dependencies": { | ||
"base64-stream": "^0.1.2", | ||
"byline": "^4.2.1", | ||
"object-assign": "^4.0.1", | ||
"parse-cookie-phantomjs": "^1.0.0", | ||
"phantom-bridge": "^2.0.0" | ||
}, | ||
"devDependencies": { | ||
"ava": "*", | ||
"cookie": "^0.3.1", | ||
"get-port": "^3.1.0", | ||
"get-stream": "^3.0.0", | ||
"image-size": "^0.5.4", | ||
"is-jpg": "^1.0.0", | ||
"is-png": "^1.0.0", | ||
"pify": "^3.0.0", | ||
"png-js": "^0.1.1", | ||
"xo": "*" | ||
}, | ||
"xo": { | ||
"esnext": true, | ||
"overrides": [ | ||
{ | ||
"files": "stream.js", | ||
"esnext": false, | ||
"rules": { | ||
"import/no-extraneous-dependencies": 0, | ||
"import/no-unresolved": 0, | ||
"no-multi-assign": 0 | ||
} | ||
} | ||
] | ||
} | ||
"name": "screenshot-stream", | ||
"version": "4.2.0", | ||
"description": "Capture screenshot of a website and return it as a stream", | ||
"license": "MIT", | ||
"repository": "kevva/screenshot-stream", | ||
"author": { | ||
"name": "Kevin Mårtensson", | ||
"email": "[email protected]", | ||
"url": "github.com/kevva" | ||
}, | ||
"engines": { | ||
"node": ">=6.4" | ||
}, | ||
"scripts": { | ||
"test": "xo && ava" | ||
}, | ||
"files": [ | ||
"index.js" | ||
], | ||
"keywords": [ | ||
"image", | ||
"page", | ||
"phantom", | ||
"phantomjs", | ||
"resolution", | ||
"screen", | ||
"screenshot", | ||
"size", | ||
"stream", | ||
"url" | ||
], | ||
"dependencies": { | ||
"file-url": "^2.0.2", | ||
"is-url-superb": "^2.0.0", | ||
"parse-resolution": "^1.0.0", | ||
"puppeteer": "^0.11.0", | ||
"tough-cookie": "^2.3.3" | ||
}, | ||
"devDependencies": { | ||
"ava": "*", | ||
"image-size": "^0.6.1", | ||
"is-jpg": "^1.0.0", | ||
"is-png": "^1.1.0", | ||
"pify": "^3.0.0", | ||
"png-js": "^0.1.1", | ||
"xo": "*" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,9 +16,9 @@ $ npm install --save screenshot-stream | |
const fs = require('fs'); | ||
const screenshot = require('screenshot-stream'); | ||
|
||
const stream = screenshot('http://google.com', '1024x768', {crop: true}); | ||
|
||
stream.pipe(fs.createWriteStream('google.com-1024x768.png')); | ||
screenshot('http://google.com', '1024x768', {crop: true}).then(data => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if we should just make the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yup, let's do that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 👌 |
||
fs.writeFileSync('google.com-1024x768.png'); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
}); | ||
``` | ||
|
||
|
||
|
@@ -51,38 +51,19 @@ Default: `false` | |
|
||
Crop to the set height. | ||
|
||
##### delay | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why was this removed? We could use delay to manually delay the capturing after opening the url, no? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was added as a hack to wait for elements or other stuff to finish. We do this automatically now. Although there probably is some option for this in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It could be used by users as well to make sure an animation has ended. If you know a certain animation takes 2 seconds, someone could use I never used it to be honest, but just finding a use case :). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would use this. For example, on my website, I load the "latest blog post" widget async, so it's loaded a second after There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we should switch to a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, good idea, but I also like to keep the delay. Sometimes I just want a screenshot of a random website and ensure it’s fully loaded first. |
||
|
||
Type: `number` *(seconds)*<br> | ||
Default: `0` | ||
|
||
Delay capturing the screenshot. Useful when the site does things after load that you want to capture. | ||
|
||
##### timeout | ||
|
||
Type: `number` *(seconds)*<br> | ||
Default: `60` | ||
|
||
Number of seconds after which PhantomJS aborts the request. | ||
Number of seconds after which Google Chrome aborts the request. | ||
|
||
##### selector | ||
|
||
Type: `string` | ||
|
||
Capture a specific DOM element. | ||
|
||
##### css | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Weird that we don't have a replacement for this... Should we open a new issue in Puppeteer? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, open an issue on Puppeteer. In the meantime, we can use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can probably do this in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Filed an issue puppeteer/puppeteer#892. Just tested things out, we could do something like this with const headHandle = await page.$('head');
const result = await page.evaluate(head => {
head.innerHTML += '<style>body { color: red; }</style>';
}, headHandle); If There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. PR sent puppeteer/puppeteer#947 |
||
|
||
Type: `string` | ||
|
||
Apply custom CSS to the webpage. Specify some CSS or the path to a CSS file. | ||
|
||
##### script | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We could use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yup. In conjunction with |
||
|
||
Type: `string` | ||
|
||
Apply custom JavaScript to the webpage. Specify some JavaScript or the path to a file. | ||
|
||
##### hide | ||
|
||
Type: `Array` | ||
|
@@ -141,18 +122,6 @@ Default: `false` | |
|
||
Set background color to `transparent` instead of `white` if no background is set. | ||
|
||
#### .on('error', callback) | ||
|
||
Type: `Function` | ||
|
||
PhantomJS errors. | ||
|
||
#### .on('warning', callback) | ||
|
||
Type: `Function` | ||
|
||
Warnings with for example page errors. | ||
|
||
|
||
## CLI | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Add an extra
)
at the end