Use slightly customised exec allocator for the Headless Chrome execution (#3)

* Use slightly customised exec allocator for the Headless Chrome execution

* Minor code tweaks

* Headless chrome task tweaks

* Bumped chromedp, added ability to provide a custom user agent

* CHANGELOG

* Version bump

* Pass ua option in the CLI mode

Author: smeshkov

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## 0.14.0
+- bumped Go to 1.22;
+- bumped `github.com/chromedp/chromedp` to v0.9.2;
+- new argument `-ua` to provide custome User Agent for the HTTP headless calls.
+
 ## 0.13.0
  - bumped Go to 1.20;
  - bumped `github.com/chromedp/chromedp` to v0.9.1;

Makefile

@@ -1,6 +1,6 @@
 .PHONY: deps clean build
 
-TAG=0.13.0
+TAG=0.14.0
 BINARY=inout
 DIST_DIR=_dist
 OS=darwin

cmd/cli/cli.go

@@ -17,6 +17,7 @@ var (
 	ready   = flag.String("r", "", "DOM CSS query, waits until element available, returns the whole HTML document")
 	until   = flag.Duration("u", 0, "duration to wait before getting HTML contents, handy for SPAs, because they keep loading in browsers for some time")
 	img     = flag.String("i", "", "enables capturing screenshot in the provided path")
+	ua      = flag.String("ua", "", "provide a custom user agent for headless HTTP calls")
 	timeout = flag.Duration("t", 5*time.Second, "timeout for the whole fetch, e.g. \"-d 1s\" sets timeout to 1 second")
 	verbose = flag.Bool("v", false, "\"-v\" enables verbose mode")
 )
@@ -37,7 +38,9 @@ func main() {
 		inout.WaitUntil(*until),
 		inout.Screenshot(len(*img) > 0),
 		inout.Timeout(ttl),
-		inout.Verbose(*verbose))
+		inout.Verbose(*verbose),
+		inout.UserAgent(*ua),
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to create reader: %v\n", err)
 		os.Exit(1)

go.mod

@@ -1,10 +1,10 @@
 module github.com/zoomio/inout
 
-go 1.20
+go 1.22
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9
-	github.com/chromedp/chromedp v0.9.1
+	github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89
+	github.com/chromedp/chromedp v0.9.2
 	github.com/stretchr/testify v1.8.4
 )
 
@@ -13,7 +13,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/gobwas/ws v1.2.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -1,7 +1,7 @@
-github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY=
-github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
-github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA=
-github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ=
+github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 h1:aPflPkRFkVwbW6dmcVqfgwp1i+UWGFH6VgR1Jim5Ygc=
+github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.2 h1:dKtNz4kApb06KuSXoTQIyUC2TrA0fhGDwNZf3bcgfKw=
+github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
 github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
 github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -10,8 +10,8 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
-github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
+github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
+github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
@@ -29,7 +29,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

headless.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/dom"
+	"github.com/chromedp/cdproto/target"
 	"github.com/chromedp/chromedp"
 )
 
@@ -16,13 +17,26 @@ type headlesResult struct {
 }
 
 func headless(ctx context.Context, c *config) (*headlesResult, error) {
-	// create context
-	childCtx, cancel := chromedp.NewContext(ctx)
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.DisableGPU,
+	)
+	if c.userAgent != "" {
+		opts = append(opts, chromedp.UserAgent(c.userAgent))
+	}
+	// Create an allocator
+	allocatorCtx, allocatorCancel := chromedp.NewExecAllocator(ctx, opts...)
+	defer allocatorCancel()
+
+	// Create a new context with the allocator
+	childCtx, cancel := chromedp.NewContext(allocatorCtx)
 	defer cancel()
 
 	var res strings.Builder
 	var img []byte
-	if err := chromedp.Run(childCtx, chromeTasks(c, &res, 90, &img)); err != nil {
+	if err := chromedp.Run(
+		childCtx,
+		// chromedp.Emulate(device.IPhone7), uncomment to emulate devices
+		chromeTasks(c, &res, 90, &img)); err != nil {
 		err = fmt.Errorf("error in running headless to %s: %w", c.source, err)
 		return nil, err
 	}
@@ -32,7 +46,12 @@ func headless(ctx context.Context, c *config) (*headlesResult, error) {
 
 // chromeTasks ...
 func chromeTasks(c *config, res *strings.Builder, quality int, buf *[]byte) chromedp.Tasks {
-	tasks := []chromedp.Action{}
+	tasks := []chromedp.Action{chromedp.ActionFunc(func(ctx context.Context) error {
+		c := chromedp.FromContext(ctx)
+		_, err := target.CreateBrowserContext().Do(cdp.WithExecutor(ctx, c.Browser))
+		return err
+	})}
+
 	if c.screenshot {
 		tasks = append(tasks, chromedp.EmulateViewport(1920, 2000))
 	}
@@ -60,12 +79,12 @@ func chromeTasks(c *config, res *strings.Builder, quality int, buf *[]byte) chro
 			}),
 		)
 	} else {
-		tasks = append(tasks, chromedp.ActionFunc(func(c context.Context) error {
-			node, err := dom.GetDocument().Do(c)
+		tasks = append(tasks, chromedp.ActionFunc(func(ctx context.Context) error {
+			node, err := dom.GetDocument().Do(ctx)
 			if err != nil {
 				return err
 			}
-			str, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(c)
+			str, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
 			if err != nil {
 				return err
 			}

options.go

@@ -54,4 +54,11 @@ var (
 			c.verbose = verbose
 		}
 	}
+
+	// UserAgent custom user agent for healess Chrome operations.
+	UserAgent = func(ua string) Option {
+		return func(c *config) {
+			c.userAgent = ua
+		}
+	}
 )

reader.go

@@ -34,27 +34,27 @@ func New(ctx context.Context, source string) (Reader, error) {
 // source - the filename or web page name, reads from STDIN if name is empty.
 // Panics on errors.
 func NewInOut(ctx context.Context, options ...Option) (Reader, error) {
-	c := &config{}
+	cfg := &config{}
 
 	// apply custom configuration
 	for _, option := range options {
-		option(c)
+		option(cfg)
 	}
 
 	childCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	start := time.Now()
-	end := start.Add(c.timeout)
+	end := start.Add(cfg.timeout)
 	go func() {
 		ticker := time.NewTicker(30 * time.Millisecond)
 		for {
 			select {
 			case <-ctx.Done():
 				return
 			case <-ticker.C:
-				if c.timeout > 0 && end.Before(time.Now()) {
-					if c.verbose {
-						fmt.Printf("timeout of %v passed, stopping...\n", c.timeout)
+				if cfg.timeout > 0 && end.Before(time.Now()) {
+					if cfg.verbose {
+						fmt.Printf("timeout of %v passed, stopping...\n", cfg.timeout)
 					}
 					cancel()
 					return
@@ -67,32 +67,32 @@ func NewInOut(ctx context.Context, options ...Option) (Reader, error) {
 	var err error
 
 	// STDIN
-	if c.source == "" {
+	if cfg.source == "" {
 		r.reader, err = handleSTDIN()
 		if err != nil {
 			return *r, err
 		}
 
 		// HTTP
-	} else if c.isHTTP() {
-		if c.verbose {
+	} else if cfg.isHTTP() {
+		if cfg.verbose {
 			fmt.Println("source is HTTP/HTTPS")
 		}
-		r.reader, r.ImgBytes, err = handleHTTP(childCtx, c)
+		r.reader, r.ImgBytes, err = handleHTTP(childCtx, cfg)
 		if err != nil {
 			return *r, err
 		}
 
 		// File system
-	} else if c.isFS() {
-		r.reader, err = handleFS(c.source)
+	} else if cfg.isFS() {
+		r.reader, err = handleFS(cfg.source)
 		if err != nil {
 			return *r, err
 		}
 
 		// Unresolvable "source"
 	} else {
-		return *r, fmt.Errorf("unknown type of provided input source: %s", c.source)
+		return *r, fmt.Errorf("unknown type of provided input source: %s", cfg.source)
 	}
 
 	r.buffer = make([]byte, 64*1024)
@@ -174,6 +174,7 @@ type config struct {
 	screenshot bool
 	timeout    time.Duration
 	verbose    bool
+	userAgent  string
 }
 
 func (c *config) isHTTP() bool {