0.10.0 - improved -q option, it is now acts as document.querySelectorAll() so it will return all the matching nodes.

Author: smeshkov

.travis.yml

@@ -1,20 +1,29 @@
 language: go
+
 go:
-- 1.13.x
+  - 1.14.x
+
 env:
-- GO111MODULE=on
-install:
-- go get -v ./...
+  - GO111MODULE=on
+
+before_install:
+  - go get -t -v ./...
+
 script:
-- "./_bin/test.sh"
+  - go test -coverprofile=coverage.txt -covermode=atomic
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
+
 before_deploy:
-- "./_bin/build.sh darwin ${TRAVIS_TAG}"
-- "./_bin/build.sh linux ${TRAVIS_TAG}"
+  - "./_bin/build.sh darwin ${TRAVIS_TAG}"
+  - "./_bin/build.sh linux ${TRAVIS_TAG}"
+
 deploy:
   provider: releases
   api_key:
     secure: dDafGtm1CpthsN3pHyciGvq+n7OiFJfiq0A32/FujYz9aBx8w4ZxMH8X4wiqw1Cy0SVRKx7Z35gVBBrYM0PZQ/Anpkm0epnZ6RfoGOkpgklKvK6T27pLYlwMnHBVPDbMVluD3I0XRvVT1dRuT0eMWsdnGy6Dh07BJtnKlsRdAbMaTlWI4OkdpS1/WoOUkxs5WuGtJ3qY7KjF0EJj21yC/8kYqrKOhjKGknggJWByvcVUNA8N6xb28TM+yDHiYAuRkUrCgnM64gFlSr8C/Tt5F/I+BaOlC7U4kdcyX+6HM06vZHN2yxLKtsKjuyhZK8D7F1Xa1heisH44jTVCg7hnwRasGKzkc9PHB+eeCTcnKjFTREgMmkEVQHD8YyJKYPxtp0ouS7Srtb/5yylUbijd8JyY8wXbUdjbnRWTYuauDajUiPM6rq/iXIpzbsM5emWRhQ14H7DMzlZnEI8DvovB0/WEGpd+Z3u182VAlgJrbdgyqNqeTRyPEjo1jOLT8hUM6lKSDXI4mMm+DjUGMV9mCkVQgFdhXlkMjRcNranNi3aa6QddwLnb5O5SOgNwkEeRuzX/opG0tC3tkbiMbijIgeE9CMZbJWJGGrGl/6KzcB3D2H/SEMlSx8ONLCAWmkRniFsEwDOc/4OJDA2x4lEhkpprzKonryqoJ4a9yBgHOBo=
-  file: 
+  file:
     - "inout_darwin_${TRAVIS_TAG}"
     - "inout_linux_${TRAVIS_TAG}"
   on:

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## 0.10.0
+ - improved `-q` option, it is now acts as `document.querySelectorAll()` so it will return all the matching nodes.
+
 ## 0.9.1
  - added retries with backoff to HTML calls;
  - fixed `-q` option, which allows querying with CSS selectors.

client_test.go

@@ -1,16 +1,16 @@
 package inout
 
 import (
-	"fmt"
 	"io/ioutil"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_fetch(t *testing.T) {
-	defer stopServer(startServer(fmt.Sprintf(":%d", 8666), clientHTML, 200))
-	res := fetch("http://localhost:8666")
+	ts := startServer(clientHTML, 200)
+	defer ts.Close()
+	res := fetch(ts.URL)
 
 	assert.NotNil(t, res)
 
@@ -24,8 +24,9 @@ func Test_fetch(t *testing.T) {
 }
 
 func Test_fetch_retries(t *testing.T) {
-	defer stopServer(startServer(fmt.Sprintf(":%d", 8666), "", 429))
-	res := fetch("http://localhost:8666")
+	ts := startServer("", 429)
+	defer ts.Close()
+	res := fetch(ts.URL)
 	assert.NotNil(t, res)
 	assert.NotNil(t, res.err)
 	assert.Equal(t, 3, res.attempts)

go.mod

@@ -1,14 +1,14 @@
 module github.com/zoomio/inout
 
-go 1.12
+go 1.14
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9
-	github.com/chromedp/chromedp v0.3.1
+	github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac
+	github.com/chromedp/chromedp v0.5.3
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/stretchr/testify v1.5.1
-	golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect
+	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 )

go.sum

@@ -1,36 +1,34 @@
-github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9 h1:ARnDd2vEk91rLNra8yk1hF40H8z+1HrD6juNpe7FsI0=
-github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9/go.mod h1:xquOK9dIGFlLaIGI4c6IyfLI/Gz0LiYYuJtzhsUODgI=
-github.com/chromedp/chromedp v0.3.1 h1:QV2NmprlDOMssqCjxemgHehVldLB2wp+Spa+bRbQGmc=
-github.com/chromedp/chromedp v0.3.1/go.mod h1:EktsZcC2iycVrRhC9fDmshBpCK9lNnZYi6x2q9uE7zI=
+github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac h1:T7V5BXqnYd55Hj/g5uhDYumg9Fp3rMTS6bykYtTIFX4=
+github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
+github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg=
+github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.0.0 h1:1WdyfgUcImUfVBvYbsW2krIsnko+1QU2t45soaF8v1M=
-github.com/gobwas/ws v1.0.0/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiMjXsGrFVK25aOWLq7n+3gh2ls=
-github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
+github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
-golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

headless.go

@@ -2,28 +2,35 @@ package inout
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
+	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/chromedp"
 )
 
-func waitForDomElement(ctx context.Context, selector, host string, verbose bool) (string, error) {
+func waitForDomElement(ctx context.Context, selector, addr string, verbose bool) (string, error) {
 	// create context
 	childCtx, cancel := chromedp.NewContext(ctx)
 	defer cancel()
 
-	var content string
-	err := chromedp.Run(childCtx, visible(host, selector, &content))
-	if err != nil {
-		return "", err
+	// navigate
+	if err := chromedp.Run(childCtx, chromedp.Navigate(addr)); err != nil {
+		return "", fmt.Errorf("could not navigate to %s: %v", addr, err)
 	}
 
-	return content, nil
-}
+	var nodes []*cdp.Node
+	err := chromedp.Run(childCtx, chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll, chromedp.NodeVisible))
+	if err != nil {
+		return "", fmt.Errorf("could not get nodes %s: %v", selector, err)
+	}
 
-func visible(host, selector string, content *string) chromedp.Tasks {
-	return chromedp.Tasks{
-		chromedp.Navigate(host),
-		chromedp.WaitVisible(selector),
-		chromedp.OuterHTML(selector, content),
+	var sb strings.Builder
+	for _, n := range nodes {
+		var cnt string
+		if err := chromedp.Run(childCtx, chromedp.OuterHTML(n.FullXPath(), &cnt)); err == nil {
+			sb.WriteString(cnt)
+		}
 	}
+	return strings.TrimSpace(sb.String()), nil
 }

headless_test.go

@@ -3,58 +3,43 @@ package inout
 import (
 	"context"
 	"fmt"
-	"log"
 	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_waitForDomElement(t *testing.T) {
-	defer stopServer(startServer(fmt.Sprintf(":%d", port), headlessIndexHTML, 200))
-	innerContents, err := waitForDomElement(context.TODO(), "div p", fmt.Sprintf("http://localhost:%d", port), false)
+	ts := startServer(headlessIndexHTML, 200)
+	defer ts.Close()
+	innerContents, err := waitForDomElement(context.TODO(), "div p", ts.URL, false)
 	assert.Nil(t, err)
 	assert.Equal(t, headlessExpectedHTML, innerContents)
 }
 
 // startServer is a simple HTTP server that displays the passed headers in the html.
-func startServer(addr, document string, status int) *http.Server {
+func startServer(document string, status int) *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(res http.ResponseWriter, _ *http.Request) {
 		res.WriteHeader(status)
 		res.Header().Set("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprint(res, document)
 	})
-	srv := &http.Server{Addr: addr, Handler: mux}
-	go func() {
-		// returns ErrServerClosed on graceful close
-		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
-			log.Fatalf("ListenAndServe(): %s", err)
-		}
-	}()
-	return srv
-}
-
-func stopServer(srv *http.Server) {
-	// close the server gracefully
-	if err := srv.Shutdown(context.TODO()); err != nil {
-		panic(err) // failure/timeout shutting down the server gracefully
-	}
+	return httptest.NewServer(mux)
 }
 
 const (
-	port = 8655
-
-	headlessExpectedHTML = "<p>There was a Boy whose name was Jim;</p>" +
-		"<p>His Friends were very good to him.</p>" +
-		"<p>They gave him Tea, and Cakes, and Jam,</p>" +
-		"<p>And slices of delicious Ham,</p>" +
-		"<p>And Chocolate with pink inside,</p>" +
-		"<p>And little Tricycles to ride,</p>" +
-		"<p>Andread him Stories through and through,</p>" +
-		"<p>And even took him to the Zoo—</p>" +
-		"<p>But there it was the dreadful Fate</p>" +
-		"<p>Befell him, which I now relate.</p>"
+	headlessExpectedHTML = "<p class=\"line\">There was a Boy whose name was Jim;</p>" +
+		"<p class=\"line\">His Friends were very good to him.</p>" +
+		"<p class=\"line\">They gave him Tea, and Cakes, and Jam,</p>" +
+		"<p class=\"line\">And slices of delicious Ham,</p>" +
+		"<p class=\"line\">And Chocolate with pink inside,</p>" +
+		"<p class=\"line\">And little Tricycles to ride,</p>" +
+		"<p class=\"line\">Andread him Stories through and through,</p>" +
+		"<p class=\"line\">And even took him to the Zoo—</p>" +
+		"<p class=\"line\">But there it was the dreadful Fate</p>" +
+		"<p class=\"line\">Befell him, which I now relate.</p>"
 
 	headlessIndexHTML = `<!doctype html>
 <html>

reader.go

@@ -51,7 +51,7 @@ func NewInOut(ctx context.Context, options ...Option) (Reader, error) {
 	start := time.Now()
 	end := start.Add(r.timeout)
 	go func() {
-		ticker := time.NewTicker(10 * time.Millisecond)
+		ticker := time.NewTicker(30 * time.Millisecond)
 		for {
 			select {
 			case <-ctx.Done():

reader_test.go

@@ -3,16 +3,16 @@ package inout
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_handleHTTP(t *testing.T) {
-	defer stopServer(startServer(fmt.Sprintf(":%d", port), readerIndexHTML, 200))
-	reader, err := handleHTTP(context.TODO(), fmt.Sprintf("http://localhost:%d", port), "", false)
+	ts := startServer(readerIndexHTML, 200)
+	defer ts.Close()
+	reader, err := handleHTTP(context.TODO(), ts.URL, "", false)
 	assert.Nil(t, err)
 	assert.NotNil(t, reader)