File watch (#588)

* remove concurrent test specs from https redirect test
* code formatting
* watch, initial
* eskip file watch client
* add watching to the cmd line option routes file
* documentation

cmd/skipper/main.go

@@ -67,7 +67,7 @@ const (
 	oauthURLUsage                  = "OAuth2 URL for Innkeeper authentication"
 	oauthCredentialsDirUsage       = "directory where oauth credentials are stored: client.json and user.json"
 	oauthScopeUsage                = "the whitespace separated list of oauth scopes"
-	routesFileUsage                = "file containing static route definitions"
+	routesFileUsage                = "file containing route definitions"
 	inlineRoutesUsage              = "inline routes in eskip format"
 	sourcePollTimeoutUsage         = "polling timeout of the routing data sources, in milliseconds"
 	insecureUsage                  = "flag indicating to ignore the verification of the TLS certificates of the backend services"
@@ -357,7 +357,7 @@ func main() {
 		KubernetesIngressClass:              kubernetesIngressClass,
 		InnkeeperUrl:                        innkeeperURL,
 		SourcePollTimeout:                   time.Duration(sourcePollTimeout) * time.Millisecond,
-		RoutesFile:                          routesFile,
+		WatchRoutesFile:                     routesFile,
 		InlineRoutes:                        inlineRoutes,
 		IdleConnectionsPerHost:              idleConnsPerHost,
 		CloseIdleConnsPeriod:                time.Duration(clsic) * time.Second,

eskipfile/doc.go

@@ -0,0 +1,11 @@
+/*
+Package eskipfile implements the DataClient interface for reading the skipper route definitions from an eskip
+formatted file.
+
+(See the DataClient interface in the skipper/routing package and the eskip
+format in the skipper/eskip package.)
+
+The package provides two implementations: one without file watch (legacy version) and one with file watch. When
+running the skipper command, the one with watch is used.
+*/
+package eskipfile

eskipfile/example_test.go

@@ -1,17 +1,3 @@
-// Copyright 2015 Zalando SE
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package eskipfile_test
 
 import (
@@ -22,18 +8,16 @@ import (
 
 func Example() {
 	// open file with a routing table:
-	dataClient, err := eskipfile.Open("/some/path/to/routing-table.eskip")
-	if err != nil {
-		// log.Fatal(err)
-		return
-	}
+	dataClient := eskipfile.Watch("/some/path/to/routing-table.eskip")
+	defer dataClient.Close()
 
-	// create routing object:
+	// create a routing object:
 	rt := routing.New(routing.Options{
-		DataClients: []routing.DataClient{dataClient}})
+		DataClients: []routing.DataClient{dataClient},
+	})
 	defer rt.Close()
 
-	// create http.Handler:
+	// create an http.Handler:
 	p := proxy.New(rt, proxy.OptionsNone)
 	defer p.Close()
 }

eskipfile/file.go

@@ -1,36 +1,17 @@
-// Copyright 2015 Zalando SE
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/*
-Package eskipfile implements a DataClient for reading the skipper route
-definitions from an eskip formatted file when opened.
-
-(See the DataClient interface in the skipper/routing package and the eskip
-format in the skipper/eskip package.)
-*/
 package eskipfile
 
 import (
-	"github.com/zalando/skipper/eskip"
 	"io/ioutil"
+
+	"github.com/zalando/skipper/eskip"
 )
 
-// A Client contains the route definitions from an eskip file.
+// Client contains the route definitions from an eskip file, not implementing file watch. Use the Open function
+// to create instances of it.
 type Client struct{ routes []*eskip.Route }
 
-// Opens an eskip file and parses it, returning a DataClient implementation.
-// If reading or parsing the file fails, returns an error.
+// Opens an eskip file and parses it, returning a DataClient implementation. If reading or parsing the file
+// fails, returns an error. This implementation doesn't provide file watch.
 func Open(path string) (*Client, error) {
 	content, err := ioutil.ReadFile(path)
 	if err != nil {
@@ -52,9 +33,8 @@ func (c Client) LoadAndParseAll() (routeInfos []*eskip.RouteInfo, err error) {
 	return
 }
 
-// Returns the parsed route definitions found in the file.
+// LoadAll returns the parsed route definitions found in the file.
 func (c Client) LoadAll() ([]*eskip.Route, error) { return c.routes, nil }
 
-// Noop. The current implementation doesn't support watching the eskip
-// file for changes.
+// LoadUpdate: noop. The current implementation doesn't support watching the eskip file for changes.
 func (c Client) LoadUpdate() ([]*eskip.Route, []string, error) { return nil, nil, nil }

eskipfile/fixtures/test.eskip

@@ -0,0 +1,2 @@
+foo: Path("/foo") -> setPath("/") -> "https://foo.example.org";
+bar: Path("/bar") -> setPath("/") -> "https://bar.example.org";

eskipfile/open_test.go

@@ -0,0 +1,55 @@
+package eskipfile
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/zalando/skipper/filters/builtin"
+	"github.com/zalando/skipper/logging/loggingtest"
+	"github.com/zalando/skipper/routing"
+)
+
+func TestOpenFails(t *testing.T) {
+	_, err := Open("notexisting.eskip")
+	if err == nil {
+		t.Error("failed to fail")
+	}
+}
+
+func TestOpenSucceeds(t *testing.T) {
+	f, err := Open("fixtures/test.eskip")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	l := loggingtest.New()
+	defer l.Close()
+
+	rt := routing.New(routing.Options{
+		FilterRegistry: builtin.MakeRegistry(),
+		DataClients:    []routing.DataClient{f},
+		Log:            l,
+		PollTimeout:    180 * time.Millisecond,
+	})
+	defer rt.Close()
+
+	if err := l.WaitFor("route settings applied", 120*time.Millisecond); err != nil {
+		t.Error(err)
+		return
+	}
+
+	check := func(id, path string) {
+		r, _ := rt.Route(&http.Request{URL: &url.URL{Path: path}})
+		if r == nil || r.Id != id {
+			t.Error("failed to load file")
+			t.FailNow()
+			return
+		}
+	}
+
+	check("foo", "/foo")
+	check("bar", "/bar")
+}

eskipfile/watch.go

@@ -0,0 +1,149 @@
+package eskipfile
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+
+	"github.com/zalando/skipper/eskip"
+)
+
+type watchResponse struct {
+	routes     []*eskip.Route
+	deletedIDs []string
+	err        error
+}
+
+// WatchClient implements a route configuration client with file watching. Use the Watch function to initialize
+// instances of it.
+type WatchClient struct {
+	fileName   string
+	routes     map[string]*eskip.Route
+	getAll     chan (chan<- watchResponse)
+	getUpdates chan (chan<- watchResponse)
+	quit       chan struct{}
+}
+
+// Watch creates a route configuration client with file watching. Watch doesn't follow file system nodes, it
+// always reads from the file identified by the initially provided file name.
+func Watch(name string) *WatchClient {
+	c := &WatchClient{
+		fileName:   name,
+		getAll:     make(chan (chan<- watchResponse)),
+		getUpdates: make(chan (chan<- watchResponse)),
+		quit:       make(chan struct{}),
+	}
+
+	go c.watch()
+	return c
+}
+
+func mapRoutes(r []*eskip.Route) map[string]*eskip.Route {
+	m := make(map[string]*eskip.Route)
+	for i := range r {
+		m[r[i].Id] = r[i]
+	}
+
+	return m
+}
+
+func (c *WatchClient) storeRoutes(r []*eskip.Route) {
+	c.routes = mapRoutes(r)
+}
+
+func (c *WatchClient) diffStoreRoutes(r []*eskip.Route) (upsert []*eskip.Route, deletedIDs []string) {
+	for i := range r {
+		if !reflect.DeepEqual(r[i], c.routes[r[i].Id]) {
+			upsert = append(upsert, r[i])
+		}
+	}
+
+	m := mapRoutes(r)
+	for id := range c.routes {
+		if _, keep := m[id]; !keep {
+			deletedIDs = append(deletedIDs, id)
+		}
+	}
+
+	c.routes = m
+	return
+}
+
+func (c *WatchClient) deleteAllListIDs() []string {
+	var ids []string
+	for id := range c.routes {
+		ids = append(ids, id)
+	}
+
+	c.routes = nil
+	return ids
+}
+
+func (c *WatchClient) loadAll() watchResponse {
+	content, err := ioutil.ReadFile(c.fileName)
+	if err != nil {
+		return watchResponse{err: err}
+	}
+
+	r, err := eskip.Parse(string(content))
+	if err != nil {
+		return watchResponse{err: err}
+	}
+
+	c.storeRoutes(r)
+	return watchResponse{routes: r}
+}
+
+func (c *WatchClient) loadUpdates() watchResponse {
+	content, err := ioutil.ReadFile(c.fileName)
+	if err != nil {
+		if _, isPerr := err.(*os.PathError); isPerr {
+			deletedIDs := c.deleteAllListIDs()
+			return watchResponse{deletedIDs: deletedIDs}
+		}
+
+		return watchResponse{err: err}
+	}
+
+	r, err := eskip.Parse(string(content))
+	if err != nil {
+		return watchResponse{err: err}
+	}
+
+	upsert, del := c.diffStoreRoutes(r)
+	return watchResponse{routes: upsert, deletedIDs: del}
+}
+
+func (c *WatchClient) watch() {
+	for {
+		select {
+		case req := <-c.getAll:
+			req <- c.loadAll()
+		case req := <-c.getUpdates:
+			req <- c.loadUpdates()
+		case <-c.quit:
+			return
+		}
+	}
+}
+
+// LoadAll returns the parsed route definitions found in the file.
+func (c *WatchClient) LoadAll() ([]*eskip.Route, error) {
+	req := make(chan watchResponse)
+	c.getAll <- req
+	rsp := <-req
+	return rsp.routes, rsp.err
+}
+
+// LoadUpdate returns differential updates when a watched file has changed.
+func (c *WatchClient) LoadUpdate() ([]*eskip.Route, []string, error) {
+	req := make(chan watchResponse)
+	c.getUpdates <- req
+	rsp := <-req
+	return rsp.routes, rsp.deletedIDs, rsp.err
+}
+
+// Close stops watching the configured file and providing updates.
+func (c *WatchClient) Close() {
+	close(c.quit)
+}