add internal network regexp for stationxml (GeoNet#213)

* add internal network regexp for stationxml

* simplified regexp matching and added a negative match option

build.sh

@@ -17,6 +17,7 @@ go build ./tools/stationxml || exit 255
     -active \
     -installed \
     -operational \
+    -networks '!(SB|.X)' \
     -channels '([EHB][HN][ZNE12])'
 
 ./stationxml -base . -output .tmp/geonet-meta/stationxml/iris.xml \

tools/stationxml/build.go

@@ -2,10 +2,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"regexp"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/GeoNet/delta/internal/metadb"
@@ -20,33 +17,14 @@ func (c Channels) Len() int           { return len(c) }
 func (c Channels) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
 func (c Channels) Less(i, j int) bool { return c[i].StartDate.Time.Before(c[j].StartDate.Time) }
 
-func matcher(path, def string) (*regexp.Regexp, error) {
-
-	// no list given
-	if path == "" {
-		return regexp.Compile(func() string {
-			if def == "" {
-				return "[A-Z0-9]+"
-			}
-			return def
-		}())
-	}
-
-	buf, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	return regexp.Compile("^(" + strings.Join(strings.Fields(string(buf)), "|") + ")$")
-}
-
 type Builder struct {
 	operational *time.Time
-	networks    *regexp.Regexp
-	stations    *regexp.Regexp
-	channels    *regexp.Regexp
-	sensors     *regexp.Regexp
-	dataloggers *regexp.Regexp
+	networks    Matcher
+	external    Matcher
+	stations    Matcher
+	channels    Matcher
+	sensors     Matcher
+	dataloggers Matcher
 	installed   bool
 	active      bool
 }
@@ -75,49 +53,59 @@ func SetOperational(operational bool, offset time.Duration) func(*Builder) error
 	}
 }
 
-func SetNetworks(list, match string) func(*Builder) error {
+func SetNetworks(match string) func(*Builder) error {
 	return func(b *Builder) error {
-		re, err := matcher(list, match)
+		re, err := Match(match)
 		if err != nil {
 			return err
 		}
 		b.networks = re
 		return nil
 	}
 }
-func SetStations(list, match string) func(*Builder) error {
+func SetExternal(match string) func(*Builder) error {
 	return func(b *Builder) error {
-		re, err := matcher(list, match)
+		re, err := Match(match)
+		if err != nil {
+			return err
+		}
+		b.external = re
+		return nil
+	}
+}
+func SetStations(match string) func(*Builder) error {
+	return func(b *Builder) error {
+		re, err := Match(match)
 		if err != nil {
 			return err
 		}
 		b.stations = re
 		return nil
 	}
 }
-func SetChannels(list, match string) func(*Builder) error {
+func SetChannels(match string) func(*Builder) error {
 	return func(b *Builder) error {
-		re, err := matcher(list, match)
+		re, err := Match(match)
 		if err != nil {
 			return err
 		}
 		b.channels = re
 		return nil
 	}
 }
-func SetSensors(list, match string) func(*Builder) error {
+func SetSensors(match string) func(*Builder) error {
 	return func(b *Builder) error {
-		re, err := matcher(list, match)
+		re, err := Match(match)
 		if err != nil {
 			return err
 		}
 		b.sensors = re
 		return nil
 	}
 }
-func SetDataloggers(list, match string) func(*Builder) error {
+func SetDataloggers(match string) func(*Builder) error {
 	return func(b *Builder) error {
-		re, err := matcher(list, match)
+		re, err := Match(match)
 		if err != nil {
 			return err
 		}
@@ -150,6 +138,12 @@ func (b *Builder) MatchNetwork(net string) bool {
 	}
 	return b.networks.MatchString(net)
 }
+func (b *Builder) MatchExternal(net string) bool {
+	if b.external == nil {
+		return true
+	}
+	return b.external.MatchString(net)
+}
 func (b *Builder) MatchStation(sta string) bool {
 	if b.stations == nil {
 		return true
@@ -205,7 +199,11 @@ func (b *Builder) Construct(base string) ([]stationxml.Network, error) {
 			continue
 		}
 
-		if !b.MatchNetwork(network.External) {
+		if !b.MatchNetwork(network.Code) {
+			continue
+		}
+
+		if !b.MatchExternal(network.External) {
 			continue
 		}
 

tools/stationxml/main.go

@@ -36,34 +36,22 @@ func main() {
 	flag.StringVar(&install, "install", "../../install", "base installs directory")
 
 	var stationRegexp string
-	flag.StringVar(&stationRegexp, "stations", "[A-Z0-9]+", "regex selection of stations")
-
-	var stationList string
-	flag.StringVar(&stationList, "station-list", "", "regex selection of stations from file")
+	flag.StringVar(&stationRegexp, "stations", "[A-Z0-9]+", "regexp selection of stations")
 
 	var channelRegexp string
-	flag.StringVar(&channelRegexp, "channels", "[A-Z0-9]+", "regex selection of channels")
-
-	var channelList string
-	flag.StringVar(&channelList, "channel-list", "", "regex selection of channels from file")
+	flag.StringVar(&channelRegexp, "channels", "[A-Z0-9]+", "regexp selection of channels")
 
 	var networkRegexp string
-	flag.StringVar(&networkRegexp, "networks", "[A-Z0-9]+", "regex selection of networks")
+	flag.StringVar(&networkRegexp, "networks", "[A-Z0-9]+", "regexp selection of internal networks")
 
-	var networkList string
-	flag.StringVar(&networkList, "network-list", "", "regex selection of networks from file")
+	var externalRegexp string
+	flag.StringVar(&externalRegexp, "external", "[A-Z0-9]+", "regexp selection of external networks")
 
 	var sensorRegexp string
-	flag.StringVar(&sensorRegexp, "sensors", ".*", "regex selection of sensors")
-
-	var sensorList string
-	flag.StringVar(&sensorList, "sensor-list", "", "regex selection of sensors from file")
+	flag.StringVar(&sensorRegexp, "sensors", ".*", "regexp selection of sensors")
 
 	var dataloggerRegexp string
-	flag.StringVar(&dataloggerRegexp, "dataloggers", ".*", "regex selection of dataloggers")
-
-	var dataloggerList string
-	flag.StringVar(&dataloggerList, "datalogger-list", "", "regex selection of dataloggers from file")
+	flag.StringVar(&dataloggerRegexp, "dataloggers", ".*", "regexp selection of dataloggers")
 
 	var installed bool
 	flag.BoolVar(&installed, "installed", false, "set station times based on installation dates")
@@ -89,6 +77,8 @@ func main() {
 		fmt.Fprintf(os.Stderr, "\n")
 		flag.PrintDefaults()
 		fmt.Fprintf(os.Stderr, "\n")
+		fmt.Fprintf(os.Stderr, "  Use a \"!\" prefix to indicate that the regexp should not match the expression\n")
+		fmt.Fprintf(os.Stderr, "\n")
 	}
 
 	flag.Parse()
@@ -97,11 +87,12 @@ func main() {
 		SetInstalled(installed),
 		SetActive(active),
 		SetOperational(operational, offset),
-		SetNetworks(networkList, networkRegexp),
-		SetStations(stationList, stationRegexp),
-		SetChannels(channelList, channelRegexp),
-		SetSensors(sensorList, sensorRegexp),
-		SetDataloggers(dataloggerList, dataloggerRegexp),
+		SetNetworks(networkRegexp),
+		SetExternal(externalRegexp),
+		SetStations(stationRegexp),
+		SetChannels(channelRegexp),
+		SetSensors(sensorRegexp),
+		SetDataloggers(dataloggerRegexp),
 	)
 	if err != nil {
 		log.Fatalf("unable to make builder: %v", err)

tools/stationxml/match.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"regexp"
+	"strings"
+)
+
+type Matcher interface {
+	MatchString(string) bool
+}
+
+type positiveMatch struct {
+	re *regexp.Regexp
+}
+
+func (p positiveMatch) MatchString(s string) bool {
+	if p.re != nil {
+		return p.re.MatchString(s)
+	}
+	return true
+}
+
+type negativeMatch struct {
+	re *regexp.Regexp
+}
+
+func (n negativeMatch) MatchString(s string) bool {
+	if n.re != nil {
+		return !n.re.MatchString(s)
+	}
+	return false
+}
+
+func Match(str string) (Matcher, error) {
+	// prepare the matching string
+	s := strings.TrimPrefix(strings.TrimSpace(str), "!")
+
+	// compile the expression
+	re, err := regexp.Compile(s)
+	if err != nil {
+		return nil, err
+	}
+
+	// a negative match if it starts with an "!" symbol
+	if strings.HasPrefix(strings.TrimSpace(str), "!") {
+		return negativeMatch{re}, nil
+	}
+
+	return positiveMatch{re}, nil
+}
+
+func MustMatch(str string) Matcher {
+	m, err := Match(str)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}

tools/stationxml/match_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestMatcher(t *testing.T) {
+
+	var tests = map[string]struct {
+		m Matcher
+		s string
+		r bool
+	}{
+		"empty positive":         {positiveMatch{}, "", true},
+		"empty negative":         {negativeMatch{}, "", false},
+		"valid positive match":   {MustMatch("AB"), "AB", true},
+		"invalid positive match": {MustMatch("AB"), "BB", false},
+		"valid negative match":   {MustMatch("!AB"), "BB", true},
+		"invalid negative match": {MustMatch("!AB"), "AB", false},
+	}
+
+	for k, v := range tests {
+		t.Log("test matcher for " + k)
+		if v.m.MatchString(v.s) != v.r {
+			t.Errorf("%s: unable to match string: %s", k, v.s)
+		}
+	}
+}