From 6735fb9fe0cedeca07d06c08ea8ccc0b562031dc Mon Sep 17 00:00:00 2001 From: Chris Reeves Date: Mon, 25 May 2020 20:15:58 +0100 Subject: [PATCH] refactor: allow the loader to pass the delimiter to the parsers --- README.md | 11 ++-- loader.go | 19 ++++-- notify.go | 46 ++++++-------- parser.go | 136 +++++++++++++++++++++++++++++------------- parsers/env/env.go | 3 +- parsers/env/parser.go | 12 +++- parsers/json/json.go | 21 +++++-- parsers/toml/toml.go | 21 +++++-- parsers/yaml/yaml.go | 21 +++++-- set.go | 8 +-- 10 files changed, 194 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index ab7941a..debd517 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,14 @@ func main() { var cfg Config // Initialise gofig with the destination struct - gfg, err := gofig.New(&cfg) + gfg, err := gofig.New(&cfg, gofig.WithDebug()) gofig.Must(err) // Setsup a yaml parser with file notification support - yml := gofig.FromFileWithNotify(yaml.New(), fsnotify.New("./config.yaml")) + yml := gofig.FromFileAndNotify(yaml.New(), fsnotify.New("./config.yaml")) // Parse the yaml file and then environment variables - gofig.Must(gfg.Parse(yml, env.New(env.HasAndTrimPrefix("GOFIG")))) + gofig.Must(gfg.Parse(yml, env.New(env.WithPrefix("GOFIG")))) // Setup gofig notification channel to send notification of configuration updates notifyCh := make(chan error, 1) @@ -91,13 +91,14 @@ GoFig implements it's parsers as sub modules. Currently it supports: # Roadmap +* [x] (PoC) Support notification of config changes via `Notifier` interface +* [x] (PoC) Implement File notifier on changes to files via `fsnotify` +* [ ] Parser Order Priority on Notify events, e.g file changes should not override env var config * [ ] Test Suite / Code Coverage reporting * [ ] Helpful errors * [ ] Support pointer values * [ ] Default Values via a struct tag, e.g: `gofig:"foo,default=bar"` * [ ] Support `omitempty` for pointer values which should not be initialised to their zero value. -* [x] (PoC) Support notification of config changes via `Notifier` interface -* [x] (PoC) Implement File notifier on changes to files via `fsnotify` * [ ] Add support for: * [ ] ETCD Parser / Notifier * [ ] Consul Parser / Notifier diff --git a/loader.go b/loader.go index a3d6c8e..15a119c 100644 --- a/loader.go +++ b/loader.go @@ -25,6 +25,7 @@ type Loader struct { debug bool keyFormatter Formatter structTag string + delimiter string } // New constructs a new Loader @@ -50,6 +51,7 @@ func New(dst interface{}, opts ...Option) (*Loader, error) { return key })), structTag: DefaultStructTag, + delimiter: ".", } for _, opt := range opts { @@ -83,6 +85,9 @@ func (l *Loader) log() Logger { // parse parses an single parser. func (l *Loader) parse(p Parser) error { + // Set the delimiter + p.SetDelimeter(l.delimiter) + // Send the keys errCh := make(chan error, 1) keyCh := make(chan string, len(l.fields)) @@ -125,8 +130,8 @@ func (l *Loader) parse(p Parser) error { field, ok := l.find(key) if ok { if field.Value.Kind() == reflect.Map { - key = strings.Trim(strings.Replace(key, field.Key, "", -1), ".") - if err := setMap(field, key, val); err != nil { + key = strings.Trim(strings.Replace(key, field.Key, "", -1), l.delimiter) + if err := setMap(field, key, val, l.delimiter); err != nil { return err } @@ -152,7 +157,11 @@ func (l *Loader) flatten(rv reflect.Value, rt reflect.Type, key string) { if fv.CanSet() { tag := TagFromStructField(ft, l.structTag) - k := l.keyFormatter.Format(strings.Trim(strings.Join(append(strings.Split(key, "."), tag.Name), "."), ".")) + k := l.keyFormatter.Format( + strings.Trim( + strings.Join( + append(strings.Split(key, l.delimiter), tag.Name), l.delimiter), + l.delimiter)) l.log().Printf("", ft.Name, fv.Kind(), k, tag) @@ -172,9 +181,9 @@ func (l *Loader) find(key string) (Field, bool) { return f, ok } - elms := strings.Split(key, ".") + elms := strings.Split(key, l.delimiter) - key = strings.Join(elms[:len(elms)-1], ".") + key = strings.Join(elms[:len(elms)-1], l.delimiter) if key == "" { return f, false } diff --git a/notify.go b/notify.go index e9763ea..bf5dbc4 100644 --- a/notify.go +++ b/notify.go @@ -73,43 +73,33 @@ func (l *Loader) Close() error { return err.NilOrError() } -// ParseNotifierFunc implements the Notifier and Parser interface. -type ParseNotifierFunc func() (Parser, Notifier) - -// Keys consumes the keys but does nothing with them. -func (fn ParseNotifierFunc) Keys(c <-chan string) error { - for { - _, ok := <-c - if !ok { - return nil - } - } +// FileNotifyParser parses and watches for notifications from a notifier. +type FileNotifyParser struct { + *FileParser + + notifier FileNotifier } -// Values calls the wrapped function returning the values from the returned Parser Values method. -func (fn ParseNotifierFunc) Values() (<-chan func() (string, interface{}), error) { - p, _ := fn() +// NewFileNotifyParser constructs a new FileNotifyParser. +func NewFileNotifyParser(parser ParseReadCloser, notifier FileNotifier) *FileNotifyParser { + return &FileNotifyParser{ + FileParser: NewFileParser(parser, notifier.Path()), - return p.Values() + notifier: notifier, + } } // Notify calls the wrapped function returning the values from the returned Notifier Notify method. -func (fn ParseNotifierFunc) Notify() <-chan error { - _, n := fn() - - return n.Notify() +func (p *FileNotifyParser) Notify() <-chan error { + return p.notifier.Notify() } // Close calls the wrapped function returning the values from the returned Notifier Close method. -func (fn ParseNotifierFunc) Close() error { - _, n := fn() - - return n.Close() +func (p *FileNotifyParser) Close() error { + return p.notifier.Close() } -// FromFileWithNotify reads a file anf notifies of changes. -func FromFileWithNotify(parser ReaderParser, notifier FileNotifier) NotifyParser { - return ParseNotifierFunc(func() (Parser, Notifier) { - return FromFile(parser, notifier.Path()), notifier - }) +// FromFileAndNotify reads a file anf notifies of changes. +func FromFileAndNotify(parser ParseReadCloser, notifier FileNotifier) NotifyParser { + return NewFileNotifyParser(parser, notifier) } diff --git a/parser.go b/parser.go index 2c70e8b..6853b64 100644 --- a/parser.go +++ b/parser.go @@ -8,13 +8,20 @@ import ( "strings" ) +// A DelimeterSetter sets the key delimeter used for flattened keys, default is . +type DelimeterSetter interface { + SetDelimeter(string) +} + // A Parser parses configuration. type Parser interface { + DelimeterSetter + // Keys sends flattened keys (e.g foo.bar.fizz_buzz) to the parser. The Parser then can then decide, if // it wishes to format the key and store internal mapping or not. // This is useful for parsers like environment variables where keys such as foo.bar.fizz_buzz would need to be // converted too FOO_BAR_FIZZ_BUZZ with a mapping to the original key. - // This allows us to maintain case sensitity in key lookups within the laoder. + // This allows us to maintain case sensitivity in key lookups within the loader. // Most parsers such as YAML, TOML and JSON will not process these keys. Keys(keys <-chan string) error @@ -39,26 +46,10 @@ type Parser interface { Values() (<-chan func() (key string, value interface{}), error) } -// A ParserFunc is an adapter allowing regular methods to act as Parser's. -type ParserFunc func() (<-chan func() (key string, value interface{}), error) +// A ParseReadCloser parses configuration from an io.ReadCloser. +type ParseReadCloser interface { + DelimeterSetter -// Keys consumes the keys but does nothing with them. -func (fn ParserFunc) Keys(c <-chan string) error { - for { - _, ok := <-c - if !ok { - return nil - } - } -} - -// Values calls the wrapped fn returning it's values. -func (fn ParserFunc) Values() (<-chan func() (string, interface{}), error) { - return fn() -} - -// A ReaderParser parses configuration from an io.Reader. -type ReaderParser interface { Values(src io.ReadCloser) (<-chan func() (key string, value interface{}), error) } @@ -67,6 +58,13 @@ type InMemoryParser struct { values map[string]interface{} } +// NewInMemoryParser constructs a new InMemoryParser. +func NewInMemoryParser() *InMemoryParser { + return &InMemoryParser{ + values: make(map[string]interface{}), + } +} + // Add adds a value to the in memory values. func (p *InMemoryParser) Add(k string, v interface{}) { p.values[k] = v @@ -77,6 +75,9 @@ func (p *InMemoryParser) Delete(k string) { delete(p.values, k) } +// SetDelimeter is a no-op. +func (p *InMemoryParser) SetDelimeter(string) {} + // Keys consumes the keys but does nothing with them. func (p *InMemoryParser) Keys(c <-chan string) error { for { @@ -106,35 +107,90 @@ func (p *InMemoryParser) Values() (<-chan func() (string, interface{}), error) { return ch, nil } -// NewInMemoryParser constructs a new InMemoryParser. -func NewInMemoryParser() *InMemoryParser { - return &InMemoryParser{ - values: make(map[string]interface{}), +// ReadCloseParser parses config from io.ReadCloser's. +type ReadCloseParser struct { + parser ParseReadCloser + src io.ReadCloser +} + +// NewReadCloseParser constructs a new ReadCloseParser. +func NewReadCloseParser(parser ParseReadCloser, src io.ReadCloser) *ReadCloseParser { + return &ReadCloseParser{ + parser: parser, + src: src, } } +// SetDelimeter sets the parsers delimeter. +func (p *ReadCloseParser) SetDelimeter(d string) { + p.parser.SetDelimeter(d) +} + +// Keys is a no-op key consumer. +func (p *ReadCloseParser) Keys(c <-chan string) error { + for { + _, ok := <-c + if !ok { + return nil + } + } +} + +// Values returns values from the parser back to gofig. +func (p *ReadCloseParser) Values() (<-chan func() (string, interface{}), error) { + return p.parser.Values(p.src) +} + // FromString parsers configuration from a string. -func FromString(parser ReaderParser, v string) Parser { - return ParserFunc(func() (<-chan func() (string, interface{}), error) { - return parser.Values(ioutil.NopCloser(strings.NewReader(v))) - }) +func FromString(parser ParseReadCloser, v string) Parser { + return NewReadCloseParser(parser, ioutil.NopCloser(strings.NewReader(v))) } // FromBytes parsers configuration from a byte slice. -func FromBytes(parser ReaderParser, b []byte) Parser { - return ParserFunc(func() (<-chan func() (string, interface{}), error) { - return parser.Values(ioutil.NopCloser(bytes.NewReader(b))) - }) +func FromBytes(parser ParseReadCloser, b []byte) Parser { + return NewReadCloseParser(parser, ioutil.NopCloser(bytes.NewReader(b))) } -// FromFile reads a file. -func FromFile(parser ReaderParser, path string) Parser { - return ParserFunc(func() (<-chan func() (string, interface{}), error) { - f, err := os.Open(path) - if err != nil { - return nil, err +// FileParser parsers configuration from a file. +type FileParser struct { + parser ParseReadCloser + path string +} + +// NewFileParser constructs a new FileParser. +func NewFileParser(parser ParseReadCloser, path string) *FileParser { + return &FileParser{ + parser: parser, + path: path, + } +} + +// SetDelimeter sets the parsers delimeter. +func (p *FileParser) SetDelimeter(d string) { + p.parser.SetDelimeter(d) +} + +// Keys is a no-op key consumer. +func (p *FileParser) Keys(c <-chan string) error { + for { + _, ok := <-c + if !ok { + return nil } + } +} + +// Values opens the file for reading and passed it to the parser to return values back to gofig. +func (p *FileParser) Values() (<-chan func() (string, interface{}), error) { + f, err := os.Open(p.path) + if err != nil { + return nil, err + } - return parser.Values(f) - }) + return p.parser.Values(f) +} + +// FromFile reads a file. +func FromFile(parser ParseReadCloser, path string) Parser { + return NewFileParser(parser, path) } diff --git a/parsers/env/env.go b/parsers/env/env.go index e59000f..450c017 100644 --- a/parsers/env/env.go +++ b/parsers/env/env.go @@ -4,7 +4,8 @@ package env // Use Option methods to configure the parsers behaviour. func New(opts ...Option) *Parser { p := &Parser{ - keys: map[string]string{}, + keys: map[string]string{}, + delimiter: ".", } for _, opt := range opts { diff --git a/parsers/env/parser.go b/parsers/env/parser.go index b8cf665..d02a628 100644 --- a/parsers/env/parser.go +++ b/parsers/env/parser.go @@ -10,15 +10,21 @@ type Parser struct { prefix string suffix string - keys map[string]string + delimiter string + keys map[string]string +} + +// SetDelimeter sets the key delimiter. +func (p *Parser) SetDelimeter(v string) { + p.delimiter = v } // Keys consumes the keys from the channel. func (p *Parser) Keys(c <-chan string) error { - // Range over the keys we need to look for and convert to env vars formats. + // Range over the keys we need to look for and convert to env variables formats. for key := range c { // Break the key at the . delimiter - elms := strings.Split(key, ".") + elms := strings.Split(key, p.delimiter) // Add prefix / suffix elms = append([]string{p.prefix}, elms...) diff --git a/parsers/json/json.go b/parsers/json/json.go index 2ab6377..e0ba09f 100644 --- a/parsers/json/json.go +++ b/parsers/json/json.go @@ -8,11 +8,20 @@ import ( ) // Parser parses YAML documents. -type Parser struct{} +type Parser struct { + delimiter string +} // New constructs a new Parser. func New() *Parser { - return &Parser{} + return &Parser{ + delimiter: ".", + } +} + +// SetDelimeter sets the key delimiter. +func (p *Parser) SetDelimeter(v string) { + p.delimiter = v } // Values parses yaml configuration, iterating over each key value pair and returning them until @@ -29,18 +38,18 @@ func (p *Parser) Values(src io.ReadCloser) (<-chan func() (string, interface{}), go func() { defer close(ch) - recurse("", dst, ch) + p.recurse("", dst, ch) }() return ch, src.Close() } -func recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { +func (p *Parser) recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { for k, v := range m { - name := strings.Trim(strings.Join(append(strings.Split(key, "."), k), "."), ".") + name := strings.Trim(strings.Join(append(strings.Split(key, p.delimiter), k), p.delimiter), p.delimiter) if reflect.ValueOf(v).Kind() == reflect.Map { - recurse(name, v.(map[string]interface{}), ch) + p.recurse(name, v.(map[string]interface{}), ch) continue } diff --git a/parsers/toml/toml.go b/parsers/toml/toml.go index 7bcfaeb..b62696f 100644 --- a/parsers/toml/toml.go +++ b/parsers/toml/toml.go @@ -9,11 +9,20 @@ import ( ) // Parser parses TOML documents. -type Parser struct{} +type Parser struct { + delimiter string +} // New constructs a new Parser. func New() *Parser { - return &Parser{} + return &Parser{ + delimiter: ".", + } +} + +// SetDelimeter sets the key delimiter. +func (p *Parser) SetDelimeter(v string) { + p.delimiter = v } // Values parses yaml configuration, iterating over each key value pair and returning them until @@ -30,18 +39,18 @@ func (p *Parser) Values(src io.ReadCloser) (<-chan func() (string, interface{}), go func() { defer close(ch) - recurse("", dst, ch) + p.recurse("", dst, ch) }() return ch, src.Close() } -func recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { +func (p *Parser) recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { for k, v := range m { - name := strings.Trim(strings.Join(append(strings.Split(key, "."), k), "."), ".") + name := strings.Trim(strings.Join(append(strings.Split(key, p.delimiter), k), p.delimiter), p.delimiter) if reflect.ValueOf(v).Kind() == reflect.Map { - recurse(name, v.(map[string]interface{}), ch) + p.recurse(name, v.(map[string]interface{}), ch) continue } diff --git a/parsers/yaml/yaml.go b/parsers/yaml/yaml.go index 4c6ddf3..0132d65 100644 --- a/parsers/yaml/yaml.go +++ b/parsers/yaml/yaml.go @@ -9,11 +9,20 @@ import ( ) // Parser parses YAML documents. -type Parser struct{} +type Parser struct { + delimiter string +} // New constructs a new Parser. func New() *Parser { - return &Parser{} + return &Parser{ + delimiter: ".", + } +} + +// SetDelimeter sets the key delimiter. +func (p *Parser) SetDelimeter(v string) { + p.delimiter = v } // Values parses yaml configuration, iterating over each key value pair and returning them until @@ -30,18 +39,18 @@ func (p *Parser) Values(src io.ReadCloser) (<-chan func() (string, interface{}), go func() { defer close(ch) - recurse("", dst, ch) + p.recurse("", dst, ch) }() return ch, src.Close() } -func recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { +func (p *Parser) recurse(key string, m map[string]interface{}, ch chan func() (string, interface{})) { for k, v := range m { - name := strings.Trim(strings.Join(append(strings.Split(key, "."), k), "."), ".") + name := strings.Trim(strings.Join(append(strings.Split(key, p.delimiter), k), p.delimiter), p.delimiter) if reflect.ValueOf(v).Kind() == reflect.Map { - recurse(name, v.(map[string]interface{}), ch) + p.recurse(name, v.(map[string]interface{}), ch) continue } diff --git a/set.go b/set.go index 1d31223..9fdefe9 100644 --- a/set.go +++ b/set.go @@ -177,7 +177,7 @@ func setSlice(field Field, value interface{}) error { } // setMap sets a field to a map, also handles nested maps. -func setMap(field Field, key string, value interface{}) error { +func setMap(field Field, key string, value interface{}, delimiter string) error { if field.Value.IsNil() { if field.Value.Type().Key().Kind() != reflect.String { return ErrInvalidValue{ @@ -192,10 +192,10 @@ func setMap(field Field, key string, value interface{}) error { // Nested map if field.Value.Type().Elem().Kind() == reflect.Map { - elms := strings.Split(key, ".") + elms := strings.Split(key, delimiter) parent, children := elms[0], elms[1:] - key = strings.Join(children, ".") + key = strings.Join(children, delimiter) if key == "" { return nil } @@ -205,7 +205,7 @@ func setMap(field Field, key string, value interface{}) error { m = reflect.New(field.Value.Type().Elem()).Elem() } - if err := setMap(Field{key, m}, key, value); err != nil { + if err := setMap(Field{key, m}, key, value, delimiter); err != nil { return err }