From ed0bf601c504158953dd3e72178eacf1d1150275 Mon Sep 17 00:00:00 2001 From: Robert Liebowitz Date: Sat, 6 Apr 2024 21:48:27 -0400 Subject: [PATCH] Add support for dotenv files While I'm not sure there is a formal dotenv specification, I've used a library that's meant to be compatible with other language libraries, which should make things pretty familiar and compatible. I've also attempted to model the semantics used by Docker Compose for file configuration, although 100% compatibility is probably not a guarantee this project will ever make. --- CHANGELOG.md | 2 + docs/spec.md | 49 +++++++++++ example/example.yml | 8 ++ go.mod | 3 +- go.sum | 6 +- runner/config.go | 7 +- runner/env_file.go | 78 ++++++++++++++++++ runner/env_file_test.go | 178 ++++++++++++++++++++++++++++++++++++++++ runner/parse.go | 8 +- runner/parse_test.go | 45 ++++++++++ tusk.schema.json | 44 ++++++++++ tusk.schema.yaml | 36 ++++++++ 12 files changed, 458 insertions(+), 6 deletions(-) create mode 100644 runner/env_file.go create mode 100644 runner/env_file_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c970a2..1ff7941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html - Arguments may now specify a `type`, which works the same way as options. - Boolean options may now be rewritten into strings using `rewrite`. +- Environment variables are now automatically parsed from `.env` files. +- Additional environment variable files can be specified with `env-file`. ### Fixed diff --git a/docs/spec.md b/docs/spec.md index 859f347..12fdd09 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -674,6 +674,55 @@ It is invalid to split the configuration; if the `include` clause is used, no other keys can be specified in the `tusk.yml`, and the full task must be defined in the included file. +### Environment Files + +Environment variables are also automatically read from a `.env` file in the +same directory as `tusk.yml` before task execution. This file is optional by +default, and supports typical "dotenv" extended syntax such as quotation marks, +comments, variable substitution, and the `export` keyword. + +A typical file might look like this: + +```sh +FOO=foovalue +BAR=barvalue +``` + +Environment files can be explicitly specified as configuration at the +top-level: + +```yaml +env-file: .local.env +``` + +This is shorthand syntax for the following: + +```yaml +env-file: + - path: .local.env + required: true +``` + +Multiple environment files can be specified. Entries are evaluted in order, so +environment variables from later files override values specified in previous +entries. + +Specifying any value for `env-file` will disable the default behavior of +auto-loading an optional `.env`. To re-enable it, specify it explicitly: + +```yaml +env-file: + - path: .env + required: false + - .local.env +``` + +To disable loading environment files completely, pass `[]` or `/dev/null`: + +```yaml +env-file: [] +``` + ### Interpreter By default, any command run will default to using `sh -c` as its interpreter. diff --git a/example/example.yml b/example/example.yml index ae4c5ce..60910d0 100644 --- a/example/example.yml +++ b/example/example.yml @@ -1,5 +1,13 @@ # yaml-language-server: $schema=../tusk.schema.yaml --- +# Environment variables can be read from a file. +env-file: + # Environment files specified as strings are automatically required. + - .default.env + # Multiple environment files or optional files may be specified as well. + - path: .local.env + required: false + # Options can be defined at a global or task-specific level. options: global: diff --git a/go.mod b/go.mod index 50f41ed..c4ca1f3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,8 @@ require ( github.com/golangci/golangci-lint v1.59.1 github.com/google/go-cmp v0.6.0 github.com/goreleaser/goreleaser v1.24.0 - github.com/rliebz/ghost v0.0.0-20240127160735-e335ae595622 + github.com/joho/godotenv v1.5.1 + github.com/rliebz/ghost v0.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/urfave/cli v1.22.15 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 986cbd4..46fa9a3 100644 --- a/go.sum +++ b/go.sum @@ -579,6 +579,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -759,8 +761,8 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rliebz/ghost v0.0.0-20240127160735-e335ae595622 h1:cHo+/eVOLKq94+NtDF7MEc9KNgDO+WDwzMuavhkV54o= -github.com/rliebz/ghost v0.0.0-20240127160735-e335ae595622/go.mod h1:Vn62ofdLNRXMmBoEKXB93ivfiUxLDQz+134JsjoGXlY= +github.com/rliebz/ghost v0.1.0 h1:W34Z8dTGgX34tmaz1TpkLMRfPrLafkvzrgSPcbdd7+w= +github.com/rliebz/ghost v0.1.0/go.mod h1:Vn62ofdLNRXMmBoEKXB93ivfiUxLDQz+134JsjoGXlY= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= diff --git a/runner/config.go b/runner/config.go index d89f5c1..f60bed7 100644 --- a/runner/config.go +++ b/runner/config.go @@ -1,9 +1,12 @@ package runner +import "github.com/rliebz/tusk/marshal" + // Config is a struct representing the format for configuration settings. type Config struct { - Name string `yaml:"name"` - Usage string `yaml:"usage"` + Name string `yaml:"name"` + Usage string `yaml:"usage"` + EnvFile marshal.Slice[EnvFile] `yaml:"env-file"` // The Interpreter field must be read before the config struct can be parsed // completely from YAML. To do so, the config text parses it elsewhere in the // code base independently from this struct. diff --git a/runner/env_file.go b/runner/env_file.go new file mode 100644 index 0000000..8edf360 --- /dev/null +++ b/runner/env_file.go @@ -0,0 +1,78 @@ +package runner + +import ( + "maps" + "os" + + "github.com/joho/godotenv" + + "github.com/rliebz/tusk/marshal" +) + +// EnvFile is a dotenv file that should be parsed during configuration start. +type EnvFile struct { + Path string `yaml:"path"` + Required bool `yaml:"required"` +} + +// UnmarshalYAML allows a string to represent a required path. +func (f *EnvFile) UnmarshalYAML(unmarshal func(any) error) error { + var path string + pathCandidate := marshal.UnmarshalCandidate{ + Unmarshal: func() error { return unmarshal(&path) }, + Assign: func() { *f = EnvFile{Path: path, Required: true} }, + } + + type envFileType EnvFile // Use new type to avoid recursion + var envFileItem envFileType + envFileCandidate := marshal.UnmarshalCandidate{ + Unmarshal: func() error { return unmarshal(&envFileItem) }, + Assign: func() { *f = EnvFile(envFileItem) }, + } + + return marshal.UnmarshalOneOf(pathCandidate, envFileCandidate) +} + +// loadEnvFiles sets env vars from a set of file configs. Values are not +// overridden. +// +// If no files are specified, it will load from an optional default of .env. +// If an empty list is specified, no files will be loaded. +func loadEnvFiles(envFiles []EnvFile) error { + // An explicit [] is an obvious attempt to remove the default, so check only + // for nilness. + if envFiles == nil { + envFiles = []EnvFile{{Path: ".env", Required: false}} + } + + envMap := make(map[string]string) + for _, envFile := range envFiles { + m, err := readEnvFile(envFile) + if err != nil { + return err + } + + maps.Copy(envMap, m) + } + + for k, v := range envMap { + if _, ok := os.LookupEnv(k); !ok { + if err := os.Setenv(k, v); err != nil { + return err + } + } + } + + return nil +} + +func readEnvFile(envFile EnvFile) (map[string]string, error) { + m, err := godotenv.Read(envFile.Path) + switch { + case !envFile.Required && os.IsNotExist(err): + case err != nil: + return nil, err + } + + return m, nil +} diff --git a/runner/env_file_test.go b/runner/env_file_test.go new file mode 100644 index 0000000..e4f1ff4 --- /dev/null +++ b/runner/env_file_test.go @@ -0,0 +1,178 @@ +package runner + +import ( + "os" + "path/filepath" + "testing" + + "github.com/rliebz/ghost" + "github.com/rliebz/ghost/be" + "gopkg.in/yaml.v2" +) + +func TestEnvFile_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + data string + want EnvFile + }{ + { + name: "empty", + data: ``, + want: EnvFile{}, + }, + { + name: "string", + data: `"dot.env"`, + want: EnvFile{ + Path: "dot.env", + Required: true, + }, + }, + { + name: "object required", + data: `{path: dot.env, required: true}`, + want: EnvFile{ + Path: "dot.env", + Required: true, + }, + }, + { + name: "object not required", + data: `{path: dot.env, required: false}`, + want: EnvFile{ + Path: "dot.env", + Required: false, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := ghost.New(t) + + var envFile EnvFile + err := yaml.UnmarshalStrict([]byte(tt.data), &envFile) + g.NoError(err) + + g.Should(be.Equal(envFile, tt.want)) + }) + } +} + +func Test_loadEnvFiles(t *testing.T) { + t.Run("default used", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + tmpdir := useTempDir(t) + + t.Setenv("BAZ", "bazvalue") + + // Just use a little bit of all the fancy syntax + data := []byte(` +# comment +FOO=foovalue +export BAR=barvalue +BAZ=newvalue +QUUX=${FOO} +`) + + err := os.WriteFile(filepath.Join(tmpdir, ".env"), data, 0o644) + g.NoError(err) + + err = loadEnvFiles(nil) + g.NoError(err) + + g.Should(be.All( + be.Equal(os.Getenv("FOO"), "foovalue"), + be.Equal(os.Getenv("BAR"), "barvalue"), + be.Equal(os.Getenv("BAZ"), "bazvalue"), + be.Equal(os.Getenv("QUUX"), "foovalue"), + )) + }) + + t.Run("default not found", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + useTempDir(t) + + err := loadEnvFiles(nil) + g.NoError(err) + }) + + t.Run("dev null", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + useTempDir(t) + + err := loadEnvFiles([]EnvFile{{Path: "/dev/null"}}) + g.NoError(err) + }) + + t.Run("empty list", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + tmpdir := useTempDir(t) + + t.Setenv("BAZ", "bazvalue") + + err := os.WriteFile(filepath.Join(tmpdir, ".env"), []byte(`FOO=foovalue`), 0o644) + g.NoError(err) + + err = loadEnvFiles([]EnvFile{}) + g.NoError(err) + + g.Should(be.Zero(os.Getenv("FOO"))) + }) + + t.Run("required found", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + tmpdir := useTempDir(t) + + err := os.WriteFile(filepath.Join(tmpdir, ".env"), []byte("FOO=foovalue"), 0o644) + g.NoError(err) + + err = loadEnvFiles([]EnvFile{ + {Path: ".env", Required: true}, + }) + g.NoError(err) + + g.Should(be.Equal(os.Getenv("FOO"), "foovalue")) + }) + + t.Run("required not found", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + useTempDir(t) + + err := loadEnvFiles([]EnvFile{ + {Path: ".env", Required: true}, + }) + g.Should(be.ErrorContaining(err, "open .env:")) + g.Should(be.True(os.IsNotExist(err))) + }) + + t.Run("overrides earlier values", func(t *testing.T) { + g := ghost.New(t) + stashEnv(t) + tmpdir := useTempDir(t) + + err := os.WriteFile(filepath.Join(tmpdir, "1.env"), []byte("FOO=one"), 0o644) + g.NoError(err) + err = os.WriteFile(filepath.Join(tmpdir, "2.env"), []byte("FOO=two"), 0o644) + g.NoError(err) + err = os.WriteFile(filepath.Join(tmpdir, "3.env"), []byte("BAR=three"), 0o644) + g.NoError(err) + + err = loadEnvFiles([]EnvFile{ + {Path: "1.env"}, + {Path: "2.env"}, + {Path: "3.env"}, + }) + g.NoError(err) + + g.Should(be.Equal(os.Getenv("FOO"), "two")) + g.Should(be.Equal(os.Getenv("BAR"), "three")) + }) +} diff --git a/runner/parse.go b/runner/parse.go index 6b26c29..9bd945c 100644 --- a/runner/parse.go +++ b/runner/parse.go @@ -18,7 +18,8 @@ func Parse(text []byte) (*Config, error) { return &cfg, nil } -// ParseComplete parses the file completely with interpolation. +// ParseComplete parses the file completely with env file parsing and +// interpolation. func ParseComplete( meta *Metadata, taskName string, @@ -30,6 +31,11 @@ func ParseComplete( return nil, err } + err = loadEnvFiles(cfg.EnvFile) + if err != nil { + return nil, err + } + t, isTaskSet := cfg.Tasks[taskName] if !isTaskSet { return cfg, nil diff --git a/runner/parse_test.go b/runner/parse_test.go index d845856..032902b 100644 --- a/runner/parse_test.go +++ b/runner/parse_test.go @@ -1,6 +1,8 @@ package runner import ( + "os" + "path/filepath" "testing" "github.com/rliebz/ghost" @@ -831,9 +833,52 @@ tasks: }}, }}, }, + + { + "env-file", + ` +env-file: example.env +options: + foo: + environment: FOO + bar: + environment: BAR +tasks: + mytask: + options: + baz: + environment: BAZ + run: echo ${foo} ${bar} ${baz} +`, + []string{}, + map[string]string{ + "baz": "bazclivalue", + }, + "mytask", + marshal.Slice[*Run]{{ + Command: marshal.Slice[*Command]{{ + Exec: "echo foovalue barvalue bazclivalue", + Print: "echo foovalue barvalue bazclivalue", + }}, + }}, + }, } func TestParseComplete_interpolates(t *testing.T) { + g := ghost.New(t) + + stashEnv(t) + tmpdir := useTempDir(t) + + envFileData := []byte(` +FOO=foovalue +BAR=barvalue +BAZ=bazvalue +`) + + err := os.WriteFile(filepath.Join(tmpdir, "example.env"), envFileData, 0o644) + g.NoError(err) + for _, tt := range interpolatetests { t.Run(tt.name, func(t *testing.T) { g := ghost.New(t) diff --git a/tusk.schema.json b/tusk.schema.json index be733c9..169f198 100644 --- a/tusk.schema.json +++ b/tusk.schema.json @@ -136,6 +136,46 @@ } ] }, + "envFile": { + "description": "A file to load environment variables from.\nFile paths specified are relative to the configuration file.\n", + "oneOf": [ + { + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "path": { + "description": "The path to an environment file relative to the configuration file.\n", + "type": "string" + }, + "required": { + "default": true, + "description": "Whether the file is required to exist.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + } + ] + }, + "envFileClause": { + "description": "The files to load environment variables from.\nIf no value is specified, environment variables will be read from an optional `.env` file automatically.\n", + "oneOf": [ + { + "$ref": "#/$defs/envFile" + }, + { + "items": { + "$ref": "#/$defs/envFile" + }, + "type": "array" + } + ] + }, "option": { "additionalProperties": false, "allOf": [ @@ -560,6 +600,10 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { + "env-file": { + "$ref": "#/$defs/envFileClause", + "title": "env-file" + }, "interpreter": { "default": "sh -c", "description": "The interpreter to use for commands.\nThe interpreter is specified as an executable, which can either be an absolute path or available on the user's PATH, followed by a series of optional arguments.\nThe commands specified in individual tasks will be passed as the final argument.\n", diff --git a/tusk.schema.yaml b/tusk.schema.yaml index 64db980..f97fa81 100644 --- a/tusk.schema.yaml +++ b/tusk.schema.yaml @@ -21,6 +21,9 @@ properties: The usage text to display in help text when using shell aliases to create a custom named CLI application. default: the modern task runner + env-file: + title: env-file + $ref: "#/$defs/envFileClause" interpreter: title: interpreter type: string @@ -159,6 +162,39 @@ $defs: - required: [command] - required: [value] + envFile: + description: > + A file to load environment variables from. + + File paths specified are relative to the configuration file. + oneOf: + - type: string + - type: object + additionalProperties: false + required: + - path + properties: + path: + description: > + The path to an environment file relative to the configuration file. + type: string + required: + description: Whether the file is required to exist. + type: boolean + default: true + + envFileClause: + description: > + The files to load environment variables from. + + If no value is specified, environment variables will be read from an + optional `.env` file automatically. + oneOf: + - $ref: "#/$defs/envFile" + - type: array + items: + $ref: "#/$defs/envFile" + type: description: > The type of the value.