diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a1e2c702..43fe490ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,6 +100,8 @@ jobs: - run: mkdir test_results - run: name: Run tests + environment: + test_env_var: test_env_var_value command: | C:\Users\circleci\go\bin\gotestsum.exe --junitfile test_results/windows.xml - store_test_results: diff --git a/.circleci/pack.sh b/.circleci/pack similarity index 100% rename from .circleci/pack.sh rename to .circleci/pack diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..abea34be9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +Dockerfile* +docker-compose*.yml +.vagrant diff --git a/.gitignore b/.gitignore index fe774dfed..450328771 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ packrd/ bin/golangci-lint integration_tests/tmp *.exe +/issues +/.vagrant +/bash_history diff --git a/Makefile b/Makefile index f50603470..5a17e2da9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) build: always - GO111MODULE=on .circleci/pack.sh + .circleci/pack go build -o build/$(GOOS)/$(GOARCH)/circleci build-all: build/linux/amd64/circleci build/darwin/amd64/circleci @@ -16,15 +16,15 @@ build/%/amd64/circleci: always clean: GO111MODULE=off go clean -i rm -rf build out docs dist - .circleci/pack.sh clean + .circleci/pack clean .PHONY: test test: - go test -v ./... + test_env_var=test_env_var_value go test -v ./... .PHONY: cover cover: - go test -race -coverprofile=coverage.txt ./... + test_env_var=test_env_var_value go test -race -coverprofile=coverage.txt ./... .PHONY: lint lint: @@ -40,7 +40,7 @@ install-packr: .PHONY: pack pack: - bash .circleci/pack.sh + bash .circleci/pack .PHONY: install-lint install-lint: diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..5a1d86399 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,18 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'etc' + +Vagrant.require_version ">= 2.0.0", "< 3.0.0" + +Vagrant.configure("2") do |config| + config.vm.box = "bento/ubuntu-20.04" + + config.vm.provider "virtualbox" do |vb| + vb.memory = "2000" + vb.cpus = Etc.nprocessors + end + + config.vm.synced_folder "~/.circleci", '/home/vagrant/.circleci' + config.vm.provision "shell", path: "bin/provision" +end diff --git a/bin/provision b/bin/provision new file mode 100755 index 000000000..e95b452f2 --- /dev/null +++ b/bin/provision @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +apt_update() { + if [[ -z $apt_updated ]] + then + apt-get -qq update && + apt_updated=true || exit + fi +} + +if ! sudo -u vagrant docker run --rm hello-world +then + apt_update && + apt-get -qq install apt-transport-https ca-certificates curl software-properties-common && + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" && + apt-get -qq update && + apt-get -qq install docker-ce && + usermod -aG docker vagrant && + sudo -u vagrant docker run --rm hello-world || + exit +fi + +if ! docker-compose --version +then + curl --silent --show-error --fail --location \ + "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" \ + -o /usr/local/bin/docker-compose && + chmod +x /usr/local/bin/docker-compose && + docker-compose --version || + exit +fi + +if ! grep '^cd /vagrant$' ~vagrant/.bashrc +then + sudo -u vagrant <<<'cd /vagrant' tee -a ~vagrant/.bashrc || + exit +fi + +if ! grep '^HISTFILE=/vagrant/bash_history$' ~vagrant/.bashrc +then + sudo -u vagrant tee -a <<<'HISTFILE=/vagrant/bash_history' ~vagrant/.bashrc || + exit +fi + +if ! sudo -u vagrant -i go version +then + curl --silent --show-error --fail --location \ + "https://golang.org/dl/go1.15.2.linux-amd64.tar.gz" | + tar -C /usr/local -xz && + sudo -u vagrant tee -a <<<'export PATH=$PATH:/usr/local/go/bin' ~vagrant/.bash_profile && + sudo -u vagrant -i go version || + exit 1 +fi + +if ! dpkg-query -s build-essential +then + apt_update && + apt-get -qq install build-essential && + dpkg-query -s build-essential || + exit +fi diff --git a/local/local.go b/local/local.go index 2141400d4..e21136b7f 100644 --- a/local/local.go +++ b/local/local.go @@ -10,6 +10,7 @@ import ( "path" "regexp" "syscall" + "strings" "github.com/CircleCI-Public/circleci-cli/api" "github.com/CircleCI-Public/circleci-cli/api/graphql" @@ -41,7 +42,11 @@ func UpdateBuildAgent() error { func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { - processedArgs, configPath := buildAgentArguments(flags) + processedArgs, configPath, err := buildAgentArguments(flags) + if err != nil { + return err + } + cl := graphql.NewClient(cfg.Host, cfg.Endpoint, cfg.Token, cfg.Debug) configResponse, err := api.ConfigQuery(cl, configPath, pipeline.FabricatedValues()) @@ -87,7 +92,7 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { arguments := generateDockerCommand(processedConfigPath, image, pwd, processedArgs...) if cfg.Debug { - _, err = fmt.Fprintf(os.Stderr, "Starting docker with args: %s", arguments) + _, err = fmt.Fprintf(os.Stderr, "Starting docker with args: %s\n", arguments) if err != nil { return err } @@ -97,6 +102,11 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { return errors.Wrap(err, "Could not find a `docker` executable on $PATH; please ensure that docker installed") } + for _, e := range os.Environ() { + pair := strings.SplitN(e, "=", 2) + fmt.Fprintf(os.Stderr, "%s=%s\n", pair[0], pair[1]) + } + err = syscall.Exec(dockerPath, arguments, os.Environ()) // #nosec return errors.Wrap(err, "failed to execute docker") } @@ -118,7 +128,7 @@ func AddFlagsForDocumentation(flags *pflag.FlagSet) { flags.String("revision", "", "Git Revision") flags.String("branch", "", "Git branch") flags.String("repo-url", "", "Git Url") - flags.StringArrayP("env", "e", nil, "Set environment variables, e.g. `-e VAR=VAL`") + flags.StringArrayP("env", "e", nil, "Set environment variables, e.g. `-e VAR=VAL` or `-e VAR`") } // Given the full set of flags that were passed to this command, return the path @@ -129,21 +139,33 @@ func AddFlagsForDocumentation(flags *pflag.FlagSet) { // GraphQL API, and feed the result of that into `build-agent`. The first step of // that process is to find the local path to the config file. This is supplied with // the `config` flag. -func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { +func buildAgentArguments(flags *pflag.FlagSet) ([]string, string, error) { var result []string = []string{} + var outerErr error // build a list of all supplied flags, that we will pass on to build-agent flags.Visit(func(flag *pflag.Flag) { if flag.Name != "config" && flag.Name != "debug" { - result = append(result, unparseFlag(flags, flag)...) + unparsedFlag, err := unparseFlag(flags, flag) + + if err != nil { + outerErr = errors.Wrap(err, "Failed to build agent arguments") + } + + result = append(result, unparsedFlag...) } }) + + if outerErr != nil { + return nil, "", outerErr + } + result = append(result, flags.Args()...) configPath, _ := flags.GetString("config") - return result, configPath + return result, configPath, nil } func picardImage(output io.Writer) (string, error) { @@ -302,7 +324,7 @@ func generateDockerCommand(configPath, image, pwd string, arguments ...string) [ // Convert the given flag back into a list of strings suitable to be passed on // the command line to run docker. // https://github.com/CircleCI-Public/circleci-cli/issues/391 -func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) []string { +func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) ([]string, error) { flagName := "--" + flag.Name result := []string{} switch flag.Value.Type() { @@ -310,10 +332,22 @@ func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) []string { // `--foo 1 --foo 2` will result in a single `foo` flag with an array of values. case "stringArray": for _, value := range flag.Value.(pflag.SliceValue).GetSlice() { + if flag.Name == "env" && ! strings.Contains(value, "=") { + // Bare env arg passed. Resolve from environment. + variableValue, ok := os.LookupEnv(value) + if !ok { + return nil, errors.New(fmt.Sprintf( + "Failed to resolve environment variable ā€˜%sā€™ in the environment.\n", + value, + )) + } else { + value = fmt.Sprintf("%s=%s", value, variableValue) + } + } result = append(result, flagName, value) } default: result = append(result, flagName, flag.Value.String()) } - return result + return result, nil } diff --git a/local/local_test.go b/local/local_test.go index 7dc4b5e42..522f1ab3a 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -67,7 +67,7 @@ var _ = Describe("build", func() { if testCase.expectedError != "" { Expect(err).To(MatchError(testCase.expectedError)) } - args, configPath := buildAgentArguments(flags) + args, configPath, _ := buildAgentArguments(flags) Expect(args).To(Equal(testCase.expectedArgs)) Expect(configPath).To(Equal(testCase.expectedConfigPath)) @@ -98,15 +98,21 @@ var _ = Describe("build", func() { }), Entry("many args, multiple envs", TestCase{ - input: []string{"--env", "foo", "--env", "bar", "--env", "baz"}, + input: []string{"--env", "foo=foo", "--env", "bar=bar", "--env", "baz=baz"}, expectedConfigPath: ".circleci/config.yml", - expectedArgs: []string{"--env", "foo", "--env", "bar", "--env", "baz"}, + expectedArgs: []string{"--env", "foo=foo", "--env", "bar=bar", "--env", "baz=baz"}, }), Entry("comma in env value (issue #440)", TestCase{ - input: []string{"--env", "{\"json\":[\"like\",\"value\"]}"}, + input: []string{"--env", "JSON={\"json\":[\"like\",\"value\"]}"}, expectedConfigPath: ".circleci/config.yml", - expectedArgs: []string{"--env", "{\"json\":[\"like\",\"value\"]}"}, + expectedArgs: []string{"--env", "JSON={\"json\":[\"like\",\"value\"]}"}, + }), + + Entry("bare env value (issue #440)", TestCase{ + input: []string{"--env", "test_env_var"}, + expectedConfigPath: ".circleci/config.yml", + expectedArgs: []string{"--env", "test_env_var=test_env_var_value"}, }), Entry("args that are not flags", TestCase{