From 684b182b0e60af1860a76953c70d53eed534016b Mon Sep 17 00:00:00 2001 From: Christopher Eaton Date: Wed, 17 May 2023 14:17:29 -0700 Subject: [PATCH] Add support for .env files. --- .gitignore | 3 ++- CONTRIBUTING.md | 45 ------------------------------------------- README.md | 40 +++++++++++++++++++++++++++++++++----- go.mod | 2 ++ go.sum | 4 ++++ localbuilder_main.go | 35 +++++++++++++++++++++++++++++---- tests/cloudbuild.yaml | 16 +++++++-------- 7 files changed, 82 insertions(+), 63 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/.gitignore b/.gitignore index d744360..2095e1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cloud-build-local vendor/ -node_modules \ No newline at end of file +node_modules +.env \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 2e72e0e..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,45 +0,0 @@ -## Contributor License Agreements - -**This project is not yet set up to accept external contributions.** - -## Contributor License Agreement - -Before we can accept your pull requests you'll need to sign a Contributor -License Agreement (CLA): - -* If you are an individual writing original source code and you own the - intellectual property, then you'll need to sign an - [individual CLA](https://developers.google.com/open-source/cla/individual). - -* If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a - [corporate CLA](https://developers.google.com/open-source/cla/corporate>). - -You can sign these electronically (just scroll to the bottom). After that, we'll -be able to accept your pull requests. - -## Developing the Local Builder - -To build and test the Local Builder, you need a working -[Go environment](https://golang.org/doc/install). You should also install -[gcloud](https://cloud.google.com/sdk/docs/quickstarts) and -[Docker](https://www.docker.com/). - -Run the following commands to install the Local Builder tool: - -``` -go get github.com/GoogleCloudPlatform/cloud-build-local -go install github.com/GoogleCloudPlatform/cloud-build-local -``` - -To run a build: - -``` -./bin/cloud-build-local --dryrun=false --config=path/to/cloudbuild.yaml path/to/code -``` - -To run the tests for Local Builder (without the vendored libraries): - -``` -go test $(go list github.com/GoogleCloudPlatform/cloud-build-local/... | grep -v vendor) -``` diff --git a/README.md b/README.md index e80d978..1d87b1e 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,38 @@ # Google Cloud Build Local -**Local Builder** runs [Google Cloud Build](https://cloud.google.com/cloud-build/) locally, allowing easier debugging, +**GCPL** runs [Google Cloud Build](https://cloud.google.com/cloud-build/) locally, allowing easier debugging, execution of builds on your own hardware, and integration into local build and test workflows. *Please note that the -Local Builder is not 100% feature-compatible with the hosted GCB service.* +Local Builder is not 100% feature-compatible with the hosted Google Cloud Build service.* ### Community Fork -This Cloud Build local builder (fork) is maintained by volunteers, which at best, makes this ok for a local debugging tool for Google Cloud Build. It does not support 100% feature parity with the hosted Cloud Build service and should not be used for production workloads. +This Cloud Build Local Builder (fork) is maintained by volunteers, which at best, makes this ok for a local debugging +tool for Google Cloud Build. It does not support 100% feature parity with the hosted Cloud Build service and should +not be used for production workloads. -Unfortunately, as of 2023, the [original repository](https://github.com/GoogleCloudPlatform/cloud-build-local) from +As of 2023, the [original repository](https://github.com/GoogleCloudPlatform/cloud-build-local) from Google has been archived. This fork is an attempt to keep the project alive, at least, in some form close to the original, and maybe improve on it a bit. +The following features have been implemented: +- Add support for loading `.env` file secrets for `secretEnv` replacements. + This will convert only the matching `secretEnv` into a corresponding `env` with a value. + *This is not supported in the cloud, and only works for GCPL. Which means the cloud will simply treat a secretEnv as + intended :)* +- No contributor agreements. Just code! + ## Usage -To run a local build: +To run a local build you should make sure you've got credentials to GCP if using any resources (`gcloud auth login`), +then specify a `false` dryrun, the config, and the source code/content directory path. ```sh ./cloud-build-local --dryrun=false --config=path/to/cloudbuild.yaml path/to/code ``` ## Development +To build and test the GCPL, you need a working +[Go environment](https://golang.org/doc/install). You should also install +[gcloud](https://cloud.google.com/sdk/docs/quickstarts) and +[Docker](https://www.docker.com/). ### Setup ```sh @@ -31,7 +45,23 @@ go get go build -o cloud-build-local github.com/GoogleCloudPlatform/cloud-build-local ``` +Optionally, create a system-wide link to the built executable: +```sh +sudo ln -s "$(pwd)/cloud-build-local" /usr/bin/cloud-build-local +``` + ### Test +If you'd like to run the manual test build... +1. Create a `tests/.env` file with a secret replacement value: + ```ini + HELLO_BUILD=my secret env variable replacement + ``` +2. Run the test locally: + ```sh + ./cloud-build-local --dryrun=false --config=./tests/cloudbuild.yaml --env=./tests/.env ./tests/src + ``` + +Other tests (legacy): ```sh go test $(go list github.com/GoogleCloudPlatform/cloud-build-local/... | grep -v vendor) ``` \ No newline at end of file diff --git a/go.mod b/go.mod index 1ee04c0..76d2141 100644 --- a/go.mod +++ b/go.mod @@ -23,9 +23,11 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.1.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect diff --git a/go.sum b/go.sum index dea7d93..2ffe301 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -214,6 +216,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/localbuilder_main.go b/localbuilder_main.go index ac09ae5..f6aeb0b 100644 --- a/localbuilder_main.go +++ b/localbuilder_main.go @@ -28,10 +28,12 @@ import ( "strings" "time" - "github.com/spf13/afero" + "golang.org/x/exp/slices" + computeMetadata "cloud.google.com/go/compute/metadata" - "golang.org/x/oauth2" "github.com/pborman/uuid" + "github.com/spf13/afero" + "golang.org/x/oauth2" "github.com/GoogleCloudPlatform/cloud-build-local/build" "github.com/GoogleCloudPlatform/cloud-build-local/common" @@ -41,6 +43,7 @@ import ( "github.com/GoogleCloudPlatform/cloud-build-local/runner" "github.com/GoogleCloudPlatform/cloud-build-local/validate" "github.com/GoogleCloudPlatform/cloud-build-local/volume" + "github.com/joho/godotenv" ) const ( @@ -51,6 +54,7 @@ const ( var ( configFile = flag.String("config", "cloudbuild.yaml", "File path of the config file") + envFile = flag.String("env", ".env", "File path to an optional .env") substitutions = flag.String("substitutions", "", `key=value pairs where the key is already defined in the build request; separate multiple substitutions with a comma, for example: _FOO=bar,_BAZ=baz`) dryRun = flag.Bool("dryrun", true, "Lints the config file and prints but does not run the commands; Local Builder runs the commands only when dryrun is set to false") push = flag.Bool("push", false, "Pushes the images to the registry") @@ -63,7 +67,7 @@ var ( ) func exitUsage(msg string) { - log.Fatalf("%s\nUsage: %s --config=cloudbuild.yaml [--substitutions=_FOO=bar] [--dryrun=true/false] [--push=true/false] [--bind-mount-source=true/false] source", msg, os.Args[0]) + log.Fatalf("%s\nUsage: %s --config=cloudbuild.yaml [--env=/path/to/.env] [--substitutions=_FOO=bar] [--dryrun=true/false] [--push=true/false] [--bind-mount-source=true/false] source", msg, os.Args[0]) } func main() { @@ -139,12 +143,35 @@ func run(ctx context.Context, source string) error { log.Printf("Warning: The client docker version installed (%s) is different from the one used in GCB (%s)", dockerClientVersion, gcbDockerVersion) } } - + // Load specified env file. + var envMap map[string]string + if envFile != nil { + var err error + if envMap, err = godotenv.Read(*envFile); err != nil { + return fmt.Errorf("Error loading env file: %v", err) + } + } + log.Printf("envMap: %+v", envMap) // Load config file into a build struct. buildConfig, err := config.Load(*configFile) if err != nil { return fmt.Errorf("Error loading config file: %v", err) } + // Replace .env secrets + if len(envMap) > 0 && len(buildConfig.Steps) > 0 { + for _, s := range buildConfig.Steps { + if len(s.SecretEnv) > 0 { + for k, v := range envMap { + index := slices.Index(s.SecretEnv, k) + if index >= 0 { + s.Env = append(s.Env, fmt.Sprintf("%s=%s", k, v)) + s.SecretEnv = slices.Delete(s.SecretEnv, index, index+1) + log.Printf("Found, Replaced: %s %v", s.Id, s) + } + } + } + } + } // When the build is run locally, there will be no build ID. Assign a unique value. buildConfig.Id = "localbuild_" + uuid.New() diff --git a/tests/cloudbuild.yaml b/tests/cloudbuild.yaml index f64e93a..e49f6dd 100644 --- a/tests/cloudbuild.yaml +++ b/tests/cloudbuild.yaml @@ -12,11 +12,11 @@ steps: id: test entrypoint: npm args: ['test'] -# - name: node -# id: build -# secretEnv: ['SOME_ENV'] -# entrypoint: bash -# args: -# - -ceu -# - | -# echo '$$SOME_ENV' +- name: node + id: build + secretEnv: ['HELLO_BUILD'] + entrypoint: bash + args: + - -ceu + - | + echo "$$HELLO_BUILD"