From ad724f0038322b9b311ce4ca0e8d4b9620b1b3ab Mon Sep 17 00:00:00 2001 From: Elliot Forbes Date: Wed, 5 Oct 2022 10:19:48 +0100 Subject: [PATCH] Migrates away from API-Service and calls orb-service directly --- .circleci/config.yml | 6 +- Dockerfile | 2 +- api/api.go | 120 ----- api/rest/client.go | 38 +- cmd/config.go | 69 ++- cmd/config_test.go | 467 +++--------------- cmd/root.go | 14 +- config/config.go | 114 +++++ .../deprecated_images.go | 7 +- go.mod | 2 +- local/local.go | 13 +- settings/settings.go | 1 + 12 files changed, 307 insertions(+), 546 deletions(-) create mode 100644 config/config.go rename cmd/deprecated-images.go => config/deprecated_images.go (93%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d7c27638..3f8297e46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ orbs: executors: go: docker: - - image: cimg/go:1.18 + - image: cimg/go:1.19 environment: CGO_ENABLED: 0 mac: @@ -105,8 +105,8 @@ jobs: - checkout - run: | brew update - brew install go@1.18 - echo 'export PATH="/usr/local/opt/go@1.18/bin:$PATH"' >> ~/.bash_profile + brew install go@1.19 + echo 'export PATH="/usr/local/opt/go@1.19/bin:$PATH"' >> ~/.bash_profile - gomod - run: make test build: diff --git a/Dockerfile b/Dockerfile index c94a4659f..0d6d09f17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM cimg/go:1.18.3 +FROM cimg/go:1.19.1 ENV CIRCLECI_CLI_SKIP_UPDATE_CHECK true diff --git a/api/api.go b/api/api.go index 674ae112c..eabbc5ef0 100644 --- a/api/api.go +++ b/api/api.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/CircleCI-Public/circleci-cli/api/graphql" - "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/references" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/Masterminds/semver" @@ -513,125 +512,6 @@ func WhoamiQuery(cl *graphql.Client) (*WhoamiResponse, error) { return &response, nil } -// ConfigQueryLegacy calls the GQL API to validate and process config with the legacy orgSlug -func ConfigQueryLegacy(cl *graphql.Client, configPath string, orgSlug string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { - var response BuildConfigResponse - var query string - config, err := loadYaml(configPath) - if err != nil { - return nil, err - } - // GraphQL isn't forwards-compatible, so we are unusually selective here about - // passing only non-empty fields on to the API, to minimize user impact if the - // backend is out of date. - var fieldAddendums string - if orgSlug != "" { - fieldAddendums += ", orgSlug: $orgSlug" - } - if len(params) > 0 { - fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" - } - query = fmt.Sprintf( - `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }`, - fieldAddendums) - - request := graphql.NewRequest(query) - request.Var("config", config) - - if values != nil { - request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) - } - if params != nil { - pipelineParameters, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) - } - request.Var("pipelineParametersJson", string(pipelineParameters)) - } - - if orgSlug != "" { - request.Var("orgSlug", orgSlug) - } - - request.SetToken(cl.Token) - - err = cl.Run(request, &response) - if err != nil { - return nil, errors.Wrap(err, "Unable to validate config") - } - if len(response.BuildConfig.ConfigResponse.Errors) > 0 { - return nil, &response.BuildConfig.ConfigResponse.Errors - } - - return &response.BuildConfig.ConfigResponse, nil -} - -// ConfigQuery calls the GQL API to validate and process config with the org id -func ConfigQuery(cl *graphql.Client, configPath string, orgId string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { - var response BuildConfigResponse - var query string - config, err := loadYaml(configPath) - if err != nil { - return nil, err - } - // GraphQL isn't forwards-compatible, so we are unusually selective here about - // passing only non-empty fields on to the API, to minimize user impact if the - // backend is out of date. - var fieldAddendums string - if orgId != "" { - fieldAddendums += ", orgId: $orgId" - } - if len(params) > 0 { - fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" - } - query = fmt.Sprintf( - `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }`, - fieldAddendums) - - request := graphql.NewRequest(query) - request.Var("config", config) - - if values != nil { - request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) - } - if params != nil { - pipelineParameters, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) - } - request.Var("pipelineParametersJson", string(pipelineParameters)) - } - - if orgId != "" { - request.Var("orgId", orgId) - } - request.SetToken(cl.Token) - - err = cl.Run(request, &response) - if err != nil { - return nil, errors.Wrap(err, "Unable to validate config") - } - if len(response.BuildConfig.ConfigResponse.Errors) > 0 { - return nil, &response.BuildConfig.ConfigResponse.Errors - } - - return &response.BuildConfig.ConfigResponse, nil -} - // OrbQuery validated and processes an orb. func OrbQuery(cl *graphql.Client, configPath string) (*ConfigResponse, error) { var response OrbConfigResponse diff --git a/api/rest/client.go b/api/rest/client.go index d745b887e..6fe2bed8c 100644 --- a/api/rest/client.go +++ b/api/rest/client.go @@ -18,6 +18,7 @@ import ( type Client struct { baseURL *url.URL + apiURL *url.URL circleToken string client *http.Client } @@ -29,13 +30,14 @@ func New(host string, config *settings.Config) *Client { endpoint += "/" } - u, _ := url.Parse(host) - + baseURL, _ := url.Parse(host) + apiURL, _ := url.Parse(config.ConfigAPIHost) client := config.HTTPClient client.Timeout = 10 * time.Second return &Client{ - baseURL: u.ResolveReference(&url.URL{Path: endpoint}), + apiURL: apiURL.ResolveReference(&url.URL{Path: endpoint}), + baseURL: baseURL.ResolveReference(&url.URL{Path: endpoint}), circleToken: config.Token, client: client, } @@ -70,6 +72,36 @@ func (c *Client) NewRequest(method string, u *url.URL, payload interface{}) (req return req, nil } +// NewAPIRequest - similar to NewRequest except it uses the apiURL as the base URL. +func (c *Client) NewAPIRequest(method string, u *url.URL, payload interface{}) (req *http.Request, err error) { + var r io.Reader + if payload != nil { + buf := &bytes.Buffer{} + r = buf + err = json.NewEncoder(buf).Encode(payload) + if err != nil { + return nil, err + } + } + + req, err = http.NewRequest(method, c.apiURL.ResolveReference(u).String(), r) + if err != nil { + return nil, err + } + + req.Header.Set("Circle-Token", c.circleToken) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", version.UserAgent()) + commandStr := header.GetCommandStr() + if commandStr != "" { + req.Header.Set("Circleci-Cli-Command", commandStr) + } + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + return req, nil +} + func (c *Client) DoRequest(req *http.Request, resp interface{}) (statusCode int, err error) { httpResp, err := c.client.Do(req) if err != nil { diff --git a/cmd/config.go b/cmd/config.go index 40344a079..52d05683c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "io/ioutil" + "net/url" "strings" - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/CircleCI-Public/circleci-cli/api/graphql" + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/config" "github.com/CircleCI-Public/circleci-cli/filetree" "github.com/CircleCI-Public/circleci-cli/local" "github.com/CircleCI-Public/circleci-cli/pipeline" @@ -20,7 +21,7 @@ import ( type configOptions struct { cfg *settings.Config - cl *graphql.Client + rest *rest.Client args []string } @@ -63,7 +64,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Check that the config file is well formed.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) + opts.rest = rest.New(config.Host, config) }, RunE: func(cmd *cobra.Command, _ []string) error { return validateConfig(opts, cmd.Flags()) @@ -85,7 +86,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Validate config and display expanded configuration.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) + opts.rest = rest.New(config.Host, config) }, RunE: func(cmd *cobra.Command, _ []string) error { return processConfig(opts, cmd.Flags()) @@ -125,7 +126,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { // The arg is actually optional, in order to support compatibility with the --path flag. func validateConfig(opts configOptions, flags *pflag.FlagSet) error { var err error - var response *api.ConfigResponse + var response *config.ConfigResponse path := local.DefaultConfigPath // First, set the path to configPath set by --path flag for compatibility if configPath != "" { @@ -138,15 +139,22 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { } //if no orgId provided use org slug - orgID, _ := flags.GetString("org-id") + var orgID string + orgID, _ = flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = api.ConfigQuery(opts.cl, path, orgID, nil, pipeline.LocalPipelineValues()) + orgID, _ = flags.GetString("org-id") + response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - response, err = api.ConfigQueryLegacy(opts.cl, path, orgSlug, nil, pipeline.LocalPipelineValues()) + orgs, err := GetOrgCollaborations(opts.rest) + if err != nil { + fmt.Println(err.Error()) + } + orgID = GetOrgIdFromSlug(orgSlug, orgs) + response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } @@ -156,7 +164,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { // link here to blog post when available // returns an error if a deprecated image is used if !ignoreDeprecatedImages { - err := deprecatedImageCheck(response) + err := config.DeprecatedImageCheck(response) if err != nil { return err } @@ -173,7 +181,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { func processConfig(opts configOptions, flags *pflag.FlagSet) error { paramsYaml, _ := flags.GetString("pipeline-parameters") - var response *api.ConfigResponse + var response *config.ConfigResponse var params pipeline.Parameters var err error @@ -194,13 +202,18 @@ func processConfig(opts configOptions, flags *pflag.FlagSet) error { //if no orgId provided use org slug orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = api.ConfigQuery(opts.cl, opts.args[0], orgID, params, pipeline.LocalPipelineValues()) + response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - response, err = api.ConfigQueryLegacy(opts.cl, opts.args[0], orgSlug, params, pipeline.LocalPipelineValues()) + orgs, err := GetOrgCollaborations(opts.rest) + if err != nil { + fmt.Println(err.Error()) + } + orgID = GetOrgIdFromSlug(orgSlug, orgs) + response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, pipeline.LocalPipelineValues()) if err != nil { return err } @@ -227,3 +240,33 @@ func packConfig(opts configOptions) error { func migrateConfig(opts configOptions) error { return proxy.Exec([]string{"config", "migrate"}, opts.args) } + +type CollaborationResult struct { + VcsTye string `json:"vcs_type"` + OrgSlug string `json:"slug"` + OrgName string `json:"name"` + OrgId string `json:"id"` + AvatarUrl string `json:"avatar_url"` +} + +// GetOrgCollaborations - fetches all the collaborations for a given user. +func GetOrgCollaborations(client *rest.Client) ([]CollaborationResult, error) { + req, err := client.NewRequest("GET", &url.URL{Path: "me/collaborations"}, nil) + if err != nil { + return nil, err + } + + var resp []CollaborationResult + _, err = client.DoRequest(req, &resp) + return resp, err +} + +// GetOrgIdFromSlug - converts a slug into an orgID. +func GetOrgIdFromSlug(slug string, collaborations []CollaborationResult) string { + for _, v := range collaborations { + if v.OrgSlug == slug { + return v.OrgId + } + } + return "" +} diff --git a/cmd/config_test.go b/cmd/config_test.go index e03177f8d..6c622f564 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -1,20 +1,13 @@ package cmd_test import ( - "encoding/json" "fmt" - "io" - "net/http" "os/exec" "path/filepath" - "time" - "github.com/CircleCI-Public/circleci-cli/api/graphql" "github.com/CircleCI-Public/circleci-cli/clitest" - "github.com/CircleCI-Public/circleci-cli/pipeline" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "gotest.tools/v3/golden" ) @@ -25,7 +18,7 @@ var _ = Describe("Config", func() { command *exec.Cmd results []byte tempSettings *clitest.TempSettings - token string = "testtoken" + // token string = "testtoken" ) BeforeEach(func() { @@ -36,395 +29,95 @@ var _ = Describe("Config", func() { tempSettings.Close() }) - Describe("a .circleci folder with config.yml and local orbs folder containing the hugo orb", func() { - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - "testdata/hugo-pack/.circleci") - results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) - }) - - It("pack all YAML contents as expected", func() { - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - }) - - Describe("local orbs folder with mixed inline and local commands, jobs, etc", func() { - BeforeEach(func() { - var path string = "nested-orbs-and-local-commands-etc" - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - filepath.Join("testdata", path, "test")) - results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) - }) - - It("pack all YAML contents as expected", func() { - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) + It("packs all YAML contents as expected", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/hugo-pack/.circleci") + results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) }) - Describe("an orb containing local executors and commands in folder", func() { - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - "testdata/myorb/test") - - results = golden.Get(GinkgoT(), filepath.FromSlash("myorb/result.yml")) - }) - - It("pack all YAML contents as expected", func() { - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) + It("given a .circleci folder with config.ymla nd local orb, packs all YAML contents as expected", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/hugo-pack/.circleci") + results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) }) - Describe("with a large nested config including rails orb", func() { - BeforeEach(func() { - var path string = "test-with-large-nested-rails-orb" - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - filepath.Join("testdata", path, "test")) - results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) - }) - - It("pack all YAML contents as expected", func() { - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - }) - - Context("config is a list and not a map", func() { - var config *clitest.TmpFile - - BeforeEach(func() { - config = clitest.OpenTmpFile(filepath.Join(tempSettings.Home, "myorb"), "config.yaml") - - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - config.RootDir, - ) - }) - - AfterEach(func() { - config.Close() - }) - - It("prints an error about invalid YAML", func() { - config.Write([]byte(`[]`)) - - expected := fmt.Sprintf("Error: Failed trying to marshal the tree to YAML : expected a map, got a `[]interface {}` which is not supported at this time for \"%s\"\n", config.Path) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - - Expect(err).ShouldNot(HaveOccurred()) - - stderr := session.Wait().Err.Contents() - Expect(string(stderr)).To(Equal(expected)) - Eventually(session).Should(clitest.ShouldFail()) - }) - }) - - Describe("validating configs", func() { - config := "version: 2.1" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.Variables["config"] = config - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "error1"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: error1")) - Eventually(session).Should(clitest.ShouldFail()) - }) - - It("returns successfully when validating a config", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) - Eventually(session).Should(gexec.Exit(0)) - }) + It("given a local orbs folder with mixed inline and local commands, jobs, etc, packs all YAML contents as expected", func() { + var path string = "nested-orbs-and-local-commands-etc" + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + filepath.Join("testdata", path, "test")) + results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) }) - Describe("validating configs with pipeline parameters", func() { - config := "version: 2.1" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "process", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--pipeline-parameters", `{"foo": "test", "bar": true, "baz": 10}`, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, pipelineParametersJson: $pipelineParametersJson) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.Variables["config"] = config - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - pipelineParams, err := json.Marshal(pipeline.Parameters{ - "foo": "test", - "bar": true, - "baz": 10, - }) - Expect(err).ToNot(HaveOccurred()) - r.Variables["pipelineParametersJson"] = string(pipelineParams) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns successfully when validating a config", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session).Should(gexec.Exit(0)) - }) + It("packs successfully given an orb containing local executors and commands in folder", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/myorb/test") + + results = golden.Get(GinkgoT(), filepath.FromSlash("myorb/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) }) - Describe("validating configs with private orbs", func() { - config := "version: 2.1" - orgId := "bb604b45-b6b0-4b81-ad80-796f15eddf87" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--org-id", orgId, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgId: $orgId) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.Variables["config"] = config - r.Variables["orgId"] = orgId - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config with a private orb", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "permission denied"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) - Eventually(session).Should(clitest.ShouldFail()) - }) + It("packs as expected given a large nested config including rails orbs", func() { + var path string = "test-with-large-nested-rails-orb" + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + filepath.Join("testdata", path, "test")) + results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) }) - Describe("validating configs with private orbs Legacy", func() { - config := "version: 2.1" - orgSlug := "circleci" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--org-slug", orgSlug, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgSlug: $orgSlug) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.Variables["config"] = config - r.Variables["orgSlug"] = orgSlug - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config with a private orb", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "permission denied"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) - Eventually(session).Should(clitest.ShouldFail()) - }) - - It("returns successfully when validating a config with private orbs", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) - Eventually(session).Should(gexec.Exit(0)) - }) + It("prints an error given a config which is a list and not a map", func() { + config := clitest.OpenTmpFile(filepath.Join(tempSettings.Home, "myorb"), "config.yaml") + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + config.RootDir, + ) + config.Write([]byte(`[]`)) + + expected := fmt.Sprintf("Error: Failed trying to marshal the tree to YAML : expected a map, got a `[]interface {}` which is not supported at this time for \"%s\"\n", config.Path) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + stderr := session.Wait().Err.Contents() + Expect(string(stderr)).To(Equal(expected)) + Eventually(session).Should(clitest.ShouldFail()) + config.Close() }) }) }) diff --git a/cmd/root.go b/cmd/root.go index f47ff96ea..a51f10ee4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,7 @@ import ( var defaultEndpoint = "graphql-unstable" var defaultHost = "https://circleci.com" +var defaultConfigApiHost = "https://api.circleci.com" var defaultRestEndpoint = "api/v2" var trueString = "true" @@ -96,12 +97,13 @@ Use "{{.CommandPath}} [command] --help" for more information about a command.{{e // MakeCommands creates the top level commands func MakeCommands() *cobra.Command { rootOptions = &settings.Config{ - Debug: false, - Token: "", - Host: defaultHost, - RestEndpoint: defaultRestEndpoint, - Endpoint: defaultEndpoint, - GitHubAPI: "https://api.github.com/", + Debug: false, + Token: "", + Host: defaultHost, + ConfigAPIHost: defaultConfigApiHost, + RestEndpoint: defaultRestEndpoint, + Endpoint: defaultEndpoint, + GitHubAPI: "https://api.github.com/", } if err := rootOptions.Load(); err != nil { diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..cd3ec9d8d --- /dev/null +++ b/config/config.go @@ -0,0 +1,114 @@ +package config + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "os" + + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/pipeline" + "github.com/pkg/errors" +) + +type ConfigError struct { + Message string `json:"message"` +} + +type ConfigResponse struct { + Valid bool `json:"valid"` + SourceYaml string `json:"source-yaml"` + OutputYaml string `json:"output-yaml"` + Errors []ConfigError `json:"errors"` +} + +type CompileConfigRequest struct { + ConfigYaml string `json:"config_yaml"` + Options Options `json:"options"` +} + +type Options struct { + OwnerID string `json:"owner_id,omitempty"` + PipelineParameters string `json:"pipeline_parameters,omitempty"` + PipelineValues map[string]string `json:"pipeline_values,omitempty"` +} + +// #nosec +func loadYaml(path string) (string, error) { + var err error + var config []byte + if path == "-" { + config, err = ioutil.ReadAll(os.Stdin) + } else { + config, err = ioutil.ReadFile(path) + } + + if err != nil { + return "", errors.Wrapf(err, "Could not load config file at %s", path) + } + + return string(config), nil +} + +// ConfigQuery - attempts to compile or validate a given config file with the +// passed in params/values. +// If the orgID is passed in, the config-compilation with private orbs should work. +func ConfigQuery( + rest *rest.Client, + configPath string, + orgID string, + params pipeline.Parameters, + values pipeline.Values, +) (*ConfigResponse, error) { + + configString, err := loadYaml(configPath) + if err != nil { + return nil, err + } + + compileRequest := CompileConfigRequest{ + ConfigYaml: configString, + Options: Options{ + PipelineValues: values, + }, + } + + if orgID != "" { + compileRequest.Options.OwnerID = orgID + } + + if len(params) >= 1 { + pipelineParamsString, err := json.Marshal(params) + if err != nil { + return nil, err + } + compileRequest.Options.PipelineParameters = string(pipelineParamsString) + } + + req, err := rest.NewAPIRequest( + "POST", + &url.URL{ + Path: "compile-config-with-defaults", + }, + compileRequest, + ) + if err != nil { + return nil, err + } + + configCompilationResp := &ConfigResponse{} + statusCode, err := rest.DoRequest(req, configCompilationResp) + if err != nil { + return nil, err + } + if statusCode != 200 { + return nil, errors.New("non 200 status code") + } + + if len(configCompilationResp.Errors) > 0 { + return nil, errors.New(fmt.Sprintf("config compilation contains errors: %s", configCompilationResp.Errors)) + } + + return configCompilationResp, nil +} diff --git a/cmd/deprecated-images.go b/config/deprecated_images.go similarity index 93% rename from cmd/deprecated-images.go rename to config/deprecated_images.go index 607edfc88..1579af3de 100644 --- a/cmd/deprecated-images.go +++ b/config/deprecated_images.go @@ -1,8 +1,6 @@ -package cmd +package config import ( - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/pkg/errors" "gopkg.in/yaml.v3" ) @@ -40,8 +38,7 @@ type processedConfig struct { } // Processes the config down to v2.0, then checks image used against the block list -func deprecatedImageCheck(response *api.ConfigResponse) error { - +func DeprecatedImageCheck(response *ConfigResponse) error { aConfig := processedConfig{} err := yaml.Unmarshal([]byte(response.OutputYaml), &aConfig) if err != nil { diff --git a/go.mod b/go.mod index d515fd50c..fc76cc3ef 100644 --- a/go.mod +++ b/go.mod @@ -90,4 +90,4 @@ require ( // fix vulnerability: CVE-2020-15114 in etcd v3.3.10+incompatible replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible -go 1.18 +go 1.19 diff --git a/local/local.go b/local/local.go index e139be8f4..0177ba7a1 100644 --- a/local/local.go +++ b/local/local.go @@ -10,8 +10,8 @@ import ( "strings" "syscall" - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/CircleCI-Public/circleci-cli/api/graphql" + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/config" "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" @@ -24,21 +24,20 @@ const DefaultConfigPath = ".circleci/config.yml" func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { var err error - var configResponse *api.ConfigResponse - cl := graphql.NewClient(cfg.HTTPClient, cfg.Host, cfg.Endpoint, cfg.Token, cfg.Debug) - + var configResponse *config.ConfigResponse + restClient := rest.New(cfg.Host, cfg) processedArgs, configPath := buildAgentArguments(flags) //if no orgId provided use org slug orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - configResponse, err = api.ConfigQuery(cl, configPath, orgID, nil, pipeline.LocalPipelineValues()) + configResponse, err = config.ConfigQuery(restClient, configPath, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - configResponse, err = api.ConfigQueryLegacy(cl, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) + configResponse, err = config.ConfigQuery(restClient, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) if err != nil { return err } diff --git a/settings/settings.go b/settings/settings.go index 24bc07f33..1459725d6 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -35,6 +35,7 @@ type Config struct { GitHubAPI string `yaml:"-"` SkipUpdateCheck bool `yaml:"-"` OrbPublishing OrbPublishingInfo `yaml:"orb_publishing"` + ConfigAPIHost string `yaml:"-"` } type OrbPublishingInfo struct {