Merge pull request #408 from CircleCI-Public/integrations-tests

Try using cucumber for bdd tests
CircleCI-Public · May 18, 2020 · a1db173 · a1db173
2 parents 977fab8 + 0700d33
commit a1db173
Show file tree

Hide file tree

Showing 8 changed files with 157 additions and 221 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -99,6 +99,33 @@ jobs:
       - gomod
       - run: make test
       - slack-notify-on-failure
+  build:
+    executor: go
+    steps:
+      - checkout
+      - gomod
+      - run: make
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "build"
+  cucumber:
+    docker:
+      - image: cimg/ruby:2.7
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: "Install CLI tool from workspace"
+          command: sudo cp ~/project/build/linux/amd64/circleci /usr/local/bin/
+      - run:
+          command: bundle install
+          working_directory: integration_tests
+      - run:
+          command: bundle exec cucumber
+          working_directory: integration_tests
+
   test:
     executor: go
     steps:
@@ -253,6 +280,9 @@ jobs:
 workflows:
   ci:
     jobs:
+      - build
+      - cucumber:
+          requires: [build]
       - shellcheck/check:
           exclude: ./vendor/*
       - test

diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,5 @@ packrd/
 
 # golangci-lint
 # We expect to install the binary each time in CI
-bin/golangci-lint
+bin/golangci-lint
+integration_tests/tmp
diff --git a/cmd/config_test.go b/cmd/config_test.go
@@ -2,236 +2,17 @@ package cmd_test
 
 import (
 	"fmt"
-	"net/http"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/CircleCI-Public/circleci-cli/clitest"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/gbytes"
 	"github.com/onsi/gomega/gexec"
 	"gotest.tools/golden"
 )
 
-func predictValidateConfigRequest() string {
-	return `
-{
-		"query": "\n\t\tquery ValidateConfig ($config: String!, $pipelineValues: [StringKeyVal!]) {\n\t\t\tbuildConfig(configYaml: $config, pipelineValues: $pipelineValues) {\n\t\t\t\tvalid,\n\t\t\t\terrors { message },\n\t\t\t\tsourceYaml,\n\t\t\t\toutputYaml\n\t\t\t}\n\t\t}",
-		"variables": {
-				"config": "some config",
-				"pipelineValues": [
-						{"key": "git.base_revision", "val": "0123456789abcdef0123456789abcdef0123"},
-						{"key": "git.branch", "val": "test_git_branch"},
-						{"key": "git.revision", "val": "0123456789abcdef0123456789abcdef0123"},
-						{"key": "git.tag", "val": "test_git_tag"},
-						{"key": "id", "val": "00000000-0000-0000-0000-000000000001"},
-						{"key": "number", "val": "1"},
-						{"key": "project.git_url", "val": "https://test.vcs/test/test"},
-						{"key": "project.type", "val": "vcs_type"}
-				]
-		}
-}`
-}
-
 var _ = Describe("Config", func() {
-	Describe("with an api and config.yml", func() {
-		var tempSettings *clitest.TempSettings
-
-		BeforeEach(func() {
-			tempSettings = clitest.WithTempSettings()
-		})
-
-		AfterEach(func() {
-			tempSettings.Cleanup()
-		})
-
-		Describe("when validating config", func() {
-			var (
-				token   string
-				config  *clitest.TmpFile
-				command *exec.Cmd
-			)
-
-			BeforeEach(func() {
-				config = clitest.OpenTmpFile(tempSettings.Home, ".circleci/config.yaml")
-
-				token = "testtoken"
-				command = exec.Command(pathCLI,
-					"config", "validate",
-					"--skip-update-check",
-					"--token", token,
-					"--host", tempSettings.TestServer.URL(),
-					config.Path,
-				)
-			})
-
-			It("works", func() {
-				config.Write([]byte(`some config`))
-
-				gqlResponse := `{
-							"buildConfig": {
-								"sourceYaml": "hello world",
-								"valid": true,
-								"errors": []
-							}
-						}`
-
-				expectedRequestJson := predictValidateConfigRequest()
-
-				tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{
-					Status:   http.StatusOK,
-					Request:  expectedRequestJson,
-					Response: gqlResponse,
-				})
-
-				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-
-				Expect(err).ShouldNot(HaveOccurred())
-				Eventually(session.Out).Should(gbytes.Say("Config file at .*circleci/config.yaml is valid"))
-				Eventually(session).Should(gexec.Exit(0))
-			})
-
-			It("prints errors if invalid", func() {
-				config.Write([]byte(`some config`))
-
-				gqlResponse := `{
-							"buildConfig": {
-								"sourceYaml": "hello world",
-								"valid": false,
-								"errors": [
-									{"message": "invalid_config"}
-								]
-							}
-						}`
-
-
-				expectedRequestJson := predictValidateConfigRequest()
-
-				tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{
-					Status:   http.StatusOK,
-					Request:  expectedRequestJson,
-					Response: gqlResponse,
-				})
-
-				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-
-				Expect(err).ShouldNot(HaveOccurred())
-				Eventually(session.Err).Should(gbytes.Say("Error:"))
-				Eventually(session.Err).Should(gbytes.Say("invalid_config"))
-				Eventually(session).ShouldNot(gexec.Exit(0))
-			})
-		})
-
-		Describe("when processing config", func() {
-			var (
-				token   string
-				config  *clitest.TmpFile
-				command *exec.Cmd
-			)
-
-			BeforeEach(func() {
-				config = clitest.OpenTmpFile(tempSettings.Home, ".circleci/config.yaml")
-
-				token = "testtoken"
-				command = exec.Command(pathCLI,
-					"config", "process",
-					"--skip-update-check",
-					"--token", token,
-					"--host", tempSettings.TestServer.URL(),
-					config.Path,
-				)
-			})
-
-			It("works", func() {
-				config.Write([]byte(`some config`))
-
-				gqlResponse := `{
-							"buildConfig": {
-								"outputYaml": "hello world",
-								"valid": true,
-								"errors": []
-							}
-						}`
-
-				expectedRequestJson := ` {
-					"query": "\n\t\tquery ValidateConfig ($config: String!, $pipelineValues: [StringKeyVal!]) {\n\t\t\tbuildConfig(configYaml: $config, pipelineValues: $pipelineValues) {\n\t\t\t\tvalid,\n\t\t\t\terrors { message },\n\t\t\t\tsourceYaml,\n\t\t\t\toutputYaml\n\t\t\t}\n\t\t}",
-					"variables": {
-					  "config": "some config"
-					}
-				  }`
-
-				tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{
-					Status:   http.StatusOK,
-					Request:  expectedRequestJson,
-					Response: gqlResponse,
-				})
-
-				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-
-				Expect(err).ShouldNot(HaveOccurred())
-				Eventually(session.Out).Should(gbytes.Say("hello world"))
-				Eventually(session).Should(gexec.Exit(0))
-			})
-
-			It("prints errors if invalid", func() {
-				config.Write([]byte(`some config`))
-
-				gqlResponse := `{
-							"buildConfig": {
-								"outputYaml": "hello world",
-								"valid": false,
-								"errors": [
-									{"message": "error1"},
-									{"message": "error2"}
-								]
-							}
-						}`
-
-				expectedRequestJson := ` {
-					"query": "\n\t\tquery ValidateConfig ($config: String!, $pipelineValues: [StringKeyVal!]) {\n\t\t\tbuildConfig(configYaml: $config, pipelineValues: $pipelineValues) {\n\t\t\t\tvalid,\n\t\t\t\terrors { message },\n\t\t\t\tsourceYaml,\n\t\t\t\toutputYaml\n\t\t\t}\n\t\t}",
-					"variables": {
-					  "config": "some config"
-					}
-				  }`
-
-				tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{
-					Status:   http.StatusOK,
-					Request:  expectedRequestJson,
-					Response: gqlResponse,
-				})
-
-				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-
-				Expect(err).ShouldNot(HaveOccurred())
-				Eventually(session.Err).Should(gbytes.Say("Error: error1\nerror2"))
-				Eventually(session).ShouldNot(gexec.Exit(0))
-			})
-
-		})
-
-		Describe("When processing a config that has special characters", func() {
-			var (
-				command *exec.Cmd
-				results []byte
-			)
-			BeforeEach(func() {
-				command = exec.Command(pathCLI,
-					"config", "process",
-					"--skip-update-check",
-					"testdata/formatting/with_percent.yml")
-				results = golden.Get(GinkgoT(), filepath.FromSlash("formatting/result.yml"))
-			})
-			It("respects percent signs", func() {
-				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
-				session.Wait()
-				Expect(err).ShouldNot(HaveOccurred())
-				Eventually(session.Err.Contents()).Should(BeEmpty())
-				Eventually(session.Out.Contents()).Should(MatchYAML(results))
-				Eventually(session).Should(gexec.Exit(0))
-			})
-		})
-	})
 
 	Describe("pack", func() {
 		var (

diff --git a/integration_tests/Gemfile b/integration_tests/Gemfile
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+source "https://rubygems.org"
+gem 'aruba'
diff --git a/integration_tests/Gemfile.lock b/integration_tests/Gemfile.lock
@@ -0,0 +1,49 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (1.0.0)
+      childprocess (~> 3.0)
+      contracts (~> 0.16.0)
+      cucumber (>= 2.4, < 4.0)
+      ffi (~> 1.9)
+      rspec-expectations (~> 3.4)
+      thor (~> 1.0)
+    backports (3.17.1)
+    builder (3.2.4)
+    childprocess (3.0.0)
+    contracts (0.16.0)
+    cucumber (3.1.2)
+      builder (>= 2.1.2)
+      cucumber-core (~> 3.2.0)
+      cucumber-expressions (~> 6.0.1)
+      cucumber-wire (~> 0.0.1)
+      diff-lcs (~> 1.3)
+      gherkin (~> 5.1.0)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (3.2.1)
+      backports (>= 3.8.0)
+      cucumber-tag_expressions (~> 1.1.0)
+      gherkin (~> 5.0)
+    cucumber-expressions (6.0.1)
+    cucumber-tag_expressions (1.1.1)
+    cucumber-wire (0.0.1)
+    diff-lcs (1.3)
+    ffi (1.12.2)
+    gherkin (5.1.0)
+    multi_json (1.14.1)
+    multi_test (0.1.2)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    thor (1.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+
+BUNDLED WITH
+   2.1.2