Orb pack (#432)

Co-authored-by: Stella Lok <[email protected]>
Co-authored-by: Kyle Tryon <[email protected]>

api/api.go

@@ -14,8 +14,8 @@ import (
 	"github.com/CircleCI-Public/circleci-cli/pipeline"
 	"github.com/CircleCI-Public/circleci-cli/references"
 	"github.com/Masterminds/semver"
-	"github.com/go-yaml/yaml"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
 )
 
 // GQLErrorsCollection is a slice of errors returned by the GraphQL server.

cmd/config.go

@@ -10,9 +10,9 @@ import (
 	"github.com/CircleCI-Public/circleci-cli/pipeline"
 	"github.com/CircleCI-Public/circleci-cli/proxy"
 	"github.com/CircleCI-Public/circleci-cli/settings"
-	"github.com/go-yaml/yaml"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 )
 
 type configOptions struct {

cmd/orb.go

@@ -4,16 +4,22 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/CircleCI-Public/circleci-cli/api"
 	"github.com/CircleCI-Public/circleci-cli/client"
+	"github.com/CircleCI-Public/circleci-cli/filetree"
 	"github.com/CircleCI-Public/circleci-cli/prompt"
 	"github.com/CircleCI-Public/circleci-cli/references"
 	"github.com/CircleCI-Public/circleci-cli/settings"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
 
 	"github.com/spf13/cobra"
 )
@@ -240,6 +246,18 @@ Please note that at this time all orbs created in the registry are world-readabl
 		},
 		Args: cobra.ExactArgs(1),
 	}
+
+	orbPack := &cobra.Command{
+		Use:   "pack <path>",
+		Short: "Pack an Orb with local scripts.",
+		Long:  ``,
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return packOrb(opts)
+		},
+		Args:   cobra.ExactArgs(1),
+		Hidden: true,
+	}
+
 	orbCreate.Flags().BoolVar(&opts.integrationTesting, "integration-testing", false, "Enable test mode to bypass interactive UI.")
 	if err := orbCreate.Flags().MarkHidden("integration-testing"); err != nil {
 		panic(err)
@@ -269,6 +287,7 @@ Please note that at this time all orbs created in the registry are world-readabl
 	orbCommand.AddCommand(unlistCmd)
 	orbCommand.AddCommand(sourceCommand)
 	orbCommand.AddCommand(orbInfoCmd)
+	orbCommand.AddCommand(orbPack)
 
 	return orbCommand
 }
@@ -740,3 +759,110 @@ https://circleci.com/orbs/registry/orb/%s
 
 	return nil
 }
+
+type OrbSchema struct {
+	Version     float32                  `yaml:"version,omitempty"`
+	Description string                   `yaml:"description,omitempty"`
+	Display     yaml.Node                `yaml:"display,omitempty"`
+	Orbs        yaml.Node                `yaml:"orbs,omitempty"`
+	Commands    yaml.Node                `yaml:"commands,omitempty"`
+	Executors   yaml.Node                `yaml:"executors,omitempty"`
+	Jobs        yaml.Node                `yaml:"jobs,omitempty"`
+	Examples    map[string]ExampleSchema `yaml:"examples,omitempty"`
+}
+
+type ExampleUsageSchema struct {
+	Version   string      `yaml:"version,omitempty"`
+	Orbs      interface{} `yaml:"orbs,omitempty"`
+	Jobs      interface{} `yaml:"jobs,omitempty"`
+	Workflows interface{} `yaml:"workflows"`
+}
+
+type ExampleSchema struct {
+	Description string             `yaml:"description,omitempty"`
+	Usage       ExampleUsageSchema `yaml:"usage,omitempty"`
+	Result      ExampleUsageSchema `yaml:"result,omitempty"`
+}
+
+func packOrb(opts orbOptions) error {
+	// Travel our Orb and build a tree from the YAML files.
+	// Non-YAML files will be ignored here.
+	_, err := os.Stat(filepath.Join(opts.args[0], "@orb.yml"))
+	if err != nil {
+		return errors.New("@orb.yml file not found, are you sure this is the Orb root?")
+	}
+
+	tree, err := filetree.NewTree(opts.args[0], "executors", "jobs", "commands", "examples")
+	if err != nil {
+		return errors.Wrap(err, "An unexpected error occurred")
+	}
+
+	y, err := yaml.Marshal(&tree)
+	if err != nil {
+		return errors.Wrap(err, "An unexpected error occurred")
+	}
+
+	var orbSchema OrbSchema
+	err = yaml.Unmarshal(y, &orbSchema)
+	if err != nil {
+		return errors.Wrap(err, "An unexpected error occurred")
+	}
+
+	err = func(nodes ...*yaml.Node) error {
+		for _, node := range nodes {
+			err = inlineIncludes(node, opts.args[0])
+			if err != nil {
+				return errors.Wrap(err, "An unexpected error occurred")
+			}
+		}
+		return nil
+	}(&orbSchema.Jobs, &orbSchema.Commands, &orbSchema.Executors)
+	if err != nil {
+		return err
+	}
+
+	final, err := yaml.Marshal(&orbSchema)
+	if err != nil {
+		return errors.Wrap(err, "Failed trying to marshal Orb YAML")
+	}
+
+	fmt.Println(string(final))
+
+	return nil
+}
+
+// Travel down a YAML node, replacing values as we go.
+func inlineIncludes(node *yaml.Node, orbRoot string) error {
+	// View: https://regexr.com/582gb
+	includeRegex, err := regexp.Compile(`(?U)^<<\s*include\((.*\/*[^\/]+)\)\s*?>>$`)
+	if err != nil {
+		return err
+	}
+
+	// If we're dealing with a ScalarNode, we can replace the contents.
+	// Otherwise, we recurse into the children of the Node in search of
+	// a matching regex.
+	if node.Kind == yaml.ScalarNode && node.Value != "" {
+		includeMatches := includeRegex.FindStringSubmatch(node.Value)
+		if len(includeMatches) > 0 {
+			filepath := filepath.Join(orbRoot, includeMatches[1])
+			file, err := ioutil.ReadFile(filepath)
+			if err != nil {
+				return errors.New(fmt.Sprintf("Could not open %s for inclusion in Orb", filepath))
+			}
+
+			node.Value = string(file)
+		}
+
+	} else {
+		// I am *slightly* worried about performance related to this approach, but don't have any
+		// larger Orbs to test against.
+		for _, child := range node.Content {
+			err := inlineIncludes(child, orbRoot)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

cmd/orb_test.go

@@ -2581,4 +2581,51 @@ https://circleci.com/orbs/registry/orb/my/orb
 			})
 		})
 	})
+
+	Describe("Orb pack", func() {
+		var (
+			tempSettings *clitest.TempSettings
+			orb          *clitest.TmpFile
+			script       *clitest.TmpFile
+			command      *exec.Cmd
+		)
+		BeforeEach(func() {
+			tempSettings = clitest.WithTempSettings()
+			orb = clitest.OpenTmpFile(tempSettings.Home, filepath.Join("commands", "orb.yml"))
+			clitest.OpenTmpFile(tempSettings.Home, "@orb.yml")
+			orb.Write([]byte(`steps:
+    - run:
+        name: Say hello
+        command: <<include(scripts/script.sh)>>
+`))
+			script = clitest.OpenTmpFile(tempSettings.Home, filepath.Join("scripts", "script.sh"))
+			script.Write([]byte(`echo Hello, world!`))
+			command = exec.Command(pathCLI,
+				"orb", "pack",
+				"--skip-update-check",
+				tempSettings.Home,
+			)
+		})
+
+		AfterEach(func() {
+			tempSettings.Close()
+			orb.Close()
+			script.Close()
+		})
+
+		It("Includes a script in the packed Orb file", func() {
+			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			Eventually(session.Out).Should(gbytes.Say(`commands:
+    orb:
+        steps:
+            - run:
+                command: echo Hello, world!
+                name: Say hello
+`))
+			Eventually(session).Should(gexec.Exit(0))
+		})
+
+	})
 })

data/data.go

@@ -1,8 +1,8 @@
 package data
 
 import (
-	"github.com/go-yaml/yaml"
 	packr "github.com/gobuffalo/packr/v2"
+	"gopkg.in/yaml.v3"
 )
 
 // YML maps the YAML found in _data/data.yml

filetree/filetree.go

@@ -8,8 +8,8 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/go-yaml/yaml"
 	"github.com/mitchellh/mapstructure"
+	"gopkg.in/yaml.v3"
 )
 
 // This is a quick hack of a function to convert interfaces
@@ -172,7 +172,7 @@ func buildTree(absRootPath string, pathNodes PathNodes) *Node {
 	return rootNode
 }
 
-func collectNodes(absRootPath string) (PathNodes, error) {
+func collectNodes(absRootPath string, allowedDirs map[string]string) (PathNodes, error) {
 	pathNodes := PathNodes{}
 	pathNodes.Map = make(map[string]*Node)
 	pathNodes.Keys = []string{}
@@ -182,8 +182,12 @@ func collectNodes(absRootPath string) (PathNodes, error) {
 			return err
 		}
 
-		// Skip any dotfolders that aren't the root path
-		if absRootPath != path && dotfolder(info) {
+		// Skip any dotfolders or dirs not explicitly allowed, if available.
+		isAllowed := true
+		if len(allowedDirs) > 0 && info.IsDir() {
+			_, isAllowed = allowedDirs[info.Name()]
+		}
+		if absRootPath != path && (dotfolder(info) || !isAllowed) {
 			// Turn off logging to stdout in this package
 			//fmt.Printf("Skipping dotfolder: %+v\n", path)
 			return filepath.SkipDir
@@ -207,13 +211,19 @@ func collectNodes(absRootPath string) (PathNodes, error) {
 }
 
 // NewTree creates a new filetree starting at the root
-func NewTree(rootPath string) (*Node, error) {
+func NewTree(rootPath string, allowedDirectories ...string) (*Node, error) {
+	allowedDirs := make(map[string]string)
+
+	for _, dir := range allowedDirectories {
+		allowedDirs[dir] = "1"
+	}
+
 	absRootPath, err := filepath.Abs(rootPath)
 	if err != nil {
 		return nil, err
 	}
 
-	pathNodes, err := collectNodes(absRootPath)
+	pathNodes, err := collectNodes(absRootPath, allowedDirs)
 	if err != nil {
 		return nil, err
 	}

filetree/filetree_test.go

@@ -7,16 +7,17 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/go-yaml/yaml"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
 
 	"github.com/CircleCI-Public/circleci-cli/filetree"
 )
 
 var _ = Describe("filetree", func() {
 	var (
 		tempRoot string
+		subDir   string
 	)
 
 	BeforeEach(func() {
@@ -30,7 +31,7 @@ var _ = Describe("filetree", func() {
 	})
 
 	Describe("NewTree", func() {
-		var subDir, subDirFile, emptyDir string
+		var subDirFile, emptyDir string
 
 		BeforeEach(func() {
 			subDir = filepath.Join(tempRoot, "sub_dir")
@@ -90,6 +91,20 @@ sub_dir:
     foo:
       bar:
         baz
+`))
+		})
+
+		It("Only checks specified directories", func() {
+			tree, err := filetree.NewTree(tempRoot, "sub_dir")
+			Expect(err).ToNot(HaveOccurred())
+
+			out, err := yaml.Marshal(tree)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(MatchYAML(`sub_dir:
+  sub_dir_file:
+    foo:
+      bar:
+        baz
 `))
 		})
 	})

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
 	github.com/fatih/color v1.9.0 // indirect
-	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/gobuffalo/buffalo-plugins v1.9.3 // indirect
 	github.com/gobuffalo/flect v0.0.0-20181210151238-24a2b68e0316 // indirect
 	github.com/gobuffalo/packr/v2 v2.0.0-rc.13
@@ -35,7 +34,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc // indirect
 	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
 	golang.org/x/tools v0.0.0-20181221235234-d00ac6d27372 // indirect
-	gopkg.in/yaml.v2 v2.2.2
+	gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c
 	gotest.tools v2.1.0+incompatible
 )
 

go.sum

@@ -35,8 +35,6 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
-github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gobuffalo/buffalo v0.12.8-0.20181004233540-fac9bb505aa8/go.mod h1:sLyT7/dceRXJUxSsE813JTQtA3Eb1vjxWfo/N//vXIY=
 github.com/gobuffalo/buffalo v0.13.0/go.mod h1:Mjn1Ba9wpIbpbrD+lIDMy99pQ0H0LiddMIIDGse7qT4=
 github.com/gobuffalo/buffalo-plugins v1.0.2/go.mod h1:pOp/uF7X3IShFHyobahTkTLZaeUXwb0GrUTb9ngJWTs=
@@ -452,9 +450,10 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkp
 gopkg.in/mail.v2 v2.0.0-20180731213649-a0242b2233b4/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=