initial protobufs && grpc (#1416)

.circleci/config.yml

@@ -15,7 +15,13 @@ jobs:
       - run: sudo apt-get update
       - run: sudo apt-get install qemu-system-x86
       - run: go install github.com/jstemmer/go-junit-report@latest
+      - run: wget https://github.com/bufbuild/buf/releases/download/v1.13.1/buf-Linux-x86_64 && sudo mv buf-Linux-x86_64 /usr/bin/buf && sudo chmod +x /usr/bin/buf
+      - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+      - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-openapiv2-v2.15.0-linux-x86_64 -o ~/go/bin/protoc-gen-openapiv2 && chmod +x ~/go/bin/protoc-gen-openapiv2
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-grpc-gateway-v2.15.0-linux-x86_64 -o ~/go/bin/protoc-gen-grpc-gateway && chmod +x ~/go/bin/protoc-gen-grpc-gateway
       - run: make deps
+      - run: make generate
       - run: ./hack/verify-gofmt.sh
       - run: ./hack/verify-govet.sh
       - run: ./hack/verify-golint.sh
@@ -34,9 +40,15 @@ jobs:
     steps:
       - checkout
       - run: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-      - run: brew update && brew install go && brew tap nanovms/homebrew-qemu && brew install nanovms/homebrew-qemu/qemu
+      - run: brew update && brew install go && brew tap nanovms/homebrew-qemu && brew install nanovms/homebrew-qemu/qemu && brew install wget && brew install protoc-gen-go-grpc
       - run: go install github.com/jstemmer/go-junit-report@latest
+      - run: brew install bufbuild/buf/buf
+      - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+      - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-openapiv2-v2.15.0-darwin-x86_64 -o ~/go/bin/protoc-gen-openapiv2 && chmod +x ~/go/bin/protoc-gen-openapiv2
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-grpc-gateway-v2.15.0-darwin-x86_64 -o ~/go/bin/protoc-gen-grpc-gateway && chmod +x ~/go/bin/protoc-gen-grpc-gateway
       - run: make deps
+      - run: make generate PATH=$PATH:~/go/bin
       - run: mkdir -p $TEST_RESULTS
       - run: go run ops.go update
       - run:
@@ -49,7 +61,13 @@ jobs:
       - image: cimg/go:1.19.5
     steps:
       - checkout
+      - run: wget https://github.com/bufbuild/buf/releases/download/v1.13.1/buf-Linux-x86_64 && sudo mv buf-Linux-x86_64 /usr/bin/buf && sudo chmod +x /usr/bin/buf
+      - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+      - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-openapiv2-v2.15.0-linux-x86_64 -o ~/go/bin/protoc-gen-openapiv2 && chmod +x ~/go/bin/protoc-gen-openapiv2
+      - run: curl -L https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.15.0/protoc-gen-grpc-gateway-v2.15.0-linux-x86_64 -o ~/go/bin/protoc-gen-grpc-gateway && chmod +x ~/go/bin/protoc-gen-grpc-gateway
       - run: go build
+      - run: make generate
       - run: echo "deb http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
       - run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
       - run: sudo apt-get update && sudo apt-get install google-cloud-sdk

.gitignore

@@ -1,3 +1,6 @@
+*.pb.go
+*.pb.gw.go
+*.swagger.json
 *.dmg
 *.zip
 *.md5

Makefile

@@ -27,9 +27,14 @@ post-test: do-test
 
 test: post-test
 
+generate:
+	buf generate --path ./protos/imageservice/imageservice.proto
+
 clean:
 	$(GOCLEAN)
 	rm -f $(BINARY_NAME)
+	rm -rf proto/imageservice/*.go
+	rm -rf proto/imageservice/*.json
 
 run:
 	$(GOBUILD) -o $(BINARY_NAME) -v .

README.md

@@ -76,6 +76,14 @@ macOS notes:
 GO111MODULE=on go build -ldflags "-w"
 ```
 
+For protobufs/grpc we use https://buf.build/ .
+
+To generate protobufs (but do not check in to vcs):
+
+```
+make generate
+```
+
 For [detailed instructions](https://nanovms.gitbook.io/ops/developer/prerequisites), please consult the documentation.
 
 # Hello World

buf.gen.yaml

@@ -0,0 +1,17 @@
+version: v1beta1
+plugins:
+  - name: go
+    out: protos
+    opt:
+      - paths=source_relative
+  - name: go-grpc
+    out: protos
+    opt:
+      - paths=source_relative
+      - require_unimplemented_servers=false
+  - name: grpc-gateway
+    out: protos
+    opt:
+      - paths=source_relative
+  - name: openapiv2
+    out: protos

buf.yaml

@@ -0,0 +1,7 @@
+version: v1beta1
+build:
+  roots:
+    - protos
+lint:
+  use:
+    - DEFAULT

cmd/cmd_daemon.go

@@ -2,7 +2,22 @@ package cmd
 
 import (
 	"fmt"
+
+	"context"
+	"log"
+	"net"
+	"net/http"
+	"os"
+
+	api "github.com/nanovms/ops/lepton"
+	"github.com/nanovms/ops/provider"
+	"github.com/nanovms/ops/types"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/nanovms/ops/protos/imageservice"
 	"github.com/spf13/cobra"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 // DaemonizeCommand turns ops into a daemon
@@ -16,10 +31,97 @@ func DaemonizeCommand() *cobra.Command {
 	return cmdDaemonize
 }
 
+// prob belongs in a root grpc-server folder
+// not in the cmds folder
+type server struct{}
+
+func (*server) GetImages(_ context.Context, in *imageservice.ImageListRequest) (*imageservice.ImagesResponse, error) {
+
+	// stubbed for now - could conceivablly store creds in server and
+	// target any provider which would be nice
+	c := &types.Config{}
+	pc := &types.ProviderConfig{}
+
+	p, err := provider.CloudProvider("onprem", pc)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	ctx := api.NewContext(c)
+	images, err := p.GetImages(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	pb := &imageservice.ImagesResponse{
+		Count: int32(len(images)),
+	}
+
+	for i := 0; i < len(images); i++ {
+		img := &imageservice.Image{
+			Name:    images[i].Name,
+			Path:    images[i].Path,
+			Size:    images[i].Size,
+			Created: images[i].Created.String(),
+		}
+
+		pb.Images = append(pb.Images, img)
+	}
+
+	return pb, nil
+}
+
 func daemonizeCommandHandler(cmd *cobra.Command, args []string) {
 	fmt.Println("Note: If on a mac this expects ops to have suid bit set for networking.")
 	fmt.Println("if you used the installer you are set otherwise run the following command\n" +
 		"\tsudo chown -R root /usr/local/bin/qemu-system-x86_64\n" +
 		"\tsudo chmod u+s /usr/local/bin/qemu-system-x86_64")
-	fmt.Println("daemonizing")
+
+	lis, err := net.Listen("tcp", ":8080")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	s := grpc.NewServer()
+	imageservice.RegisterImagesServer(s, &server{})
+	log.Println("Serving gRPC on 0.0.0.0:8080")
+	go func() {
+		err := s.Serve(lis)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+	}()
+
+	conn, err := grpc.DialContext(
+		context.Background(),
+		"0.0.0.0:8080",
+		grpc.WithBlock(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	gwmux := runtime.NewServeMux()
+	err = imageservice.RegisterImagesHandler(context.Background(), gwmux, conn)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	gwServer := &http.Server{
+		Addr:    ":8090",
+		Handler: gwmux,
+	}
+
+	log.Println("Serving json on http://0.0.0.0:8090")
+	fmt.Println("try issuing a request:\tcurl -XGET -k http://localhost:8090/v1/images | jq")
+	err = gwServer.ListenAndServe()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
 }

go.mod

@@ -3,7 +3,7 @@ module github.com/nanovms/ops
 go 1.19
 
 require (
-	cloud.google.com/go/storage v1.10.0
+	cloud.google.com/go/storage v1.27.0
 	github.com/Azure/azure-sdk-for-go v63.0.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.10.0
 	github.com/Azure/go-autorest/autorest v0.11.24
@@ -20,6 +20,7 @@ require (
 	github.com/go-errors/errors v1.0.1
 	github.com/golang/mock v1.6.0
 	github.com/gophercloud/gophercloud v0.12.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0
 	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297
 	github.com/olekukonko/tablewriter v0.0.4
@@ -30,22 +31,29 @@ require (
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/terra-farm/go-virtualbox v0.0.4
 	github.com/tj/go-spin v1.1.0
 	github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31
 	github.com/vmware/govmomi v0.22.2
 	github.com/vultr/govultr/v2 v2.10.0
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
-	google.golang.org/api v0.56.0
+	golang.org/x/oauth2 v0.3.0
+	golang.org/x/sys v0.3.0
+	google.golang.org/api v0.103.0
+	google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37
+	google.golang.org/grpc v1.51.0
+	google.golang.org/protobuf v1.28.1
+	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 	k8s.io/client-go v0.23.1
 )
 
 require (
-	cloud.google.com/go v0.93.3 // indirect
+	cloud.google.com/go v0.105.0 // indirect
+	cloud.google.com/go/compute v1.13.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.8.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -68,11 +76,12 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
@@ -96,17 +105,14 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.6.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/net v0.3.0 // indirect
+	golang.org/x/term v0.3.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220614165028-45ed7f3ff16e // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.23.1 // indirect
 	k8s.io/apimachinery v0.23.1 // indirect