From 60050038b7769bc1b2c740aa64b2a8c57b0956aa Mon Sep 17 00:00:00 2001 From: Ian Eyberg Date: Tue, 7 Feb 2023 17:47:40 -0800 Subject: [PATCH] instances/volumes (#1427) * instances/volumes * . --- Makefile | 6 + README.md | 15 ++ cmd/cmd_daemon.go | 102 +-------- daemon/daemon.go | 213 +++++++++++++++++++ lepton/cloud.go | 1 + lepton/volume.go | 2 +- protos/instanceservice/instanceservice.proto | 34 +++ protos/volumeservice/volumeservice.proto | 31 +++ provider/onprem/instance.go | 12 +- provider/onprem/onprem_instance.go | 144 ++++++++++++- qemu/qemu_unix.go | 9 +- release.sh | 2 +- 12 files changed, 458 insertions(+), 113 deletions(-) create mode 100644 daemon/daemon.go create mode 100644 protos/instanceservice/instanceservice.proto create mode 100644 protos/volumeservice/volumeservice.proto diff --git a/Makefile b/Makefile index 5b0a5afc..0c6fdef8 100644 --- a/Makefile +++ b/Makefile @@ -29,12 +29,18 @@ test: post-test generate: buf generate --path ./protos/imageservice/imageservice.proto + buf generate --path ./protos/instanceservice/instanceservice.proto + buf generate --path ./protos/volumeservice/volumeservice.proto clean: $(GOCLEAN) rm -f $(BINARY_NAME) rm -rf protos/imageservice/*.go rm -rf protos/imageservice/*.json + rm -rf protos/instanceservice/*.go + rm -rf protos/instanceservice/*.json + rm -rf protos/volumeservice/*.go + rm -rf protos/volumeservice/*.json run: $(GOBUILD) -o $(BINARY_NAME) -v . diff --git a/README.md b/README.md index bc4445b8..f277d95c 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,21 @@ You can find more examples and tutorial on youtube as well: [https://www.youtube.com/channel/UC3mqDqCVu3moVKzmP2YNmlg](https://www.youtube.com/channel/UC3mqDqCVu3moVKzmP2YNmlg) +## Daemon + +OPS started out as a daemon-less cli tool to build and run unikernels +locally and to also interact with the various clouds. We will keep that +functionality as-is, however, ops can also run as a daemon locally for +software that is a composition of multiple services. The daemon expects +to have elevated privileges (currently via suid bit) in order to place +the various programs on their class c network (vs relying on user-mode). +This is not necessary for 'ops run', 'ops pkg load' or 'ops instance +create' but only for multipl services ran locally that expect to +communicate to each other vs just the host. + +For now the daemon and 'ops instance create' share metadata but that is +expected to change in the future. + ## Apple M1/M2 Users The Apple M1 and M2 are ARM based. OPS is built for users primarily diff --git a/cmd/cmd_daemon.go b/cmd/cmd_daemon.go index 4578d772..df294d4c 100644 --- a/cmd/cmd_daemon.go +++ b/cmd/cmd_daemon.go @@ -3,21 +3,9 @@ package cmd import ( "fmt" - "context" - "log" - "net" - "net/http" - "os" + "github.com/nanovms/ops/daemon" - api "github.com/nanovms/ops/lepton" - "github.com/nanovms/ops/provider" - "github.com/nanovms/ops/types" - - "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "github.com/nanovms/ops/protos/imageservice" "github.com/spf13/cobra" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) // DaemonizeCommand turns ops into a daemon @@ -31,97 +19,11 @@ func DaemonizeCommand() *cobra.Command { return cmdDaemonize } -// prob belongs in a root grpc-server folder -// not in the cmds folder -type server struct{} - -func (*server) GetImages(_ context.Context, in *imageservice.ImageListRequest) (*imageservice.ImagesResponse, error) { - - // stubbed for now - could conceivablly store creds in server and - // target any provider which would be nice - c := &types.Config{} - pc := &types.ProviderConfig{} - - p, err := provider.CloudProvider("onprem", pc) - if err != nil { - fmt.Println(err) - } - - ctx := api.NewContext(c) - images, err := p.GetImages(ctx) - if err != nil { - return nil, err - } - - pb := &imageservice.ImagesResponse{ - Count: int32(len(images)), - } - - for i := 0; i < len(images); i++ { - img := &imageservice.Image{ - Name: images[i].Name, - Path: images[i].Path, - Size: images[i].Size, - Created: images[i].Created.String(), - } - - pb.Images = append(pb.Images, img) - } - - return pb, nil -} - func daemonizeCommandHandler(cmd *cobra.Command, args []string) { fmt.Println("Note: If on a mac this expects ops to have suid bit set for networking.") fmt.Println("if you used the installer you are set otherwise run the following command\n" + "\tsudo chown -R root /usr/local/bin/qemu-system-x86_64\n" + "\tsudo chmod u+s /usr/local/bin/qemu-system-x86_64") - lis, err := net.Listen("tcp", ":8080") - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - s := grpc.NewServer() - imageservice.RegisterImagesServer(s, &server{}) - log.Println("Serving gRPC on 0.0.0.0:8080") - go func() { - err := s.Serve(lis) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - }() - - conn, err := grpc.DialContext( - context.Background(), - "0.0.0.0:8080", - grpc.WithBlock(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - gwmux := runtime.NewServeMux() - err = imageservice.RegisterImagesHandler(context.Background(), gwmux, conn) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - gwServer := &http.Server{ - Addr: ":8090", - Handler: gwmux, - } - - log.Println("Serving json on http://0.0.0.0:8090") - fmt.Println("try issuing a request:\tcurl -XGET -k http://localhost:8090/v1/images | jq") - err = gwServer.ListenAndServe() - if err != nil { - fmt.Println(err) - os.Exit(1) - } + daemon.Daemonize() } diff --git a/daemon/daemon.go b/daemon/daemon.go new file mode 100644 index 00000000..eeb326d7 --- /dev/null +++ b/daemon/daemon.go @@ -0,0 +1,213 @@ +package daemon + +import ( + "fmt" + + "context" + "log" + "net" + "net/http" + "os" + + api "github.com/nanovms/ops/lepton" + "github.com/nanovms/ops/provider" + "github.com/nanovms/ops/types" + + "github.com/nanovms/ops/protos/imageservice" + "github.com/nanovms/ops/protos/instanceservice" + "github.com/nanovms/ops/protos/volumeservice" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type server struct{} + +func (*server) GetInstances(_ context.Context, in *instanceservice.InstanceListRequest) (*instanceservice.InstancesResponse, error) { + // stubbed for now - could conceivablly store creds in server and + // target any provider which would be nice + c := &types.Config{} + pc := &types.ProviderConfig{} + + p, err := provider.CloudProvider("onprem", pc) + if err != nil { + fmt.Println(err) + } + + // for now we read from old 'ops run' output stored in + // ~/.ops/instances/{pid} but can def. re-factor to be in-mem for + // future since this is coming from daemon + ctx := api.NewContext(c) + instances, err := p.GetInstances(ctx) + if err != nil { + return nil, err + } + + pb := &instanceservice.InstancesResponse{ + Count: int32(len(instances)), + } + + for i := 0; i < len(instances); i++ { + instance := &instanceservice.Instance{ + Name: instances[i].Name, + Image: instances[i].Image, + Pid: instances[i].ID, + Status: instances[i].Status, + PrivateIp: instances[i].PrivateIps[0], + Created: instances[i].Created, + } + + pb.Instances = append(pb.Instances, instance) + } + + return pb, nil +} + +func (*server) GetImages(_ context.Context, in *imageservice.ImageListRequest) (*imageservice.ImagesResponse, error) { + + // stubbed for now - could conceivablly store creds in server and + // target any provider which would be nice + c := &types.Config{} + pc := &types.ProviderConfig{} + + p, err := provider.CloudProvider("onprem", pc) + if err != nil { + fmt.Println(err) + } + + ctx := api.NewContext(c) + images, err := p.GetImages(ctx) + if err != nil { + return nil, err + } + + pb := &imageservice.ImagesResponse{ + Count: int32(len(images)), + } + + for i := 0; i < len(images); i++ { + img := &imageservice.Image{ + Name: images[i].Name, + Path: images[i].Path, + Size: images[i].Size, + Created: images[i].Created.String(), + } + + pb.Images = append(pb.Images, img) + } + + return pb, nil +} + +func (*server) GetVolumes(_ context.Context, in *volumeservice.VolumeListRequest) (*volumeservice.VolumesResponse, error) { + + // stubbed for now - could conceivablly store creds in server and + // target any provider which would be nice + c := &types.Config{} + pc := &types.ProviderConfig{} + + p, err := provider.CloudProvider("onprem", pc) + if err != nil { + fmt.Println(err) + } + + ctx := api.NewContext(c) + // no clue why this is passed around like this + ctx.Config().VolumesDir = api.LocalVolumeDir + + volumes, err := p.GetAllVolumes(ctx) + if err != nil { + return nil, err + } + + rvols := *volumes + + pb := &volumeservice.VolumesResponse{ + Count: int32(len(rvols)), + } + + for i := 0; i < len(rvols); i++ { + if err != nil { + return nil, err + } + + vol := &volumeservice.Volume{ + Name: rvols[i].Name, + Path: rvols[i].Path, + Size: rvols[i].Size, // unfort this has extra meta such as 'mb' + Created: rvols[i].CreatedAt, + } + + pb.Volumes = append(pb.Volumes, vol) + } + + return pb, nil +} + +// Daemonize starts a grpc server along with a json frontend to interact +// with local/'onprem' installations. +func Daemonize() { + lis, err := net.Listen("tcp", ":8080") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + s := grpc.NewServer() + imageservice.RegisterImagesServer(s, &server{}) + instanceservice.RegisterInstancesServer(s, &server{}) + volumeservice.RegisterVolumesServer(s, &server{}) + + log.Println("Serving gRPC on 0.0.0.0:8080") + go func() { + err := s.Serve(lis) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + }() + + conn, err := grpc.DialContext( + context.Background(), + "0.0.0.0:8080", + grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + gwmux := runtime.NewServeMux() + err = imageservice.RegisterImagesHandler(context.Background(), gwmux, conn) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = instanceservice.RegisterInstancesHandler(context.Background(), gwmux, conn) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = volumeservice.RegisterVolumesHandler(context.Background(), gwmux, conn) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + gwServer := &http.Server{ + Addr: ":8090", + Handler: gwmux, + } + + log.Println("Serving json on http://0.0.0.0:8090") + fmt.Println("try issuing a request:\tcurl -XGET -k http://localhost:8090/v1/images | jq") + err = gwServer.ListenAndServe() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/lepton/cloud.go b/lepton/cloud.go index a1583542..ed6f2ebc 100644 --- a/lepton/cloud.go +++ b/lepton/cloud.go @@ -16,6 +16,7 @@ type CloudImage struct { // CloudInstance represents the instance that widely use in different // Cloud Providers. +// mainly used for formatting standard response from any cloud provider type CloudInstance struct { ID string Name string diff --git a/lepton/volume.go b/lepton/volume.go index 89f6ff88..9317e74b 100644 --- a/lepton/volume.go +++ b/lepton/volume.go @@ -20,7 +20,7 @@ type NanosVolume struct { Name string `json:"name"` Label string `json:"label"` Data string `json:"data"` - Size string `json:"size"` + Size string `json:"size"` // this has extra meta in it that should be converted to just bytes Path string `json:"path"` AttachedTo string `json:"attached_to"` CreatedAt string `json:"created_at"` diff --git a/protos/instanceservice/instanceservice.proto b/protos/instanceservice/instanceservice.proto new file mode 100644 index 00000000..805437fb --- /dev/null +++ b/protos/instanceservice/instanceservice.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package instanceservice; + +import "google/api/annotations.proto"; + +option go_package = "github.com/nanovms/ops/protos;instanceservice"; + +service Instances { + rpc GetInstances (InstanceListRequest) returns (InstancesResponse) { + option (google.api.http) = { + get: "/v1/instances" + }; + } +} + +message InstanceListRequest {} + +message InstancesResponse { + int32 count = 1; + repeated Instance instances = 2; +} + +message Instance { + string Name = 1; + string Image = 2; + repeated string ports = 3; + bool Bridged = 4; + string PrivateIp = 5; + string Mac = 6; + string Pid = 7; + string Status = 8; + string Created = 9; +} diff --git a/protos/volumeservice/volumeservice.proto b/protos/volumeservice/volumeservice.proto new file mode 100644 index 00000000..853293bb --- /dev/null +++ b/protos/volumeservice/volumeservice.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package volumeservice; + +import "google/api/annotations.proto"; + +option go_package = "github.com/nanovms/ops/protos;volumeservice"; + +service Volumes { + rpc GetVolumes (VolumeListRequest) returns (VolumesResponse) { + option (google.api.http) = { + get: "/v1/volumes" + }; + } +} + +message VolumeListRequest {} + +message VolumesResponse { + int32 count = 1; + repeated Volume volumes = 2; +} + +message Volume { + string Id = 1; + string Name = 2; + string Label = 3; + string Size = 4; // FIXME: should be int64 in bytes + string Path = 5; + string Created = 6; +} diff --git a/provider/onprem/instance.go b/provider/onprem/instance.go index c73ddd81..45ba3432 100644 --- a/provider/onprem/instance.go +++ b/provider/onprem/instance.go @@ -4,10 +4,16 @@ import ( "strings" ) +// for now only assumes a single interface +// TOOD: get from protos type instance struct { - Instance string `json:"instance"` - Image string `json:"image"` - Ports []string `json:"ports"` + Instance string `json:"instance"` + Image string `json:"image"` + Ports []string `json:"ports"` + Bridged bool `json:"bridged"` + PrivateIP string `json:"private_ip"` // assume only loopback unless bridged is set, only set if bridged is set + Mac string `json:"mac"` + Pid string `json:"pid"` } func (in *instance) portList() string { diff --git a/provider/onprem/onprem_instance.go b/provider/onprem/onprem_instance.go index 59a9cf75..36b6290d 100644 --- a/provider/onprem/onprem_instance.go +++ b/provider/onprem/onprem_instance.go @@ -4,15 +4,19 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "path" "strconv" "strings" "syscall" + "time" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/log" "github.com/nanovms/ops/qemu" "github.com/olekukonko/tablewriter" + + "golang.org/x/sys/unix" ) // CreateInstance on premise @@ -71,6 +75,10 @@ func (p *OnPrem) CreateInstance(ctx *lepton.Context) error { Ports: c.RunConfig.Ports, } + if qemu.OPSD != "" { + i.Bridged = true + } + d1, err := json.Marshal(i) if err != nil { log.Error(err) @@ -100,6 +108,111 @@ func (p *OnPrem) GetInstanceByName(ctx *lepton.Context, name string) (*lepton.Cl return nil, lepton.ErrInstanceNotFound(name) } +// super hacky mac extraction, arp resolution; revisit in future +// could also potentially extract from logs || could have nanos ping +// upon boot +func findBridgedIP(instanceID string) string { + out, err := execCmd("ps aux | grep " + instanceID) //fixme: cmd injection + if err != nil { + fmt.Println(err) + } + + oo := strings.Split(out, "netdev=vmnet,mac=") + ooz := strings.Split(oo[1], " ") + mac := ooz[0] + + out, err = execCmd("ps aux |grep -a " + instanceID + " | grep -v grep | awk {'print $2'}") + if err != nil { + fmt.Println(err) + } + pid := strings.TrimSpace(out) + + /// only use for resolution not for storage + dmac := formatOctet(mac) + + // needs recent activity to register + out, err = execCmd("arp -a | grep " + dmac) + if err != nil { + fmt.Println(err) + } + + if strings.Contains(out, "(") { + oo = strings.Split(out, "(") + ooz = strings.Split(oo[1], ")") + ip := ooz[0] + + logMac(instanceID, pid, mac, ip) + + return ip + } + + return "" +} + +// returns a mac with leading zeros dropped which is what mac does +// d6:3f:9b:0f:0c:c8 +// d6:3f:9b:f:c:c8 +func formatOctet(mac string) string { + octets := strings.Split(mac, ":") + newmac := "" + for i := 0; i < len(octets); i++ { + oi := strings.TrimLeft(octets[i], "0") + newmac += oi + ":" + } + + newmac = strings.TrimRight(newmac, ":") + return newmac +} + +// log our mac,ip,pid so we don't have to lookup again +// FIXME: for bridged we can store in a more proper fashion +// we still want to support daemon-less as much as we can +// +// also should throw a lock on this at some point unless we migrate to +// something else that is a bit more industrial +func logMac(instanceID string, pid string, mac string, ip string) { + + opshome := lepton.GetOpsHome() + instancesPath := path.Join(opshome, "instances") + + fullpath := path.Join(instancesPath, pid) + + body, err := os.ReadFile(fullpath) + if err != nil { + fmt.Println(err) + } + + var i instance + if err := json.Unmarshal(body, &i); err != nil { + fmt.Println(err) + } + + i.PrivateIP = ip + i.Pid = pid + i.Mac = mac + + d1, err := json.Marshal(i) + if err != nil { + fmt.Println(err) + } + + err = os.WriteFile(instancesPath+"/"+pid, d1, 0644) + if err != nil { + fmt.Println(err) + } +} + +func execCmd(cmdStr string) (output string, err error) { + cmd := exec.Command("/bin/bash", "-c", cmdStr) + out, err := cmd.CombinedOutput() + if err != nil { + return + } + + output = string(out) + return +} + // GetInstances return all instances on prem func (p *OnPrem) GetInstances(ctx *lepton.Context) (instances []lepton.CloudInstance, err error) { opshome := lepton.GetOpsHome() @@ -141,18 +254,41 @@ func (p *OnPrem) GetInstances(ctx *lepton.Context) (instances []lepton.CloudInst return nil, err } - fi, err := f.Info() + pips := []string{} + if i.Bridged { + if i.PrivateIP == "" || i.Mac == "" { + i.PrivateIP = findBridgedIP(i.Instance) + } + + pips = append(pips, i.PrivateIP) + } else { + pips = append(pips, "127.0.0.1") + } + + file, err := os.Open(fullpath) + if err != nil { + return nil, err + } + defer file.Close() + + stat := &unix.Stat_t{} + err = unix.Fstat(int(file.Fd()), stat) if err != nil { return nil, err } + // rather un-helpful if we're just overwriting it though + ctime := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) + + // perhaps return proto'd version here instead then wrap + // w/cloudinstance for cli instances = append(instances, lepton.CloudInstance{ - ID: f.Name(), + ID: f.Name(), // pid Name: i.Instance, Image: i.Image, Status: "Running", - Created: lepton.Time2Human(fi.ModTime()), - PrivateIps: []string{"127.0.0.1"}, + Created: lepton.Time2Human(ctime), + PrivateIps: pips, PublicIps: strings.Split(i.portList(), ","), }) } diff --git a/qemu/qemu_unix.go b/qemu/qemu_unix.go index d3a39be5..c75b2532 100644 --- a/qemu/qemu_unix.go +++ b/qemu/qemu_unix.go @@ -117,9 +117,10 @@ func (q *qemu) addSerial(serialType string) { q.serial = serial{serialtype: serialType} } -// we inject at release build time to enable virtio-net-pci -// this is only for a packaged macos release -var opsD = "" +// OPSD is injected at release build time to enable vmnet-bridged +// this is only for a packaged macos release that wish to run N +// instances locally. +var OPSD = "" // addDevice adds a device to the qemu for rendering to string arguments. If the // devType is "user" then the ifaceName is ignored and host forward ports are @@ -127,7 +128,7 @@ var opsD = "" // Backend interface are created for each device and their ids are auto // incremented. func (q *qemu) addNetDevice(devType, ifaceName, mac string, hostPorts []string, udpPorts []string) { - if opsD != "" { + if OPSD != "" { dv := device{ driver: "virtio-net-pci", devtype: "netdev=vmnet", diff --git a/release.sh b/release.sh index 61fa3da8..9c781554 100755 --- a/release.sh +++ b/release.sh @@ -28,7 +28,7 @@ gsutil cp "$hash" gs://cli/linux/release/"$VERSION"/"$hash" gsutil setacl public-read gs://cli/linux/release/"$VERSION"/"$hash" # TODO: -# flag here with "-X github.com/nanovms/ops/qemu.opsD=true" for signed/packaged mac binaries +# flag here with "-X github.com/nanovms/ops/qemu.OPSD=true" for signed/packaged mac binaries GO111MODULE=on GOOS=darwin go build -ldflags "-w -X github.com/nanovms/ops/lepton.Version=$VERSION" gsutil cp ops gs://cli/darwin