add ability to build/run different archs on hosts with different arch…

…s (eg: x86 on arm and arm on x86) (#1475)

* default to host arch but allow ops to cross-build/run on other arches as well

* nightlies

* .

* .

* .

* .

cmd/cmd_run.go

@@ -50,10 +50,12 @@ func runCommandHandler(cmd *cobra.Command, args []string) {
 	globalFlags := NewGlobalCommandFlags(flags)
 	nightlyFlags := NewNightlyCommandFlags(flags)
 	nanosVersionFlags := NewNanosVersionCommandFlags(flags)
+
 	buildImageFlags := NewBuildImageCommandFlags(flags)
 	runLocalInstanceFlags := NewRunLocalInstanceCommandFlags(flags)
 
 	mergeContainer := NewMergeConfigContainer(configFlags, globalFlags, nightlyFlags, nanosVersionFlags, buildImageFlags, runLocalInstanceFlags)
+
 	err = mergeContainer.Merge(c)
 	if err != nil {
 		exitWithError(err.Error())

cmd/flags_build_image.go

@@ -199,6 +199,7 @@ func NewBuildImageCommandFlags(cmdFlags *pflag.FlagSet) (flags *BuildImageComman
 	if err != nil {
 		exitWithError(err.Error())
 	}
+
 	return
 }
 

cmd/flags_nanos_version.go

@@ -14,6 +14,14 @@ type NanosVersionCommandFlags struct {
 	NanosVersion string
 }
 
+func archPath() string {
+	if runtime.GOARCH == "arm64" || lepton.AltGOARCH == "arm64" {
+		return "arm"
+	}
+
+	return ""
+}
+
 // MergeToConfig downloads specified nanos version build and change configuration nanos tools paths
 func (flags *NanosVersionCommandFlags) MergeToConfig(config *types.Config) (err error) {
 	nanosVersion := flags.NanosVersion
@@ -26,10 +34,7 @@ func (flags *NanosVersionCommandFlags) MergeToConfig(config *types.Config) (err
 		}
 
 		if !exists {
-			arch := ""
-			if runtime.GOARCH == "arm64" {
-				arch = "arm"
-			}
+			arch := archPath()
 			err = lepton.DownloadReleaseImages(nanosVersion, arch)
 			if err != nil {
 				return

cmd/flags_nightly.go

@@ -18,12 +18,20 @@ import (
 // NightlyCommandFlags is used to change configuration to use nightly build tools paths
 type NightlyCommandFlags struct {
 	Nightly bool
+	Arch    string
 }
 
 // MergeToConfig downloads nightly build and change configuration nanos tools paths
 func (flags *NightlyCommandFlags) MergeToConfig(config *types.Config) (err error) {
 	config.NightlyBuild = flags.Nightly
 
+	if flags.Arch != "" {
+		api.AltGOARCH = flags.Arch
+		// this is dumb, FIXME
+		api.NightlyLocalFolderm = api.NightlyLocalFolder()
+		api.NightlyReleaseURLm = api.NightlyReleaseURL()
+	}
+
 	if config.NightlyBuild {
 		var version string
 		version, err = downloadNightlyImages(config)
@@ -46,19 +54,44 @@ func NewNightlyCommandFlags(cmdFlags *pflag.FlagSet) (flags *NightlyCommandFlags
 		exitWithError(err.Error())
 	}
 
+	flags.Arch, err = cmdFlags.GetString("arch")
+	if err != nil {
+		exitWithError(err.Error())
+	}
+
 	return
 }
 
 // PersistNightlyCommandFlags append nightly flag to a command
 func PersistNightlyCommandFlags(cmdFlags *pflag.FlagSet) {
 	cmdFlags.BoolP("nightly", "n", false, "nightly build")
+	cmdFlags.StringP("arch", "", "", "set different architecture")
+}
+
+func getKernelVersion(version string) string {
+	return path.Join(api.GetOpsHome(), version, "kernel.img")
+}
+
+func setKernelVersion(version string) string {
+	if api.AltGOARCH != "" {
+		if runtime.GOARCH == "amd64" {
+			return version + "-arm"
+		}
+		return version
+	}
+	if runtime.GOARCH == "arm64" {
+		return version + "-arm"
+	}
+	return version
 }
 
 func updateNanosToolsPaths(c *types.Config, version string) {
 	if c.NightlyBuild {
 		version = "nightly"
 	}
 
+	version = setKernelVersion(version)
+
 	if c.Boot == "" {
 		bootPath := path.Join(api.GetOpsHome(), version, "boot.img")
 		if _, err := os.Stat(bootPath); err == nil {
@@ -69,11 +102,7 @@ func updateNanosToolsPaths(c *types.Config, version string) {
 	c.UefiBoot = api.GetUefiBoot(version)
 
 	if c.Kernel == "" {
-		if runtime.GOARCH == "arm64" {
-			c.Kernel = path.Join(api.GetOpsHome(), version+"-arm", "kernel.img")
-		} else {
-			c.Kernel = path.Join(api.GetOpsHome(), version, "kernel.img")
-		}
+		c.Kernel = getKernelVersion(version)
 	}
 
 	if _, err := os.Stat(c.Kernel); os.IsNotExist(err) {
@@ -95,7 +124,12 @@ func updateNanosToolsPaths(c *types.Config, version string) {
 func downloadNightlyImages(c *types.Config) (string, error) {
 	var err error
 	err = api.DownloadNightlyImages(c)
+
+	if runtime.GOARCH == "arm64" || api.AltGOARCH == "arm64" {
+		return "nightly-arm", err
+	}
 	return "nightly", err
+
 }
 
 func getCurrentVersion() (string, error) {

cmd/flags_run_local_instance.go

@@ -35,6 +35,7 @@ type RunLocalInstanceCommandFlags struct {
 	TapName        string
 	Trace          bool
 	Verbose        bool
+	Arch           string
 }
 
 // MergeToConfig overrides configuration passed by argument with command flags values

cmd/run_local_instance.go

@@ -45,6 +45,8 @@ func RunLocalInstance(c *types.Config) (err error) {
 		}
 	}
 
+	fmt.Println("running local instance")
+
 	c.RunConfig.Kernel = c.Kernel
 
 	fmt.Printf("booting %s ...\n", c.RunConfig.ImageName)

lepton/const.go

@@ -128,12 +128,16 @@ func getImageTempDir(c *types.Config) string {
 	return c.BuildDir
 }
 
-// NightlyReleaseURL give URL for nightly build
-var NightlyReleaseURL = nightlyReleaseURL()
+// NightlyReleaseURLm give URL for nightly build.
+var NightlyReleaseURLm = NightlyReleaseURL()
 
 var realGOOS = getGOOS()
 
-var realGOARCH = getGOARCH()
+// RealGOARCH is the underlying host architecture.
+var RealGOARCH = getGOARCH()
+
+// AltGOARCH is an optional user-supplied cross-build arch for both build and run.
+var AltGOARCH = ""
 
 func getGOARCH() string {
 	return runtime.GOARCH
@@ -149,41 +153,44 @@ func getGOOS() string {
 }
 
 func nightlyFileName() string {
-	a := realGOARCH
-	if a == "arm64" {
+	if RealGOARCH == "arm64" || AltGOARCH == "arm64" {
 		return "nanos-nightly-linux-virt.tar.gz"
 	}
 	return "nanos-nightly-linux.tar.gz"
 }
 
 func nightlyTimestamp() string {
-	if realGOARCH == "arm64" {
+	if RealGOARCH == "arm64" || AltGOARCH == "arm64" {
 		return "nanos-nightly-linux-virt.timestamp"
 	}
 	return "nanos-nightly-linux.timestamp"
 }
 
-func nightlyReleaseURL() string {
+// NightlyReleaseURL points to the latest nightly release url that is
+// arch dependent upon flag set.
+func NightlyReleaseURL() string {
 	var sb strings.Builder
 	sb.WriteString(nightlyReleaseBaseURL)
 	sb.WriteString(nightlyFileName())
 	return sb.String()
 }
 
-func nightlyLocalFolder() string {
-	if realGOARCH == "arm64" {
+// NightlyLocalFolder points to the latest nightly release url that is
+// arch dependent upon flag set.
+func NightlyLocalFolder() string {
+	if RealGOARCH == "arm64" || AltGOARCH == "arm64" {
 		return path.Join(GetOpsHome(), "nightly-arm")
 	}
 	return path.Join(GetOpsHome(), "nightly")
 }
 
-// NightlyLocalFolder is directory path where nightly builds are stored
-var NightlyLocalFolder = nightlyLocalFolder()
+// NightlyLocalFolderm is directory path where nightly builds are stored
+var NightlyLocalFolderm = NightlyLocalFolder()
 
 // LocalTimeStamp gives local timestamp from download nightly build
 func LocalTimeStamp() (string, error) {
 	timestamp := nightlyTimestamp()
-	data, err := os.ReadFile(path.Join(NightlyLocalFolder, timestamp))
+	data, err := os.ReadFile(path.Join(NightlyLocalFolderm, timestamp))
 
 	// first time download?
 	if os.IsNotExist(err) {
@@ -213,7 +220,7 @@ func RemoteTimeStamp() (string, error) {
 
 func updateLocalTimestamp(timestamp string) error {
 	fname := nightlyTimestamp()
-	return os.WriteFile(path.Join(NightlyLocalFolder, fname), []byte(timestamp), 0755)
+	return os.WriteFile(path.Join(NightlyLocalFolderm, fname), []byte(timestamp), 0755)
 }
 
 // UpdateLocalRelease updates nanos version used on ops operations
@@ -275,7 +282,7 @@ func getLastReleaseLocalFolder() string {
 
 func getKlibsDir(nightly bool, nanosVersion string) string {
 	if nightly {
-		return nightlyLocalFolder() + "/klibs"
+		return NightlyLocalFolder() + "/klibs"
 	} else if nanosVersion != "" && nanosVersion != "0.0" {
 		return getReleaseLocalFolder(nanosVersion) + "/klibs"
 	}

lepton/image.go

@@ -512,18 +512,19 @@ func DownloadNightlyImages(c *types.Config) error {
 		return err
 	}
 
-	if _, err := os.Stat(NightlyLocalFolder); os.IsNotExist(err) {
-		os.MkdirAll(NightlyLocalFolder, 0755)
+	if _, err := os.Stat(NightlyLocalFolderm); os.IsNotExist(err) {
+		os.MkdirAll(NightlyLocalFolderm, 0755)
 	}
-	localtar := path.Join(NightlyLocalFolder, nightlyFileName())
+	localtar := path.Join(NightlyLocalFolderm, nightlyFileName())
 	// we have an update, let's download since it's nightly
+
 	if remote != local || c.Force {
-		if err = DownloadFileWithProgress(localtar, NightlyReleaseURL, 600); err != nil {
+		if err = DownloadFileWithProgress(localtar, NightlyReleaseURLm, 600); err != nil {
 			return errors.Wrap(err, 1)
 		}
 		// update local timestamp
 		updateLocalTimestamp(remote)
-		ExtractPackage(localtar, NightlyLocalFolder, c)
+		ExtractPackage(localtar, NightlyLocalFolderm, c)
 	}
 
 	return nil
@@ -563,7 +564,7 @@ func CheckNanosVersionExists(version string) (bool, error) {
 // arch defaults to x86-64 if empty
 func DownloadReleaseImages(version string, arch string) error {
 	url := getReleaseURL(version)
-	if arch == "arm" {
+	if arch == "arm" || AltGOARCH == "arm64" {
 		url = strings.Replace(url, ".tar.gz", "-virt.tar.gz", -1)
 	}
 
@@ -572,7 +573,7 @@ func DownloadReleaseImages(version string, arch string) error {
 
 	localFolder := getReleaseLocalFolder(version)
 
-	if arch == "arm" {
+	if arch == "arm" || AltGOARCH == "arm64" {
 		localFolder = localFolder + "-arm"
 	}
 

qemu/qemu_options_unix_amd64.go

@@ -3,8 +3,6 @@
 
 package qemu
 
-const qemuBaseCommand = "qemu-system-x86_64"
-
 var hypervisors = map[string]func() Hypervisor{
 	"qemu-system-x86_64": newQemu,
 }

qemu/qemu_options_unix_arm64.go

@@ -3,8 +3,6 @@
 
 package qemu
 
-const qemuBaseCommand = "qemu-system-aarch64"
-
 var hypervisors = map[string]func() Hypervisor{
 	"qemu-system-aarch64": newQemu,
 }