Merge pull request #49 from openucx/topic/overlap_config

Add support for a configuration file to tune overlap; fix overlap env…

etc/examples/overlap_conf.json

@@ -0,0 +1,44 @@
+{
+    "benchmarks": [
+        {
+            "name": "iallgather",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "iallgatherv",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "iallreduce",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "ialltoall",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "ialltoallv",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "ibarrier",
+            "max_num_elts": 10000000
+        },
+        {
+            "name": "ibcast",
+            "max_num_elts": 10000000
+        },
+        {
+            "name": "igather",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "igatherv",
+            "max_num_elts": 1000000
+        },
+        {
+            "name": "ireduce",
+            "max_num_elts": 1000000
+        }
+    ]
+}

tools/cmd/openhpca_run/openhpca_run.go

@@ -15,10 +15,12 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/gvallee/go_benchmark/pkg/benchmark"
 	"github.com/gvallee/go_hpc_jobmgr/pkg/implem"
+	"github.com/gvallee/go_hpc_jobmgr/pkg/mpi"
 	"github.com/gvallee/go_software_build/pkg/app"
 	"github.com/gvallee/go_util/pkg/util"
 	"github.com/gvallee/validation_tool/pkg/experiments"
@@ -192,6 +194,7 @@ func main() {
 	osuNonContigMemSelectFlag := flag.Bool("osu-noncontigmem", false, "Explicitly select OSU for non-contiguous memory for execution. Only selected benchmarks will be executed")
 	smbSelectFlag := flag.Bool("smb", false, "Explicitly select SMB for execution. Only selected benchmarks will be executed")
 	overlapSelectFlag := flag.Bool("overlap", false, "Explicitly select the overlap benchmark suite for execution. Only selected benchmarks will be executed")
+	overlapConfigFilePathFlag := flag.String("overlap-config", "", "Path to the overlap configuration file. An example is available there: 'etc/examples/overlap_conf.json'")
 
 	flag.Parse()
 
@@ -289,6 +292,22 @@ func main() {
 		}
 	}
 
+	overlapConfig := new(overlap.Config)
+	if *overlapConfigFilePathFlag != "" {
+		err := overlapConfig.LoadConfig(*overlapConfigFilePathFlag)
+		if err != nil {
+			fmt.Printf("ERROR: unable to load overlap configuration: %s\n", err)
+			os.Exit(1)
+		}
+	}
+
+	// Detect the MPI implementation so we can properly customize the environment
+	localMPI, err := mpi.DetectFromDir(cfg.WP.MpiDir)
+	if err != nil {
+		fmt.Printf("unable to detect the MPI implementation installed in %s: %s\n", cfg.WP.MpiDir, err)
+		os.Exit(1)
+	}
+
 	for benchmarkName, installedBenchmark := range benchmarksToRun {
 		for _, subBenchmark := range installedBenchmark.SubBenchmarks {
 			e := new(experiments.Experiment)
@@ -316,9 +335,19 @@ func main() {
 
 			// Make sure to set special environment variables
 			// todo: find a better way to abtract this, i.e., make sure it is set correctly for all MPI implementations
-			overlapNumElts := os.Getenv("OPENHPCA_OVERLAP_MAX_NUM_ELTS")
+			// Data from the overlap configuration file always prevail on the environment variable from the calling
+			// process
+			overlapNumElts := os.Getenv(overlap.MaxNumEltsEnvVar)
+			if overlapConfig.MaxNumEltsLookupTable != nil {
+				overlapNumElts = strconv.Itoa(overlapConfig.MaxNumEltsLookupTable[subBenchmark.BinName])
+			}
 			if overlapNumElts != "" && benchmarkName == "overlap" {
-				e.MpirunArgs = append(e.MpirunArgs, "-x OPENHPCA_OVERLAP_MAX_NUM_ELTS="+overlapNumElts)
+				if localMPI.ID == implem.OMPI {
+					e.MpirunArgs = append(e.MpirunArgs, "-x "+overlap.MaxNumEltsEnvVar+"="+overlapNumElts)
+				}
+				if localMPI.ID == implem.MPICH || localMPI.ID == implem.MVAPICH2 {
+					e.MpirunArgs = append(e.MpirunArgs, "-genv "+overlap.MaxNumEltsEnvVar+"="+overlapNumElts)
+				}
 			}
 
 			exps.List = append(exps.List, e)

tools/go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/gvallee/go_benchmark v1.0.0
-	github.com/gvallee/go_hpc_jobmgr v1.4.1
+	github.com/gvallee/go_hpc_jobmgr v1.4.2
 	github.com/gvallee/go_osu v1.8.2
 	github.com/gvallee/go_software_build v1.2.9
 	github.com/gvallee/go_util v1.5.1

tools/go.sum

@@ -29,8 +29,8 @@ github.com/gvallee/go_exec v0.0.3/go.mod h1:b8ZYmmYCaHo/g92ZHvYK8LnTp06sdTU3u2Cj
 github.com/gvallee/go_exec v1.1.0 h1:3xhwLIcxY4VNGGPL5YZYxnWeTgLYU0PSOEMcO0gfi+Q=
 github.com/gvallee/go_exec v1.1.0/go.mod h1:s9fSkVniLYlQUZb99zuioVH45nkvnYxw6vmGEaZCUPk=
 github.com/gvallee/go_hpc_jobmgr v1.4.0/go.mod h1:gsikmgiQ+su5Cbaj48ErVtXyyTxXSiJf8r6CEUiN2wI=
-github.com/gvallee/go_hpc_jobmgr v1.4.1 h1:iG6Q2xQxV4FWljEO2wBAzIl1XZ8FO1swYwYLIxkKYW4=
-github.com/gvallee/go_hpc_jobmgr v1.4.1/go.mod h1:gsikmgiQ+su5Cbaj48ErVtXyyTxXSiJf8r6CEUiN2wI=
+github.com/gvallee/go_hpc_jobmgr v1.4.2 h1:mUc5TtftN8XQFzT05Em7WfqoeOb24vu9T5VY6xseXrA=
+github.com/gvallee/go_hpc_jobmgr v1.4.2/go.mod h1:gsikmgiQ+su5Cbaj48ErVtXyyTxXSiJf8r6CEUiN2wI=
 github.com/gvallee/go_osu v1.8.2 h1:QFr+zzzWx8eSLuyTQHzcCZXyOO7T9x8msq1rHf+18y0=
 github.com/gvallee/go_osu v1.8.2/go.mod h1:MWDdnxDuRK73XptsTl3TRy3mcUepFVu2weEobf4E25I=
 github.com/gvallee/go_software_build v0.0.17/go.mod h1:xpG+JoRV2XgxWOWmwuySzdKs2+jhWIeLu3MtTrWfQ2Y=

tools/internal/pkg/overlap/overlap.go

@@ -7,7 +7,9 @@
 package overlap
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -22,8 +24,21 @@ import (
 	"github.com/openucx/openhpca/tools/internal/pkg/util"
 )
 
+type Benchmark struct {
+	Name       string `json:"name"`
+	MaxNumElts int    `json:"max_num_elts"`
+}
+
+type Benchmarks struct {
+	Benchmarks []Benchmark
+}
+
 // Config represents the configuration of the overlap suite
 type Config struct {
+	Benchs *Benchmarks
+
+	MaxNumEltsLookupTable map[string]int
+
 	URL string
 }
 
@@ -53,6 +68,8 @@ const (
 	overlapIgatherBinName     = "overlap_igather"
 	overlapIgathervID         = "overlap_igatherv"
 	overlapIgathervBinName    = "overlap_igatherv"
+
+	MaxNumEltsEnvVar = "OPENHPCA_OVERLAP_MAX_NUM_ELTS"
 )
 
 var RequiredBenchmarks = []string{overlapIallreduceID, overlapIreduceID, overlapIallgatherID, overlapIallgathervID,
@@ -269,3 +286,31 @@ func DetectInstall(cfg *benchmark.Config, wp *workspace.Config) *benchmark.Insta
 func Display(cfg *benchmark.Config) {
 	fmt.Printf("\toverlap benchmark suite URL: %s\n", cfg.URL)
 }
+
+func (c *Config) LoadConfig(configFilePath string) error {
+	f, err := os.Open(configFilePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	content, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+
+	c.Benchs = new(Benchmarks)
+	err = json.Unmarshal(content, &c.Benchs)
+	if err != nil {
+		return err
+	}
+
+	// Create the lookup tables based on the data from the config file
+	if c.MaxNumEltsLookupTable == nil {
+		c.MaxNumEltsLookupTable = make(map[string]int)
+	}
+	for _, b := range c.Benchs.Benchmarks {
+		c.MaxNumEltsLookupTable[b.Name] = b.MaxNumElts
+	}
+	return nil
+}