supporting numactl

supporting numactl

Author: FerencBodon-Kx

README.md

@@ -106,7 +106,7 @@ This takes three arguments:
    is kept on the filesystem. Useful for testing performance on a fuller
    filesystem, which could be modeled through running multiple iterations
    of `mthread.sh`.
-1. Optional: date. This test assumes that data was already generated (`keep` flag was used the previous test). Format of `%m%dD%H%M` is expected like `0404D1232`.
+1. Optional: date. This test assumes that data was already generated (`keep` flag was used the previous test). Format of `%m%d_H%M%S` is expected like `0408_152349`.
 
 If you would like the data to be compressed then pass the environment variable `COMPRESS` with the [kdb+ compression parameters](https://code.kx.com/q/kb/file-compression/#compression-parameters).
 
@@ -115,7 +115,7 @@ Example usages
 ```bash
 $ ./mthread.sh $(nproc) full delete
 $ COMPRESS="17 2 6" ./mthread.sh 8 full keep
-$ ./mthread.sh 8 readonly keep 0404:1232
+$ ./mthread.sh 8 readonly keep 0408_152349
 ```
 
 

common.sh

@@ -1,10 +1,12 @@
 if [[ $(uname) == "Linux" ]]; then
-    COREPERSOCKET=$(lscpu | grep "Core(s) per socket" | cut -d":" -f 2 |xargs)
     SOCKETNR=$(lscpu | grep "Socket(s)" | cut -d":" -f 2 |xargs)
+    COREPERSOCKET=$(lscpu | grep "Core(s) per socket" | cut -d":" -f 2 |xargs)
+    THREADPERCORE=$(lscpu | grep "Thread(s) per core" | cut -d":" -f 2 |xargs)
     CPUMOODEL=$(lscpu | grep "Model name" | cut -d":" -f 2 |xargs)
 else
-    COREPERSOCKET=$(sysctl -n hw.ncpu)
     SOCKETNR=1
+    COREPERSOCKET=$(sysctl -n hw.ncpu)
+    THREADPERCORE=1
     CPUMOODEL=$(sysctl -n machdep.cpu.brand_string)
 fi
-CORECOUNT=$((COREPERSOCKET * SOCKETNR))
+COMPUTECOUNT=$((COREPERSOCKET * SOCKETNR * THREADPERCORE))

config/env

@@ -14,6 +14,11 @@ export EXCLUDETESTS=""
 
 export SUDO="sudo"  # "" or "sudo" if sudo is required for some commands e.g. to flush page cache
 
+# Worker processes can be started with a numactl command.
+# Option `roundrobin` means that worker `i` is started on numa node `i mod N` where N is the number of numa nodes.
+# If N is 1 or NUMA is set to `nonuma` then no numactl is used.
+export NUMA="nonuma" # nonuma or roundrobin
+
 ##############
 # Settings for object storage
 # See this for more details: https://code.kx.com/insights/1.4/core/objstor/main.html

mthread.sh

@@ -93,10 +93,11 @@ yq -i ".env.THREADNR=$THREADNR" $CONFIG
 yq -i ".env.PROCNR=$NUMPROCESSES" $CONFIG
 yq -i ".env.FLUSH=\"$(basename $FLUSH)\"" $CONFIG
 yq -i ".env.DBDIR=\"$(cat $PARFILE)\"" $CONFIG
-yq -i ".nano.VERSION=\"$(yq '.dev' version.yaml)\"" $CONFIG
-yq -i ".kdb.MAJOR=$($QBIN -q  <<< ".z.K" | tr -d f)" $CONFIG
-yq -i ".kdb.MINOR=\"$($QBIN -q  <<< ".z.k")\"" $CONFIG
-yq -i ".kdb.QBIN=\"$QBIN\"" $CONFIG
+yq -i ".env.NUMA=\"$NUMA\"" $CONFIG
+yq -i ".nano.version=\"$(yq '.dev' version.yaml)\"" $CONFIG
+yq -i ".kdb.major=$($QBIN -q  <<< ".z.K" | tr -d f)" $CONFIG
+yq -i ".kdb.minor=\"$($QBIN -q  <<< ".z.k")\"" $CONFIG
+yq -i ".kdb.qbin=\"$QBIN\"" $CONFIG
 yq -i ".dbize.SEQWRITETESTLIMIT=$SEQWRITETESTLIMIT" $CONFIG
 yq -i ".dbize.RANDREADNUMBER=$RANDREADNUMBER" $CONFIG
 yq -i ".dbize.RANDREADFILESIZE=$RANDREADFILESIZE" $CONFIG
@@ -105,15 +106,27 @@ yq -i ".system.os.name=\"$(uname)\"" $CONFIG
 yq -i ".system.os.kernel=\"$(uname -r)\"" $CONFIG
 yq -i ".system.cpu.arch=\"$(arch)\"" $CONFIG
 yq -i ".system.cpu.model=\"$CPUMOODEL\"" $CONFIG
-yq -i ".system.cpu.corepersocket=$COREPERSOCKET" $CONFIG
 yq -i ".system.cpu.socketnr=$SOCKETNR" $CONFIG
+yq -i ".system.cpu.corepersocket=$COREPERSOCKET" $CONFIG
+yq -i ".system.cpu.threadpercore=$THREADPERCORE" $CONFIG
 yq -i ".system.memsizeGB=$($QBIN -q <<<'.Q.w[][`mphy] div 1024 * 1024 * 1024')" $CONFIG
 
+function getNuma {
+    echo ""
+}
+
 if [[ $(uname) == "Linux" ]]; then
     lscpu > ${RESDIR}/lscpu.out
     ${SUDO} dmidecode -t memory > ${RESDIR}/dmidecode.out
     if command -v numactl 2>&1 >/dev/null; then
       numactl --hardware > ${RESDIR}/numactl.out
+      NUMANODES=$(lscpu|grep "NUMA node(s)"|cut -d":" -f 2|xargs)
+      if [[ ${NUMA} == "roundrobin" ]] && [[ ${NUMANODES} -gt 1 ]]; then
+        function getNuma {
+          local IDX=$(((${1}-1) % NUMANODES))
+          echo "numactl -N $IDX -m $IDX"
+        }
+      fi
     fi
 fi
 
@@ -174,7 +187,8 @@ function runTest {
   ${QBIN} ./src/controller.q -iostatfile ${IOSTATFILE} -s $NUMPROCESSES -q -p ${CONTROLLERPORT} > ${CURRENTLOGDIR}/controller_${TESTER%.*}.log 2 >&1 &
   j=0
   for i in $(seq $NUMPROCESSES); do
-  	${QBIN} ./src/${TESTER} -processes $NUMPROCESSES -db ${array[$j]}/${HOST}.${i}/${DATE} -result ${RESFILEPREFIX}${i}.psv -controller ${CONTROLLERPORT} -s ${THREADNR} -p $((WORKERBASEPORT + i)) > ${LOGFILEPREFIX}${i}_${TESTER%.*}.log 2>&1 &
+    local NUMAPREFIX=$(getNuma $i)
+  	${NUMAPREFIX} ${QBIN} ./src/${TESTER} -processes $NUMPROCESSES -db ${array[$j]}/${HOST}.${i}/${DATE} -result ${RESFILEPREFIX}${i}.psv -controller ${CONTROLLERPORT} -s ${THREADNR} -p $((WORKERBASEPORT + i)) > ${LOGFILEPREFIX}${i}_${TESTER%.*}.log 2>&1 &
     j=$(( ($j + 1) % $NUMSEGS ))
   done
   wait -n

runSeveral.sh

@@ -6,10 +6,10 @@ source common.sh
 
 if [ $# -lt 1 ]; then
    readonly OUTPUT=./results/throughput_total.psv
-   readonly LIMIT=$CORECOUNT
+   readonly LIMIT=$COMPUTECOUNT
 elif [ $# -lt 2 ]; then
    readonly OUTPUT=$1
-   readonly LIMIT=$CORECOUNT
+   readonly LIMIT=$COMPUTECOUNT
 else
    readonly OUTPUT=$1
    readonly LIMIT=$2

version.yaml

@@ -1,2 +1,2 @@
-pub: 3.3.3
-dev: 3.3.3
+pub: 3.4.0
+dev: 3.4.0