Updates to Jeff's test scripts on Falcon - supports numactl bindings.

Author: jyoung3131

eval/jy_scripts_falcon/1_multi_test.sh

@@ -8,23 +8,32 @@ fi
 gpus=(1)
 #scale factors
 #msize=(15)
-msize=(15 16 17 18 19 20 21 22 23 24 25 26 27)
+#msize=(15 16 17 18 19 20 21 22 23 24 25 26 27)
+msize=(22 27)
 #square root of nodes
-snodes=(1 2 3 4 5 6 7 8)
+snodes=(8 7 6 5)
 #snodes=(3 4)
 #BTR
 btrs=(64)
 #btcs=(128 256 512)
 
-declare -a g500ver=("g500_CompressionNoOptimizedNoOMP" "g500_CompressionOptimizedNoOMP" "g500_NoCompressionNoOptimizedNoOMP" "g500_NoCompressionOptimizedNoOMP" "g500_CompressionNoOptimizeOMP" "g500_CompressionOptimizedOMP" "g500_NoCompressionNoOptimizedOMP" "g500_NoCompressionOptimizedOMP")
+#All test variations
+#declare -a g500ver=("g500_CompressionNoOptimizedNoOMP" "g500_CompressionOptimizedNoOMP" "g500_NoCompressionNoOptimizedNoOMP" "g500_NoCompressionOptimizedNoOMP" "g500_CompressionNoOptimizeOMP" "g500_CompressionOptimizedOMP" "g500_NoCompressionNoOptimizedOMP" "g500_NoCompressionOptimizedOMP")
+
+#Restrict tests for ScoreP
+#declare -a g500ver=("g500_CompressionNoOptimizedNoOMP" "g500_CompressionOptimizedNoOMP" "g500_NoCompressionNoOptimizedNoOMP" "g500_NoCompressionOptimizedNoOMP")
+#declare -a g500ver=("scp_CompOptOMP" "scp_CompOptNoOMP" "scp_NoCompOptNoOMP" "scp_NoCompOptOMP" "scp_CompNoOptOMP" "scp_CompNoOptNoOMP" "scp_NoCompNoOptNoOMP" "scp_NoCompNoOptOMP")
+declare -a g500ver=("scp_CompOptOMP")
 
 for g500 in "${g500ver[@]}"
 do
 echo "Copying over a new g500 version, $g500ver"
 cd ../cpu_2d/
 cp -f g500_versions/${g500} g500
+#Sleep to try and get NFS to sync properly across nodes
+sleep 20
 cd -
-log="${g500}_mar16"
+log="${g500}_may16_numa"
 
 for msc in ${msize[@]}
 do 
@@ -53,15 +62,27 @@ do
 
             for btr in ${btrs[@]}
             do 
+                # score-p
+		# before script execution:
+		export G500_ENABLE_RUNTIME_SCALASCA=yes
+
+		# pass score-p's $SCOREP_EXPERIMENT_DIRECTORY to mpirun
+                
+		#mpirun -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREP_EXPERIMENT_DIRECTORY -np ${np} -hostfile ${hostfile} --display-map -bynode ./test2.sh ${sf} ${c} "s4-bp128-d4" 64 | tee ${file}
                 #for btc in ${btcs[@]}
                 #do
 	            srp=$i
                     n=`expr $i \* $i` 
                     logfile="result/falcon_${log}_nd${n}_scale${msc}_btr${btr}_btc${btr}.log"
-                    echo "./2_test_falcon.sh nd$n srp$srp msc$msc gpu$ngpus btr$btr btc$btr $logfile"
+                    scorepdir="scorep_results/falcon_${log}_nd${n}_scale${msc}_btr${btr}.scorep"
+		    if [ "x$G500_ENABLE_RUNTIME_SCALASCA" = "xyes" ]; then
+			export SCOREP_EXPERIMENT_DIRECTORY=${scorepdir}
+                        echo "SCOREPDIR is ${SCOREP_EXPERIMENT_DIRECTORY}"
+         	    fi	
+                    echo "./2_test_falcon.sh nd$n srp$srp msc$msc gpu$ngpus btr$btr btc$btr $logfile $scorepdir"
                     echo ""
-                    ./2_test_falcon.sh $n $srp $msc $ngpus $btr $btr $logfile
-                    sleep 2
+                    ./2_test_falcon.sh $n $srp $msc $ngpus $btr $btr $logfile $scorepdir
+                    sleep 1 
                 #done
 
             done

eval/jy_scripts_falcon/2_test_falcon.sh

@@ -6,14 +6,21 @@ GPUS=$4
 BTR=$5
 BTC=$6
 LOGFILE=$7
+SCOREPDIR=$8
 HOSTS=~/scripts/hostfile_falcon_64
 
 
 LD_LIBRARY_PATH=/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/lib64:/usr/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/lib:/usr/lib64/boost148/
 PATH=/opt/rh/devtoolset-2/root/usr/bin:/net/rd6/jyoung9/mpi/ompi184/bin:/usr/bin:/usr/sbin:/bin:/usr/local/bin:/bin:/usr/local/sbin:/usr/local/cuda/bin:/usr/include/boost148/
 MPIRUN=/net/rd6/jyoung9/mpi/ompi184/bin/mpirun
 
+#Control for GPU and socket
+#CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
+#Bind the CPU and memory to the socket with the GPU being used
+cudavisible="CUDA_VISIBLE_DEVICES=0"
+
+
 source /net/rd6/jyoung9/.bashrc
-echo "$MPIRUN -v -x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -hostfile $HOSTS -np $N -map-by node --display-map -mca btl openib,self -mca mpi_warn_on_fork 0 ./3_launch.sh $SF $SRP $GPUS $BTR $BTC &> ${LOGFILE}"
-$MPIRUN -v -x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -hostfile $HOSTS -np $N -map-by node --display-map -mca btl openib,self -mca mpi_warn_on_fork 0 ./3_launch.sh $SF $SRP $GPUS $BTR $BTC &> ${LOGFILE}
+echo "$MPIRUN -v -x $cudavisible -x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR -hostfile $HOSTS -np $N -map-by node --display-map -mca btl openib,self -mca mpi_warn_on_fork 0 ./3_launch.sh $SF $SRP $GPUS $BTR $BTC &> ${LOGFILE}"
+$MPIRUN -v -x $cudavisible -x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR -hostfile $HOSTS -np $N -map-by node --display-map -mca btl openib,self -mca mpi_warn_on_fork 0 ./3_launch.sh $SF $SRP $GPUS $BTR $BTC &> ${LOGFILE}
 #$MPIRUN -v -x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH -hostfile $HOSTS -np $N -map-by slot --display-map -mca btl openib,self -mca mpi_warn_on_fork 0 ./3_launch.sh $SF $SRP $GPUS $BTR $BTC &> ${LOGFILE}

eval/jy_scripts_falcon/3_launch.sh

@@ -5,6 +5,8 @@ GPUS=$3
 BTR=$4
 BTC=$5
 
-export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
-#echo "../cpu_2d/g500 -s $SF -C $SRP -gpus $GPUS -qs 1 -be s4-bp128-d4 -btr $BTR -btc $BTC"
-../cpu_2d/g500 -s $SF -C $SRP -gpus $GPUS -qs 1 -be s4-bp128-d4 -btr $BTR -btc $BTC 
+export CUDA_VISIBLE_DEVICES=0
+NUMAEXEC="numactl --membind=0 --cpunodebind=0"
+
+#echo "$NUMAEXEC ../cpu_2d/g500 -s $SF -C $SRP -gpus $GPUS -qs 1 -be s4-bp128-d4 -btr $BTR -btc $BTC"
+$NUMAEXEC ../cpu_2d/g500 -s $SF -C $SRP -gpus $GPUS -qs 1 -be s4-bp128-d4 -btr $BTR -btc $BTC