From e4bc77a0b8119b7f0da7d2f8730cd355f4348534 Mon Sep 17 00:00:00 2001 From: Cloud User Date: Thu, 29 Sep 2022 16:10:18 +0000 Subject: [PATCH 1/6] adding call out if GPUs are still set --- pyglidein/glidein_start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyglidein/glidein_start.sh b/pyglidein/glidein_start.sh index 487aed9..bc222ca 100755 --- a/pyglidein/glidein_start.sh +++ b/pyglidein/glidein_start.sh @@ -76,6 +76,8 @@ if [ $GPUS != 0 ]; then fi fi +echo "Starting pyglidein with GPUS=$GPUS" + ## # Done with config ## From 93e8bbaa596994b24d23e53c885862d4cfabebb1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Sep 2022 16:11:53 +0000 Subject: [PATCH 2/6] 1.1.17 Automatically generated by python-semantic-release --- CHANGELOG.md | 3 +++ pyglidein/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdda6bf..46d6cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ +## v1.1.17 (2022-09-29) + + ## v1.1.16 (2022-09-29) diff --git a/pyglidein/__init__.py b/pyglidein/__init__.py index 43dcbc8..00ea415 100644 --- a/pyglidein/__init__.py +++ b/pyglidein/__init__.py @@ -5,7 +5,7 @@ # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.1.16" +__version__ = "1.1.17" version_info = ( int(__version__.split(".")[0]), int(__version__.split(".")[1]), From 76ed70c835184f2959e79448bedfa10af1e735cc Mon Sep 17 00:00:00 2001 From: Benedikt Riedel Date: Thu, 29 Sep 2022 18:16:56 +0000 Subject: [PATCH 3/6] Making GPU detection more robust --- pyglidein/glidein_start.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pyglidein/glidein_start.sh b/pyglidein/glidein_start.sh index bc222ca..39fc834 100755 --- a/pyglidein/glidein_start.sh +++ b/pyglidein/glidein_start.sh @@ -61,22 +61,35 @@ fi CVMFS="True" # GPU type detection +OLD_GPUS=$GPUS GPU_NAMES="" if [ $GPUS != 0 ]; then if command -v nvidia-smi >/dev/null; then if [ "$GPUS" = "all" ]; then GPU_NAMES=$(nvidia-smi --query-gpu=name --format=csv,noheader|sed ':a;N;$!ba;s/\n/,/g'); else - GPU2=$(echo "$GPUS"|sed 's/CUDA//g'|sed 's/OCL//g'); + if [ $GPUS = 1 ]; then + GPU2=$(nvidia-smi --query-gpu=index --format=csv,noheader); + else + GPU2=$(echo "$GPUS"|sed 's/CUDA//g'|sed 's/OCL//g'); + fi GPU_NAMES=$(nvidia-smi --query-gpu=name --format=csv,noheader --id=$GPU2|sed ':a;N;$!ba;s/\n/,/g'); fi else # GPUs might exist but nvidia-smi is not available. re-set $GPUS GPUS=0 - fi + fi +fi + +if [ [ "$GPUS" = 0 ] && [ "$OLD_GPUS" != "$GPUS" ] ]; then + echo "Attemping at GPU_NAMES detection has failed" + exit 1 fi -echo "Starting pyglidein with GPUS=$GPUS" + +if [ "$GPU_NAMES" != "" ]; then + echo "Starting pyglidein with GPUS=$GPUS with GPU_NAMES=$GPU_NAMES" +fi ## # Done with config @@ -136,7 +149,7 @@ export _condor_NUM_SLOTS_TYPE_1=1 export _condor_SLOT_TYPE_1_PARTITIONABLE="True" #export _condor_SLOT_TYPE_1_CONSUMPTION_POLICY="True" #export _condor_SLOT_TYPE_1_CONSUMPTION_GPUs="quantize(ifThenElse(target.RequestGpus =!= undefined,target.RequestGpus,0),{0})"; -if [ $SITE -eq "Anvil" ]; then +if [ "$SITE" = "Anvil" ]; then export _condor_NETWORK_INTERFACE='172.18.*' export _condor_MASTER_DEBUG=D_HOSTNAME:2,D_ALWAYS:2 fi From 907241cfad798464c90639076d42f1ef1fd93330 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Sep 2022 18:18:30 +0000 Subject: [PATCH 4/6] 1.1.18 Automatically generated by python-semantic-release --- CHANGELOG.md | 3 +++ pyglidein/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46d6cce..c7a3274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ +## v1.1.18 (2022-09-29) + + ## v1.1.17 (2022-09-29) diff --git a/pyglidein/__init__.py b/pyglidein/__init__.py index 00ea415..ae28829 100644 --- a/pyglidein/__init__.py +++ b/pyglidein/__init__.py @@ -5,7 +5,7 @@ # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.1.17" +__version__ = "1.1.18" version_info = ( int(__version__.split(".")[0]), int(__version__.split(".")[1]), From df5287c362f9645e86d93d09c441bd517715961b Mon Sep 17 00:00:00 2001 From: Benedikt Riedel Date: Thu, 29 Sep 2022 20:56:10 +0000 Subject: [PATCH 5/6] making sure that CUDA is set properly --- pyglidein/glidein_start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pyglidein/glidein_start.sh b/pyglidein/glidein_start.sh index 39fc834..5a33c25 100755 --- a/pyglidein/glidein_start.sh +++ b/pyglidein/glidein_start.sh @@ -70,6 +70,7 @@ if [ $GPUS != 0 ]; then else if [ $GPUS = 1 ]; then GPU2=$(nvidia-smi --query-gpu=index --format=csv,noheader); + GPUS=CUDA$GPU2; else GPU2=$(echo "$GPUS"|sed 's/CUDA//g'|sed 's/OCL//g'); fi From 1477950fa8f6fa40139c88741620f1f2248ec8fc Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Sep 2022 20:57:23 +0000 Subject: [PATCH 6/6] 1.1.19 Automatically generated by python-semantic-release --- CHANGELOG.md | 3 +++ pyglidein/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7a3274..fd88205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ +## v1.1.19 (2022-09-29) + + ## v1.1.18 (2022-09-29) diff --git a/pyglidein/__init__.py b/pyglidein/__init__.py index ae28829..959712f 100644 --- a/pyglidein/__init__.py +++ b/pyglidein/__init__.py @@ -5,7 +5,7 @@ # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.1.18" +__version__ = "1.1.19" version_info = ( int(__version__.split(".")[0]), int(__version__.split(".")[1]),