diff --git a/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py b/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py index 324b9c1f..d7ddac30 100644 --- a/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py +++ b/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py @@ -122,7 +122,7 @@ def __getCSConfig( self ): return S_ERROR( "/LocalSite/VirtualMachineName is not defined" ) #Variables coming from the vm imgPath = "/Resources/VirtualMachines/RunningPods/%s" % self.runningPod - for csOption, csDefault, varName in ( ( "MinWorkingLoad", 0.01, "vmMinWorkingLoad" ), + for csOption, csDefault, varName in ( ( "MinWorkingLoad", 0.03, "vmMinWorkingLoad" ), ( "LoadAverageTimespan", 60, "vmLoadAvgTimespan" ), ( "HaltPeriod", 600, "haltPeriod" ), ( "HaltBeforeMargin", 300, "haltBeforeMargin" ), diff --git a/WorkloadManagementSystem/Client/AmazonInstance.py b/WorkloadManagementSystem/Client/AmazonInstance.py index e3cca914..dcf2d23f 100644 --- a/WorkloadManagementSystem/Client/AmazonInstance.py +++ b/WorkloadManagementSystem/Client/AmazonInstance.py @@ -169,14 +169,25 @@ def create_VMInstance( self, runningPodRequirements ): instanceType ) ) try: if contextMethod == 'cloudinit': - reservation = self.__vmImage.run( min_count = 1, + if ( keyName == None or keyName == 'nouse' ): + reservation = self.__vmImage.run( min_count = 1, + max_count = 1, + user_data = userdata, + instance_type = instanceType ) + else: + reservation = self.__vmImage.run( min_count = 1, max_count = 1, user_data = userdata, key_name= keyName, instance_type = instanceType ) else: #adhoc instance - reservation = self.__vmImage.run( min_count = numImages, + if ( keyName == None or keyName == 'nouse' ): + reservation = self.__vmImage.run( min_count = numImages, + max_count = 1, + instance_type = instanceType ) + else: + reservation = self.__vmImage.run( min_count = numImages, max_count = 1, key_name= keyName, instance_type = instanceType ) diff --git a/WorkloadManagementSystem/Client/BuildCloudinitScript.py b/WorkloadManagementSystem/Client/BuildCloudinitScript.py index d31cc979..ac54d850 100644 --- a/WorkloadManagementSystem/Client/BuildCloudinitScript.py +++ b/WorkloadManagementSystem/Client/BuildCloudinitScript.py @@ -112,7 +112,7 @@ def __buildCloudinitScript( self, file.write('vmRunLogAgentURL=%s\n' % (vmRunLogAgentURL)) file.write('vmCvmfsContextURL=%s\n' % (vmCvmfsContextURL)) file.write('vmDiracContextURL=%s\n' % (vmDiracContextURL)) - file.write('cvmfs_http_proxy=%s\n' % (cvmfs_http_proxy)) + file.write('cvmfs_http_proxy=\"%s\"\n' % (cvmfs_http_proxy)) file.write('siteName=%s\n' % (siteName)) file.write('cloudDriver=%s\n' % (cloudDriver)) file.write('vmStopPolicy=%s\n' % (vmStopPolicy)) diff --git a/WorkloadManagementSystem/DB/VirtualMachineDB.py b/WorkloadManagementSystem/DB/VirtualMachineDB.py index 26ccc307..e539dccf 100755 --- a/WorkloadManagementSystem/DB/VirtualMachineDB.py +++ b/WorkloadManagementSystem/DB/VirtualMachineDB.py @@ -342,19 +342,16 @@ def getUniqueID( self, instanceID ): """ tableName, _validStates, idName = self.__getTypeTuple( 'Instance' ) - print instanceID - print "instanceID" - sqlQuery = "SELECT UniqueID FROM `%s` WHERE %s = %s" % ( tableName, idName, instanceID ) uniqueID = self._query( sqlQuery ) - print "uniqueID" - print uniqueID - if not uniqueID[ 'OK' ]: return uniqueID uniqueID = uniqueID[ 'Value' ] + if not uniqueID: + return S_ERROR( ' Unregistered VM, uniqueID not found for instanceID %s' % (instanceID) ) + return S_OK( uniqueID[ 0 ][ 0 ] ) def getInstanceID( self, uniqueID ): diff --git a/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py b/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py index c876133c..b37815b6 100644 --- a/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py +++ b/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py @@ -102,7 +102,7 @@ def haltInstances(vmList): continue imageName = imageName[ 'Value' ] - gLogger.info( 'Attemping to halt Stalled instance: %s, endpoint: %s imageName: %s' % (str(uniqueID),endpoint,imageName) ) + gLogger.info( 'Attemping to halt instance: %s, endpoint: %s imageName: %s' % (str(uniqueID),endpoint,imageName) ) if ( cloudDriver == 'occi-0.9' or cloudDriver == 'occi-0.8' or cloudDriver == 'rocci-1.1' ): oima = OcciImage( imageName, endpoint ) diff --git a/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-eiscat-DIRAC.context.sh b/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-eiscat-DIRAC.context.sh new file mode 100644 index 00000000..0e0aeabd --- /dev/null +++ b/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-eiscat-DIRAC.context.sh @@ -0,0 +1,293 @@ +#!/bin/bash +# +# dirac contextualization script +# To be run as root on VM +# + + +get_packaging_system() { + YUM_CMD=$(which yum) + APT_GET_CMD=$(which apt-get) + + if [ ! -z $YUM_CMD ] + then + echo "RedHat based" + PACKAGE_MANAGER="yum" + elif [ ! -z $APT_GET_CMD ] + then + echo "Debian based" + PACKAGE_MANAGER="apt-get" + else + echo "Package manager not implemented." + fi +} + +install_unzip() { + get_packaging_system + [ ! -z $PACKAGE_MANAGER ] && $PACKAGE_MANAGER -y update + [ ! -z $PACKAGE_MANAGER ] && $PACKAGE_MANAGER -y install unzip +} + +install_easy_install() { + get_packaging_system + [ ! -z $PACKAGE_MANAGER ] && $PACKAGE_MANAGER -y install python-setuptools +} + +install_eiscat_software_stack() { + #ubuntu based + sudo apt-add-repository -y ppa:octave/stable + sudo apt-get -y update + sudo apt-get -y install octave + cd /usr/local/bin + wget http://dirac1.grid.cyfronet.pl:8088/repo/remtg.tgz + #eiscat CVS + #wget http://www.eiscat.se/raw/rtg/remtg.tgz + tar xzvf remtg.tgz + chown -R dirac:dirac remtg + export export PATH=/usr/local/bin/remtg:$PATH + cd $OLDPWD +} + + echo "Starting dirac-context-script.sh" > /var/log/dirac-context-script.log 2>&1 + + +if [ $# -ne 9 ] +then + echo "ERROR: Given $# parameters" >> /var/log/dirac-context-script.log 2>&1 + echo " Given parameters: $@" >> /var/log/dirac-context-script.log 2>&1 + echo " Required parameters: general-DIRAC-context.sh '' '' '' '' '' '' '' '' ''" >> /var/log/dirac-context-script.log 2>&1 + exit 1 +fi + +siteName=${1} +vmStopPolicy=${2} +putCertPath=${3} +putKeyPath=${4} +localVmRunJobAgent=${5} +localVmRunVmMonitorAgent=${6} +localVmRunVmUpdaterAgent=${7} +localVmRunLogAgent=${8} +cloudDriver=${9} + +echo "Running dirac-contex.sh '' '' '' '' '' '' '' '' ''" >> /var/log/dirac-context-script.log 2>&1 +echo "1 $siteName" >> /var/log/dirac-context-script.log 2>&1 +echo "2 $vmStopPolicy" >> /var/log/dirac-context-script.log 2>&1 +echo "3 $putCertPath" >> /var/log/dirac-context-script.log 2>&1 +echo "4 $putKeyPath" >> /var/log/dirac-context-script.log 2>&1 +echo "5 $localVmRunJobAgent" >> /var/log/dirac-context-script.log 2>&1 +echo "6 $localVmRunVmMonitorAgent" >> /var/log/dirac-context-script.log 2>&1 +echo "7 $localVmRunVmUpdaterAgent" >> /var/log/dirac-context-script.log 2>&1 +echo "8 $localVmRunLogAgent" >> /var/log/dirac-context-script.log 2>&1 +echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 + +# dirac user: + /usr/sbin/useradd -m -s /bin/bash -d /opt/dirac dirac >> /var/log/dirac-context-script.log 2>&1 + chown -R dirac.dirac /opt/dirac + +# servercert/serverkey previouslly to this script copied +# + cd /opt/dirac + su dirac -c'mkdir -p etc/grid-security' >> /var/log/dirac-context-script.log 2>&1 + chmod -R 755 etc >> /var/log/dirac-context-script.log 2>&1 + mv ${putCertPath} etc/grid-security/servercert.pem >> /var/log/dirac-context-script.log 2>&1 + mv ${putKeyPath} etc/grid-security/serverkey.pem >> /var/log/dirac-context-script.log 2>&1 + + sleep 1 + + # If there is no key, is because the cert is a user proxy + if [ ! -s etc/grid-security/serverkey.pem ] + then + isproxy="Y" + diracuid=`id -u dirac` + proxyname=`echo "x509up_u${diracuid}"` + echo "User proxy: ${proxyname}" >> /var/log/dirac-context-script.log 2>&1 + mv etc/grid-security/servercert.pem /tmp/${proxyname} + chmod 600 /tmp/${proxyname} + cp /tmp/${proxyname} /tmp/x509up_u0 + chown dirac.dirac /tmp/${proxyname} + ls -l /tmp/${proxyname} >> /var/log/dirac-context-script.log 2>&1 + else + isproxy="N" + chmod 444 etc/grid-security/servercert.pem >> /var/log/dirac-context-script.log 2>&1 + chmod 400 etc/grid-security/serverkey.pem >> /var/log/dirac-context-script.log 2>&1 + fi + + + chown -R dirac:dirac etc >> /var/log/dirac-context-script.log 2>&1 + +# +# Installing EISCAT software stack +# +install_eiscat_software_stack >> /var/log/dirac-context-script.log 2>&1 + +# +# Installing DIRAC +# + cd /opt/dirac + wget --no-check-certificate -O dirac-install 'http://dirac1.grid.cyfronet.pl:8088/repo/integration/DIRAC/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 + + su dirac -c'python dirac-install -V "VMEGI"' >> /var/log/dirac-context-script.log 2>&1 + + # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez folder: + rm -rf VMDIRAC + wget --no-check-certificate -O vmdirac.zip 'http://dirac1.grid.cyfronet.pl:8088/repo/vmendez/master.zip' >> /var/log/dirac-context-script.log 2>&1 + # checking if unzip installed + if [ ! `which unzip` ] + then + install_unzip + fi + unzip vmdirac.zip >> /var/log/dirac-context-script.log 2>&1 + mv VMDIRAC-master VMDIRAC + chown -R dirac:dirac VMDIRAC + cd VMDIRAC + for i in `find . -name "*pyo"` + do + chown root:root $i + done + cd /opt/dirac + + source bashrc >> /var/log/dirac-context-script.log 2>&1 + env >> /var/log/dirac-context-script.log 2>&1 + chmod ugo+w /var/log/dirac-context-script.log + + # to the runsvdir stuff: + export PATH + export LD_LIBRARY_PATH + platform=`dirac-platform` + # for the VM Monitor + # checking if easy_install installed + if [ ! `which easy_install` ] + then + echo "easy_install not installed. Installing">> /var/log/dirac-context-script.log 2>&1 + install_easy_install + fi + echo "Installing easy_install simplejson for the VM Monitor" >> /var/log/dirac-context-script.log 2>&1 + `which python` `which easy_install` simplejson >> /var/log/dirac-context-script.log 2>&1 + # getting RunningPodRequirements + requirements='' + while read keyval + do + if [ `echo $keyval | grep '^Tag' ` ] + then + tagval=`echo $keyval|cut -f2 -d"="` + requirements=`echo "$requirements -o /Resources/Computing/CEDefaults/Tag=$tagval"` + requirements=`echo "$requirements -o /AgentJobRequirements/RequiredTag=$tagval"` + else + requirements=`echo "$requirements -o /LocalSite/$keyval"` + fi + done > /var/log/dirac-context-script.log 2>&1 + # options H: SkipCAChecks, dd: debug level 2 + # options only for debuging D: SkipCADownload + else + #hostcert credentials (compatibility previous v6r14) + su dirac -c"source bashrc;dirac-configure -UHddd $requirements -o /LocalSite/CloudDriver=$cloudDriver -o /LocalSite/Site=$siteName -o /LocalSite/VMStopPolicy=$vmStopPolicy -o /LocalSite/CE=CE-nouse defaults-VMEGI.cfg" >> /var/log/dirac-context-script.log 2>&1 + # options H: SkipCAChecks, dd: debug level 2, U: UseServerCertificateCredentials + # options only for debuging D: SkipCADownload + # after configuration with UseServerCertificate = yes for the configuration with CS + #  we have to change to allow user proxy delegation for agents: + su dirac -c'sed "s/UseServerCertificate = yes/#UseServerCertificate = yes/" etc/dirac.cfg > dirac.cfg.aux' + fi + if [ `ls /opt/dirac/etc/grid-security/certificates | wc -l` -ne 0 ] + then + echo "certificates download in dirac-configure at retry: $retry" >> /var/log/dirac-context-script.log 2>&1 + break + fi + echo "certificates was not download in dirac-configure at retry: $retry" >> /var/log/dirac-context-script.log 2>&1 + done + su dirac -c'cp etc/dirac.cfg dirac.cfg.postconfigure' + su dirac -c'mv dirac.cfg.aux etc/dirac.cfg' + if [ -n "$TAGVAL" ] + then + # Tag is going to Resource Computing CE section + su dirac -c'echo "Resources" >> etc/dirac.cfg' + su dirac -c'echo "{" >> etc/dirac.cfg' + su dirac -c'echo " Computing" >> etc/dirac.cfg' + su dirac -c'echo " {" >> etc/dirac.cfg' + su dirac -c'echo " CEDefaults" >> etc/dirac.cfg' + su dirac -c'echo " {" >> etc/dirac.cfg' + su dirac -c'echo -n " " >> etc/dirac.cfg' + su dirac -c'echo "$TAGVAL" >> etc/dirac.cfg' + su dirac -c'echo " }" >> etc/dirac.cfg' + su dirac -c'echo " }" >> etc/dirac.cfg' + su dirac -c'echo "}" >> etc/dirac.cfg' + fi + echo "etc/dirac.cfg content previous to agents run: " >> /var/log/dirac-context-script.log 2>&1 + cat etc/dirac.cfg >> /var/log/dirac-context-script.log 2>&1 + echo >> /var/log/dirac-context-script.log 2>&1 + + +# start the agents: VirtualMachineMonitor, JobAgent, VirtualMachineConfigUpdater + + cd /opt/dirac + if [ ${localVmRunJobAgent} != 'nouse' ] + then + mkdir -p startup/WorkloadManagement_JobAgent/log >> /var/log/dirac-context-script.log 2>&1 + mv ${localVmRunJobAgent} startup/WorkloadManagement_JobAgent/run >> /var/log/dirac-context-script.log 2>&1 + cp ${localVmRunLogAgent} startup/WorkloadManagement_JobAgent/log/run >> /var/log/dirac-context-script.log 2>&1 + chmod 755 startup/WorkloadManagement_JobAgent/log/run + chmod 755 startup/WorkloadManagement_JobAgent/run + + echo "rights and permissions to control and work JobAgent dirs" >> /var/log/dirac-context-script.log 2>&1 + mkdir -p /opt/dirac/control/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + mkdir -p /opt/dirac/work/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + chmod 775 /opt/dirac/control/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + chmod 775 /opt/dirac/work/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + chown root:dirac /opt/dirac/work/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + chown root:dirac /opt/dirac/control/WorkloadManagement/JobAgent >> /var/log/dirac-context-script.log 2>&1 + echo "/opt/dirac/control/WorkloadManagement content" >> /var/log/dirac-context-script.log 2>&1 + ls -l /opt/dirac/control/WorkloadManagement >> /var/log/dirac-context-script.log 2>&1 + echo "/opt/dirac/work/WorkloadManagement content" >> /var/log/dirac-context-script.log 2>&1 + ls -l /opt/dirac/work/WorkloadManagement >> /var/log/dirac-context-script.log 2>&1 + echo >> /var/log/dirac-context-script.log 2>&1 + fi + + if [ ${localVmRunVmUpdaterAgent} != 'nouse' ] + then + mkdir -p startup/WorkloadManagement_VirtualMachineConfigUpdater/log >> /var/log/dirac-context-script.log 2>&1 + mv ${localVmRunVmUpdaterAgent} startup/WorkloadManagement_VirtualMachineConfigUpdater/run >> /var/log/dirac-context-script.log 2>&1 + cp ${localVmRunLogAgent} startup/WorkloadManagement_VirtualMachineConfigUpdater/log/run >> /var/log/dirac-context-script.log 2>&1 + chmod 755 startup/WorkloadManagement_VirtualMachineConfigUpdater/log/run + chmod 755 startup/WorkloadManagement_VirtualMachineConfigUpdater/run + fi + + mkdir -p startup/WorkloadManagement_VirtualMachineMonitorAgent/log >> /var/log/dirac-context-script.log 2>&1 + mv ${localVmRunVmMonitorAgent} startup/WorkloadManagement_VirtualMachineMonitorAgent/run >> /var/log/dirac-context-script.log 2>&1 + mv ${localVmRunLogAgent} startup/WorkloadManagement_VirtualMachineMonitorAgent/log/run >> /var/log/dirac-context-script.log 2>&1 + chmod 755 startup/WorkloadManagement_VirtualMachineMonitorAgent/log/run + chmod 755 startup/WorkloadManagement_VirtualMachineMonitorAgent/run + + echo "runsvdir startup, have a look to DIRAC JobAgent, VirtualMachineMonitorAgent and VirtualMachineConfigUpdater logs" >> /var/log/dirac-context-script.log 2>&1 + runsvdir -P /opt/dirac/startup 'log: DIRAC runsv' & + +# +# END installing DIRAC +# + +# avoiding ssh conection refused: +# echo "After DIRAC install:" >> /var/log/dirac-context-script.log 2>&1 +# ls -l /etc/ssh >> /var/log/dirac-context-script.log 2>&1 +# chmod 600 /etc/ssh/* >> /var/log/dirac-context-script.log 2>&1 +# chmod go+r /etc/ssh/ssh_config /etc/ssh/ssh_host_dsa_key.pub /etc/ssh/ssh_host_key.pub /etc/ssh/ssh_host_rsa_key.pub >> /var/log/dirac-context-script.log 2>&1 +# echo "After restoring rights:" >> /var/log/dirac-context-script.log 2>&1 +# ls -l /etc/ssh >> /var/log/dirac-context-script.log 2>&1 + + # + # STOPING DIRAC AGENTS: + # +# cd /opt/dirac +# killall runsvdir +# runsvctrl d startup/* +# killall runsv + + echo "END dirac-context-script.sh" >> /var/log/dirac-context-script.log 2>&1 + +exit $RETVAL diff --git a/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-general-DIRAC.context.sh b/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-general-DIRAC.context.sh index e9181de2..a1b50085 100644 --- a/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-general-DIRAC.context.sh +++ b/WorkloadManagementSystem/private/bootstrap/EGI-Fedcloud-general-DIRAC.context.sh @@ -68,6 +68,7 @@ echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 # dirac user: /usr/sbin/useradd -m -s /bin/bash -d /opt/dirac dirac >> /var/log/dirac-context-script.log 2>&1 + chown -R dirac.dirac /opt/dirac # servercert/serverkey previouslly to this script copied # @@ -104,13 +105,13 @@ echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 # Installing DIRAC # cd /opt/dirac - wget --no-check-certificate -O dirac-install 'https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O dirac-install 'http://dirac1.grid.cyfronet.pl:8088/repo/integration/DIRAC/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 su dirac -c'python dirac-install -V "VMEGI"' >> /var/log/dirac-context-script.log 2>&1 - # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez git folder: + # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez folder: rm -rf VMDIRAC - wget --no-check-certificate -O vmdirac.zip 'https://github.com/vmendez/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O vmdirac.zip 'http://dirac1.grid.cyfronet.pl:8088/repo/vmendez/master.zip' >> /var/log/dirac-context-script.log 2>&1 # checking if unzip installed if [ ! `which unzip` ] then @@ -146,8 +147,15 @@ echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 # getting RunningPodRequirements requirements='' while read keyval - do - requirements=`echo "$requirements -o /LocalSite/$keyval"` + do + if [ `echo $keyval | grep '^Tag' ` ] + then + tagval=`echo $keyval|cut -f2 -d"="` + requirements=`echo "$requirements -o /Resources/Computing/CEDefaults/Tag=$tagval"` + requirements=`echo "$requirements -o /AgentJobRequirements/RequiredTag=$tagval"` + else + requirements=`echo "$requirements -o /LocalSite/$keyval"` + fi done > /var/log/dirac-context-script.log 2>&1 # # Installing DIRAC -# FOR DEBUGGIN PURPOSES installing debuggin github version instead of cvmfs repository released DIRAC: # cd /opt/dirac - wget --no-check-certificate -O dirac-install 'https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O dirac-install 'https://yourreposerverhere-notGit/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 su dirac -c'python dirac-install -V "VMFranceGrilles"' >> /var/log/dirac-context-script.log 2>&1 - # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez git folder: + # FOR DEBUGGIN PURPOSES overwriting with last released in your folder: rm -rf VMDIRAC - wget --no-check-certificate -O vmdirac.zip 'https://github.com/vmendez/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O vmdirac.zip 'https://yourreposerverhere-notGit/yourfolder/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 unzip vmdirac.zip >> /var/log/dirac-context-script.log 2>&1 mv VMDIRAC-master VMDIRAC chown -R dirac:dirac VMDIRAC diff --git a/WorkloadManagementSystem/private/bootstrap/cvmfs-enmr-context.sh b/WorkloadManagementSystem/private/bootstrap/cvmfs-enmr-context.sh new file mode 100644 index 00000000..e37a25af --- /dev/null +++ b/WorkloadManagementSystem/private/bootstrap/cvmfs-enmr-context.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# cvmfs configuration for lhcb repository: +# to be run as root on the VM +# +# + +if [ $# -ne 1 ] +then + echo "cvmfs-enmr-context.sh " +fi + +echo "received cvmfs_http_proxy parameter $1" >> /var/log/cvmfs-context-script.log 2>&1 +rpm --import https://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM >> /var/log/cvmfs-context-script.log 2>&1 +yum -y install https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm >> /var/log/cvmfs-context-script.log 2>&1 +yum -y update >> /var/log/cvmfs-context-script.log 2>&1 +echo "install cvmfs and cvmfs-config-default" >> /var/log/cvmfs-context-script.log 2>&1 +yum -y install cvmfs cvmfs-config-default >> /var/log/cvmfs-context-script.log 2>&1 + + + + cat</etc/cvmfs/default.local +CVMFS_SERVER_URL="http://cvmfs-egi.gridpp.rl.ac.uk:8000/cvmfs/@fqrn@;http://klei.nikhef.nl:8000/cvmfs/@fqrn@;http://cvmfsrepo.lcg.triumf.ca:8000/cvmfs/@fqrn@;http://cvmfsrep.grid.sinica.edu.tw:8000/cvmfs/@fqrn@" +CVMFS_KEYS_DIR=/etc/cvmfs/keys/egi.eu +CVMFS_CACHE_BASE=/home/cache +CVMFS_REPOSITORIES=wenmr.egi.eu +CVMFS_QUOTA_LIMIT=4000 +CVMFS_HTTP_PROXY="$1" +EOF + + +#alternative setup +#CVMFS_SERVER_URL="http://cvmfs-egi.gridpp.rl.ac.uk:8000/cvmfs/@org@.gridpp.ac.uk;http://cvmfs01.nikhef.nl/cvmfs/@org@.gridpp.ac.uk" +#CVMFS_PUBLIC_KEY=/etc/cvmfs/keys/gridpp.ac.uk.pub + +#alternative setup needs this: +# cat</etc/cvmfs/keys/gridpp.ac.uk.pub +#-----BEGIN PUBLIC KEY----- +#MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp7C4KDvOIEVJepuAHjxE +#EES1sDdohz0hiU6uvSqxVYjKVR4Y4/0I/D/zLijQI+MHR7859RN0/6fsZ3b3At3l +#UbvNfqq6DN1zVjjd0xagC6SMBhSfj/iQKQSsG8MXSyiNmM8YalVHJSPqoova6CPE +#EgLEjnHKTNEogTNjKBwbP2ELPLkfVoNoxxrXPSox7aln8JdgyZzZlBwm98gnFa1v +#JTVAl0HQnUJ6cjMwO31wIGVMdvZ+P962t+2bPGfOCm6Ly6BusXcLoIIeez5SBerB +#aHz//NSTZDbHVNPEqpoo1AQVVOo4XJmqo64jBa3G4Dr0zSda1bkZMVhsyUtjhfEB +#DwIDAQAB +#-----END PUBLIC KEY----- +#EOF + + cat</etc/fuse.conf +user_allow_other +EOF + +# reaload configuration to activate new setup: +cvmfs_config reload >> /var/log/cvmfs-context-script.log 2>&1 +cvmfs_config showconfig >> /var/log/cvmfs-context-script.log 2>&1 +service autofs start >> /var/log/cvmfs-context-script.log 2>&1 +chkconfig autofs on >> /var/log/cvmfs-context-script.log 2>&1 +# just for testing: +#cvmfs_config chksetup >> /var/log/cvmfs-context-script.log 2>&1 +mkdir -p /cvmfs/wenmr.egi.eu +mount -t cvmfs wenmr.egi.eu /cvmfs/wenmr.egi.eu >> /var/log/cvmfs-context-script.log 2>&1 +export VO_ENMR_EU_SW_DIR=/cvmfs/wenmr.egi.eu + + +exit 0 diff --git a/WorkloadManagementSystem/private/bootstrap/general-DIRAC-context.sh b/WorkloadManagementSystem/private/bootstrap/general-DIRAC-context.sh index a6561f5e..7ea17964 100644 --- a/WorkloadManagementSystem/private/bootstrap/general-DIRAC-context.sh +++ b/WorkloadManagementSystem/private/bootstrap/general-DIRAC-context.sh @@ -86,16 +86,15 @@ echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 # # Installing DIRAC -# FOR DEBUGGIN PURPOSES installing debuggin github version instead of cvmfs repository released DIRAC: # cd /opt/dirac - wget --no-check-certificate -O dirac-install 'https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O dirac-install 'http://dirac1.grid.cyfronet.pl:8088/repo/integration/DIRAC/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 su dirac -c'python dirac-install -V "VMDIRAC"' >> /var/log/dirac-context-script.log 2>&1 - # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez git folder: + # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez folder: rm -rf VMDIRAC - wget --no-check-certificate -O vmdirac.zip 'https://github.com/vmendez/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O vmdirac.zip 'http://dirac1.grid.cyfronet.pl:8088/repo/vmendez/master.zip' >> /var/log/dirac-context-script.log 2>&1 # checking if unzip installed if [ ! `which unzip` ] then diff --git a/WorkloadManagementSystem/private/bootstrap/genome-DIRAC-context.sh b/WorkloadManagementSystem/private/bootstrap/genome-DIRAC-context.sh index 6ddbf71f..7e0ee0b2 100644 --- a/WorkloadManagementSystem/private/bootstrap/genome-DIRAC-context.sh +++ b/WorkloadManagementSystem/private/bootstrap/genome-DIRAC-context.sh @@ -107,16 +107,14 @@ echo "9 $cloudDriver" >> /var/log/dirac-context-script.log 2>&1 # # Installing DIRAC -# FOR DEBUGGIN PURPOSES installing debuggin github version instead of cvmfs repository released DIRAC: # cd /opt/dirac - wget --no-check-certificate -O dirac-install 'https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 - + wget --no-check-certificate -O dirac-install 'http://dirac1.grid.cyfronet.pl:8088/repo/integration/DIRAC/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 su dirac -c'python dirac-install -V "VMEGI"' >> /var/log/dirac-context-script.log 2>&1 - # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez git folder: + # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez folder: rm -rf VMDIRAC - wget --no-check-certificate -O vmdirac.zip 'https://github.com/vmendez/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O vmdirac.zip 'http://dirac1.grid.cyfronet.pl:8088/repo/vmendez/master.zip' >> /var/log/dirac-context-script.log 2>&1 # checking if unzip installed if [ ! `which unzip` ] then diff --git a/WorkloadManagementSystem/private/bootstrap/genome-mountAWSvol-DIRAC-context.sh b/WorkloadManagementSystem/private/bootstrap/genome-mountAWSvol-DIRAC-context.sh index c289d6e8..ec4022ee 100644 --- a/WorkloadManagementSystem/private/bootstrap/genome-mountAWSvol-DIRAC-context.sh +++ b/WorkloadManagementSystem/private/bootstrap/genome-mountAWSvol-DIRAC-context.sh @@ -101,6 +101,9 @@ chown dirac.dirac /mnt/dirac >> /var/log/dirac-context-script.log 2>&1 # here /opt/dirac and /opt/breakseq ad others are linked to /mnt rm -f /home/ubuntu/genome-opt.tgz >> /var/log/dirac-context-script.log 2>&1 +mkdir /mnt/tmp >> /var/log/dirac-context-script.log 2>&1 +rm -rf /tmp >> /var/log/dirac-context-script.log 2>&1 +ln -s /mnt/tmp /tmp >> /var/log/dirac-context-script.log 2>&1 # servercert/serverkey previouslly to this script copied # @@ -121,16 +124,14 @@ rm -f /home/ubuntu/genome-opt.tgz >> /var/log/dirac-context-script.log 2>&1 # # Installing DIRAC -# FOR DEBUGGIN PURPOSES installing debuggin github version instead of cvmfs repository released DIRAC: # cd /opt/dirac - wget --no-check-certificate -O dirac-install 'https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 - + wget --no-check-certificate -O dirac-install 'http://dirac1.grid.cyfronet.pl:8088/repo/integration/DIRAC/Core/scripts/dirac-install.py' >> /var/log/dirac-context-script.log 2>&1 su dirac -c'python dirac-install -V "VMEGI"' >> /var/log/dirac-context-script.log 2>&1 - # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez git folder: + # FOR DEBUGGIN PURPOSES overwriting with last released in the local vmendez folder: rm -rf VMDIRAC - wget --no-check-certificate -O vmdirac.zip 'https://github.com/vmendez/VMDIRAC/archive/master.zip' >> /var/log/dirac-context-script.log 2>&1 + wget --no-check-certificate -O vmdirac.zip 'http://dirac1.grid.cyfronet.pl:8088/repo/vmendez/master.zip' >> /var/log/dirac-context-script.log 2>&1 # checking if unzip installed if [ ! `which unzip` ] then diff --git a/release.notes b/release.notes index 69b5cbe9..c124b719 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,8 @@ +[rel-v1r5] +NEW: New Tag logic, tested with v6r15p2 +NEW: cvmfs for enrmr VO (non CERN LHC repos) +Minor bugfixes + [rel-v1r4] NEW: A image can have different bootImageName and Flavor depending in the cloud endpoint