Skip to content
This repository has been archived by the owner on Jul 21, 2024. It is now read-only.

BN-83 GPU support integration #118

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

BN-83 GPU support integration #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
695be04
cuda gpu spike using custom images
slatunje Oct 22, 2020
902f3f4
remove unused code
slatunje Oct 26, 2020
b2b8bf5
use default image as custom for now
slatunje Oct 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 51 additions & 6 deletions BlendNet/providers/gcp/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,17 +428,52 @@ def createInstanceAgent(cfg):
# TODO: option to specify prefix/suffix for the name
machine = 'zones/%s/machineTypes/%s' % (configs['zone'], cfg['instance_type'])
# TODO: add option to specify the image to use
#image_res = compute.images().getFromFamily(project='ubuntu-os-cloud', family='ubuntu-minimal-1804-lts').execute()
image_res = compute.images().getFromFamily(project='debian-cloud', family='debian-10').execute()
# image_res = compute.images().getFromFamily(project='ubuntu-os-cloud', family='ubuntu-minimal-1804-lts').execute()
# image_res = compute.images().getFromFamily(project='debian-cloud', family='debian-10').execute()

# image_custom = 'ubuntu-os-cuda' # TODO should be from ui
image_custom = 'ubuntu-os-cloud' # TODO temporary workaround but there need to be some logic to select image
image_res = compute.images().get(project=configs['project'], image=image_custom).execute()
# image_res = compute.images().getFromFamily(project='ubuntu-os-cloud', family='ubuntu-2004-lts').execute()

accelerator_type = 'nvidia-tesla-p4' # TODO: should be define from ui.
gpu_acc_type = compute.acceleratorTypes().get(project=configs['project'], zone=configs['zone'], acceleratorType=accelerator_type).execute()
gpu_acc_count = 1

# TODO: make script overridable
# TODO: too much hardcode here
startup_script = '''#!/bin/sh
echo '--> GPU CUDA ====================================================================='

if [ ! -x /usr/bin/nvidia-smi ]; then
echo '--> Installing CUDA for GPU dependencies'

curl -O https://developer.download.nvidia.com/compute/cuda/repos/{cuda_ubuntu_version}/x86_64/cuda-{cuda_ubuntu_version}.pin
mv cuda-{cuda_ubuntu_version}.pin /etc/apt/preferences.d/cuda-repository-pin-600
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/{cuda_ubuntu_version}/x86_64/{cuda_pub_key}.pub
add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/{cuda_ubuntu_version}/x86_64/ /"

apt-get -qq update
apt-get -yq install cuda

nvidia-smi

echo '--> CUDA Driver Installed -- Shutting Down Now!'

shutdown -r now
else
echo '--> CUDA Driver OK'

nvidia-smi
fi

echo '--> BLENDER ======================================================================'

echo '--> Check for blender dependencies'
dpkg -l libxrender1 libxi6 libgl1
if [ $? -gt 0 ]; then
apt update
apt install --no-install-recommends -y libxrender1 libxi6 libgl1
apt -qq update
apt install --no-install-recommends -yq libxrender1 libxi6 libgl1
fi

if [ ! -x /srv/blender/blender ]; then
Expand All @@ -463,7 +498,7 @@ def createInstanceAgent(cfg):
User=blendnet-user
WorkingDirectory=~
Type=simple
ExecStart=/srv/blender/blender -b -noaudio -P /srv/blendnet/agent.py
ExecStart=/srv/blender/blender -b -noaudio -P /srv/blendnet/agent.py --debug-gpu --debug-memory --debug-cycles --debug-gpumem
Restart=always
TimeoutStopSec=20
StandardOutput=syslog
Expand All @@ -475,12 +510,16 @@ def createInstanceAgent(cfg):

systemctl daemon-reload
systemctl start blendnet-agent.service # We don't need "enable" here

echo '--> GPU BLENDNET COMPLETE ========================================================'
'''.format(
blender_url=cfg['dist_url'],
blender_sha256=cfg['dist_checksum'],
project=configs['project'],
session_id=cfg['session_id'],
name=cfg['instance_name'],
cuda_ubuntu_version='ubuntu2004',
cuda_pub_key='7fa2af80',
)
#su -l -s /bin/sh -c '/srv/blender/blender -b -noaudio -P /srv/blendnet/agent.py' blendnet-user

Expand All @@ -491,6 +530,8 @@ def createInstanceAgent(cfg):
'description': 'BlendNet Agent worker',
'scheduling': {
'preemptible': cfg['use_cheap_instance'],
'onHostMaintenance': 'TERMINATE',
'automaticRestart': True,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually Manager should decide what to do with the agent instance.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes were simply added to allow me to get gpu working.
Feel free to move it to where is fits best as I do not know the code base very well

},
'labels': {
'app': 'blendnet',
Expand All @@ -508,6 +549,10 @@ def createInstanceAgent(cfg):
'diskSizeGb': '200',
},
}],
'guestAccelerators': [{
'acceleratorCount': gpu_acc_count,
'acceleratorType': gpu_acc_type['selfLink'],
}],
'networkInterfaces': [{
'network': 'global/networks/default', # TODO: a way to specify network to use
'accessConfigs': [{
Expand All @@ -522,8 +567,8 @@ def createInstanceAgent(cfg):
'https://www.googleapis.com/auth/devstorage.read_only',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/trace.append',
],
}],
Expand Down
18 changes: 17 additions & 1 deletion BlendNet/script-render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,23 @@ def eprint(*args, **kwargs):
# Switch to use maximum threads possible on the worker
scene.render.threads_mode = 'AUTO'

scene.cycles.device = 'CPU' # The only one supported right now
# scene.cycles.device = 'CPU' # The only one supported right now

eprint('--------------- CUDA, DEVICES & GPU ------------------')

eprint('INFO: Enable CUDA to nvidia driver')
bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA'

eprint('INFO: Enable and list all devices, or optionally disable CPU')
for devices in bpy.context.preferences.addons['cycles'].preferences.get_devices():
for device in devices:
device.use = True
if device.type == 'CPU':
device.use = False
eprint("Device '{}' type {} : {}" . format(device.name, device.type, device.use))

eprint('INFO: Set Scene Devices as GPU for Cycles')
scene.cycles.device = 'GPU'

# Disabling square samples - script is getting the real number of samples to render
scene.cycles.use_square_samples = False
Expand Down