CUDA out of memory with pieapp on NVIDIA GeForce RTX 5090 #263
Description
CUDA out of memory with pieapp on NVIDIA GeForce RTX 5090 Command-line info below:
root@bf9e3a8b80bd:/# nvidia-smi
Tue Jun 3 12:10:13 2025
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 575.51.03 Driver Version: 575.51.03 CUDA Version: 12.9 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA GeForce RTX 5090 On | 00000000:02:00.0 Off | N/A |
| 0% 23C P8 7W / 575W | 2MiB / 32607MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| No running processes found |
+-----------------------------------------------------------------------------------------+
root@bf9e3a8b80bd:/# cd workspace/
root@bf9e3a8b80bd:/workspace# source pytorch_env/bin/activate
(pytorch_env) root@bf9e3a8b80bd:/workspace# cd test/png/
(pytorch_env) root@bf9e3a8b80bd:/workspace/test/png# python
Python 3.12.3 (main, Feb 4 2025, 14:48:35) [GCC 13.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
import torch
print(torch.version)
2.8.0.dev20250602+cu128
print(torch.cuda.is_available())
True
print(torch.cuda.get_device_name(0))
NVIDIA GeForce RTX 5090
import pyiqa
print(pyiqa.list_models())
['ahiq', 'arniqa', 'arniqa-clive', 'arniqa-csiq', 'arniqa-flive', 'arniqa-kadid', 'arniqa-live', 'arniqa-spaq', 'arniqa-tid', 'brisque', 'brisque_matlab', 'ckdn', 'clipiqa', 'clipiqa+', 'clipiqa+_rn50_512', 'clipiqa+_vitL14_512', 'clipscore', 'cnniqa', 'compare2score', 'cw_ssim', 'dbcnn', 'deepdc', 'dists', 'entropy', 'fid', 'fid_dinov2', 'fsim', 'gmsd', 'hyperiqa', 'ilniqe', 'inception_score', 'laion_aes', 'liqe', 'liqe_mix', 'lpips', 'lpips+', 'lpips-vgg', 'lpips-vgg+', 'mad', 'maniqa', 'maniqa-kadid', 'maniqa-pipal', 'ms_ssim', 'msswd', 'musiq', 'musiq-ava', 'musiq-paq2piq', 'musiq-spaq', 'nima', 'nima-koniq', 'nima-spaq', 'nima-vgg16-ava', 'niqe', 'niqe_matlab', 'nlpd', 'nrqm', 'paq2piq', 'pi', 'pieapp', 'piqe', 'psnr', 'psnry', 'qalign', 'qalign_4bit', 'qalign_8bit', 'qualiclip', 'qualiclip+', 'qualiclip+-clive', 'qualiclip+-flive', 'qualiclip+-spaq', 'ssim', 'ssimc', 'stlpips', 'stlpips-vgg', 'topiq_fr', 'topiq_fr-pipal', 'topiq_iaa', 'topiq_iaa_res50', 'topiq_nr', 'topiq_nr-face', 'topiq_nr-face-v1', 'topiq_nr-flive', 'topiq_nr-spaq', 'topiq_nr_swin-face', 'tres', 'tres-flive', 'unique', 'uranker', 'vif', 'vsi', 'wadiqam_fr', 'wadiqam_nr']
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
device
device(type='cuda')
iqa_metric = pyiqa.create_metric('pieapp', device=device)
Loading pretrained model PieAPP from /root/.cache/torch/hub/pyiqa/PieAPPv0.1-0937b014.pth
dist = "ReadySetGo_video2x_3840x2160_30fps_420_10bit_YUV_5fps_1.png"
ref = "ReadySetGo_3840x2160_30fps_420_10bit_YUV_5fps_1.png"
print(iqa_metric.lower_better)
True
score = iqa_metric(dist, ref)
Traceback (most recent call last):
File "", line 1, in
File "/workspace/pytorch_env/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1767, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/pytorch_env/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/software/IQA-PyTorch/pyiqa/models/inference_model.py", line 115, in forward
output = self.net(target.to(device), ref.to(device), **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/pytorch_env/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1767, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/pytorch_env/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/software/IQA-PyTorch/pyiqa/archs/pieapp_arch.py", line 151, in forward
ref_multi_scale, ref_coarse = self.compute_features(image_ref_patches)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/software/IQA-PyTorch/pyiqa/archs/pieapp_arch.py", line 97, in compute_features
self.conv3(self.pool2(F.relu(self.conv2(F.relu(self.conv1(input))))))
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/pytorch_env/lib/python3.12/site-packages/torch/nn/functional.py", line 1713, in relu
result = torch.relu(input)
^^^^^^^^^^^^^^^^^
torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 10.66 GiB. GPU 0 has a total capacity of 31.36 GiB of which 7.10 GiB is free. Including non-PyTorch memory, this process has 24.25 GiB memory in use. Of the allocated memory 17.29 GiB is allocated by PyTorch, and 6.37 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)