diff --git a/cluster/deploy.sh b/cluster/deploy.sh
index 0cb59eb99595..e166c11e2bba 100755
--- a/cluster/deploy.sh
+++ b/cluster/deploy.sh
@@ -50,17 +50,14 @@ EOF
 
 echo "Cleaning up ..."
 # Work around https://github.com/kubernetes/kubernetes/issues/33517
-cluster/kubectl.sh --core delete -f manifests/virt-handler.yaml --cascade=false --grace-period 0 2>/dev/null || :
-cluster/kubectl.sh --core delete pods -l=daemon=virt-handler --force --grace-period 0 2>/dev/null || :
+$KUBECTL delete -f manifests/virt-handler.yaml --cascade=false --grace-period 0 2>/dev/null || :
+$KUBECTL delete pods -n kube-system -l=daemon=virt-handler --force --grace-period 0 2>/dev/null || :
 
-cluster/kubectl.sh --core delete -f manifests/libvirt.yaml --cascade=false --grace-period 0 2>/dev/null || :
-cluster/kubectl.sh --core delete pods -l=daemon=libvirt --force --grace-period 0 2>/dev/null || :
-
-# Make sure TPRs are deleted, we use CRDs now
-cluster/kubectl.sh --core delete thirdpartyresources --all  || :
+$KUBECTL delete -f manifests/libvirt.yaml --cascade=false --grace-period 0 2>/dev/null || :
+$KUBECTL delete pods -n kube-system -l=daemon=libvirt --force --grace-period 0 2>/dev/null || :
 
 # Make sure that the vms CRD is deleted, we use virtualmachines now
-cluster/kubectl.sh --core delete customresourcedefinitions vms.kubevirt.io  || :
+$KUBECTL delete customresourcedefinitions vms.kubevirt.io  || :
 
 # Remove all external facing services
 externalServiceManifests | cluster/kubectl.sh --core delete -f - || :