Use weakref for automatic deletion of tmpfiles (to fix pyscf#2318) (pyscf#2320)

* Use weakref for automatic deletion of tmpfiles

* github actions tests check number of leftover temporary files

* temporary file check pytest exit status

* Avoid h5f.flush() on closed file

---------

Co-authored-by: Qiming Sun <[email protected]>

Author: chillenb

.github/workflows/run_tests.sh

@@ -3,9 +3,11 @@ export OMP_NUM_THREADS=1
 export PYTHONPATH=$(pwd):$PYTHONPATH 
 ulimit -s 20000
 
+mkdir -p pyscftmpdir
 echo 'pbc_tools_pbc_fft_engine = "NUMPY"' > .pyscf_conf.py
 echo "dftd3_DFTD3PATH = './pyscf/lib/deps/lib'" >> .pyscf_conf.py
 echo "scf_hf_SCF_mute_chkfile = True" >> .pyscf_conf.py
+echo 'TMPDIR = "./pyscftmpdir"' >> .pyscf_conf.py
 
 version=$(python -c 'import sys; print("{0}.{1}".format(*sys.version_info[:2]))')
 # pytest-cov on Python 3.12 consumes huge memory
@@ -15,3 +17,14 @@ if [ "$RUNNER_OS" == "Linux" ] && [ $version != "3.12" ]; then
 else
   pytest pyscf/ -s -c pytest.ini pyscf
 fi
+
+pytest_status=$?
+
+num_tmpfiles="$(ls -1 pyscftmpdir | wc -l)"
+echo "There are "$num_tmpfiles" leftover temporary files"
+rm -rf pyscftmpdir
+
+# Test fails if pytest failed or if temporary files were left over.
+if test "$num_tmpfiles" -gt 0 || test "$pytest_status" -ne 0; then
+  exit 1
+fi

pyscf/lib/misc.py

@@ -31,6 +31,7 @@
 import itertools
 import inspect
 import collections
+import weakref
 import ctypes
 import numpy
 import scipy
@@ -1144,7 +1145,7 @@ def _finished(self):
         causes outcore DF to hang on an NFS filesystem.
         '''
         try:
-            if super().id:
+            if super().id and super().id.valid:
                 super().flush()
             super().close()
         except AttributeError:  # close not defined in old h5py
@@ -1181,6 +1182,14 @@ def __init__(self, filename=None, mode='a', prefix='', suffix='',
         if filename is None:
             filename = H5TmpFile._gen_unique_name(dir, pre=prefix, suf=suffix)
             self.delete_on_close = True
+
+        def _delete_with_check(fname, should_delete):
+            if should_delete and os.path.exists(fname):
+                os.remove(fname)
+
+        self._finalizer = weakref.finalize(self, _delete_with_check,
+                                           filename, self.delete_on_close)
+
         super().__init__(filename, mode, *args, **kwargs)
 
     # Python 3 stdlib does not have a way to just generate
@@ -1201,11 +1210,9 @@ def _gen_unique_name(directory, pre='', suf=''):
         raise FileExistsError("No usable temporary file name found")
 
     def close(self):
-        filename = self.filename
         self._finished()
-        if self.delete_on_close:
-            if os.path.exists(filename):
-                os.remove(filename)
+        self._finalizer()
+
     def __exit__(self, type, value, traceback):
         self.close()