Merge pull request #10 from NiftyPET/devel

Author: casperdcl

niftypet/ninst/install_tools.py

@@ -8,17 +8,23 @@
 import os
 import platform
 import re
-import shutil
 import sys
+from functools import wraps
 from os import fspath, path
 from pathlib import Path
-from subprocess import PIPE, CalledProcessError, check_output, run
+from shutil import copyfileobj, rmtree
+from subprocess import CalledProcessError, check_output, run
 from textwrap import dedent
+from urllib import request
+from urllib.parse import urlparse
+from zipfile import ZipFile
+
+from tqdm.auto import tqdm
+from tqdm.utils import CallbackIOWrapper
 
 from . import cudasetup as cs
 
 if os.getenv("DISPLAY", False):
-    from functools import wraps
     from tkinter import Tk
     from tkinter.filedialog import askdirectory as ask
 
@@ -37,21 +43,6 @@ def askdir(title, initialdir):
         return initialdir if res == "" else res
 
 
-def askdirectory(title="Folder", initialdir="~", name=""):
-    """
-    Args:
-      initialdir (str):  default: "~"
-    Returns (str):
-      one of (decreasing Precedence):
-      - `os.getenv(name)`
-      - `input()` or `tkinter.filedialog.askdirectory()`
-      - `initialdir`
-    """
-    initialdir = path.expanduser(initialdir)
-    res = os.getenv(name, None)
-    return askdir(title, initialdir) if res is None else res
-
-
 log = logging.getLogger(__name__)
 
 # NiftyReg
@@ -82,6 +73,7 @@ def askdirectory(title="Folder", initialdir="~", name=""):
 # 'dcm2niix_27-Jun-2018_win.zip'
 # sha1_dcm =  '4b641113273d86ad73123816993092fc643ac62f'
 # dcm_ver = '1.0.20180622'
+http_dcm = {"Windows": http_dcm_win, "Linux": http_dcm_lin, "Darwin": http_dcm_mac}
 
 # source and build folder names
 dirsrc = "_src"
@@ -91,6 +83,7 @@ def askdirectory(title="Folder", initialdir="~", name=""):
 ncpu = multiprocessing.cpu_count()
 
 LOG_FORMAT = "%(levelname)s:%(asctime)s:%(name)s:%(funcName)s\n> %(message)s"
+CHUNK_SIZE = 2 ** 15  # 32 kiB
 
 
 class LogHandler(logging.StreamHandler):
@@ -102,6 +95,73 @@ def __init__(self, *args, **kwargs):
         self.setFormatter(fmt)
 
 
+def askdirectory(title="Folder", initialdir="~", name=""):
+    """
+    Args:
+      initialdir (str):  default: "~"
+    Returns (str):
+      one of (decreasing Precedence):
+      - `os.getenv(name)`
+      - `input()` or `tkinter.filedialog.askdirectory()`
+      - `initialdir`
+    """
+    initialdir = path.expanduser(initialdir)
+    res = os.getenv(name, None)
+    return askdir(title, initialdir) if res is None else res
+
+
+def urlopen_cached(url, outdir, fname=None, mode="rb"):
+    """
+    Download `url` to `outdir/fname`.
+    Cache based on `url` at `outdir/fname`.url
+
+    Args:
+      url (str): source
+      outdir (path-like): destination
+      fname (str): optional, auto-detected from `url` if not given
+      mode (str): for returned file object
+    Returns:
+      file
+    """
+    outdir = Path(outdir).expanduser()
+    outdir.mkdir(exist_ok=True)
+    if fname is None:
+        fname = Path(urlparse(url).path).name
+    fout = outdir / fname
+    cache = outdir / f"{fname}.url"
+    if not fout.is_file() or not cache.is_file() or cache.read_text().strip() != url:
+        req = request.Request(url=url)
+        with request.urlopen(req) as raw:
+            with tqdm.wrapattr(raw, "read", total=getattr(raw, "length", None)) as fd:
+                with fout.open("wb") as fo:
+                    i = fd.read(CHUNK_SIZE)
+                    while i:
+                        fo.write(i)
+                        i = fd.read(CHUNK_SIZE)
+        cache.write_text(url)
+    return fout.open(mode)
+
+
+def extractall(fzip, dest, desc="Extracting"):
+    """zipfile.Zipfile(fzip).extractall(dest) with progress"""
+    dest = Path(dest).expanduser()
+    with ZipFile(fzip) as zipf:
+        with tqdm(
+            desc=desc,
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+            total=sum(getattr(i, "file_size", 0) for i in zipf.infolist()),
+        ) as pbar:
+            for i in zipf.infolist():
+                if not getattr(i, "file_size", 0):  # directory
+                    zipf.extract(i, fspath(dest))
+                else:
+                    with zipf.open(i) as fi:
+                        with open(fspath(dest / i.filename), "wb") as fo:
+                            copyfileobj(CallbackIOWrapper(pbar.update, fi), fo)
+
+
 def query_yesno(question):
     valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
     prompt = " [Y/n]: "
@@ -243,32 +303,29 @@ def check_version(Cnt, chcklst=None):
     # niftyreg reg_resample first
     if "RESPATH" in chcklst and "RESPATH" in Cnt:
         try:
-            p = run([Cnt["RESPATH"], "--version"], stdout=PIPE)
-            out = p.stdout.decode("utf-8")
+            out = check_output([Cnt["RESPATH"], "--version"]).decode("U8")
             if reg_ver in out:
                 output["RESPATH"] = True
-        except OSError:
+        except (CalledProcessError, FileNotFoundError):
             log.error("NiftyReg (reg_resample) either is NOT installed or is corrupt.")
 
     # niftyreg reg_aladin
     if "REGPATH" in chcklst and "REGPATH" in Cnt:
         try:
-            p = run([Cnt["REGPATH"], "--version"], stdout=PIPE)
-            out = p.stdout.decode("utf-8")
+            out = check_output([Cnt["REGPATH"], "--version"]).decode("U8")
             if reg_ver in out:
                 output["REGPATH"] = True
-        except OSError:
+        except (CalledProcessError, FileNotFoundError):
             log.error("NiftyReg (reg_aladin) either is NOT installed or is corrupt.")
 
     # dcm2niix
     if "DCM2NIIX" in chcklst and "DCM2NIIX" in Cnt:
         try:
-            p = run([Cnt["DCM2NIIX"], "-h"], stdout=PIPE)
-            out = p.stdout.decode("utf-8")
+            out = check_output([Cnt["DCM2NIIX"], "-h"]).decode("U8")
             ver_str = re.search(r"(?<=dcm2niiX version v)\d{1,2}.\d{1,2}.\d*", out)
             if ver_str and dcm_ver in ver_str.group(0):
                 output["DCM2NIIX"] = True
-        except OSError:
+        except (CalledProcessError, FileNotFoundError):
             log.error("dcm2niix either is NOT installed or is corrupt.")
 
     # hdw mu-map list
@@ -295,25 +352,13 @@ def download_dcm2niix(Cnt, dest):
     )
 
     # -create the installation folder
-    if not dest.is_dir():
-        dest.mkdir()
+    dest.mkdir(exist_ok=True)
     binpath = dest / "bin"
-    if not binpath.is_dir():
-        binpath.mkdir()
+    binpath.mkdir(exist_ok=True)
 
-    import urllib.error
-    import urllib.parse
-    import urllib.request
-    import zipfile
-
-    http_dcm = {"Windows": http_dcm_win, "Linux": http_dcm_lin, "Darwin": http_dcm_mac}
-    urllib.request.urlretrieve(
-        http_dcm[platform.system()], fspath(dest / "dcm2niix.zip")
-    )
+    with urlopen_cached(http_dcm[platform.system()], dest) as fd:
+        extractall(fd, binpath)
 
-    zipf = zipfile.ZipFile(fspath(dest / "dcm2niix.zip"), "r")
-    zipf.extractall(fspath(binpath))
-    zipf.close()
     Cnt["DCM2NIIX"] = fspath(next(binpath.glob("dcm2niix*")))
     # ensure the permissions are given to the executable
     os.chmod(Cnt["DCM2NIIX"], 755)
@@ -333,14 +378,12 @@ def install_tool(app, Cnt):
     # pick the target installation folder for tools
     if Cnt.get("PATHTOOLS", None):
         path_tools = Path(Cnt["PATHTOOLS"])
-        if not path_tools.is_dir():
-            path_tools.mkdir()
+        path_tools.mkdir(exist_ok=True)
     else:
         path_tools = Path(
             askdirectory(title="Path to place NiftyPET tools", name="PATHTOOLS")
         )
-        if not path_tools.is_dir():
-            path_tools.mkdir()
+        path_tools.mkdir(exist_ok=True)
         if path_tools.name != Cnt["DIRTOOLS"]:
             path_tools /= Cnt["DIRTOOLS"]
         Cnt["PATHTOOLS"] = fspath(path_tools)
@@ -357,8 +400,7 @@ def install_tool(app, Cnt):
         )
 
     # create the main tools folder
-    if not path_tools.is_dir():
-        path_tools.mkdir()
+    path_tools.mkdir(exist_ok=True)
     # identify the specific path for the requested app
     if app == "niftyreg":
         repo = repo_reg
@@ -378,7 +420,7 @@ def install_tool(app, Cnt):
 
     # Check if the source folder exists and delete it, if it does
     if dest.is_dir():
-        shutil.rmtree(fspath(dest))
+        rmtree(fspath(dest))
     dest.mkdir()
     os.chdir(dest)
 
@@ -389,8 +431,7 @@ def install_tool(app, Cnt):
     run(["git", "checkout", sha1])
     os.chdir(cwd)
 
-    if not dirbld.is_dir():
-        os.mkdir(dirbld)
+    dirbld.mkdir(exist_ok=True)
     os.chdir(dirbld)
     # run cmake with arguments
     if platform.system() == "Windows":

niftypet/ninst/raw/resources.py

@@ -1,11 +1,6 @@
 """Resources file for NiftyPET NIPET and NIMPA etc."""
-# ---------------------------------------------------------------
 __author__ = ("Pawel J. Markiewicz", "Casper O. da Costa-Luis")
 __copyright__ = "Copyright 2018-20"
-# ---------------------------------------------------------------
-
-import errno
-import os
 from math import ceil, pi
 
 try:
@@ -437,129 +432,3 @@ def get_mmr_constants():
     Cnt = get_setup(Cnt=Cnt)
 
     return Cnt
-
-
-def get_refimg(pathsrc):
-    """Reference images reconstructed for testing future versions and editions of NiftyPET.
-    It is based on the open source list-mode amyloid PET data available at:
-    https://doi.org/10.5281/zenodo.1472951
-    """
-
-    if (
-        isinstance(pathsrc, dict)
-        and "REFPATH" in pathsrc
-        and os.path.exists(pathsrc["REFPATH"])
-    ):
-        folder_ref = pathsrc["REFPATH"]
-    elif isinstance(pathsrc, str) and os.path.exists(pathsrc):
-        folder_ref = pathsrc
-    else:
-        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), folder_ref)
-
-    # ------------------------------------------------------------------------------------------------
-    # > predetermined structure of the reference folder as a dictionary:
-    refpaths = {
-        "histo": {"p": 1570707830, "d": 817785422},
-        "basic": {},
-        "aligned": {"spm": {}, "niftyreg": {}},
-    }
-
-    refpaths["basic"] = {
-        "pet": os.path.join(
-            folder_ref, "basic", "17598013_t-3000-3600sec_itr-4_suvr.nii.gz"
-        ),
-        "omu": os.path.join(
-            folder_ref, "basic", "mumap-from-DICOM_no-alignment.nii.gz"
-        ),
-        "hmu": os.path.join(folder_ref, "basic", "hardware_umap.nii.gz"),
-    }
-    refpaths["aligned"]["niftyreg"] = {
-        "hmu": os.path.join(
-            folder_ref, "dyn_aligned", "niftyreg", "hardware_umap.nii.gz"
-        ),
-        "omu": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "niftyreg",
-            "mumap-PCT-aligned-to_t0-3600_AC.nii.gz",
-        ),
-        "pos": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "niftyreg",
-            "17598013_t0-3600sec_itr2_AC-UTE.nii.gz",
-        ),
-        "pet": os.path.join(
-            folder_ref, "dyn_aligned", "niftyreg", "17598013_nfrm-2_itr-4.nii.gz"
-        ),
-        "trm": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "niftyreg",
-            "17598013_nfrm-2_itr-4_trimmed-upsampled-scale-2.nii.gz",
-        ),
-        "pvc": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "niftyreg",
-            "17598013_nfrm-2_itr-4_trimmed-upsampled-scale-2_PVC.nii.gz",
-        ),
-    }
-    refpaths["aligned"]["spm"] = {
-        "hmu": os.path.join(folder_ref, "dyn_aligned", "spm", "hardware_umap.nii.gz"),
-        "omu": os.path.join(
-            folder_ref, "dyn_aligned", "spm", "mumap-PCT-aligned-to_t0-3600_AC.nii.gz"
-        ),
-        "pos": os.path.join(
-            folder_ref, "dyn_aligned", "spm", "17598013_t0-3600sec_itr2_AC-UTE.nii.gz"
-        ),
-        "pet": os.path.join(
-            folder_ref, "dyn_aligned", "spm", "17598013_nfrm-2_itr-4.nii.gz"
-        ),
-        "trm": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "spm",
-            "17598013_nfrm-2_itr-4_trimmed-upsampled-scale-2.nii.gz",
-        ),
-        "pvc": os.path.join(
-            folder_ref,
-            "dyn_aligned",
-            "spm",
-            "17598013_nfrm-2_itr-4_trimmed-upsampled-scale-2_PVC.nii.gz",
-        ),
-    }
-    # ------------------------------------------------------------------------------------------------
-
-    testext = {"basic": {}, "aligned": {}}
-    testext["basic"]["pet"] = "static reconstruction with unaligned UTE mu-map"
-    testext["basic"]["hmu"] = "hardware mu-map for the static unaligned reconstruction"
-    testext["basic"]["omu"] = "object mu-map for the static unaligned reconstruction"
-    # ---
-    testext["aligned"]["pet"] = "2-frame scan with aligned UTE mu-map"
-    testext["aligned"]["hmu"] = "hardware mu-map for the 2-frame aligned reconstruction"
-    testext["aligned"]["omu"] = "object mu-map for the 2-frame aligned reconstruction"
-    testext["aligned"][
-        "pos"
-    ] = "AC reconstruction for positioning (full acquisition used)"
-    testext["aligned"]["trm"] = "trimming post reconstruction"
-    testext["aligned"]["pvc"] = "PVC post reconstruction"
-    # ------------------------------------------------------------------------------------------------
-
-    # > check if all files are in
-    # > basic
-    frefs = refpaths["basic"]
-    for k in frefs:
-        if not os.path.isfile(frefs[k]):
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), frefs[k])
-
-    # > reg tools: niftyreg and spm
-    frefs = refpaths["aligned"]
-    for r in frefs:
-        for k in frefs[r]:
-            if not os.path.isfile(frefs[r][k]):
-                raise FileNotFoundError(
-                    errno.ENOENT, os.strerror(errno.ENOENT), frefs[r][k]
-                )
-
-    return refpaths, testext