Add timeouts

drms/client.py

@@ -2,6 +2,7 @@
 import re
 import time
 import shutil
+import socket
 from pathlib import Path
 from collections import OrderedDict
 from urllib.error import URLError, HTTPError
@@ -409,7 +410,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
         ----------
         timeout : int or None
             Maximum number of seconds until this method times out. If
-            set to None (the default), the status will be updated
+            set to `None` (the default), the status will be updated
             indefinitely until the request succeeded or failed.
         sleep : int or None
             Time in seconds between status updates (defaults to 5
@@ -470,7 +471,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
                 logger.info(f"Request not found on server, {retries_notfound} retries left.")
                 retries_notfound -= 1
 
-    def download(self, directory, *, index=None, fname_from_rec=None):
+    def download(self, directory, *, index=None, fname_from_rec=None, timeout=60):
         """
         Download data files.
 
@@ -506,6 +507,10 @@ def download(self, directory, *, index=None, fname_from_rec=None):
             generated. This also applies to movie files from exports
             with protocols 'mpg' or 'mp4', where the original filename
             is used locally.
+        timeout: float, optional
+            Sets the timeout to "urlopen", this defaults to 60 seconds.
+            This can be overridden if you set the socket timeout using
+            `socket.setdefaulttimeout`.
 
         Returns
         -------
@@ -555,7 +560,8 @@ def download(self, directory, *, index=None, fname_from_rec=None):
             logger.info(f"    record: {di.record}")
             logger.info(f"    filename: {di.filename}")
             try:
-                with urlopen(di.url) as response, open(fpath_tmp, "wb") as out_file:
+                timeout = socket.getdefaulttimeout() or timeout
+                with urlopen(di.url, timeout=timeout) as response, open(fpath_tmp, "wb") as out_file:
                     shutil.copyfileobj(response, out_file)
             except (HTTPError, URLError):
                 fpath_new = None

drms/json.py

@@ -1,5 +1,6 @@
 import sys
 import json as _json
+import socket
 from enum import Enum
 from urllib.parse import urlencode, quote_plus
 from urllib.request import Request, HTTPError, urlopen
@@ -38,12 +39,13 @@ class HttpJsonRequest:
     Use `HttpJsonClient` to create an instance.
     """
 
-    def __init__(self, url, encoding):
+    def __init__(self, url, encoding, timeout=60):
+        timeout = socket.getdefaulttimeout() or timeout
         self._encoding = encoding
         try:
             req = Request(url)
             req.add_header("User-Agent", f"drms/{drms.__version__}, python/{sys.version[:5]}")
-            self._http = urlopen(req)
+            self._http = urlopen(req, timeout=timeout)
         except HTTPError as e:
             e.msg = f"Failed to open URL: {e.url} with {e.code} - {e.msg}"
             raise e

drms/tests/conftest.py

@@ -23,7 +23,7 @@ def __call__(self):
         return self.result
 
 
-def site_reachable(url, timeout=15):
+def site_reachable(url, timeout=60):
     """
     Checks if the given URL is accessible.
     """