Robust glider search (#104)

* update pre-commits

* use py313

* add tenacity for retrying

* retry glider data fecthing 3x, while waiting 5 s.

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ exclude: |
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
     - id: trailing-whitespace
     - id: check-ast
@@ -20,7 +20,7 @@ repos:
     - id: check-added-large-files
 
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.3.0
+  rev: v2.4.0
   hooks:
     - id: codespell
       exclude: >
@@ -36,14 +36,14 @@ repos:
     - id: add-trailing-comma
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.9.3
   hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
 - repo: https://github.com/nbQA-dev/nbQA
-  rev: 1.8.7
+  rev: 1.9.1
   hooks:
     - id: nbqa-check-ast
     - id: nbqa-ruff

environment.yml

@@ -20,6 +20,7 @@ dependencies:
   - pytest
   - requests
   - suds
+  - tenacity
   - pip
   - pip:
     - git+https://github.com/ioos/ioos_metrics.git

ioos_metrics/ioos_metrics.py

@@ -12,6 +12,7 @@
 from fake_useragent import UserAgent
 from gliderpy.fetchers import GliderDataFetcher
 from shapely.geometry import LineString, Point
+from tenacity import retry, stop_after_attempt, wait_fixed
 
 from ioos_metrics.national_platforms import national_platforms
 
@@ -223,6 +224,7 @@ def _make_track_geom(df) -> "pd.DataFrame":
             )
         )
 
+    @retry(stop=stop_after_attempt(3), wait=wait_fixed(5))
     def _computed_metadata(dataset_id) -> dict:
         """Download the minimum amount of data possible for the computed
         metadata.
@@ -269,15 +271,12 @@ def _computed_metadata(dataset_id) -> dict:
     for _, row in list(df.iterrows()):
         dataset_id = row["Dataset ID"]
         info_url = row["info_url"].replace("html", "csv")
-        info_df = pd.read_csv(info_url)
-        info = _metadata(info_df)
         try:
+            info_df = pd.read_csv(info_url)
+            info = _metadata(info_df)
             info.update(_computed_metadata(dataset_id=dataset_id))
-        except (httpx.HTTPError, httpx.HTTPStatusError):
-            print(  # noqa: T201
-                f"Could not fetch glider {dataset_id=}. "
-                "This could be a server side error and the metrics will be incomplete!",
-            )
+        except (httpx.HTTPError, httpx.HTTPStatusError, ValueError) as e:
+            print(f"Could not fetch glider {dataset_id=}.\n{e=}")  # noqa: T201
             continue
         metadata.update({dataset_id: info})
     return pd.DataFrame(metadata).T

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "pyarrow",
   "pyobis",
   "requests",
+  "tenacity",
 ]
 [project.urls]
 documentation = "https://ioos.github.io/ioos_metrics"