Merge branch 'main' into refactor_atn

ioos · Jan 29, 2025 · 18b7af6 · 18b7af6
2 parents 5bad5e3 + a6aa9a3
commit 18b7af6
Show file tree

Hide file tree

Showing 39 changed files with 167 additions and 7,048 deletions.
diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml
@@ -7,7 +7,6 @@ on:
        - main
      paths:
        - '.github/workflows/metrics.yml'
-       - 'gts_regional_metrics.py'
        - 'btn_metrics.py'
   schedule:
     - cron: "0 12 5 * *"
@@ -42,7 +41,6 @@ jobs:
         shell: bash -l {0}
         run: >
           python btn_metrics.py
-          && python gts_regional_metrics.py
 
       - name: Get current date
         run: echo "NOW=$(date -u)" >> ${GITHUB_ENV}

diff --git a/.github/workflows/website_create_and_deploy.yml b/.github/workflows/website_create_and_deploy.yml
@@ -27,6 +27,9 @@ jobs:
         pip install plotly
         pip install geopandas
         pip install folium
+        pip install datetime
+        pip install fiscalyear
+        pip install erddapy
     - name: create HTML file
       run: |
         cd website

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,7 +11,7 @@ exclude: |
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
     - id: trailing-whitespace
     - id: check-ast
@@ -20,7 +20,7 @@ repos:
     - id: check-added-large-files
 
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.3.0
+  rev: v2.4.0
   hooks:
     - id: codespell
       exclude: >
@@ -36,14 +36,14 @@ repos:
     - id: add-trailing-comma
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.9.3
   hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
 - repo: https://github.com/nbQA-dev/nbQA
-  rev: 1.8.7
+  rev: 1.9.1
   hooks:
     - id: nbqa-check-ast
     - id: nbqa-ruff

diff --git a/environment.yml b/environment.yml
@@ -20,6 +20,7 @@ dependencies:
   - pytest
   - requests
   - suds
+  - tenacity
   - pip
   - pip:
     - git+https://github.com/ioos/ioos_metrics.git
diff --git a/gts/GTS_regional_totals_FY2018_Q2.csv b/gts/GTS_regional_totals_FY2018_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2018_Q3.csv b/gts/GTS_regional_totals_FY2018_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2018_Q4.csv b/gts/GTS_regional_totals_FY2018_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2019_Q1.csv b/gts/GTS_regional_totals_FY2019_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2019_Q2.csv b/gts/GTS_regional_totals_FY2019_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2019_Q3.csv b/gts/GTS_regional_totals_FY2019_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2019_Q4.csv b/gts/GTS_regional_totals_FY2019_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2020_Q1.csv b/gts/GTS_regional_totals_FY2020_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2020_Q2.csv b/gts/GTS_regional_totals_FY2020_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2020_Q3.csv b/gts/GTS_regional_totals_FY2020_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2020_Q4.csv b/gts/GTS_regional_totals_FY2020_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2021_Q1.csv b/gts/GTS_regional_totals_FY2021_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2021_Q2.csv b/gts/GTS_regional_totals_FY2021_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2021_Q3.csv b/gts/GTS_regional_totals_FY2021_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2021_Q4.csv b/gts/GTS_regional_totals_FY2021_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2022_Q1.csv b/gts/GTS_regional_totals_FY2022_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2022_Q2.csv b/gts/GTS_regional_totals_FY2022_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2022_Q3.csv b/gts/GTS_regional_totals_FY2022_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2022_Q4.csv b/gts/GTS_regional_totals_FY2022_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2023_Q1.csv b/gts/GTS_regional_totals_FY2023_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2023_Q2.csv b/gts/GTS_regional_totals_FY2023_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2023_Q3.csv b/gts/GTS_regional_totals_FY2023_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2023_Q4.csv b/gts/GTS_regional_totals_FY2023_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2024_Q1.csv b/gts/GTS_regional_totals_FY2024_Q1.csv
diff --git a/gts/GTS_regional_totals_FY2024_Q2.csv b/gts/GTS_regional_totals_FY2024_Q2.csv
diff --git a/gts/GTS_regional_totals_FY2024_Q3.csv b/gts/GTS_regional_totals_FY2024_Q3.csv
diff --git a/gts/GTS_regional_totals_FY2024_Q4.csv b/gts/GTS_regional_totals_FY2024_Q4.csv
diff --git a/gts/GTS_regional_totals_FY2025_Q1.csv b/gts/GTS_regional_totals_FY2025_Q1.csv
diff --git a/gts_regional_metrics.py b/gts_regional_metrics.py
diff --git a/ioos_btn_metrics.csv b/ioos_btn_metrics.csv
@@ -5,3 +5,4 @@ date_UTC,Federal Partners,Regional Associations,HF Radar Stations,NGDAC Glider D
 2022-10-05,17.0,11.0,165.0,59088.0,390.0,635.0,4444.0,6.0,8.0,9.0,13.0,34.0,24499.0,1.0,5.0
 2023-01-05,17.0,11.0,165.0,62042.0,768.0,635.0,4444.0,6.0,8.0,9.0,13.0,34.0,11840.0,1.0,5.0
 2024-01-26,17.0,11.0,165.0,61990.0,721.0,886.0,5190.0,5.0,8.0,11.0,13.0,34.0,42599.0,1.0,5.0
+2025-01-21,17.0,11.0,165.0,74178.0,731.0,1081.0,6134.0,10.0,13.0,11.0,13.0,34.0,37475.0,1.0,5.0
diff --git a/ioos_metrics/ioos_metrics.py b/ioos_metrics/ioos_metrics.py
@@ -1,5 +1,6 @@
 """Code extracted from IOOS_BTN.ipynb."""
 
+import datetime
 import functools
 import io
 import logging
@@ -12,6 +13,7 @@
 from fake_useragent import UserAgent
 from gliderpy.fetchers import GliderDataFetcher
 from shapely.geometry import LineString, Point
+from tenacity import retry, stop_after_attempt, wait_fixed
 
 from ioos_metrics.national_platforms import national_platforms
 
@@ -94,7 +96,7 @@ def federal_partners():
 
 
 @functools.lru_cache(maxsize=128)
-def ngdac_gliders_fast(min_time="2000-01-01T00:00:00Z", max_time="2023-12-31T23:59:59Z") -> int:
+def ngdac_gliders_fast(min_time="2000-01-01T00:00:00Z", max_time=None) -> int:
     """NGDAC Glider Days.
 
     This version uses the AllDatasets entry to compute the glider days.
@@ -121,6 +123,9 @@ def ngdac_gliders_fast(min_time="2000-01-01T00:00:00Z", max_time="2023-12-31T23:
       Note that data with NaN can be real glider day with lost data. Which is OK for this metric.
 
     """
+    if max_time is None:
+        max_time = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
     df = pd.read_csv(
         "https://gliders.ioos.us/erddap/tabledap/allDatasets.csvp?minTime,maxTime,datasetID",
     )
@@ -223,6 +228,7 @@ def _make_track_geom(df) -> "pd.DataFrame":
             )
         )
 
+    @retry(stop=stop_after_attempt(3), wait=wait_fixed(5))
     def _computed_metadata(dataset_id) -> dict:
         """Download the minimum amount of data possible for the computed
         metadata.
@@ -269,15 +275,12 @@ def _computed_metadata(dataset_id) -> dict:
     for _, row in list(df.iterrows()):
         dataset_id = row["Dataset ID"]
         info_url = row["info_url"].replace("html", "csv")
-        info_df = pd.read_csv(info_url)
-        info = _metadata(info_df)
         try:
+            info_df = pd.read_csv(info_url)
+            info = _metadata(info_df)
             info.update(_computed_metadata(dataset_id=dataset_id))
-        except (httpx.HTTPError, httpx.HTTPStatusError):
-            print(  # noqa: T201
-                f"Could not fetch glider {dataset_id=}. "
-                "This could be a server side error and the metrics will be incomplete!",
-            )
+        except (httpx.HTTPError, httpx.HTTPStatusError, ValueError) as e:
+            print(f"Could not fetch glider {dataset_id=}.\n{e=}")  # noqa: T201
             continue
         metadata.update({dataset_id: info})
     return pd.DataFrame(metadata).T