Merge pull request #55 from alan-turing-institute/update-pydantic

Update pydantic and clarify date logic

Author: Iain-S

controller_function/poetry.lock

@@ -14,17 +14,17 @@ azure-identity = "^1.16.0"
 azure-mgmt-resource = "^23.1.1"
 azure-mgmt-subscription = "^3.1.1"
 opencensus-ext-azure = "^1.1.13"
-pydantic = "^2.7.1"
-pydantic-settings = "^2.2.1"
+pydantic = "^2.11.5"
+pydantic-settings = "^2.9.1"
 PyJWT = {extras = ["crypto"], version = "^2.8.0"}
 requests = "^2.31.0"
-rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.1.0" }
+rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.2.0" }
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.5.1"
 flake8 = "^3.9.2"
 isort = "^5.13.2"
-mypy = "^1.10.0"
+mypy = "^1.15.0"
 pylint = "^3.2.0"
 pylint-absolute-imports = "^1.1.0"
 types-requests = "^2.31.0.20240406"  # Should match requests version

controller_function/pyproject.toml

@@ -15,11 +15,11 @@ azure-mgmt-authorization = "^4.0.0"
 azure-mgmt-subscription = "^3.1.1"
 msgraph-sdk = "^1.28.0"
 opencensus-ext-azure = "^1.1.14"
-pydantic = "^2.11.3"
-pydantic-settings = "^2.8.1"
+pydantic = "^2.11.5"
+pydantic-settings = "^2.9.1"
 PyJWT = {extras = ["crypto"], version = "^2.10.1"}
 requests = "^2.32.3"
-rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.1.0" }
+rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.2.0" }
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.8.0"

status_function/poetry.lock

@@ -98,7 +98,13 @@ def main(mytimer: func.TimerRequest) -> None:
                 usage_item.monthly_upload = today
 
             logger.warning("Sending usage for %s", dates)
-            send_usage(config.API_URL, usage_items, monthly_usage_upload=True)
+            send_usage(
+                config.API_URL,
+                usage_items,
+                date_from,
+                date_to,
+                monthly_usage_upload=True,
+            )
 
             logger.warning("Monthly usage function finished.")
             return

status_function/pyproject.toml

@@ -19,19 +19,18 @@ azure-mgmt-costmanagement = "^4.0.1"
 azure-mgmt-managementgroups = "^1.0.0"
 azure-mgmt-subscription = "^3.1.1"
 opencensus-ext-azure = "^1.1.13"
-pydantic = "^2.7.1"
-pydantic-settings = "^2.2.1"
+pydantic = "^2.11.5"
+pydantic-settings = "^2.9.1"
 PyJWT = {extras = ["crypto"], version = "^2.8.0"}
 requests = "^2.31.0"
-rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.1.0" }
+rctab_models = { git = "https://github.com/alan-turing-institute/rctab-models", tag = "0.2.0" }
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.5.1"
-flake8 = "^3.9.2"
 isort = "^5.13.2"
-mypy = "^1.10.0"
+mypy = "^1.15.0"
 pre-commit = "^3.7.1"
-pylint = "^3.2.0"
+pylint = "^3.3.7"
 pylint-absolute-imports = "^1.1.0"
 types-requests = "^2.31.0.20240406"  # Should match requests version
 

usage_function/monthly_usage/__init__.py

@@ -9,19 +9,11 @@ status=0
 # shellcheck disable=SC1091
 source .venv/bin/activate
 
-# Check Python coding style with flake8, using black's default max line length
-# To auto-format a file, you can run `python -m black filename.py`
-echo "Running flake8..."
-python -m flake8 --max-line-length=88 --exclude=.venv,venv
-status=$((status+$?))
-
-
 # Run here rather than as a pre-commit hook so that local imports happen last
 echo "Running isort..."
 isort . --profile=black
 status=$((status+$?))
 
-
 # Find all .py files (ignoring .venv or venv directories) and check their
 # code style with pylint, using (something close to) Google's default config
 echo "Running pylint..."

usage_function/poetry.lock

@@ -35,7 +35,7 @@ def test_main(self) -> None:
             mock_get_all_usage.return_value = ["usage1", "usage2"]
 
             with patch("usage.datetime") as mock_datetime:
-                now = datetime.now()
+                now = datetime(year=2022, month=7, day=10)
                 mock_datetime.now.return_value = now
 
                 with patch(
@@ -61,9 +61,11 @@ def test_main(self) -> None:
                             mock_timer.past_due = True
 
                             usage.main(mock_timer)
+                            start_datetime = datetime(2022, 7, 8)
+                            end_datetime = datetime(2022, 7, 9)
 
                             mock_date_range.assert_called_once_with(
-                                now - timedelta(days=2), now - timedelta(days=1)
+                                start_datetime, end_datetime
                             )
 
                         mock_get_all_usage.assert_has_calls(
@@ -87,6 +89,8 @@ def test_main(self) -> None:
                                 call(
                                     HTTP_ADAPTER.validate_python("https://my.host"),
                                     ["usage1", "usage2"],
+                                    start_datetime.date(),
+                                    end_datetime.date(),
                                 ),
                             ]
                         )

usage_function/pyproject.toml

@@ -135,23 +135,32 @@ def test_retrieve_and_send_usage(self) -> None:
         )
 
         with patch("utils.usage.BearerAuth") as mock_auth:
+            # Patch POST so that it returns an error (300 status code).
             with patch("requests.post") as mock_post:
                 mock_response = MagicMock()
                 mock_response.status_code = 300
                 mock_response.text = "some-mock-text"
                 mock_post.return_value = mock_response
 
+                sept_2021 = date(2021, 9, 1)
+
                 with patch("utils.usage.logging.warning") as mock_log:
                     with self.assertRaises(RuntimeError):
                         utils.usage.retrieve_and_send_usage(
                             HTTP_ADAPTER.validate_python("https://123.123.123.123"),
                             [example_usage_detail],  # type: ignore
+                            sept_2021,
+                            sept_2021,
                         )
 
                     usage = models.Usage(**usage_dict)
 
                     expected_data = (
-                        models.AllUsage(usage_list=[usage])
+                        models.AllUsage(
+                            usage_list=[usage],
+                            start_date=sept_2021,
+                            end_date=sept_2021,
+                        )
                         .model_dump_json()
                         .encode("utf-8")
                     )
@@ -171,6 +180,7 @@ def test_retrieve_and_send_usage(self) -> None:
                         "some-mock-text",
                     )
 
+            # Patch POST so that it returns a success (200 status code).
             with patch("requests.post") as mock_post:
                 mock_response = MagicMock()
                 mock_response.status_code = 200
@@ -180,12 +190,18 @@ def test_retrieve_and_send_usage(self) -> None:
                     utils.usage.retrieve_and_send_usage(
                         HTTP_ADAPTER.validate_python("https://123.123.123.123"),
                         [example_usage_detail],  # type: ignore
+                        sept_2021,
+                        sept_2021,
                     )
 
                     usage = models.Usage(**usage_dict)
 
                     expected_data = (
-                        models.AllUsage(usage_list=[usage])
+                        models.AllUsage(
+                            usage_list=[usage],
+                            start_date=sept_2021,
+                            end_date=sept_2021,
+                        )
                         .model_dump_json()
                         .encode("utf-8")
                     )

usage_function/run_tests.sh

@@ -32,7 +32,7 @@ def main(mytimer: func.TimerRequest) -> None:
     if mytimer.past_due:
         logger.info("The timer is past due.")
 
-    now = datetime.now()
+    now = datetime.now().replace(hour=0, microsecond=0, second=0, minute=0)
     start_datetime = (
         now
         - timedelta(days=config.USAGE_HISTORY_DAYS - 1)
@@ -58,7 +58,12 @@ def main(mytimer: func.TimerRequest) -> None:
             )
 
             try:
-                retrieve_and_send_usage(config.API_URL, usage)
+                retrieve_and_send_usage(
+                    config.API_URL,
+                    usage,
+                    start_datetime.date(),
+                    end_datetime.date(),
+                )
                 break
             except HttpResponseError as e:
                 logger.error("Request to azure failed. Trying again in 60 seconds")

usage_function/tests/test_function_app.py

@@ -2,7 +2,7 @@
 
 import copy
 import logging
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from functools import lru_cache
 from typing import Generator, Iterable, Optional
 from uuid import UUID
@@ -11,7 +11,7 @@
 from azure.identity import DefaultAzureCredential
 from azure.mgmt.consumption import ConsumptionManagementClient
 from azure.mgmt.consumption.models import UsageDetailsListResult
-from pydantic_core import Url
+from pydantic import HttpUrl
 from rctab_models import models
 
 from utils.auth import BearerAuth
@@ -186,22 +186,29 @@ def retrieve_usage(
 
 
 def retrieve_and_send_usage(
-    hostname_or_ip: Url, usage_data: Iterable[UsageDetailsListResult]
+    hostname_or_ip: HttpUrl,
+    usage_data: Iterable[UsageDetailsListResult],
+    start_date: date,
+    end_date: date,
 ) -> None:
     """Retrieve usage data from Azure and send it to the API.
 
     Args:
         hostname_or_ip: Hostname or IP of the API.
         usage_data: Usage data object.
+        start_date: The start of the date range that has been collected.
+        end_date: The inclusive end of the date range that has been collected.
     """
     usage_list = retrieve_usage(usage_data)
 
-    send_usage(hostname_or_ip, usage_list)
+    send_usage(hostname_or_ip, usage_list, start_date, end_date)
 
 
 def send_usage(
-    hostname_or_ip: Url,
+    hostname_or_ip: HttpUrl,
     all_item_list: list[models.Usage],
+    start_date: date,
+    end_date: date,
     monthly_usage_upload: bool = False,
 ) -> None:
     """Post each item of usage_data to a route."""
@@ -221,7 +228,13 @@ def get_first_run_time() -> datetime:
 
     # Note that omitting the encoding appears to work but will
     # fail server-side with some characters, such as en-dash.
-    data = models.AllUsage(usage_list=all_item_list).model_dump_json().encode("utf-8")
+    data = (
+        models.AllUsage(
+            usage_list=all_item_list, start_date=start_date, end_date=end_date
+        )
+        .model_dump_json()
+        .encode("utf-8")
+    )
 
     for _ in range(2):
         resp = requests.post(