Changes for Brian C's pr review

Author: mnazzaro

arxiv/base/__init__.py

@@ -126,6 +126,10 @@ def register_blueprint(self: Flask, blueprint: Blueprint,
         filters.register_filters(app)
         context_processors.register_context_processors(app)
 
+
+        # This piece of code is crucial to making sure sqlalchemy sessions work in flask
+        # It is the same as the flask_sqlalchemy implementation 
+        # See: https://github.com/pallets-eco/flask-sqlalchemy/blob/42a36a3cb604fd39d81d00b54ab3988bbd0ad184/src/flask_sqlalchemy/session.py#L109
         @app.teardown_appcontext
         def remove_scoped_session (response_or_exc):
             session.remove()

arxiv/config/__init__.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     curl http://127.0.0.1:5000/ -sv -H "Host: subdomain.arxiv.org"
     """
 
-    SECRET_KEY: str = "qwert2345"
+    SECRET_KEY: SecretStr = SecretStr("qwert2345")
 
     ARXIV_ACCESSIBILITY_URL: str = "mailto:[email protected]"
 
@@ -137,6 +137,8 @@ class Settings(BaseSettings):
     Timezone of the arxiv business offices.
     """
 
+    FS_TZ: str = "US/Eastern"
+
     BASE_VERSION: str = importlib.metadata.version('arxiv-base')
     """The version of the arxiv-base package."""
 

arxiv/db/__init__.py

@@ -1,5 +1,30 @@
 """
 This is the primary SQLAlchemy implementation of the main arXiv database 
+
+To use this in a simple non-flask non-fastapi script do:
+
+from arxiv.db import get_db
+
+with get_db() as session:
+    session.execute(
+        select(...)
+    )
+
+To use this in flask just do:
+
+from arxiv.db import session
+
+session.execute(
+    select(...)
+)
+
+To use this with fastapi do:
+
+with get_db() as session:
+    session.execute(
+        select(...)
+    )
+
 """
 from typing import Generator
 import logging

arxiv/db/types.py

@@ -1,3 +1,9 @@
+"""
+This file contains common database types. We have lots of integer 
+primary keys in our db, so we can include a shorthand like intpk
+to make annotating the column type in .models easier
+"""
+
 from typing import Annotated
 from sqlalchemy.orm import mapped_column
 

arxiv/document/parse_abs.py

@@ -8,7 +8,7 @@
 from zoneinfo import ZoneInfo
 from dateutil import parser
 
-from flask import current_app
+from ..config import settings
 
 from ..taxonomy import definitions
 from .metadata import Archive, AuthorList, Category, \
@@ -75,10 +75,7 @@ def parse_abs_file(filename: str) -> DocMetadata:
     try:
         with absfile.open(mode='r', encoding='latin-1') as absf:
             raw = absf.read()
-            if current_app:
-                modified = datetime.fromtimestamp(absfile.stat().st_mtime, tz=_get_tz())
-            else:
-                modified = datetime.fromtimestamp(absfile.stat().st_mtime)
+            modified = datetime.fromtimestamp(absfile.stat().st_mtime, tz=_get_tz())
             modified = modified.astimezone(ZoneInfo("UTC"))
             return parse_abs(raw, modified)
 
@@ -301,9 +298,9 @@ def alt_component_split(components: List[str]) -> List[str]:
 
 
 def _get_tz() -> ZoneInfo:
-    """Gets the timezone from the flask current_app."""
+    """Gets the timezone from the environment"""
     global _fs_tz
     if _fs_tz is None:
-        _fs_tz = ZoneInfo(current_app.config["FS_TZ"])
+        _fs_tz = ZoneInfo(settings.FS_TZ)
 
     return _fs_tz

arxiv/ops/fastly_log_ingest/app.py

@@ -63,8 +63,8 @@ def __init__(self, message: Message, arrival_time: float):
         self.arrival_time = arrival_time
         try:
             self.data: dict = json.loads(message.data.decode('utf-8').strip())
-        except:
-            logger.warning(f'Failed to log the following message: {message.data.decode("utf-8")}')
+        except Exception as e:
+            logger.warning(f'Failed to log the following message: {message.data.decode("utf-8", errors='replace')} with {str(e)}')
 
     def to_payload(self) -> dict:
         """Convert the message into a `payload` for the log entry."""

arxiv/urls/__init__.py

@@ -1,12 +1,13 @@
 from ..identifier import Identifier
+from ..config import settings
 
 def pdf_canonical_url (identifier: Identifier) -> str:
     # This won't include the version unless identifier 
     # was explicitly initialized with a version
-    return f'https://arxiv.org/pdf/{identifier.idv}'
+    return f'https://{settings.CANONICAL_SERVER}/pdf/{identifier.idv}'
 
 def html_canonical_url (identifier: Identifier) -> str:
-    return f'https://arxiv.org/html/{identifier.idv}'
+    return f'https://{settings.CANONICAL_SERVER}/html/{identifier.idv}'
 
 def abs_canonical_url (identifier: Identifier) -> str:
-    return f'https://arxiv.org/abs/{identifier.idv}'
+    return f'https://{settings.CANONICAL_SERVER}/abs/{identifier.idv}'

arxiv/util/tests/test_authors.py

@@ -1,7 +1,7 @@
 """Tests for author and affiliation parsing."""
 from unittest import TestCase
 
-from arxiv.authors import parse_author_affil, split_authors
+from ...authors import parse_author_affil, split_authors
 
 
 class TestAuthorAffiliationParsing(TestCase):

mypy.ini

@@ -1,6 +1,4 @@
 [mypy]
-# mypy_path = $MYPYPATH:./sqlalchemy-stubs
-
 #
 # Covered by --strict, with some turned off:
 #

pyproject.toml

@@ -16,7 +16,7 @@ include = [
 [tool.poetry.dependencies]
 python = "^3.11"
 pydantic = "==1.10.*"
-flask = "3.0.2"
+flask = "3.0.*"
 pytz = "*"
 boto3 = "==1.*"
 wtforms = "*"