Add PostgreSQL-backed data store for production deployments (#238)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: michaelchu

optopsy/data/_yf_helpers.py

@@ -9,13 +9,13 @@
 
 import pandas as pd
 
-from optopsy.data.providers.cache import ParquetCache
+from optopsy.data.providers.cache import get_store
 
 _log = logging.getLogger(__name__)
 
 # Cache for yfinance OHLCV data (category="yf_stocks", one file per symbol).
 # Deliberately distinct from EODHD's "stocks" category to avoid schema collisions.
-_yf_cache = ParquetCache()
+_yf_cache = get_store()
 _YF_CACHE_CATEGORY = "yf_stocks"
 _YF_DEDUP_COLS = ["date"]
 

optopsy/data/cli.py

@@ -22,16 +22,16 @@ def _cmd_cache_size(args: argparse.Namespace) -> None:
 
     import_optional_dependency("pyarrow")
 
-    from optopsy.data.providers.cache import ParquetCache
+    from optopsy.data.providers.cache import get_store
 
-    cache = ParquetCache()
-    entries = cache.size()
+    store = get_store()
+    entries = store.size()
     if not entries:
         print("Cache is empty.")
         return
     for name, nbytes in entries.items():
         print(f"  {name:<30s} {_format_bytes(nbytes):>10s}")
-    print(f"  {'Total':<30s} {_format_bytes(cache.total_size_bytes()):>10s}")
+    print(f"  {'Total':<30s} {_format_bytes(store.total_size_bytes()):>10s}")
 
 
 def _cmd_cache_clear(args: argparse.Namespace) -> None:
@@ -40,11 +40,11 @@ def _cmd_cache_clear(args: argparse.Namespace) -> None:
 
     import_optional_dependency("pyarrow")
 
-    from optopsy.data.providers.cache import ParquetCache
+    from optopsy.data.providers.cache import get_store
 
-    cache = ParquetCache()
+    store = get_store()
     symbol = args.symbol
-    count = cache.clear(symbol=symbol)
+    count = store.clear(symbol=symbol)
     if symbol:
         print(f"Cleared {count} cached file(s) for {symbol.upper()}.")
     else:
@@ -199,10 +199,10 @@ def _download_stocks_with_rich(symbol: str) -> None:
         return
 
     from optopsy.data._yf_helpers import _YF_CACHE_CATEGORY, _yf_fetch_and_cache
-    from optopsy.data.providers.cache import ParquetCache
+    from optopsy.data.providers.cache import get_store
 
-    cache = ParquetCache()
-    cached = cache.read(_YF_CACHE_CATEGORY, symbol)
+    store = get_store()
+    cached = store.read(_YF_CACHE_CATEGORY, symbol)
 
     with console.status(f"[bold green]Fetching {symbol} from yfinance…"):
         try:
@@ -219,7 +219,7 @@ def _download_stocks_with_rich(symbol: str) -> None:
     date_max = pd.to_datetime(result["date"]).dt.date.max()
     row_count = len(result)
 
-    size_bytes = cache.size().get(f"{_YF_CACHE_CATEGORY}/{symbol}", 0)
+    size_bytes = store.size().get(f"{_YF_CACHE_CATEGORY}/{symbol}", 0)
     size_str = _format_bytes(size_bytes)
 
     console.print(f"  [bold]{symbol}[/bold]  {date_min} → {date_max}")
@@ -279,6 +279,58 @@ def _render() -> None:
         )
 
 
+def _cmd_import(args: argparse.Namespace) -> None:
+    """Import local parquet files into the PostgreSQL data store."""
+    import os
+
+    import pandas as pd
+    from rich.console import Console
+
+    _load_env()
+
+    from optopsy.data.providers.cache import get_store
+
+    console = Console()
+    store = get_store()
+
+    # Verify we're targeting PostgreSQL
+    db_url = os.environ.get("DATABASE_URL", "")
+    if not db_url.startswith(("postgres://", "postgresql://")):
+        console.print(
+            "[red]DATABASE_URL is not set or is not PostgreSQL. "
+            "Import requires a PostgreSQL database.[/red]"
+        )
+        return
+
+    for fpath in args.files:
+        if not os.path.exists(fpath):
+            console.print(f"[red]File not found: {fpath}[/red]")
+            continue
+
+        # Auto-detect category from path
+        category = args.category
+        if not category:
+            if "/options/" in fpath or "\\options\\" in fpath:
+                category = "options"
+            elif "/yf_stocks/" in fpath or "\\yf_stocks\\" in fpath:
+                category = "yf_stocks"
+            else:
+                console.print(
+                    f"[yellow]Cannot detect category for {fpath}. Use -c flag.[/yellow]"
+                )
+                continue
+
+        symbol = os.path.basename(fpath).replace(".parquet", "").upper()
+
+        with console.status(f"Importing {symbol} ({category})..."):
+            df = pd.read_parquet(fpath)
+            store.write(category, symbol, df)
+
+        console.print(
+            f"  [green]Imported {symbol}: {len(df):,} rows into {category}[/green]"
+        )
+
+
 def _build_data_subparsers(
     subparsers: argparse._SubParsersAction,
 ) -> dict[str, argparse.ArgumentParser]:
@@ -340,6 +392,25 @@ def _build_data_subparsers(
     )
     clear_parser.set_defaults(func=_cmd_cache_clear)
 
+    # --- import ---
+    import_parser = subparsers.add_parser(
+        "import",
+        help="Import local parquet files into the PostgreSQL data store",
+    )
+    import_parser.add_argument(
+        "files",
+        nargs="+",
+        help="Parquet file paths (e.g. ~/.optopsy/cache/options/SPY.parquet)",
+    )
+    import_parser.add_argument(
+        "-c",
+        "--category",
+        choices=["options", "yf_stocks"],
+        default=None,
+        help="Data category (auto-detected from path if omitted)",
+    )
+    import_parser.set_defaults(func=_cmd_import)
+
     return {"cache": cache_parser}
 
 

optopsy/data/providers/cache.py

@@ -17,6 +17,7 @@
 import pandas as pd
 
 from optopsy.data.paths import CACHE_DIR
+from optopsy.data.providers.store import DataStore
 
 _log = logging.getLogger(__name__)
 
@@ -98,7 +99,7 @@ def compute_date_gaps(
     return gaps
 
 
-class ParquetCache:
+class ParquetCache(DataStore):
     """Simple parquet-based cache for immutable historical data.
 
     Each (category, symbol) pair maps to a single parquet file at:
@@ -212,3 +213,17 @@ def size(self) -> dict[str, int]:
     def total_size_bytes(self) -> int:
         """Return total cache size in bytes."""
         return sum(self.size().values())
+
+
+def get_store() -> DataStore:
+    """Return the appropriate data store backend.
+
+    Uses ``PostgresStore`` when ``DATABASE_URL`` is set to a PostgreSQL URL,
+    otherwise falls back to the file-based ``ParquetCache``.
+    """
+    db_url = os.environ.get("DATABASE_URL", "")
+    if db_url.startswith(("postgres://", "postgresql://")):
+        from optopsy.data.providers.pg_store import PostgresStore
+
+        return PostgresStore(db_url)
+    return ParquetCache()

optopsy/data/providers/eodhd.py

@@ -28,7 +28,7 @@
 import requests
 
 from .base import DataProvider
-from .cache import ParquetCache
+from .cache import get_store
 
 _log = logging.getLogger(__name__)
 
@@ -147,7 +147,7 @@ def _safe_raise_for_status(resp: requests.Response) -> None:
 
 class EODHDProvider(DataProvider):
     def __init__(self) -> None:
-        self._cache = ParquetCache()
+        self._cache = get_store()
         self._session = requests.Session()
         self._last_request_time: float = 0.0
         self._request_count: int = 0