Uncache history (#635)

We had a cache in each snapshot with pointers to all the snapshot
ancestry. This makes expiration harder, because we need to invalidate
these caches.

We are replacing the cache by a single `parent_id` link. In the future,
we may optimize this for performance.

Notice: python is still turning the stream of parents into a python
list. This is now much worse because the new ancestry version is much
slower. We'll need to change this to use an AsyncGenerator.

Author: paraseba

docs/docs/scripts/readthedocs.js

@@ -21,4 +21,4 @@ ${ config.versions.active.map(
 </div>`;
 
 document.querySelector(".md-header__topic").insertAdjacentHTML("beforeend", versioning);
-});
+});

icechunk-python/python/icechunk/__init__.py

@@ -22,7 +22,7 @@
     S3Credentials,
     S3Options,
     S3StaticCredentials,
-    SnapshotMetadata,
+    SnapshotInfo,
     Storage,
     StorageConcurrencySettings,
     StorageSettings,
@@ -98,7 +98,7 @@
     "S3Options",
     "S3StaticCredentials",
     "Session",
-    "SnapshotMetadata",
+    "SnapshotInfo",
     "Storage",
     "StorageConcurrencySettings",
     "StorageSettings",

icechunk-python/python/icechunk/_icechunk_python.pyi

@@ -219,7 +219,7 @@ class PyRepository:
         branch: str | None = None,
         tag: str | None = None,
         snapshot: str | None = None,
-    ) -> list[SnapshotMetadata]: ...
+    ) -> list[SnapshotInfo]: ...
     def create_branch(self, branch: str, snapshot_id: str) -> None: ...
     def list_branches(self) -> set[str]: ...
     def lookup_branch(self, branch: str) -> str: ...
@@ -311,13 +311,17 @@ class PyAsyncStringGenerator(AsyncGenerator[str, None], metaclass=abc.ABCMeta):
     def __aiter__(self) -> PyAsyncStringGenerator: ...
     async def __anext__(self) -> str: ...
 
-class SnapshotMetadata:
+class SnapshotInfo:
     """Metadata for a snapshot"""
     @property
     def id(self) -> str:
         """The snapshot ID"""
         ...
     @property
+    def parent_id(self) -> str | None:
+        """The snapshot ID"""
+        ...
+    @property
     def written_at(self) -> datetime.datetime:
         """
         The timestamp when the snapshot was written
@@ -330,11 +334,9 @@ class SnapshotMetadata:
         """
         ...
 
-class PyAsyncSnapshotGenerator(
-    AsyncGenerator[SnapshotMetadata, None], metaclass=abc.ABCMeta
-):
+class PyAsyncSnapshotGenerator(AsyncGenerator[SnapshotInfo, None], metaclass=abc.ABCMeta):
     def __aiter__(self) -> PyAsyncSnapshotGenerator: ...
-    async def __anext__(self) -> SnapshotMetadata: ...
+    async def __anext__(self) -> SnapshotInfo: ...
 
 class S3StaticCredentials:
     access_key_id: str

icechunk-python/python/icechunk/repository.py

@@ -3,7 +3,7 @@
 from icechunk._icechunk_python import (
     PyRepository,
     RepositoryConfig,
-    SnapshotMetadata,
+    SnapshotInfo,
     Storage,
 )
 from icechunk.credentials import AnyCredential
@@ -181,7 +181,7 @@ def ancestry(
         branch: str | None = None,
         tag: str | None = None,
         snapshot: str | None = None,
-    ) -> list[SnapshotMetadata]:
+    ) -> list[SnapshotInfo]:
         """
         Get the ancestry of a snapshot.
 
@@ -196,7 +196,7 @@ def ancestry(
 
         Returns
         -------
-        list[SnapshotMetadata]
+        list[SnapshotInfo]
             The ancestry of the snapshot, listing out the snapshots and their metadata.
 
         Notes

icechunk-python/src/config.rs

@@ -106,7 +106,7 @@ fn format_option<'a, T: AsRef<str> + 'a>(o: Option<T>) -> String {
     }
 }
 
-fn format_option_string<'a, T: AsRef<str> + 'a>(o: Option<T>) -> String {
+pub(crate) fn format_option_string<'a, T: AsRef<str> + 'a>(o: Option<T>) -> String {
     match o.as_ref() {
         None => "None".to_string(),
         Some(s) => format!(r#""{}""#, s.as_ref()),

icechunk-python/src/lib.rs

@@ -22,7 +22,7 @@ use errors::{
     PyRebaseFailedError,
 };
 use pyo3::prelude::*;
-use repository::{PyRepository, PySnapshotMetadata};
+use repository::{PyRepository, PySnapshotInfo};
 use session::PySession;
 use store::PyStore;
 
@@ -34,7 +34,7 @@ fn _icechunk_python(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyRepositoryConfig>()?;
     m.add_class::<PySession>()?;
     m.add_class::<PyStore>()?;
-    m.add_class::<PySnapshotMetadata>()?;
+    m.add_class::<PySnapshotInfo>()?;
     m.add_class::<PyConflictSolver>()?;
     m.add_class::<PyBasicConflictSolver>()?;
     m.add_class::<PyConflictDetector>()?;

icechunk-python/src/repository.rs

@@ -7,7 +7,7 @@ use chrono::{DateTime, Utc};
 use futures::TryStreamExt;
 use icechunk::{
     config::Credentials,
-    format::{snapshot::SnapshotMetadata, SnapshotId},
+    format::{snapshot::SnapshotInfo, SnapshotId},
     repository::{RepositoryError, VersionInfo},
     Repository,
 };
@@ -16,40 +16,45 @@ use tokio::sync::RwLock;
 
 use crate::{
     config::{
-        datetime_repr, PyCredentials, PyRepositoryConfig, PyStorage, PyStorageSettings,
+        datetime_repr, format_option_string, PyCredentials, PyRepositoryConfig,
+        PyStorage, PyStorageSettings,
     },
     errors::PyIcechunkStoreError,
     session::PySession,
 };
 
-#[pyclass(name = "SnapshotMetadata", eq)]
+#[pyclass(name = "SnapshotInfo", eq)]
 #[derive(Clone, Debug, PartialEq, Eq)]
-pub struct PySnapshotMetadata {
+pub struct PySnapshotInfo {
     #[pyo3(get)]
     id: String,
     #[pyo3(get)]
+    parent_id: Option<String>,
+    #[pyo3(get)]
     written_at: DateTime<Utc>,
     #[pyo3(get)]
     message: String,
 }
 
-impl From<SnapshotMetadata> for PySnapshotMetadata {
-    fn from(val: SnapshotMetadata) -> Self {
-        PySnapshotMetadata {
+impl From<SnapshotInfo> for PySnapshotInfo {
+    fn from(val: SnapshotInfo) -> Self {
+        PySnapshotInfo {
             id: val.id.to_string(),
-            written_at: val.written_at,
+            parent_id: val.parent_id.map(|id| id.to_string()),
+            written_at: val.flushed_at,
             message: val.message,
         }
     }
 }
 
 #[pymethods]
-impl PySnapshotMetadata {
+impl PySnapshotInfo {
     pub fn __repr__(&self) -> String {
         // TODO: escape
         format!(
-            r#"SnapshotMetadata(id="{id}",written_at={at},message="{message}")"#,
+            r#"SnapshotInfo(id="{id}", parent_id={parent}, written_at={at}, message="{message}")"#,
             id = self.id,
+            parent = format_option_string(self.parent_id.as_ref()),
             at = datetime_repr(&self.written_at),
             message = self.message.chars().take(10).collect::<String>() + "...",
         )
@@ -225,7 +230,7 @@ impl PyRepository {
         branch: Option<String>,
         tag: Option<String>,
         snapshot: Option<String>,
-    ) -> PyResult<Vec<PySnapshotMetadata>> {
+    ) -> PyResult<Vec<PySnapshotInfo>> {
         // This function calls block_on, so we need to allow other thread python to make progress
         py.allow_threads(move || {
             let version = args_to_version_info(branch, tag, snapshot)?;
@@ -237,7 +242,7 @@ impl PyRepository {
                     .ancestry(&version)
                     .await
                     .map_err(PyIcechunkStoreError::RepositoryError)?
-                    .map_ok(Into::<PySnapshotMetadata>::into)
+                    .map_ok(Into::<PySnapshotInfo>::into)
                     .try_collect::<Vec<_>>()
                     .await
                     .map_err(PyIcechunkStoreError::RepositoryError)?;