Upgrade pyo3 to 0.23, sync changelogs (#413)

* Bump pyo3 * Start conversion to 0.23 * A bunch of cleanup * All good except deprecations around streams * ruff * Get ruff to use preview flag in pre commit * More ruff lints * expand ruff * Allow deprecated pyobject methods for time being * Sync lock * Add new unicode licecnse * Revert small change * Update python changelog, lint * Update changelogs * Apply suggestions from code review * Update Changelog.python.md --------- Co-authored-by: Deepak Cherian <[email protected]>
earth-mover · Nov 26, 2024 · bbefbd7 · bbefbd7
1 parent f1b7693
commit bbefbd7
Show file tree

Hide file tree

Showing 13 changed files with 95 additions and 90 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Changelog.python.md b/Changelog.python.md
@@ -1,5 +1,21 @@
 # Changelog
 
+## Python Icechunk Library 0.1.0a5
+
+### Features
+
+- Sync with zarr 3.0b2. The biggest change is the `mode` param on `IcechunkStore` methods has been simplified to `read_only`.
+- Changed `IcechunkStore::distributed_commit` to `IcechunkStore::merge`, which now does *not* commit, but attempts to merge the changes from another store back into the current store.
+- Added a new `icechunk.dask.store_dask` method to write a dask array to an icechunk store. This is required for safely writing dask arrays to an icechunk store.
+- Added a new `icechunk.xarray.to_icechunk` method to write an xarray dataset to an icechunk store. This is *required* for safely writing xarray datasets with dask arrays to an icechunk store in a distributed or multi-processing context.
+
+### Fixes
+
+- The `StorageConfig` methods have been correctly typed.
+- `IcechunkStore` instances are now set to `read_only` by default after pickling.
+- When checking out a snapshot or tag, the `IcechunkStore` will be set to read-only. If you want to write to the store, you must call `IcechunkStore::set_writeable()`.
+- An error will now be raised if you try to checkout a snapshot that does not exist.
+
 ## Python Icechunk Library 0.1.0a4
 
 ### Features

diff --git a/Changelog.rust.md b/Changelog.rust.md
@@ -1,5 +1,18 @@
 # Changelog
 
+## Rust Icechunk Library 0.1.0-alpha.5
+
+### Features
+
+- Added new `Store::merge` method to merge changes from another store back into the current store.
+- Added new `garbage_collect` method to remove dangling chunks from the store.
+- Added new `Repository::rebase` method to detect and optionally fix conflicts between the current changes and the tip of a branch, allowing the user to commit the changes to the branch.
+
+### Fixes
+
+- `Store` will now be set to `ReadOnly` after checking out a snapshot or tag.
+- An error will now be raised if you try to checkout a snapshot that does not exist.
+
 ## Rust Icechunk Library 0.1.0-alpha.4
 
 ### Features

diff --git a/deny.toml b/deny.toml
@@ -22,6 +22,7 @@ allow = [
   "OpenSSL",
   "Unicode-DFS-2016",
   "CC0-1.0",
+  "Unicode-3.0",
 ]
 # The confidence threshold for detecting a license from license text.
 # The higher the value, the more closely the license text must be to the

diff --git a/icechunk-python/Cargo.toml b/icechunk-python/Cargo.toml
@@ -22,12 +22,12 @@ bytes = "1.8.0"
 chrono = { version = "0.4.38" }
 futures = "0.3.31"
 icechunk = { path = "../icechunk", version = "0.1.0-alpha.4" }
-pyo3 = { version = "0.22", features = [
+pyo3 = { version = "0.23", features = [
   "chrono",
   "extension-module",
   "experimental-async",
 ] }
-pyo3-async-runtimes = { version = "0.22.0", features = ["tokio-runtime"] }
+pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"] }
 async-stream = "0.3.6"
 thiserror = "2.0.3"
 tokio = "1.41"

diff --git a/icechunk-python/examples/smoke-test.py b/icechunk-python/examples/smoke-test.py
@@ -59,7 +59,7 @@ def create_array(*, group, name, size, dtype, fill_value) -> np.ndarray:
     return array
 
 
-async def run(store: Store) -> None:
+def run(store: Store) -> None:
     write_start = time.time()
     group = zarr.group(store=store, overwrite=True)
     group.attrs["foo"] = "foo"

diff --git a/icechunk-python/src/errors.rs b/icechunk-python/src/errors.rs
@@ -1,3 +1,5 @@
+use std::convert::Infallible;
+
 use icechunk::{
     format::IcechunkFormatError, repository::RepositoryError, zarr::StoreError,
 };
@@ -32,6 +34,12 @@ pub(crate) enum PyIcechunkStoreError {
     UnkownError(String),
 }
 
+impl From<Infallible> for PyIcechunkStoreError {
+    fn from(_: Infallible) -> Self {
+        PyIcechunkStoreError::UnkownError("Infallible".to_string())
+    }
+}
+
 impl From<StoreError> for PyIcechunkStoreError {
     fn from(error: StoreError) -> Self {
         match error {