Use string paths with open_mfdataset (#1613)

Author: jrbourbeau

tests/geospatial/test_cloud_optimize.py

@@ -19,4 +19,4 @@ def test_cloud_optimize(
     with setup_benchmark(
         **scale_kwargs[scale], **cluster_kwargs
     ) as benchmark:  # noqa: F841
-        benchmark(cloud_optimize, scale, s3fs=s3, storage_url=s3_url)
+        benchmark(cloud_optimize, scale, fs=s3, storage_url=s3_url)

tests/geospatial/workloads/cloud_optimize.py

@@ -5,7 +5,7 @@
 
 
 def cloud_optimize(
-    scale: Literal["small", "medium", "large"], s3fs: S3FileSystem, storage_url: str
+    scale: Literal["small", "medium", "large"], fs: S3FileSystem, storage_url: str
 ):
     models = [
         "ACCESS-CM2",
@@ -59,12 +59,11 @@ def cloud_optimize(
 
     # Get netCDF data files -- see https://registry.opendata.aws/nex-gddp-cmip6
     # for dataset details.
-    file_list = []
+    files = []
     for model in models:
         for variable in variables:
             data_dir = f"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1f1/{variable}/*.nc"
-            file_list += [f"s3://{path}" for path in s3fs.glob(data_dir)]
-    files = [s3fs.open(f) for f in file_list]
+            files += [f"s3://{path}" for path in fs.glob(data_dir)]
     print(f"Processing {len(files)} NetCDF files")
 
     # Load input NetCDF data files