Merge pull request #23 from cedadev/poetry2

Poetry 2 + Logs

Author: dwest77a

.github/workflows/ci.yml

@@ -30,9 +30,9 @@ jobs:
           python3 -m pip install --upgrade pip
           pip3 install poetry
           poetry install
-      - name: Run isort
-        run: |
-          poetry run isort --check --diff cfapyx
+      #- name: Run isort
+      #  run: |
+      #    poetry run isort --check --diff cfapyx
       # Test with pytest
       - name: Run pytest
         run: |

.gitignore

@@ -9,6 +9,7 @@ CFAPyX.egg-info/
 .vscode/
 testing/
 build/
+dist/
 .DS_Store
 docs/.DS_Store
 docs/source/.DS_Store

cfapyx/__init__.py

@@ -1,2 +1,3 @@
 from .backend import CFANetCDFBackendEntrypoint
 from .creator import CFANetCDF
+from .utils import set_verbose

cfapyx/backend.py

@@ -11,8 +11,13 @@
 
 from cfapyx.datastore import CFADataStore
 
+from cfapyx.utils import logstream
+
 logger = logging.getLogger(__name__)
 
+logger.addHandler(logstream)
+logger.propagate = False
+
 def open_cfa_dataset(
         filename_or_obj,
         drop_variables=None,
@@ -22,11 +27,13 @@ def open_cfa_dataset(
         decode_coords=None,
         use_cftime=None,
         decode_timedelta=None,
-        cfa_options: dict=None,
+        cfa_options: dict = None,
         group=None,
         ):
     """
-    Top-level function which opens a CFA dataset using Xarray. Creates a CFA Datastore 
+    Top-level function which opens a CFA dataset using Xarray. 
+    
+    Creates a CFA Datastore 
     from the ``filename_or_obj`` provided, then passes this to a CFA StoreBackendEntrypoint
     to create an Xarray Dataset. Most parameters are not handled by CFA, so only the 
     CFA-relevant ones are described here.
@@ -52,11 +59,7 @@ def open_cfa_dataset(
     store = CFADataStore.open(filename_or_obj, group=group)
 
     # Expands cfa_options into individual kwargs for the store.
-    store.cfa_options    = cfa_options
-
-    use_active = False
-    if hasattr(store, 'use_active'):
-        use_active = store.use_active
+    store.cfa_options = cfa_options
 
     # Xarray makes use of StoreBackendEntrypoints to provide the Dataset 'ds'
     store_entrypoint = CFAStoreBackendEntrypoint()
@@ -68,13 +71,13 @@ def open_cfa_dataset(
         decode_coords=decode_coords,
         drop_variables=drop_variables,
         use_cftime=use_cftime,
-        decode_timedelta=decode_timedelta,
-        use_active=use_active
+        decode_timedelta=decode_timedelta
     )
 
     return ds
 
 class CFANetCDFBackendEntrypoint(BackendEntrypoint):
+    """Open CFA-netCDF files (.nca) using 'cfapyx' in Xarray"""
 
     description = 'Open CFA-netCDF files (.nca) using "cfapyx" in Xarray'
     url = "https://cedadev.github.io/CFAPyX/"
@@ -115,6 +118,8 @@ def open_dataset(
             group=group)
 
 class CFAStoreBackendEntrypoint(StoreBackendEntrypoint):
+    """Open CFA-based Abstract Data Store"""
+    
     description = "Open CFA-based Abstract Data Store"
     url = "https://cedadev.github.io/CFAPyX/"
 
@@ -128,7 +133,6 @@ def open_dataset(
         drop_variables=None,
         use_cftime=None,
         decode_timedelta=None,
-        use_active=False,
     ) -> Dataset:
         """
         Takes cfa_xarray_store of type AbstractDataStore and creates an xarray.Dataset object.
@@ -162,18 +166,7 @@ def open_dataset(
         )
 
         # Create the xarray.Dataset object here.
-        if use_active:
-            try:
-                from XarrayActive import ActiveDataset
-
-                ds = ActiveDataset(vars, attrs=attrs)
-            except ImportError:
-                raise ImportError(
-                    '"ActiveDataset" from XarrayActive failed to import - please '
-                    'ensure you have the XarrayActive package installed.'
-                )
-        else:
-            ds = Dataset(vars, attrs=attrs)
+        ds = Dataset(vars, attrs=attrs)
 
         ds = ds.set_coords(coord_names.intersection(vars))
         ds.set_close(cfa_xarray_store.close)

cfapyx/creator.py

@@ -9,8 +9,13 @@
 import netCDF4
 import numpy as np
 
+from cfapyx.utils import logstream
+
 logger = logging.getLogger(__name__)
 
+logger.addHandler(logstream)
+logger.propagate = False
+
 CONCAT_MSG = 'See individual datasets for more information.'
 
 class CFACreateMixin:
@@ -93,7 +98,6 @@ def _first_pass(self, agg_dims: list = None) -> tuple:
                         f'Files contain differing numbers of dimensions. "{d}"'
                         'appears to not be present in all files.'
                     )
-                
                 new_info, arr_components = self._collect_dim_info(
                     ds, d, pure_dimensions, coord_variables, 
                     agg_dims=agg_dims, first_time=first_time)
@@ -305,7 +309,7 @@ def _arrange_dimensions(
                 sort = np.argsort(arr)
 
                 cdimarr = None
-                nds = []
+                nds, nstarts = [],[]
                 for s in sort:
 
                     if cdimarr is None:
@@ -314,21 +318,25 @@ def _arrange_dimensions(
                         cdimarr = np.concatenate((cdimarr, np.array(arrays[s])))
 
                     nds.append(sizes[s])
+                    nstarts.append(starts[s])
 
                 ndimsizes   = tuple(nds) # Removed np.array here
 
             info['size'] = cdimarr.size
             info['array'] = cdimarr
             info['sizes'] = ndimsizes
 
+            # Reorder starts like with sizes
+            info['starts'] = nstarts
+
         if agg_dims is not None:
             if len(agg_dims) != len(aggregation_dims):
                 raise ValueError(
                     'Found fewer aggregation dims than user provided value.'
                     f'User defined: ({list(agg_dims)})'
                     f'Derived: ({list(aggregation_dims)})'
                 )
-
+            
         return dim_info, aggregation_dims
 
     def _assemble_location(

cfapyx/datastore.py

@@ -17,8 +17,13 @@
 from cfapyx.group import CFAGroupWrapper
 from cfapyx.wrappers import FragmentArrayWrapper
 
+from cfapyx.utils import logstream
+
 logger = logging.getLogger(__name__)
 
+logger.addHandler(logstream)
+logger.propagate = False
+
 
 xarray_subs = {
     'file:///':'/'
@@ -34,6 +39,8 @@ class CFADataStore(NetCDF4DataStore):
     that may be un-set at time of use.
     """
 
+    wrapper = FragmentArrayWrapper
+
     @property
     def chunks(self):
         if hasattr(self,'_cfa_chunks'):
@@ -55,8 +62,7 @@ def cfa_options(self):
             'substitutions': self._substitutions,
             'decode_cfa': self._decode_cfa,
             'chunks': self.chunks,
-            'chunk_limits': self._chunk_limits,
-            'use_active': self.use_active
+            'chunk_limits': self._chunk_limits
         }
 
     @cfa_options.setter
@@ -69,7 +75,6 @@ def _set_cfa_options(
             decode_cfa=True,
             chunks={},
             chunk_limits=True,
-            use_active=False,
         ):
         """
         Method to set cfa options.
@@ -80,8 +85,6 @@ def _set_cfa_options(
         :param decode_cfa:      (bool) Optional setting to disable CFA decoding 
                                 in some cases, default is True.
 
-        :param use_active:      (bool) Enable for use with XarrayActive.
-
         :param chunks:          (dict) Not implemented in 2024.9.0
 
         :param chunk_limits:    (dict) Not implemented in 2024.9.0
@@ -91,7 +94,6 @@ def _set_cfa_options(
         self._substitutions = substitutions
         self._decode_cfa    = decode_cfa
         self._chunk_limits  = chunk_limits
-        self.use_active     = use_active
 
     def _acquire(self, needs_lock=True):
         """
@@ -360,7 +362,7 @@ def open_variable(self, name: str, var):
         :returns:       The variable object opened as either a standard store variable 
                         or CFA aggregated variable.
         """
-        if type(var) == tuple:
+        if isinstance(var, tuple):
             if var[1] and self._decode_cfa:
                 variable = self.open_cfa_variable(name, var[0])
             else:
@@ -412,7 +414,7 @@ def open_cfa_variable(self, name: str, var):
 
         ## Array-like object 
         data = indexing.LazilyIndexedArray(
-            FragmentArrayWrapper(
+            self.wrapper(
                 fragment_info,
                 fragment_space,
                 shape=array_shape,

cfapyx/decoder.py

@@ -5,8 +5,13 @@
 import logging
 from itertools import accumulate, product
 
+from cfapyx.utils import logstream
+
 logger = logging.getLogger(__name__)
 
+logger.addHandler(logstream)
+logger.propagate = False
+
 def get_fragment_positions(fragment_size_per_dim):
     """
     Get the positions in index space for each fragment.

cfapyx/group.py

@@ -4,8 +4,13 @@
 
 import logging
 
+from cfapyx.utils import logstream
+
 logger = logging.getLogger(__name__)
 
+logger.addHandler(logstream)
+logger.propagate = False
+
 class VariableWrapper:
     """
     Wrapper object for the ``ds.variables`` and ``ds.attributes`` objects which can handle

cfapyx/tests/conftest.py

@@ -4,8 +4,7 @@ def pytest_collection_modifyitems(items):
     CLASS_ORDER = [
         "TestPath"
         "TestCFAWrite",
-        "TestCFARead",
-        "TestCleanup"
+        "TestCFARead"
     ]
 
     sorted_items = items.copy()

cfapyx/tests/test_basic.py

@@ -7,9 +7,4 @@
 class TestPath:
     def testpath(self, tests=TESTDIR):
         logger.error(os.getcwd())
-        logger.error(tests)
-
-class TestCleanup:
-    def test_cleanup(self, testdir=TESTDIR):
-        os.system(f'rm {testdir}/testrain.nca')
-        print('Integration tests: Cleanup - complete')
+        logger.error(tests)