Merge pull request #382 from ecmwf/ignore_keys

Ignore keys

cfgrib/dataset.py

@@ -751,6 +751,7 @@ def open_fieldset(
  indexpath: T.Optional[str] = None,
  filter_by_keys: T.Dict[str, T.Any] = {},
  read_keys: T.Sequence[str] = (),
+ ignore_keys: T.Sequence[str] = [],
  time_dims: T.Sequence[str] = ("time", "step"),
  extra_coords: T.Dict[str, str] = {},
  computed_keys: messages.ComputedKeysType = cfmessage.COMPUTED_KEYS,
@@ -762,6 +763,7 @@ def open_fieldset(
  log.warning(f"indexpath value {indexpath} is ignored")
 
  index_keys = compute_index_keys(time_dims, extra_coords, filter_by_keys)
+ index_keys = [key for key in index_keys if key not in ignore_keys]
  index = messages.FieldsetIndex.from_fieldset(fieldset, index_keys, computed_keys)
  filtered_index = index.subindex(filter_by_keys)
  return open_from_index(filtered_index, read_keys, time_dims, extra_coords, **kwargs)
@@ -771,10 +773,12 @@ def open_fileindex(
  stream: messages.FileStream,
  indexpath: str = messages.DEFAULT_INDEXPATH,
  index_keys: T.Sequence[str] = INDEX_KEYS + ["time", "step"],
+ ignore_keys: T.Sequence[str] = [],
  filter_by_keys: T.Dict[str, T.Any] = {},
  computed_keys: messages.ComputedKeysType = cfmessage.COMPUTED_KEYS,
 ) -> messages.FileIndex:
  index_keys = sorted(set(index_keys) | set(filter_by_keys))
+ index_keys = [key for key in index_keys if key not in ignore_keys]
  index = messages.FileIndex.from_indexpath_or_filestream(
  stream, index_keys, indexpath=indexpath, computed_keys=computed_keys
  )
@@ -789,12 +793,13 @@ def open_file(
  read_keys: T.Sequence[str] = (),
  time_dims: T.Sequence[str] = ("time", "step"),
  extra_coords: T.Dict[str, str] = {},
+ ignore_keys: T.Sequence[str] = [],
  **kwargs: T.Any,
 ) -> Dataset:
  """Open a GRIB file as a ``cfgrib.Dataset``."""
  path = os.fspath(path)
  stream = messages.FileStream(path, errors=errors)
  index_keys = compute_index_keys(time_dims, extra_coords)
- index = open_fileindex(stream, indexpath, index_keys, filter_by_keys=filter_by_keys)
+ index = open_fileindex(stream, indexpath, index_keys, ignore_keys=ignore_keys, filter_by_keys=filter_by_keys)
 
  return open_from_index(index, read_keys, time_dims, extra_coords, errors=errors, **kwargs)

cfgrib/xarray_plugin.py

@@ -99,6 +99,7 @@ def open_dataset(
  indexpath: str = messages.DEFAULT_INDEXPATH,
  filter_by_keys: T.Dict[str, T.Any] = {},
  read_keys: T.Iterable[str] = (),
+ ignore_keys: T.Iterable[str] = (),
  encode_cf: T.Sequence[str] = ("parameter", "time", "geography", "vertical"),
  squeeze: bool = True,
  time_dims: T.Iterable[str] = ("time", "step"),
@@ -111,6 +112,7 @@ def open_dataset(
  indexpath=indexpath,
  filter_by_keys=filter_by_keys,
  read_keys=read_keys,
+ ignore_keys=ignore_keys,
  encode_cf=encode_cf,
  squeeze=squeeze,
  time_dims=time_dims,

tests/test_30_dataset.py

@@ -131,6 +131,13 @@ def test_build_dataset_components_time_dims() -> None:
  assert dims == {"number": 28, "indexing_time": 2, "step": 20, "latitude": 6, "longitude": 11}
 
 
+def test_build_dataset_components_ignore_keys() -> None:
+ stream = messages.FileStream(TEST_DATA_UKMO, "warn")
+ index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH, dataset.INDEX_KEYS)
+ assert "subCentre" in index.index_keys
+ index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH, dataset.INDEX_KEYS, ignore_keys=["subCentre"])
+ assert "subCentre" not in index.index_keys
+
 def test_Dataset() -> None:
  res = dataset.open_file(TEST_DATA)
  assert "Conventions" in res.attributes
@@ -172,6 +179,14 @@ def test_Dataset_encode_cf_time() -> None:
  assert res.variables["t"].data[:, :, :, :].mean() > 0.0
 
 
+def test_Dataset_encode_ignore_keys() -> None:
+ res = dataset.open_file(TEST_DATA)
+ assert res.attributes["GRIB_edition"] == 1
+
+ res = dataset.open_file(TEST_DATA, ignore_keys=["edition"])
+ assert "GRIB_edition" not in res.attributes
+
+
 def test_Dataset_encode_cf_geography() -> None:
  res = dataset.open_file(TEST_DATA, encode_cf=("geography",))
  assert "history" in res.attributes
@@ -303,6 +318,27 @@ def test_open_fieldset_computed_keys() -> None:
  assert np.array_equal(res.variables["2t"].data[()], np.array(fieldset[0]["values"]))
 
 
+def test_open_fieldset_ignore_keys() -> None:
+ fieldset = {
+ -10: {
+ "gridType": "regular_ll",
+ "Nx": 2,
+ "Ny": 3,
+ "distinctLatitudes": [-10.0, 0.0, 10.0],
+ "distinctLongitudes": [0.0, 10.0],
+ "paramId": 167,
+ "shortName": "2t",
+ "subCentre": "test",
+ "values": [[1, 2], [3, 4], [5, 6]],
+ }
+ }
+
+ res = dataset.open_fieldset(fieldset)
+ assert "GRIB_subCentre" in res.attributes
+
+ res = dataset.open_fieldset(fieldset, ignore_keys="subCentre")
+ assert "GRIB_subCentre" not in res.attributes
+
 def test_open_file() -> None:
  res = dataset.open_file(TEST_DATA, filter_by_keys={"shortName": "t"})
 

tests/test_50_xarray_plugin.py

@@ -29,6 +29,13 @@ def test_xr_open_dataset_file() -> None:
  assert list(ds.data_vars) == ["skt"]
 
 
+def test_xr_open_dataset_file_ignore_keys() -> None:
+ ds = xr.open_dataset(TEST_DATA, engine="cfgrib")
+ assert "GRIB_typeOfLevel" in ds["skt"].attrs
+ ds = xr.open_dataset(TEST_DATA, engine="cfgrib", ignore_keys=["typeOfLevel"])
+ assert "GRIB_typeOfLevel" not in ds["skt"].attrs
+
+
 def test_xr_open_dataset_dict() -> None:
  fieldset = {
  -10: {
@@ -49,6 +56,26 @@ def test_xr_open_dataset_dict() -> None:
  assert list(ds.data_vars) == ["2t"]
 
 
+def test_xr_open_dataset_dict_ignore_keys() -> None:
+ fieldset = {
+ -10: {
+ "gridType": "regular_ll",
+ "Nx": 2,
+ "Ny": 3,
+ "distinctLatitudes": [-10.0, 0.0, 10.0],
+ "distinctLongitudes": [0.0, 10.0],
+ "paramId": 167,
+ "shortName": "2t",
+ "typeOfLevel": "surface",
+ "values": [[1, 2], [3, 4], [5, 6]],
+ }
+ }
+ ds = xr.open_dataset(fieldset, engine="cfgrib")
+ assert "GRIB_typeOfLevel" in ds["2t"].attrs
+ ds = xr.open_dataset(fieldset, engine="cfgrib", ignore_keys=["typeOfLevel"])
+ assert "GRIB_typeOfLevel" not in ds["2t"].attrs
+
+
 def test_xr_open_dataset_list() -> None:
  fieldset = [
  {
@@ -73,6 +100,27 @@ def test_xr_open_dataset_list() -> None:
  assert ds_empty.equals(xr.Dataset())
 
 
+def test_xr_open_dataset_list_ignore_keys() -> None:
+ fieldset = [
+ {
+ "gridType": "regular_ll",
+ "Nx": 2,
+ "Ny": 3,
+ "distinctLatitudes": [-10.0, 0.0, 10.0],
+ "distinctLongitudes": [0.0, 10.0],
+ "paramId": 167,
+ "shortName": "2t",
+ "typeOfLevel": "surface",
+ "values": [[1, 2], [3, 4], [5, 6]],
+ }
+ ]
+
+ ds = xr.open_dataset(fieldset, engine="cfgrib")
+ assert "GRIB_typeOfLevel" in ds["2t"].attrs
+ ds = xr.open_dataset(fieldset, engine="cfgrib", ignore_keys=["typeOfLevel"])
+ assert "GRIB_typeOfLevel" not in ds["2t"].attrs
+
+
 def test_read() -> None:
  expected = {
  "latitude": 37,