Skip to content

Simple support for selective netcdf loading. #4175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion lib/iris/fileformats/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,12 @@ def _load_grib(*args, **kwargs):
#
FORMAT_AGENT.add_spec(
FormatSpecification(
"NetCDF", MagicNumber(4), 0x43444601, netcdf.load_cubes, priority=5
"NetCDF",
MagicNumber(4),
0x43444601,
netcdf.load_cubes,
priority=5,
constraint_aware_handler=True,
)
)

Expand All @@ -107,6 +112,7 @@ def _load_grib(*args, **kwargs):
0x43444602,
netcdf.load_cubes,
priority=5,
constraint_aware_handler=True,
)
)

Expand All @@ -119,6 +125,7 @@ def _load_grib(*args, **kwargs):
0x894844460D0A1A0A,
netcdf.load_cubes,
priority=5,
constraint_aware_handler=True,
)
)

Expand All @@ -129,6 +136,7 @@ def _load_grib(*args, **kwargs):
lambda protocol: protocol in ["http", "https"],
netcdf.load_cubes,
priority=6,
constraint_aware_handler=True,
)
FORMAT_AGENT.add_spec(_nc_dap)
del _nc_dap
Expand Down
36 changes: 35 additions & 1 deletion lib/iris/fileformats/netcdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
OceanSg2Factory,
)
import iris.config
import iris._constraints
import iris.coord_systems
import iris.coords
import iris.cube
Expand Down Expand Up @@ -780,7 +781,33 @@ def coord_from_term(term):
cube.add_aux_factory(factory)


def load_cubes(filenames, callback=None):
def translate_constraints_to_var_callback(constraints):
"""
Translate load constraints into a simple data-var filter function, if possible.

Returns:
* function(cf_var:CFDataVariable): --> bool,
or None.

For now, ONLY handles a single NameConstraint with no 'STASH' component.

"""
constraints = iris._constraints.list_of_constraints(constraints)
result = None
if len(constraints) == 1:
(constraint,) = constraints
if (
isinstance(constraint, iris._constraints.NameConstraint)
and constraint.STASH == "none"
):
# As long as it doesn't use a STASH match, then we can treat it as
# a testing against name properties of cf_var , so we can call its
# 'cube+func' passing the cf_var, as if the cf_var were a cube.
result = constraint._cube_func
return result


def load_cubes(filenames, callback=None, constraints=None):
"""
Loads cubes from a list of NetCDF filenames/URLs.

Expand All @@ -798,6 +825,9 @@ def load_cubes(filenames, callback=None):
Generator of loaded NetCDF :class:`iris.cube.Cube`.

"""
# Create a low-level data-var filter from the original load constraints, if they are suitable.
var_callback = translate_constraints_to_var_callback(constraints)

# TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded
# into standard behaviour.
# Deferred import to avoid circular imports.
Expand Down Expand Up @@ -836,6 +866,10 @@ def load_cubes(filenames, callback=None):
cf.cf_group.promoted.values()
)
for cf_var in data_variables:
if var_callback and not var_callback(cf_var):
# Deliver only selected results.
continue

# cf_var-specific mesh handling, if a mesh is present.
# Build the mesh_coords *before* loading the cube - avoids
# mesh-related attributes being picked up by
Expand Down