diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a18ed7..6edfa93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,28 @@ +## v1.1.1 +### 2025-01-14 + +This version of HOSS merges the feature branch that contains V1.1.0 to the main branch. +Additional updates included code quality improvements with additional unit tests, revised methodology +in functions that selected the data points from the coordinate datasets and calculation of the dimension +arrays. Functions were added to determine the dimension order for 2D variables. + + +## v1.1.0 +### 2024-11-25 + +This version of HOSS provides support for gridded products that do not contain +CF-Convention compliant grid mapping variables and 1-D dimension variables, such +as SMAP L3. Functions updated to retrieve coordinate attributes and grid mappings, +using overrides specified in the hoss_config.json configuration file. +This implementation uses the latitude/longitude values across one row, and one +column to project it to the target grid using Proj and to compute the the X-Y +dimension arrays. Functions have been added to check any fill values present in +the coordinate variable data. Check for the dimension order for 2D datasets is +done using the latitude and longitude data varying across the row versus the +column. Support for multiple grids is handled by associating the group-name into +the cache-name for coordinates already processed for dimension ranges. +Several new functions related to this implementation have been added to +a new module `coordinate_utilities.py`. ## v1.0.5 ### 2024-08-19 diff --git a/docker/service_version.txt b/docker/service_version.txt index 90a27f9..524cb55 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.0.5 +1.1.1 diff --git a/docs/requirements.txt b/docs/requirements.txt index dd7a29c..700910c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -16,5 +16,5 @@ # harmony-py~=0.4.10 netCDF4~=1.6.4 -notebook~=7.0.4 +notebook~=7.2.2 xarray~=2023.9.0 diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py new file mode 100644 index 0000000..91a33e6 --- /dev/null +++ b/hoss/coordinate_utilities.py @@ -0,0 +1,407 @@ +""" This module contains utility functions used for + coordinate variables and functions to convert the + coordinate variable data to projected x/y dimension values +""" + +import numpy as np +from netCDF4 import Dataset +from pyproj import CRS, Transformer +from varinfo import VariableFromDmr, VarInfoFromDmr + +from hoss.exceptions import ( + IncompatibleCoordinateVariables, + InvalidCoordinateData, + InvalidCoordinateDataset, + MissingCoordinateVariable, + MissingVariable, + UnsupportedDimensionOrder, +) + + +def get_coordinate_variables( + varinfo: VarInfoFromDmr, + requested_variables: list[str], +) -> tuple[list[str], list[str]]: + """This function returns latitude and longitude variable names from + latitude and longitude variables listed in the CF-Convention coordinates + metadata attribute. It returns them in a specific + order [latitude_name, longitude_name]" + """ + + coordinate_variables = varinfo.get_references_for_attribute( + requested_variables, 'coordinates' + ) + + latitude_coordinate_variables = [ + coordinate + for coordinate in coordinate_variables + if varinfo.get_variable(coordinate) is not None + and varinfo.get_variable(coordinate).is_latitude() + ] + + longitude_coordinate_variables = [ + coordinate + for coordinate in coordinate_variables + if varinfo.get_variable(coordinate) is not None + and varinfo.get_variable(coordinate).is_longitude() + ] + + return latitude_coordinate_variables, longitude_coordinate_variables + + +def get_variables_with_anonymous_dims( + varinfo: VarInfoFromDmr, variables: set[str] +) -> set[str]: + """ + returns a set of variables without any dimensions + associated with it + """ + + return set( + variable + for variable in variables + if (len(varinfo.get_variable(variable).dimensions) == 0) + or (any_absent_dimension_variables(varinfo, variable)) + ) + + +def any_absent_dimension_variables(varinfo: VarInfoFromDmr, variable: str) -> bool: + """returns variable with fake dimensions - dimensions + that have been created by opendap, but are not really + dimension variables + """ + return any( + varinfo.get_variable(dimension) is None + for dimension in varinfo.get_variable(variable).dimensions + ) + + +def get_dimension_array_names_from_coordinate_variables( + varinfo: VarInfoFromDmr, + variable_name: str, +) -> list[str]: + """ + Returns the dimensions names from coordinate variables + """ + + latitude_coordinates, longitude_coordinates = get_coordinate_variables( + varinfo, [variable_name] + ) + + # for one variable, the coordinate array length will always be 1 or 0 + if len(latitude_coordinates) == 1 and len(longitude_coordinates) == 1: + dimension_array_names = get_dimension_array_names( + varinfo, latitude_coordinates[0] + ) + # if variable does not have coordinates (len = 0) + elif ( + varinfo.get_variable(variable_name).is_latitude() + or varinfo.get_variable(variable_name).is_longitude() + ): + dimension_array_names = get_dimension_array_names(varinfo, variable_name) + else: + dimension_array_names = [] + + return dimension_array_names + + +def get_dimension_array_names(varinfo: VarInfoFromDmr, variable_name: str) -> str: + """returns the x-y variable names that would + match the group of the input variable. The 'dim_y' dimension + and 'dim_x' names are returned with the group pathname + + """ + variable = varinfo.get_variable(variable_name) + + if variable is not None: + dimension_array_names = [ + f'{variable.group_path}/dim_y', + f'{variable.group_path}/dim_x', + ] + else: + raise MissingVariable(variable_name) + + return dimension_array_names + + +def create_dimension_arrays_from_coordinates( + prefetch_dataset: Dataset, + latitude_coordinate: VariableFromDmr, + longitude_coordinate: VariableFromDmr, + crs: CRS, + projected_dimension_names: list[str], +) -> dict[str, np.ndarray]: + """Generate artificial 1D dimensions scales for each + 2D dimension or coordinate variable. + 1) Get 2 valid geo grid points + 2) convert them to a projected x-y extent + 3) Generate the x-y dimscale array and return to the calling method + + """ + lat_arr = get_2d_coordinate_array( + prefetch_dataset, + latitude_coordinate.full_name_path, + ) + lon_arr = get_2d_coordinate_array( + prefetch_dataset, + longitude_coordinate.full_name_path, + ) + + row_indices, col_indices = get_valid_sample_pts( + lat_arr, lon_arr, latitude_coordinate, longitude_coordinate + ) + + dim_order_is_y_x, row_dim_values = get_dimension_order_and_dim_values( + lat_arr, lon_arr, row_indices, crs, is_row=True + ) + dim_order, col_dim_values = get_dimension_order_and_dim_values( + lat_arr, lon_arr, col_indices, crs, is_row=False + ) + if dim_order_is_y_x != dim_order: + raise InvalidCoordinateData("the order of dimensions do not match") + + row_size, col_size = get_row_col_sizes_from_coordinates( + lat_arr, lon_arr, dim_order_is_y_x + ) + + y_dim = interpolate_dim_values_from_sample_pts( + row_dim_values, np.transpose(row_indices)[0], row_size + ) + + x_dim = interpolate_dim_values_from_sample_pts( + col_dim_values, np.transpose(col_indices)[1], col_size + ) + + projected_y, projected_x = tuple(projected_dimension_names) + + if dim_order_is_y_x: + return {projected_y: y_dim, projected_x: x_dim} + raise UnsupportedDimensionOrder('x,y') + # this is not currently supported in the calling function in spatial.py + # return {projected_x: x_dim, projected_y: y_dim} + + +def get_2d_coordinate_array( + prefetch_dataset: Dataset, + coordinate_name: str, +) -> np.ndarray: + """This function returns the `numpy` array from a + coordinate dataset. + + """ + try: + coordinate_array = prefetch_dataset[coordinate_name][:] + except IndexError as exception: + raise MissingCoordinateVariable(coordinate_name) from exception + + return coordinate_array + + +def get_valid_sample_pts( + lat_arr: np.ndarray, + lon_arr: np.ndarray, + lat_coordinate: VariableFromDmr, + lon_coordinate: VariableFromDmr, +) -> tuple[list, list]: + """ + This function finds a set of indices maximally spread across + a row, and the set maximally spread across a column, with the + indices being valid in both the latitude and longitude datasets. + When interpolating between these points, the maximal spread + ensures the greatest interpolation accuracy. + """ + valid_lat_lon_mask = np.logical_and( + get_valid_indices(lat_arr, lat_coordinate), + get_valid_indices(lon_arr, lon_coordinate), + ) + + # get maximally spread points within rows + max_x_spread_pts = get_max_spread_pts(~valid_lat_lon_mask) + + # Doing the same for the columns is done by transposing the valid_mask + # and then fixing the results from [x, y] to [y, x]. + max_y_spread_trsp = get_max_spread_pts(np.transpose(~valid_lat_lon_mask)) + max_y_spread_pts = [ + list(np.flip(max_y_spread_trsp[0])), + list(np.flip(max_y_spread_trsp[1])), + ] + + return max_y_spread_pts, max_x_spread_pts + + +def get_valid_indices( + lat_lon_array: np.ndarray, coordinate: VariableFromDmr +) -> np.ndarray: + """ + Returns an array of boolean values indicating valid values - non-fill, + within range - for a given coordinate variable. Returns an empty + ndarray of size (0,0) for any other variable. + """ + + # get_attribute_value returns a value of type `str` + coordinate_fill = coordinate.get_attribute_value('_FillValue') + if coordinate_fill is not None: + is_not_fill = ~np.isclose(lat_lon_array, float(coordinate_fill)) + else: + # Creates an entire array of `True` values. + is_not_fill = np.ones_like(lat_lon_array, dtype=bool) + + if coordinate.is_longitude(): + valid_indices = np.logical_and( + is_not_fill, + np.logical_and(lat_lon_array >= -180.0, lat_lon_array <= 360.0), + ) + elif coordinate.is_latitude(): + valid_indices = np.logical_and( + is_not_fill, + np.logical_and(lat_lon_array >= -90.0, lat_lon_array <= 90.0), + ) + else: + raise InvalidCoordinateDataset(coordinate.full_name_path) + + return valid_indices + + +def get_max_spread_pts( + valid_geospatial_mask: np.ndarray, +) -> list[list]: + """ + This function returns two data points by x, y indices that are spread farthest + from each other in the same row, i.e., have the greatest delta-x value - and + are valid data points from the valid_geospatial_mask array passed in. The input array + must be a 2D Numpy mask array providing the valid data points, e.g., filtering + out fill values and out-of-range values. + - input is Numpy Mask Array, e.g., invalid latitudes & longitudes + - returns 2 points by indices, [[y_ind, x_ind], [y_ind, x_ind] + """ + # fill a sample array with index values, arr_ind[i, j] = j + arr_indices = np.indices( + (valid_geospatial_mask.shape[-2], valid_geospatial_mask.shape[-1]) + )[1] + if valid_geospatial_mask.ndim == 2: + # mask arr_ind to hide the invalid data points + valid_indices = np.ma.array(arr_indices, mask=valid_geospatial_mask) + elif valid_geospatial_mask.ndim == 3: + # use just 2 of the dimensions + # mask arr_ind to hide the invalid data points + valid_indices = np.ma.array(arr_indices, mask=valid_geospatial_mask[0, :, :]) + else: + raise NotImplementedError + + if valid_indices.count() == 0: + raise InvalidCoordinateData("No valid coordinate data") + + # ptp (peak-to-peak) finds the greatest delta-x value amongst valid points + # for each row. Result is 1D + index_spread = valid_indices.ptp(axis=1) + + # This finds which row has the greatest spread (delta-x) + max_spread = np.argmax(index_spread) + + # Using the row reference, find the min and max + min_index = np.min(valid_indices[max_spread]) + max_index = np.max(valid_indices[max_spread]) + + # There is just one valid point + if min_index == max_index: + raise InvalidCoordinateData("Only one valid point in coordinate data") + + return [[max_spread, min_index], [max_spread, max_index]] + + +def get_dimension_order_and_dim_values( + lat_array_points: np.ndarray, + lon_array_points: np.ndarray, + grid_dimension_indices: list[tuple[int, int]], + crs: CRS, + is_row: bool, +) -> tuple[bool, np.ndarray]: + """Determines the order of dimensions based on whether the + projected y or projected_x values are varying across row or column. + Also returns a 1-D array of dimension values for the requested + projected spatial dimension. The input lat lon arrays and dimension + indices are assumed to be 2D in this implementation of the function. + """ + lat_arr_values = [lat_array_points[i][j] for i, j in grid_dimension_indices] + lon_arr_values = [lon_array_points[i][j] for i, j in grid_dimension_indices] + + from_geo_transformer = Transformer.from_crs(4326, crs) + x_values, y_values = ( # pylint: disable=unpacking-non-sequence + from_geo_transformer.transform(lat_arr_values, lon_arr_values) + ) + y_variance = np.abs(np.diff(y_values)) + x_variance = np.abs(np.diff(x_values)) + + # If input lat/lon array is row and projected y_values is varying more + # than projected x_values, then the dimensions are ordered (y,x) + # If input lat/lon array is col and projected y_values is changing more + # than projected x_values, then the dimensions are ordered (x,y) + + if y_variance > x_variance: + is_y_x_order = is_row + dimension_values = y_values + elif x_variance > y_variance: + is_y_x_order = not is_row + dimension_values = x_values + else: + raise InvalidCoordinateData("x/y values are varying by the same amount") + + return is_y_x_order, dimension_values + + +def get_row_col_sizes_from_coordinates( + lat_arr: np.ndarray, lon_arr: np.ndarray, dim_order_is_y_x: bool +) -> tuple[int, int]: + """ + This function returns the row and column sizes of the coordinate datasets + The last two dimensions of the array correspond to the spatial dimensions. + which is the recommendation from CF-Conventions. + """ + + if lat_arr.ndim >= 2 and lon_arr.shape == lat_arr.shape: + col_size = lat_arr.shape[-1] + row_size = lat_arr.shape[-2] + elif ( + lat_arr.ndim == 1 + and lon_arr.ndim == 1 + and lat_arr.size > 0 + and lon_arr.size > 0 + ): + if dim_order_is_y_x: + col_size = lon_arr.size + row_size = lat_arr.size + else: + col_size = lat_arr.size + row_size = lon_arr.size + else: + raise IncompatibleCoordinateVariables(lon_arr.shape, lat_arr.shape) + return row_size, col_size + + +def interpolate_dim_values_from_sample_pts( + dim_values: np.ndarray, + dim_indices: list[int], + dim_size: int, +) -> np.ndarray: + """ + Return a full dimension data array based upon 2 valid projected values + given in dim_values and located by dim_indices. The dim_indices need + to be between 0 and dim_size. Returns a 1D array of size = dim_size + with proper dimension array values, with linear interpolation between + the given dim_values. + """ + + if (dim_indices[1] != dim_indices[0]) and (dim_values[1] != dim_values[0]): + dim_resolution = (dim_values[1] - dim_values[0]) / ( + dim_indices[1] - dim_indices[0] + ) + else: + raise InvalidCoordinateData( + 'No distinct valid coordinate points - ' + f'dim_index={dim_indices[0]}, dim_value={dim_values[0]}' + ) + + dim_min = dim_values[0] - (dim_resolution * dim_indices[0]) + dim_max = dim_values[1] + (dim_resolution * (dim_size - 1 - dim_indices[1])) + + return np.linspace(dim_min, dim_max, dim_size) diff --git a/hoss/dimension_utilities.py b/hoss/dimension_utilities.py index 9ee8d02..15eb722 100644 --- a/hoss/dimension_utilities.py +++ b/hoss/dimension_utilities.py @@ -22,8 +22,15 @@ from numpy.ma.core import MaskedArray from varinfo import VariableFromDmr, VarInfoFromDmr -from hoss.bbox_utilities import flatten_list -from hoss.exceptions import InvalidNamedDimension, InvalidRequestedRange +from hoss.coordinate_utilities import ( + get_coordinate_variables, + get_dimension_array_names_from_coordinate_variables, +) +from hoss.exceptions import ( + InvalidIndexSubsetRequest, + InvalidNamedDimension, + InvalidRequestedRange, +) from hoss.utilities import ( format_variable_set_string, get_opendap_nc4, @@ -55,7 +62,7 @@ def is_index_subset(message: Message) -> bool: ) -def prefetch_dimension_variables( +def get_prefetch_variables( opendap_url: str, varinfo: VarInfoFromDmr, required_variables: Set[str], @@ -64,48 +71,51 @@ def prefetch_dimension_variables( access_token: str, config: Config, ) -> str: - """Determine the dimensions that need to be "pre-fetched" from OPeNDAP in + """Determine the variables that need to be "pre-fetched" from OPeNDAP in order to derive index ranges upon them. Initially, this was just spatial and temporal dimensions, but to support generic dimension subsets, all required dimensions must be prefetched, along with any associated bounds variables referred to via the "bounds" metadata - attribute. - + attribute. In cases where dimension variables do not exist, coordinate + variables will be prefetched and used to calculate dimension-scale values. + If there are no prefetch variables, the function will raise an + InvalidIndexSubsetRequest exception. """ - required_dimensions = varinfo.get_required_dimensions(required_variables) - - # Iterate through all requested dimensions and extract a list of bounds - # references for each that has any. This will produce a list of lists, - # which should be flattened into a single list and then combined into a set - # to remove duplicates. - bounds = set( - flatten_list( - [ - list(varinfo.get_variable(dimension).references.get('bounds')) - for dimension in required_dimensions - if varinfo.get_variable(dimension).references.get('bounds') is not None - ] + prefetch_variables = varinfo.get_required_dimensions(required_variables) + if prefetch_variables: + prefetch_variables.update( + varinfo.get_references_for_attribute(prefetch_variables, 'bounds') + ) + else: + latitude_coordinates, longitude_coordinates = get_coordinate_variables( + varinfo, required_variables ) - ) - required_dimensions.update(bounds) + if latitude_coordinates and longitude_coordinates: + prefetch_variables = set(latitude_coordinates + longitude_coordinates) + + if not prefetch_variables: + raise InvalidIndexSubsetRequest( + "No dimensions or coordinates exist for the requested variables" + ) logger.info( 'Variables being retrieved in prefetch request: ' - f'{format_variable_set_string(required_dimensions)}' + f'{format_variable_set_string(prefetch_variables)}' ) - required_dimensions_nc4 = get_opendap_nc4( - opendap_url, required_dimensions, output_dir, logger, access_token, config + prefetch_variables_nc4 = get_opendap_nc4( + opendap_url, prefetch_variables, output_dir, logger, access_token, config ) # Create bounds variables if necessary. - add_bounds_variables(required_dimensions_nc4, required_dimensions, varinfo, logger) - - return required_dimensions_nc4 + check_add_artificial_bounds( + prefetch_variables_nc4, prefetch_variables, varinfo, logger + ) + return prefetch_variables_nc4 -def add_bounds_variables( +def check_add_artificial_bounds( dimensions_nc4: str, required_dimensions: Set[str], varinfo: VarInfoFromDmr, @@ -409,7 +419,9 @@ def get_dimension_indices_from_bounds( def add_index_range( - variable_name: str, varinfo: VarInfoFromDmr, index_ranges: IndexRanges + variable_name: str, + varinfo: VarInfoFromDmr, + index_ranges: IndexRanges, ) -> str: """Append the index ranges of each dimension for the specified variable. If there are no dimensions with listed index ranges, then the full @@ -418,29 +430,54 @@ def add_index_range( the antimeridian or Prime Meridian) will have a minimum index greater than the maximum index. In this case the full dimension range should be requested, as the related values will be masked before returning the - output to the user. + output to the user. When a variable does not have named dimensions, + the index_ranges cache is checked for dimensions derived from the + coordinates CF-Conventions metadata attribute. """ variable = varinfo.get_variable(variable_name) - range_strings = [] - for dimension in variable.dimensions: - dimension_range = index_ranges.get(dimension) + if variable.dimensions: + variable_dimensions = variable.dimensions + else: + # Anonymous dimensions, so check for dimension derived from coordinates: + variable_dimensions = get_dimension_array_names_from_coordinate_variables( + varinfo, variable_name + ) - if dimension_range is not None and dimension_range[0] <= dimension_range[1]: - range_strings.append(f'[{dimension_range[0]}:{dimension_range[1]}]') - else: - range_strings.append('[]') + range_strings = get_range_strings(variable_dimensions, index_ranges) if all(range_string == '[]' for range_string in range_strings): indices_string = '' else: indices_string = ''.join(range_strings) - return f'{variable_name}{indices_string}' +def get_range_strings( + variable_dimensions: list, + index_ranges: IndexRanges, +) -> list: + """Retrieves index ranges which is a list of string elements + [min:max] from cache. If there is not an index range in the + cache for a dimension, the returned string is []. A bounding box + can cross the longitudinal edge of the grid. In those cases the + minimum dimension index is greater than the maximum dimension + index and this function will return []. HOSS will request the + full dimension range from OPeNDAP when the index range is []. + """ + range_strings = [] + for dimension in variable_dimensions: + dimension_range = index_ranges.get(dimension) + if dimension_range is not None and dimension_range[0] <= dimension_range[1]: + range_strings.append(f'[{dimension_range[0]}:{dimension_range[1]}]') + else: + range_strings.append('[]') + + return range_strings + + def get_fill_slice(dimension: str, fill_ranges: IndexRanges) -> slice: """Check the dictionary of dimensions that need to be filled for the given dimension. If present, the minimum index will be greater than the @@ -549,6 +586,7 @@ def get_dimension_bounds( be returned. """ + bounds = varinfo.get_variable(dimension_name).references.get('bounds') if bounds is not None: diff --git a/hoss/exceptions.py b/hoss/exceptions.py index 1cb1439..148bf4e 100644 --- a/hoss/exceptions.py +++ b/hoss/exceptions.py @@ -57,7 +57,7 @@ class InvalidRequestedRange(CustomError): def __init__(self): super().__init__( 'InvalidRequestedRange', - 'Input request specified range outside supported ' 'dimension range', + 'Input request specified range outside supported dimension range', ) @@ -108,6 +108,119 @@ def __init__(self): ) +class MissingVariable(CustomError): + """This exception is raised when HOSS tries to get variables and + they are missing or empty. + + """ + + def __init__(self, referring_variable): + super().__init__( + 'MissingVariable', + f'"{referring_variable}" is ' 'not present in source granule file.', + ) + + +class MissingCoordinateVariable(CustomError): + """This exception is raised when HOSS tries to get latitude and longitude + variables and they are missing or empty. These variables are referred to + in the science variables with coordinate attributes. + + """ + + def __init__(self, referring_variable): + super().__init__( + 'MissingCoordinateVariable', + f'Coordinate: "{referring_variable}" is ' + 'not present in source granule file.', + ) + + +class InvalidIndexSubsetRequest(CustomError): + """This exception is raised when HOSS tries to get dimensions or + coordinate variables as part of a prefetch from opendap when there is + a spatial or temporal request, and there are no prefetch variables + returned. + + """ + + def __init__(self, custom_msg): + super().__init__( + 'InvalidIndexSubsetRequest', + custom_msg, + ) + + +class InvalidCoordinateVariable(CustomError): + """This exception is raised when HOSS tries to get latitude and longitude + variables and they have fill values to the extent that it cannot be used. + These variables are referred in the science variables with coordinate attributes. + + """ + + def __init__(self, referring_variable): + super().__init__( + 'InvalidCoordinateVariable', + f'Coordinate: "{referring_variable}" is ' + 'not valid in source granule file.', + ) + + +class IncompatibleCoordinateVariables(CustomError): + """This exception is raised when HOSS tries to get latitude and longitude + coordinate variable and they do not match in shape or have a size of 0. + + """ + + def __init__(self, longitude_shape, latitude_shape): + super().__init__( + 'IncompatibleCoordinateVariables', + f'Longitude coordinate shape: "{longitude_shape}"' + f'does not match the latitude coordinate shape: "{latitude_shape}"', + ) + + +class InvalidCoordinateData(CustomError): + """This exception is raised when the data does not contain at least. + two valid points. This could occur when there are too many fill values and distinct valid + indices could not be obtained + + """ + + def __init__(self, custom_msg): + super().__init__( + 'InvalidCoordinateData', + f'{custom_msg}', + ) + + +class InvalidCoordinateDataset(CustomError): + """This exception is raised when there are too + many fill values and two distinct valid indices + could not be obtained + + """ + + def __init__(self, coordinate_name): + super().__init__( + 'InvalidCoordinateDataset', + f'Cannot get valid indices for {coordinate_name}', + ) + + +class UnsupportedDimensionOrder(CustomError): + """This exception is raised when the granule file included in the input + request is not the nominal dimension order which is 'y,x'. + + """ + + def __init__(self, dimension_order: str): + super().__init__( + 'UnsupportedDimensionOrder', + f'Dimension Order "{dimension_order}" not ' 'supported.', + ) + + class UnsupportedShapeFileFormat(CustomError): """This exception is raised when the shape file included in the input Harmony message is not GeoJSON. diff --git a/hoss/hoss_config.json b/hoss/hoss_config.json index c214d6b..c289aec 100644 --- a/hoss/hoss_config.json +++ b/hoss/hoss_config.json @@ -59,24 +59,6 @@ "Epoch": "2018-01-01T00:00:00.000000" } ], - "Grid_Mapping_Data": [ - { - "Grid_Mapping_Dataset_Name": "EASE2_Global", - "grid_mapping_name": "lambert_cylindrical_equal_area", - "standard_parallel": 30.0, - "longitude_of_central_meridian": 0.0, - "false_easting": 0.0, - "false_northing": 0.0 - }, - { - "Grid_Mapping_Dataset_Name": "EASE2_Polar", - "grid_mapping_name": "lambert_azimuthal_equal_area", - "longitude_of_projection_origin": 0.0, - "latitude_of_projection_origin": 90.0, - "false_easting": 0.0, - "false_northing": 0.0 - } - ], "CF_Overrides": [ { "Applicability": { @@ -161,34 +143,58 @@ { "Applicability": { "Mission": "SMAP", - "ShortNamePath": "SPL3FT(P|P_E)" + "ShortNamePath": "SPL3FT(P|P_E)", + "Variable_Pattern": "(?i).*global.*" }, - "Applicability_Group": [ + "Attributes": [ { - "Applicability": { - "Variable_Pattern": "(?i).*global.*" - }, - "Attributes": [ - { - "Name": "Grid_Mapping", - "Value": "EASE2_Global" - } - ], - "_Description": "Some versions of these collections omit global grid mapping information" - }, + "Name": "grid_mapping", + "Value": "/EASE2_global_projection" + } + ], + "_Description": "SMAP L3 collections omit global grid mapping information" + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3FT(P|P_E)", + "Variable_Pattern": "(?i).*polar.*" + }, + "Attributes": [ { - "Applicability": { - "Variable_Pattern": "(?i).*polar.*" - }, - "Attributes": [ - { - "Name": "Grid_Mapping", - "Value": "EASE2_Polar" - } - ], - "_Description": "Some versions of these collections omit polar grid mapping information" + "Name": "grid_mapping", + "Value": "/EASE2_polar_projection" } - ] + ], + "_Description": "SMAP L3 collections omit polar grid mapping information" + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP_E", + "Variable_Pattern": "Soil_Moisture_Retrieval_Data_(A|P)M/.*" + }, + "Attributes": [ + { + "Name": "grid_mapping", + "Value": "/EASE2_global_projection" + } + ], + "_Description": "SMAP L3 collections omit global grid mapping information" + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP_E", + "Variable_Pattern": "Soil_Moisture_Retrieval_Data_Polar_(A|P)M/.*" + }, + "Attributes": [ + { + "Name": "grid_mapping", + "Value": "/EASE2_polar_projection" + } + ], + "_Description": "SMAP L3 collections omit polar grid mapping information" }, { "Applicability": { @@ -197,11 +203,84 @@ }, "Attributes": [ { - "Name": "Grid_Mapping", - "Value": "EASE2_Polar" + "Name": "grid_mapping", + "Value": "/EASE2_polar_projection" } ], - "_Description": "Some versions of these collections omit polar grid mapping information" + "_Description": "SMAP L3 collections omit polar grid mapping information" + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SM(P|A|AP)|SPL2SMAP_S" + }, + "Attributes": [ + { + "Name": "grid_mapping", + "Value": "/EASE2_global_projection" + } + ], + "_Description": "SMAP L3 collections omit global grid mapping information" + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3FT(P|P_E)|SPL3SM(P|P_E|A|AP)|SPL2SMAP_S", + "Variable_Pattern": "/EASE2_global_projection" + }, + "Attributes": [ + { + "Name": "grid_mapping_name", + "Value": "lambert_cylindrical_equal_area" + }, + { + "Name":"standard_parallel", + "Value": 30.0 + }, + { + "Name": "longitude_of_central_meridian", + "Value": 0.0 + }, + { + "Name": "false_easting", + "Value": 0.0 + }, + { + "Name": "false_northing", + "Value": 0.0 + } + ], + "_Description": "Provide missing global grid mapping attributes for SMAP L3 collections." + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3FT(A|P|P_E)|SPL3SM(P|P_E|A|AP)|SPL2SMAP_S", + "Variable_Pattern": "/EASE2_polar_projection" + }, + "Attributes": [ + { + "Name": "grid_mapping_name", + "Value": "lambert_azimuthal_equal_area" + }, + { + "Name": "longitude_of_projection_origin", + "Value" : 0.0 + }, + { + "Name": "latitude_of_projection_origin", + "Value": 90.0 + }, + { + "Name": "false_easting", + "Value": 0.0 + }, + { + "Name": "false_northing", + "Value": 0.0 + } + ], + "_Description": "Provide missing polar grid mapping attributes for SMAP L3 collections." }, { "Applicability": { @@ -211,12 +290,40 @@ }, "Attributes": [ { - "Name": "_fill", - "Value": "-9999" + "Name": "_FillValue", + "Value": "-9999.0" } ], "_Description": "Ensure metadata fill value matches what is present in arrays." }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SM(A|P|AP|P_E)", + "Variable_Pattern": "/Soil_Moisture_Retrieval_(Data|Data_AM|Data_Polar_AM)/(latitude|longitude).*" + }, + "Attributes": [ + { + "Name": "_FillValue", + "Value": "-9999.0" + } + ], + "_Description": "Ensure metadata fill value matches what is present in arrays." + }, + { + "Applicability": { + "Mission": "SMAP", + "ShortNamePath": "SPL3SMP", + "Variable_Pattern": "/Soil_Moisture_Retrieval_Data_PM/.*" + }, + "Attributes": [ + { + "Name": "coordinates", + "Value": "/Soil_Moisture_Retrieval_Data_PM/latitude_pm, /Soil_Moisture_Retrieval_Data_PM/longitude_pm" + } + ], + "_Description": "Ensure variables in /Soil_Moisture_Retrieval_Data_PM group point to correct coordinate variables." + }, { "Applicability": { "Mission": "SMAP", diff --git a/hoss/projection_utilities.py b/hoss/projection_utilities.py index bdacdc0..cc21f9a 100644 --- a/hoss/projection_utilities.py +++ b/hoss/projection_utilities.py @@ -49,6 +49,11 @@ def get_variable_crs(variable: str, varinfo: VarInfoFromDmr) -> CRS: another are stored in the `Variable.references` dictionary attribute as sets. There should only be one reference in the `grid_mapping` attribute value, so the first element of the set is retrieved. + If the grid mapping variable, as referred to in the grid_mapping + CF-Convention metadata attribute, does not exist in the file then + the earthdata-varinfo configuration file is checked, as it may + contain metadata overrides specified for that non-existent variable + name. """ grid_mapping = next( @@ -57,9 +62,21 @@ def get_variable_crs(variable: str, varinfo: VarInfoFromDmr) -> CRS: if grid_mapping is not None: try: - crs = CRS.from_cf(varinfo.get_variable(grid_mapping).attributes) + grid_mapping_variable = varinfo.get_variable(grid_mapping) + if grid_mapping_variable is not None: + cf_attributes = grid_mapping_variable.attributes + else: + # check for any overrides + cf_attributes = varinfo.get_missing_variable_attributes(grid_mapping) + + if cf_attributes: + crs = CRS.from_cf(cf_attributes) + else: + raise MissingGridMappingVariable(grid_mapping, variable) + except AttributeError as exception: raise MissingGridMappingVariable(grid_mapping, variable) from exception + else: raise MissingGridMappingMetadata(variable) diff --git a/hoss/spatial.py b/hoss/spatial.py index 79eb5c2..cc8f163 100644 --- a/hoss/spatial.py +++ b/hoss/spatial.py @@ -27,7 +27,7 @@ from harmony.message import Message from netCDF4 import Dataset from numpy.ma.core import MaskedArray -from varinfo import VarInfoFromDmr +from varinfo import VariableFromDmr, VarInfoFromDmr from hoss.bbox_utilities import ( BBox, @@ -35,6 +35,12 @@ get_harmony_message_bbox, get_shape_file_geojson, ) +from hoss.coordinate_utilities import ( + create_dimension_arrays_from_coordinates, + get_coordinate_variables, + get_dimension_array_names_from_coordinate_variables, + get_variables_with_anonymous_dims, +) from hoss.dimension_utilities import ( IndexRange, IndexRanges, @@ -78,6 +84,10 @@ def get_spatial_index_ranges( around the exterior of the user-defined GeoJSON shape, to ensure the correct extents are derived. + If geographic and projected dimensions are not specified in the granule, + the coordinate datasets are used to calculate the x-y dimensions and the index ranges + are calculated similar to a projected grid. + """ bounding_box = get_harmony_message_bbox(harmony_message) index_ranges = {} @@ -91,7 +101,7 @@ def get_spatial_index_ranges( ) with Dataset(dimensions_path, 'r') as dimensions_file: - if len(geographic_dimensions) > 0: + if geographic_dimensions: # If there is no bounding box, but there is a shape file, calculate # a bounding box to encapsulate the GeoJSON shape: if bounding_box is None and shape_file_path is not None: @@ -103,7 +113,7 @@ def get_spatial_index_ranges( dimension, varinfo, dimensions_file, bounding_box ) - if len(projected_dimensions) > 0: + if projected_dimensions: for non_spatial_variable in non_spatial_variables: index_ranges.update( get_projected_x_y_index_ranges( @@ -115,6 +125,26 @@ def get_spatial_index_ranges( shape_file_path=shape_file_path, ) ) + variables_with_anonymous_dims = get_variables_with_anonymous_dims( + varinfo, required_variables + ) + for variable_with_anonymous_dims in variables_with_anonymous_dims: + latitude_coordinates, longitude_coordinates = get_coordinate_variables( + varinfo, [variable_with_anonymous_dims] + ) + if latitude_coordinates and longitude_coordinates: + index_ranges.update( + get_x_y_index_ranges_from_coordinates( + variable_with_anonymous_dims, + varinfo, + dimensions_file, + varinfo.get_variable(latitude_coordinates[0]), + varinfo.get_variable(longitude_coordinates[0]), + index_ranges, + bounding_box=bounding_box, + shape_file_path=shape_file_path, + ) + ) return index_ranges @@ -187,6 +217,79 @@ def get_projected_x_y_index_ranges( return x_y_index_ranges +def get_x_y_index_ranges_from_coordinates( + non_spatial_variable: str, + varinfo: VarInfoFromDmr, + prefetch_coordinate_datasets: Dataset, + latitude_coordinate: VariableFromDmr, + longitude_coordinate: VariableFromDmr, + index_ranges: IndexRanges, + bounding_box: BBox = None, + shape_file_path: str = None, +) -> IndexRanges: + """This function returns a dictionary containing the minimum and maximum + index ranges for the projected_x and projected_y recalculated dimension scales + + index_ranges = {'projected_x': (20, 42), 'projected_y': (31, 53)} + + This method is called when the CF standards are not followed + in the source granule and only coordinate datasets are provided. + The coordinate datasets along with the crs is used to calculate + the x-y projected dimension scales. The dimensions of the input, + non-spatial variable are checked for associated coordinates. If + these are present, and they have not already been added to the + `index_ranges` cache, the extents of the input spatial subset + are determined in these projected coordinates. The minimum and + maximum values are then derived from these projected coordinate + points. + + """ + + crs = get_variable_crs(non_spatial_variable, varinfo) + + projected_dimension_names = get_dimension_array_names_from_coordinate_variables( + varinfo, non_spatial_variable + ) + + dimension_arrays = create_dimension_arrays_from_coordinates( + prefetch_coordinate_datasets, + latitude_coordinate, + longitude_coordinate, + crs, + projected_dimension_names, + ) + + projected_y, projected_x = dimension_arrays.keys() + + if not set((projected_x, projected_y)).issubset(set(index_ranges.keys())): + + x_y_extents = get_projected_x_y_extents( + dimension_arrays[projected_x][:], + dimension_arrays[projected_y][:], + crs, + shape_file=shape_file_path, + bounding_box=bounding_box, + ) + + x_index_ranges = get_dimension_index_range( + dimension_arrays[projected_x][:], + x_y_extents['x_min'], + x_y_extents['x_max'], + bounds_values=None, + ) + y_index_ranges = get_dimension_index_range( + dimension_arrays[projected_y][:], + x_y_extents['y_min'], + x_y_extents['y_max'], + bounds_values=None, + ) + x_y_index_ranges = {projected_x: x_index_ranges, projected_y: y_index_ranges} + else: + x_y_index_ranges = {} + + return x_y_index_ranges + + def get_geographic_index_range( dimension: str, varinfo: VarInfoFromDmr, diff --git a/hoss/subset.py b/hoss/subset.py index 8a01321..a08e590 100644 --- a/hoss/subset.py +++ b/hoss/subset.py @@ -21,13 +21,17 @@ IndexRanges, add_index_range, get_fill_slice, + get_prefetch_variables, get_requested_index_ranges, is_index_subset, - prefetch_dimension_variables, ) from hoss.spatial import get_spatial_index_ranges from hoss.temporal import get_temporal_index_ranges -from hoss.utilities import download_url, format_variable_set_string, get_opendap_nc4 +from hoss.utilities import ( + download_url, + format_variable_set_string, + get_opendap_nc4, +) def subset_granule( @@ -87,7 +91,7 @@ def subset_granule( if request_is_index_subset: # Prefetch all dimension variables in full: - dimensions_path = prefetch_dimension_variables( + dimensions_path = get_prefetch_variables( opendap_url, varinfo, required_variables, @@ -122,6 +126,7 @@ def subset_granule( shape_file_path = get_request_shape_file( harmony_message, output_dir, logger, config ) + index_ranges.update( get_spatial_index_ranges( required_variables, diff --git a/pip_requirements.txt b/pip_requirements.txt index 41ba9c5..1bb9bce 100644 --- a/pip_requirements.txt +++ b/pip_requirements.txt @@ -1,6 +1,6 @@ # This file should contain requirements to be installed via Pip. # Open source packages available from PyPI -earthdata-varinfo ~= 1.0.0 +earthdata-varinfo ~= 2.3.0 harmony-service-lib ~= 1.0.25 netCDF4 ~= 1.6.4 numpy ~= 1.24.2 diff --git a/tests/data/SC_SPL3SMP_008.dmr b/tests/data/SC_SPL3SMP_008.dmr new file mode 100644 index 0000000..b45abae --- /dev/null +++ b/tests/data/SC_SPL3SMP_008.dmr @@ -0,0 +1,2778 @@ + + + + 3.21.0-428 + + + 3.21.0-428 + + + libdap-3.21.0-103 + + + +# TheBESKeys::get_as_config() +AllowedHosts=^https?:\/\/ +BES.Catalog.catalog.FollowSymLinks=Yes +BES.Catalog.catalog.RootDirectory=/usr/share/hyrax +BES.Catalog.catalog.TypeMatch=dmrpp:.*\.(dmrpp)$; +BES.Catalog.catalog.TypeMatch+=h5:.*(\.bz2|\.gz|\.Z)?$; +BES.Data.RootDirectory=/dev/null +BES.LogName=./bes.log +BES.UncompressCache.dir=/tmp/hyrax_ux +BES.UncompressCache.prefix=ux_ +BES.UncompressCache.size=500 +BES.module.cmd=/usr/lib64/bes/libdap_xml_module.so +BES.module.dap=/usr/lib64/bes/libdap_module.so +BES.module.dmrpp=/usr/lib64/bes/libdmrpp_module.so +BES.module.h5=/usr/lib64/bes/libhdf5_module.so +BES.modules=dap,cmd,h5,dmrpp +H5.DefaultHandleDimensions=true +H5.EnableCF=false +H5.EnableCheckNameClashing=true + + + + build_dmrpp -c /tmp/bes_conf_IAud -f /usr/share/hyrax/DATA/SMAP_L3_SM_P_20150331_R18290_002.h5 -r /tmp/dmr__0PmwFd -u OPeNDAP_DMRpp_DATA_ACCESS_URL -M + + + + + + + The SMAP observatory houses an L-band radiometer that operates at 1.414 GHz and an L-band radar that operates at 1.225 GHz. The instruments share a rotating reflector antenna with a 6 meter aperture that scans over a 1000 km swath. The bus is a 3 axis stabilized spacecraft that provides momentum compensation for the rotating antenna. + + + 14.60000038 + + + SMAP + + + + + JPL CL#14-2285, JPL 400-1567 + + + SMAP Handbook + + + 2014-07-01 + + + + + JPL CL#14-2285, JPL 400-1567 + + + SMAP Handbook + + + 2014-07-01 + + + + + The SMAP 1.414 GHz L-Band Radiometer + + + L-Band Radiometer + + + SMAP RAD + + + + + The SMAP 1.225 GHz L-Band Radar Instrument + + + L-Band Synthetic Aperture Radar + + + SMAP SAR + + + + + JPL CL#14-2285, JPL 400-1567 + + + SMAP Handbook + + + 2014-07-01 + + + + + + soil_moisture + + + + Percentage of EASE2 grid cells with Retrieved Soil Moistures outside the Acceptable Range. + + + Percentage of EASE2 grid cells with soil moisture measures that fall outside of a predefined acceptable range. + + + directInternal + + + percent + + + 100. + + + + + Percentage of EASE2 grid cells that lack soil moisture retrieval values relative to the total number of grid cells where soil moisture retrieval was attempted. + + + Percent of Missing Data + + + percent + + + 176.5011139 + + + directInternal + + + + + + eng + + + utf8 + + + 1.0 + + + Product Specification Document for the SMAP Level 3 Passive Soil Moisture Product (L3_SM_P) + + + 2013-02-08 + + + L3_SM_P + + + + + doi:10.5067/OMHVSRGFX38O + + + SPL3SMP + + + SMAP + + + utf8 + + + National Aeronautics and Space Administration (NASA) + + + eng + + + 008 + + + Daily global composite of up-to 30 half-orbit L2_SM_P soil moisture estimates based on radiometer brightness temperature measurements acquired by the SMAP radiometer during ascending and descending half-orbits at approximately 6 PM and 6 AM local solar time. + + + onGoing + + + 2021-08-31 + + + The software that generates the Level 3 Soil Moisture Passive product and the data system that automates its production were designed and implemented + at the Jet Propulsion Laboratory, California Institute of Technology in Pasadena, California. + + + geoscientificInformation + + + The SMAP L3_SM_P algorithm provides daily global composite of soil moistures based on radiometer data on a 36 km grid. + + + SMAP L3 Radiometer Global Daily 36 km EASE-Grid Soil Moisture + + + National Snow and Ice Data Center + + + R18 + + + grid + + + The Calibration and Validation Version 2 Release of the SMAP Level 3 Daily Global Composite Passive Soil Moisture Science Processing Software. + + + + + SPL3SMP + + + utf8 + + + be9694f7-6503-4c42-8423-4c824df6c1f2 + + + eng + + + 1.8.13 + + + 008 + + + Daily global composite of up-to 15 half-orbit L2_SM_P soil moisture estimates based on radiometer brightness temperature measurements acquired by the SMAP radiometer during descending half-orbits at approximately 6 AM local solar time. + + + 2022-03-11 + + + onGoing + + + The software that generates the Level 3 SM_P product and the data system that automates its production were designed and implemented at the Jet Propulsion Laboratory, California Institute of Technology in Pasadena, California. + + + geoscientificInformation + + + The SMAP L3_SM_P effort provides soil moistures based on radiometer data on a 36 km grid. + + + HDF5 + + + SMAP_L3_SM_P_20150331_R18290_002.h5 + + + asNeeded + + + R18290 + + + Jet Propulsion Laboratory + + + 2016-05-01 + + + L3_SM_P + + + grid + + + The Calibration and Validation Version 2 Release of the SMAP Level 3 Daily Global Composite Passive Soil Moisture Science Processing Software. + + + + + Soil moisture is retrieved over land targets on the descending (AM) SMAP half-orbits when the SMAP spacecraft is travelling from North to South, while the SMAP instruments are operating in the nominal mode. The L3_SM_P product represents soil moisture retrieved over the entre UTC day. Retrievals are performed but flagged as questionable over urban areas, mountainous areas with high elevation variability, and areas with high ( &gt; 5 kg/m**2) vegetation water content; for retrievals using the high-resolution radar, cells in the nadir region are also flagged. Retrievals are inhibited for permanent snow/ice, frozen ground, and excessive static or transient open water in the cell, and for excessive RFI in the sensor data. + + + 180. + + + -180. + + + -85.04450226 + + + 85.04450226 + + + 2015-03-31T00:00:00.000Z + + + 2015-03-31T23:59:59.999Z + + + + + SMAP_L3_SM_P_20150331_R18290_002.qa + + + An ASCII product that contains statistical information on data product results. These statistics enable data producers and users to assess the quality of the data in the data product granule. + + + 2022-03-11 + + + + + 0 + + + 0 + + + 0 + + + 2 + + + SMAP Fixed Earth Grids, SMAP Science Document no: 033, May 11, 2009 + + + point + + + + 406 + + + 36. + + + + + EASE-Grid 2.0 + + + EASE-Grid 2.0: Incremental but Significant Improvements for Earth-Gridded Data Sets (ISPRS Int. J. Geo-Inf. 2012, 1, 32-45; doi:10.3390/ijgi1010032) + + + 2012-03-31 + + + + + 964 + + + 36. + + + + + The Equal-Area Scalable Earth Grid (EASE-Grid 2.0) is used for gridding satellite data sets. The EASE-Grid 2.0 is defined on the WGS84 ellipsoid to allow users to import data into standard GIS software formats such as GeoTIFF without reprojection. + + + Equal-Area Scalable Earth Grid + + + + + + 869 + + + 864 + + + + + + A configuration file that specifies the complete set of elements within the input Level 2 SM_P Product that the Radiometer Level 3_SM_P Science Processing Software (SPS) needs in order to function. + + + R18290 + + + SMAP_L3_SM_P_SPS_InputConfig_L2_SM_P.xml + + + 2022-03-11 + + + + + Precomputed longitude at each EASEGrid cell on 36-km grid on Global Cylindrical Projection + + + 002 + + + EZ2Lon_M36_002.float32 + + + 2013-05-09 + + + + + Passive soil moisture estimates onto a 36-km global Earth-fixed grid, based on radiometer measurements acquired when the SMAP spacecraft is travelling from North to South at approximately 6:00 AM local time. + + + SMAP_L2_SM_P_00864_D_20150331T162851_R18290_001.h5 + SMAP_L2_SM_P_00865_A_20150331T162945_R18290_001.h5 + SMAP_L2_SM_P_00865_D_20150331T171859_R18290_001.h5 + SMAP_L2_SM_P_00866_A_20150331T180814_R18290_001.h5 + SMAP_L2_SM_P_00866_D_20150331T185725_R18290_001.h5 + SMAP_L2_SM_P_00867_A_20150331T194640_R18290_001.h5 + SMAP_L2_SM_P_00867_D_20150331T203555_R18290_001.h5 + SMAP_L2_SM_P_00868_A_20150331T212510_R18290_001.h5 + SMAP_L2_SM_P_00868_D_20150331T221420_R18290_001.h5 + SMAP_L2_SM_P_00869_A_20150331T230335_R18290_001.h5 + SMAP_L2_SM_P_00869_D_20150331T235250_R18290_001.h5 + + + doi:10.5067/LPJ8F0TAK6E0 + + + L2_SM_P + + + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + 2022-02-22 + + + 36. + + + + + A configuration file that specifies the source of the values for each of the data elements that comprise the metadata in the output Radiometer Level 3_SM_P product. + + + R18290 + + + SMAP_L3_SM_P_SPS_MetConfig_L2_SM_P.xml + + + 2022-03-11 + + + + + A configuration file that lists the entire content of the output Radiometer Level 3_SM_P_ product. + + + R18290 + + + SMAP_L3_SM_P_SPS_OutputConfig_L3_SM_P.xml + + + 2022-03-11 + + + + + A configuration file generated automatically within the SMAP data system that specifies all of the conditions required for each individual run of the Radiometer Level 3 SM P Science Processing Software (SPS). + + + R18290 + + + SMAP_L3_SM_P_SPS_RunConfig_20220311T185327319.xml + + + 2022-03-11 + + + + + + Soil moisture retrieved using default retrieval algorithm from brightness temperatures acquired by the SMAP radiometer during the spacecraft descending pass. Level 2 granule data are then mosaicked on a daily basis to form the Level 3 product. + + + 2022-03-11T18:53:27.319Z + + + Algorithm Theoretical Basis Document: SMAP L2 and L3 Radiometer Soil Moisture (Passive) Data Products: L2_SM_P &amp; L3_SM_P + + + 2000-01-01T11:58:55.816Z + + + 023 + + + L3_SM_P_SPS + + + 2021-08-17 + + + 015 + + + L3_SM_P_SPS + + + Version 1.1 + + + 2019-04-15 + + + 2015-10-30 + + + 026 + + + Level 3 soil moisture product is formed by mosaicking Level 2 soil moisture granule data acquired over one day. + + + Algorithm Theoretical Basis Document: SMAP L2 and L3 Radiometer Soil Moisture (Passive) Data Products: L2_SM_P &amp; L3_SM_P + + + 2451545. + + + J2000 + + + 2012-10-26 + + + Soil Moisture Active Passive Mission (SMAP) Science Data System (SDS) Operations Facility + + + Soil Moisture Active Passive (SMAP) Radiometer processing algorithm + + + Preliminary + + + + + + + + + Longitude of the center of the Earth based grid cell. + + + degrees_east + + + + + + + Latitude of the center of the Earth based grid cell. + + + degrees_north + + + + + + + 0. + + + The fraction of the area of the 36 km grid cell that is covered by static water based on a Digital Elevation Map. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + + + Representative SCA-V soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + + + Representative angle between the antenna boresight vector and the normal to the Earth&apos;s surface for all footprints within the cell. + + + degrees + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 90. + + + -9999. + + + + + + + + + Arithmetic average of the acquisition time of all of the brightness temperature footprints with a center that falls within the EASE grid cell in UTC. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the DCA retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + + + The measured opacity of the vegetation used in the DCA retrieval in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + + + Bit flags that represent the quality of the horizontal polarization brightness temperature within each grid cell + + + 65534 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, 16384s, 32768s + + + Horizontal_polarization_quality Horizontal_polarization_range Horizontal_polarization_RFI_detection Horizontal_polarization_RFI_correction Horizontal_polarization_NEDT Horizontal_polarization_direct_sun_correction Horizontal_polarization_reflected_sun_correction Horizontal_polarization_reflected_moon_correction Horizontal_polarization_direct_galaxy_correction Horizontal_polarization_reflected_galaxy_correction Horizontal_polarization_atmosphere_correction Horizontal_polarization_Faraday_rotation_correction Horizontal_polarization_null_value_bit Horizontal_polarization_water_correction Horizontal_polarization_RFI_check Horizontal_polarization_RFI_clean + + + + + + + + + A unitless value that is indicative of bare soil roughness used in DCA within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + + 254 + + + An enumerated type that specifies the most common landcover class in the grid cell based on the IGBP landcover map. The array order is longitude (ascending), followed by latitude (descending), and followed by IGBP land cover type descending dominance (only the first three types are listed) + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + + + The row index of the 36 km EASE grid cell that contains the associated data. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 405 + + + 65534 + + + + + + + + + Horizontal polarization brightness temperature in 36 km Earth grid cell before adjustment for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + + + Vertical polarization brightness temperature in 36 km Earth grid cell adjusted for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + + + Fourth stokes parameter for each 36 km grid cell calculated with an adjustment for the presence of water bodies + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + + + Weighted average of the longitude of the center of the brightness temperature footprints that fall within the EASE grid cell. + + + degrees + + + -180. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 179.9989929 + + + -9999. + + + + + + + + + The measured opacity of the vegetation used in the SCA-H retrieval in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the SCA-H retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + + + A unitless value that is indicative of bare soil roughness used in DCA within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + /Soil_Moisture_Retrieval_Data_AM/roughness_coefficient + + + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in SCA-H within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + + 0. + + + The fraction of the grid cell that contains the most common land cover in that area based on the IGBP landcover map. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + + + 0. + + + Diffuse reflecting power of the Earth&apos;s surface used in SCA-V within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the DCA retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + /Soil_Moisture_Retrieval_Data_AM/retrieval_qual_flag_dca + + + + + + + + + Representative measure of water in the vegetation within the 36 km grid cell. + + + kg/m**2 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 20. + + + -9999. + + + + + + + + + Vertical polarization brightness temperature in 36 km Earth grid cell before adjustment for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + + + A unitless value that is indicative of bare soil roughness used in SCA-V within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + Representative SCA-H soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + + + Gain weighted fraction of static water within the radiometer horizontal polarization brightness temperature antenna pattern in 36 km Earth grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + A unitless value that is indicative of bare soil roughness used in SCA-H within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + Representative DCA soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + + + Third stokes parameter for each 36 km grid cell calculated with an adjustment for the presence of water bodies + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + + + 65534 + + + Bit flags that represent the quality of the 3rd Stokes brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 4096s, 16384s, 32768s + + + 3rd_Stokes_quality 3rd_Stokes_range 3rd_Stokes_RFI_detection 3rd_Stokes_RFI_correction 3rd_Stokes_NEDT 3rd_Stokes_direct_sun_correction 3rd_Stokes_reflected_sun_correction 3rd_Stokes_reflected_moon_correction 3rd_Stokes_direct_galaxy_correction 3rd_Stokes_reflected_galaxy_correction 3rd_Stokes_atmosphere_correction 3rd_Stokes_null_value_bit 3rd_Stokes_RFI_check 3rd_Stokes_RFI_clean + + + + + + + + + 65534 + + + Bit flags that represent the quality of the 4th Stokes brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 4096s, 16384s, 32768s + + + 4th_Stokes_quality 4th_Stokes_range 4th_Stokes_RFI_detection 4th_Stokes_RFI_correction 4th_Stokes_NEDT 4th_Stokes_direct_sun_correction 4th_Stokes_reflected_sun_correction 4th_Stokes_reflected_moon_correction 4th_Stokes_direct_galaxy_correction 4th_Stokes_reflected_galaxy_correction 4th_Stokes_atmosphere_correction 4th_Stokes_null_value_bit 4th_Stokes_RFI_check 4th_Stokes_RFI_clean + + + + + + + + + Horizontal polarization brightness temperature in 36 km Earth grid cell adjusted for the presence of water bodies. + + + Kelvin + + + 0. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 330. + + + -9999. + + + + + + + + + Gain weighted fraction of static water within the radiometer vertical polarization brightness temperature antenna pattern in 36 km Earth grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in DCA within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + Indicates if the grid point lies on land (0) or water (1). + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 1 + + + 65534 + + + + + + + + + Arithmetic average of the acquisition time of all of the brightness temperature footprints with a center that falls within the EASE grid cell in seconds since noon on January 1, 2000 UTC. + + + seconds + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.90000000002 + + + 940000000. + + + -9999. + + + + + + + + + The measured opacity of the vegetation used in the DCA retrieval in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + /Soil_Moisture_Retrieval_Data_AM/vegetation_opacity + + + + + + + + + A unitless value that is indicative of aggregated bulk_density within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 2.650000095 + + + -9999. + + + + + + + + + Net uncertainty measure of soil moisture measure for the Earth based grid cell. - Calculation method is TBD. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 0.200000003 + + + -9999. + + + + + + + + + The fraction of the area of the 36 km grid cell that is covered by water based on the radar detection algorithm. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + 65534 + + + Bit flags that represent the quality of the vertical polarization brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, 16384s, 32768s + + + Vertical_polarization_quality Vertical_polarization_range Vertical_polarization_RFI_detection Vertical_polarization_RFI_correction Vertical_polarization_NEDT Vertical_polarization_direct_sun_correction Vertical_polarization_reflected_sun_correction Vertical_polarization_reflected_moon_correction Vertical_polarization_direct_galaxy_correction Vertical_polarization_reflected_galaxy_correction Vertical_polarization_atmosphere_correction Vertical_polarization_Faraday_rotation_correction Vertical_polarization_null_value_bit Vertical_polarization_water_correction Vertical_polarization_RFI_check Vertical_polarization_RFI_clean + + + + + + + + + Weighted average of the latitude of the center of the brightness temperature footprints that fall within the EASE grid cell. + + + degrees + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -90. + + + 90. + + + -9999. + + + + + + + + + The column index of the 36 km EASE grid cell that contains the associated data. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 963 + + + 65534 + + + + + + + + + Temperature at land surface based on GMAO GEOS-5 data. + + + Kelvins + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 350. + + + -9999. + + + + + + + + + Representative DCA soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + /Soil_Moisture_Retrieval_Data_AM/soil_moisture + + + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the SCA-V retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + + + Fraction of the 36 km grid cell that is denoted as frozen. Based on binary flag that specifies freeze thaw conditions in each of the component 3 km grid cells. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + Bit flags that record ambient surface conditions for the grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s + + + 36_km_static_water_body 36_km_radar_water_body_detection 36_km_coastal_proximity 36_km_urban_area 36_km_precipitation 36_km_snow_or_ice 36_km_permanent_snow_or_ice 36_km_radiometer_frozen_ground 36_km_model_frozen_ground 36_km_mountainous_terrain 36_km_dense_vegetation 36_km_nadir_region + + + + + + + + + The measured opacity of the vegetation used in the SCA-V retrieval in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in DCA within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + /Soil_Moisture_Retrieval_Data_AM/albedo + + + + + + + + + A unitless value that is indicative of aggregated clay fraction within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + + + + + A unitless value that is indicative of aggregated bulk density within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 2.650000095 + + + -9999. + + + + + + + Representative angle between the antenna boresight vector and the normal to the Earth&apos;s surface for all footprints within the cell. + + + degrees + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 90. + + + -9999. + + + + + + + The row index of the 36 km EASE grid cell that contains the associated data. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 405 + + + 65534 + + + + + + + The fraction of the area of the 36 km grid cell that is covered by static water based on a Digital Elevation Map. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + Fraction of the 36 km grid cell that is denoted as frozen. Based on binary flag that specifies freeze thaw conditions in each of the component 3 km grid cells. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + A unitless value that is indicative of bare soil roughness used in DCA retrievals within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + A unitless value that is indicative of bare soil roughness used in DCA retrievals within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + /Soil_Moisture_Retrieval_Data_PM/roughness_coefficient_dca_pm + + + + + + + Bit flags that record ambient surface conditions for the grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s + + + 36_km_static_water_body 36_km_radar_water_body_detection 36_km_coastal_proximity 36_km_urban_area 36_km_precipitation 36_km_snow_or_ice 36_km_permanent_snow_or_ice 36_km_radar_frozen_ground 36_km_model_frozen_ground 36_km_mountainous_terrain 36_km_dense_vegetation 36_km_nadir_region + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the DCA retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + Horizontal polarization brightness temperature in 36 km Earth grid cell adjusted for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the SCA-V retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + Representative DCA soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in SCA-V retrievals within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + 0. + + + Gain weighted fraction of static water within the radiometer vertical polarization brightness temperature antenna pattern in 36 km Earth grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + A unitless value that is indicative of bare soil roughness used in SCA-V retrievals within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + 65534 + + + Bit flags that represent the quality of the 4th Stokes brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 4096s, 16384s, 32768s + + + 4th_Stokes_quality 4th_Stokes_range 4th_Stokes_RFI_detection 4th_Stokes_RFI_correction 4th_Stokes_NEDT 4th_Stokes_direct_sun_correction 4th_Stokes_reflected_sun_correction 4th_Stokes_reflected_moon_correction 4th_Stokes_direct_galaxy_correction 4th_Stokes_reflected_galaxy_correction 4th_Stokes_atmosphere_correction 4th_Stokes_null_value_bit 4th_Stokes_RFI_check 4th_Stokes_RFI_clean + + + + + + + Third stokes parameter for each 36 km grid cell calculated with an adjustment for the presence of water bodies + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the DCA retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + /Soil_Moisture_Retrieval_Data_PM/retrieval_qual_flag_pm + + + + + + + Representative SCA-V soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + 65534 + + + 1s, 2s, 4s, 8s + + + Bit flags that record the conditions and the quality of the SCA-H retrieval algorithms that generate soil moisture for the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + Retrieval_recommended Retrieval_attempted Retrieval_success FT_retrieval_success + + + + + + + The measured opacity of the vegetation used in SCA-H retrievals in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + Kelvin + + + 0. + + + Vertical polarization brightness temperature in 36 km Earth grid cell before adjustment for the presence of water bodies. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 330. + + + -9999. + + + + + + + A unitless value that is indicative of bare soil roughness used in SCA-H retrievals within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + Bit flags that represent the quality of the horizontal polarization brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, 16384s, 32768s + + + Horizontal_polarization_quality Horizontal_polarization_range Horizontal_polarization_RFI_detection Horizontal_polarization_RFI_correction Horizontal_polarization_NEDT Horizontal_polarization_direct_sun_correction Horizontal_polarization_reflected_sun_correction Horizontal_polarization_reflected_moon_correction Horizontal_polarization_direct_galaxy_correction Horizontal_polarization_reflected_galaxy_correction Horizontal_polarization_atmosphere_correction Horizontal_polarization_Faraday_rotation_correction Horizontal_polarization_null_value_bit Horizontal_polarization_water_correction Horizontal_polarization_RFI_check Horizontal_polarization_RFI_clean + + + + + + + A unitless value that is indicative of aggregated clay fraction within the 36 km grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + Weighted average of the latitude of the center of the brightness temperature footprints that fall within the EASE grid cell. + + + degrees + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -90. + + + 90. + + + -9999. + + + + + + + The column index of the 36 km EASE grid cell that contains the associated data. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 963 + + + 65534 + + + + + + + Arithmetic average of the acquisition time of all of the brightness temperature footprints with a center that falls within the EASE grid cell in seconds since noon on January 1, 2000 UTC. + + + seconds + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.90000000002 + + + 940000000. + + + -9999. + + + + + + + Fourth stokes parameter for each 36 km grid cell calculated with an adjustment for the presence of water bodies + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + Net uncertainty measure of soil moisture measure for the Earth based grid cell. - Calculation method is TBD. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 0.200000003 + + + -9999. + + + + + + + + 254 + + + An enumerated type that specifies the most common landcover class in the grid cell based on the IGBP landcover map. The array order is longitude (ascending), followed by latitude (descending), and followed by IGBP land cover type descending dominance (only the first three types are listed) + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + Arithmetic average of the acquisition time of all of the brightness temperature footprints with a center that falls within the EASE grid cell in UTC. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in DCA retrievals within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + + + + + Longitude of the center of the Earth based grid cell. + + + degrees_east + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -180. + + + 179.9989929 + + + -9999. + + + + + + + 65534 + + + Bit flags that represent the quality of the 3rd Stokes brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 4096s, 16384s, 32768s + + + 3rd_Stokes_quality 3rd_Stokes_range 3rd_Stokes_RFI_detection 3rd_Stokes_RFI_correction 3rd_Stokes_NEDT 3rd_Stokes_direct_sun_correction 3rd_Stokes_reflected_sun_correction 3rd_Stokes_reflected_moon_correction 3rd_Stokes_direct_galaxy_correction 3rd_Stokes_reflected_galaxy_correction 3rd_Stokes_atmosphere_correction 3rd_Stokes_null_value_bit 3rd_Stokes_RFI_check 3rd_Stokes_RFI_clean + + + + + + + The measured opacity of the vegetation used in DCA retrievals in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + Bit flags that represent the quality of the vertical polarization brightness temperature within each grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, 16384s, 32768s + + + Vertical_polarization_quality Vertical_polarization_range Vertical_polarization_RFI_detection Vertical_polarization_RFI_correction Vertical_polarization_NEDT Vertical_polarization_direct_sun_correction Vertical_polarization_reflected_sun_correction Vertical_polarization_reflected_moon_correction Vertical_polarization_direct_galaxy_correction Vertical_polarization_reflected_galaxy_correction Vertical_polarization_atmosphere_correction Vertical_polarization_Faraday_rotation_correction Vertical_polarization_null_value_bit Vertical_polarization_water_correction Vertical_polarization_RFI_check Vertical_polarization_RFI_clean + + + + + + + 0. + + + The fraction of the area of the 36 km grid cell that is covered by water based on the radar detection algorithm. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + Representative SCA-H soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + Representative DCA soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + /Soil_Moisture_Retrieval_Data_PM/soil_moisture_pm + + + + + + + + 0. + + + The fraction of the grid cell that contains the most common land cover in that area based on the IGBP landcover map. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + The measured opacity of the vegetation used in DCA retrievals in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + /Soil_Moisture_Retrieval_Data_PM/vegetation_opacity_dca_pm + + + + + + + 0. + + + Gain weighted fraction of static water within the radiometer horizontal polarization brightness temperature antenna pattern in 36 km Earth grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 1. + + + -9999. + + + + + + + Weighted average of the longitude of the center of the brightness temperature footprints that fall within the EASE grid cell. + + + degrees + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -180. + + + 179.9989929 + + + -9999. + + + + + + + Diffuse reflecting power of the Earth&apos;s surface used in DCA retrievals within the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 1. + + + -9999. + + + /Soil_Moisture_Retrieval_Data_PM/albedo_dca_pm + + + + + + + Representative measure of water in the vegetation within the 36 km grid cell. + + + kg/m**2 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 20. + + + -9999. + + + + + + + 0. + + + Diffuse reflecting power of the Earth&apos;s surface used in SCA-H retrievals within the grid cell. + + + 1. + + + -9999. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + Horizontal polarization brightness temperature in 36 km Earth grid cell before adjustment for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + Latitude of the center of the Earth based grid cell. + + + degrees_north + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -90. + + + 90. + + + -9999. + + + + + + + Vertical polarization brightness temperature in 36 km Earth grid cell adjusted for the presence of water bodies. + + + Kelvin + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 330. + + + -9999. + + + + + + + The measured opacity of the vegetation used in SCA-V retrievals in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + Indicates if the grid point lies on land (0) or water (1). + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 1 + + + 65534 + + + + + + + Temperature at land surface based on GMAO GEOS-5 data. + + + Kelvins + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 350. + + + -9999. + + + + diff --git a/tests/data/SC_SPL3SMP_008_fake.dmr b/tests/data/SC_SPL3SMP_008_fake.dmr new file mode 100644 index 0000000..009e070 --- /dev/null +++ b/tests/data/SC_SPL3SMP_008_fake.dmr @@ -0,0 +1,464 @@ + + + + 3.21.0-428 + + + 3.21.0-428 + + + libdap-3.21.0-103 + + + build_dmrpp -c /tmp/bes_conf_IAud -f /usr/share/hyrax/DATA/SMAP_L3_SM_P_20150331_R18290_002.h5 -r /tmp/dmr__0PmwFd -u OPeNDAP_DMRpp_DATA_ACCESS_URL -M + + + + + + + + Longitude of the center of the Earth based grid cell. + + + degrees_east + + + + + + + Latitude of the center of the Earth based grid cell. + + + degrees_north + + + + + + + Longitude of the center of the Earth based grid cell. + + + + + + + Latitude of the center of the Earth based grid cell. + + + + + + + north polar grid longitude + + + longitude + + + degrees_east + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + -180. + + + 180. + + + referenceInformation + + + The north polar grid longitude + + + X + + + + + + + north polar grid latitude + + + latitude + + + degrees_north + + + L3B ATM ATBD, Section 2.0, Section 5.0, Table 5. + + + 60. + + + 90. + + + referenceInformation + + + The north polar grid latitude + + + + Y + + + + + + + + 254 + + + An enumerated type that specifies the most common landcover class + in the grid cell based on the IGBP landcover map. The array order is longitude (ascending), followed by latitude (descending), and followed by IGBP land cover type descending dominance (only the first three types are listed) + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + + + + + + + Weighted average of the longitude of the center of the brightness temperature footprints that fall within the EASE grid cell. + + + degrees + + + -180. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 179.9989929 + + + -9999. + + + + + + + + + Representative SCA-V soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/missing_lat /Soil_Moisture_Retrieval_Data_AM/missing_lon + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + Representative SCA-V soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/fake_lat /Soil_Moisture_Retrieval_Data_AM/fake_lon + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + The measured opacity of the vegetation used in the SCA-H retrieval in the grid cell. + + + -999999.875 + + + 999999.875 + + + -9999. + + + + + + + + 0. + + + The fraction of the grid cell that contains the most common land cover in that area based on the IGBP landcover map. + + + /Soil_Moisture_Retrieval_Data_AM/latitude + + + 1. + + + -9999. + + + + + + + + + + + Representative DCA soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + + + + + + The column index of the 36 km EASE grid cell that contains the associated data. + + + /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0 + + + 963 + + + 65534 + + + + + + + + + Temperature at land surface based on GMAO GEOS-5 data. + + + Kelvins + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 350. + + + -9999. + + + + + + + + + Bit flags that record ambient surface conditions for the grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s + + + 36_km_static_water_body 36_km_radar_water_body_detection 36_km_coastal_proximity 36_km_urban_area 36_km_precipitation 36_km_snow_or_ice 36_km_permanent_snow_or_ice 36_km_radiometer_frozen_ground 36_km_model_frozen_ground 36_km_mountainous_terrain 36_km_dense_vegetation 36_km_nadir_region + + + + + + + + + + + + + Bit flags that record ambient surface conditions for the grid cell + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 65534 + + + 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s + + + 36_km_static_water_body 36_km_radar_water_body_detection 36_km_coastal_proximity 36_km_urban_area 36_km_precipitation 36_km_snow_or_ice 36_km_permanent_snow_or_ice 36_km_radar_frozen_ground 36_km_model_frozen_ground 36_km_mountainous_terrain 36_km_dense_vegetation 36_km_nadir_region + + + + + + + Longitude of the center of the Earth based grid cell. + + + degrees_east + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -180. + + + 179.9989929 + + + -9999. + + + + + + + The measured opacity of the vegetation used in DCA retrievals in the grid cell. + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -999999.875 + + + 999999.875 + + + -9999. + + + fake_dim_1 + fake_dim_2 + + + + + + + Latitude of the center of the Earth based grid cell. + + + degrees_north + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + -90. + + + 90. + + + -9999. + + + + + + + Temperature at land surface based on GMAO GEOS-5 data. + + + Kelvins + + + /Soil_Moisture_Retrieval_Data_AM/latitude /Soil_Moisture_Retrieval_Data_AM/longitude + + + 0. + + + 350. + + + -9999. + + + + + + + Representative SCA-V soil moisture measurement for the Earth based grid cell. + + + cm**3/cm**3 + + + /Soil_Moisture_Retrieval_Data_PM/fake_lat /Soil_Moisture_Retrieval_Data_PM/fake_lon + + + 0.01999999955 + + + 0.5 + + + -9999. + + + + diff --git a/tests/data/SC_SPL3SMP_008_prefetch.nc4 b/tests/data/SC_SPL3SMP_008_prefetch.nc4 new file mode 100644 index 0000000..f866858 Binary files /dev/null and b/tests/data/SC_SPL3SMP_008_prefetch.nc4 differ diff --git a/tests/data/SC_SPL3SMP_009_prefetch.nc4 b/tests/data/SC_SPL3SMP_009_prefetch.nc4 new file mode 100644 index 0000000..f253189 Binary files /dev/null and b/tests/data/SC_SPL3SMP_009_prefetch.nc4 differ diff --git a/tests/unit/test_coordinate_utilities.py b/tests/unit/test_coordinate_utilities.py new file mode 100644 index 0000000..4beee4f --- /dev/null +++ b/tests/unit/test_coordinate_utilities.py @@ -0,0 +1,1071 @@ +from logging import getLogger +from os.path import exists +from unittest import TestCase +from unittest.mock import ANY, patch + +import numpy as np +from harmony.util import config +from netCDF4 import Dataset +from numpy.testing import assert_array_equal +from pyproj import CRS +from varinfo import VarInfoFromDmr + +from hoss.coordinate_utilities import ( + any_absent_dimension_variables, + create_dimension_arrays_from_coordinates, + get_2d_coordinate_array, + get_coordinate_variables, + get_dimension_array_names, + get_dimension_array_names_from_coordinate_variables, + get_dimension_order_and_dim_values, + get_max_spread_pts, + get_row_col_sizes_from_coordinates, + get_valid_indices, + get_valid_sample_pts, + get_variables_with_anonymous_dims, + interpolate_dim_values_from_sample_pts, +) +from hoss.exceptions import ( + IncompatibleCoordinateVariables, + InvalidCoordinateData, + InvalidCoordinateVariable, + MissingCoordinateVariable, + MissingVariable, +) + + +class TestCoordinateUtilities(TestCase): + """A class for testing functions in the `hoss.coordinate_utilities` + module. + + """ + + @classmethod + def setUpClass(cls): + """Create fixtures that can be reused for all tests.""" + cls.config = config(validate=False) + cls.logger = getLogger('tests') + cls.varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + config_file='hoss/hoss_config.json', + ) + cls.test_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008_fake.dmr', + 'SPL3SMP', + config_file='hoss/hoss_config.json', + ) + cls.nc4file = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' + cls.latitude = '/Soil_Moisture_Retrieval_Data_AM/latitude' + cls.longitude = '/Soil_Moisture_Retrieval_Data_AM/longitude' + + cls.lon_arr = np.array( + [ + [-179.3, -120.2, -60.6, -9999, -9999, -9999, 80.2, 120.6, 150.5, 178.4], + [-179.3, -120.2, -60.6, -999, 999, -9999, 80.2, 120.6, 150.5, 178.4], + [-179.3, -120.2, -60.6, -9999, -9999, -9999, 80.2, 120.6, 150.5, 178.4], + [-179.3, -120.2, -60.6, -9999, -9999, -9999, 80.2, 120.6, 150.5, 178.4], + [-179.3, -120.2, -60.6, -9999, -9999, -9999, 80.2, 120.6, 150.5, 178.4], + ] + ) + + cls.lat_arr = np.array( + [ + [89.3, 89.3, -9999, 89.3, 89.3, 89.3, -9999, 89.3, 89.3, 89.3], + [50.3, 50.3, 50.3, 50.3, 50.3, 50.3, -9999, 50.3, 50.3, 50.3], + [1.3, 1.3, 1.3, 1.3, 1.3, 1.3, -9999, -9999, 1.3, 1.3], + [-9999, -60.2, -60.2, -99, -9999, -9999, -60.2, -60.2, -60.2, -60.2], + [-88.1, -88.1, -88.1, 99, -9999, -9999, -88.1, -88.1, -88.1, -88.1], + ] + ) + + cls.lon_arr_reversed = np.array( + [ + [ + -179.3, + -179.3, + -179.3, + -179.3, + -9999, + -9999, + -179.3, + -179.3, + -179.3, + -179.3, + ], + [ + -120.2, + -120.2, + -120.2, + -9999, + -9999, + -120.2, + -120.2, + -120.2, + -120.2, + -120.2, + ], + [20.6, 20.6, 20.6, 20.6, 20.6, 20.6, 20.6, 20.6, -9999, -9999], + [ + 150.5, + 150.5, + 150.5, + 150.5, + 150.5, + 150.5, + -9999, + -9999, + 150.5, + 150.5, + ], + [ + 178.4, + 178.4, + 178.4, + 178.4, + 178.4, + 178.4, + 178.4, + -9999, + 178.4, + 178.4, + ], + ] + ) + cls.lat_arr_reversed = np.array( + [ + [89.3, 79.3, -9999, 59.3, 29.3, 2.1, -9999, -59.3, -79.3, -89.3], + [89.3, 79.3, 60.3, 59.3, 29.3, 2.1, -9999, -59.3, -79.3, -89.3], + [89.3, -9999, 60.3, 59.3, 29.3, 2.1, -9999, -9999, -9999, -89.3], + [ + -9999, + 79.3, + -60.3, + -9999, + -9999, + -9999, + -60.2, + -59.3, + -79.3, + -89.3, + ], + [ + 89.3, + 79.3, + -60.3, + -9999, + -9999, + -9999, + -60.2, + -59.3, + -79.3, + -9999, + ], + ] + ) + + def setUp(self): + """Create fixtures that should be unique per test.""" + + def tearDown(self): + """Remove per-test fixtures.""" + + def test_get_coordinate_variables(self): + """Ensure that the correct coordinate variables are + retrieved for the reqquested science variable + + """ + requested_science_variables = [ + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm', + ] + expected_coordinate_variables = ( + [ + '/Soil_Moisture_Retrieval_Data_AM/latitude', + '/Soil_Moisture_Retrieval_Data_PM/latitude_pm', + ], + [ + '/Soil_Moisture_Retrieval_Data_AM/longitude', + '/Soil_Moisture_Retrieval_Data_PM/longitude_pm', + ], + ) + + with self.subTest('Retrieves expected coordinates for the requested variables'): + actual_coordinate_variables = get_coordinate_variables( + self.varinfo, requested_science_variables + ) + # the order of the results maybe random + self.assertEqual( + len(expected_coordinate_variables), len(actual_coordinate_variables) + ) + self.assertCountEqual( + expected_coordinate_variables[0], actual_coordinate_variables[0] + ) + self.assertCountEqual( + expected_coordinate_variables[0], actual_coordinate_variables[0] + ) + for expected_variable in expected_coordinate_variables[0]: + self.assertIn(expected_variable, actual_coordinate_variables[0]) + + for expected_variable in expected_coordinate_variables[1]: + self.assertIn(expected_variable, actual_coordinate_variables[1]) + + with self.subTest('No lat coordinate variables for the requested variables'): + # should return one valid list and an empty list + actual_coordinate_variables = get_coordinate_variables( + self.test_varinfo, + ['/Soil_Moisture_Retrieval_Data_AM/no_lat_coordinate_variable'], + ) + self.assertTupleEqual( + actual_coordinate_variables, + ([], ['/Soil_Moisture_Retrieval_Data_AM/longitude']), + ) + with self.subTest('No lon coordinate variables for the requested variables'): + # should return one valid list and an empty list + actual_coordinate_variables = get_coordinate_variables( + self.test_varinfo, + ['/Soil_Moisture_Retrieval_Data_AM/no_lon_coordinate_variable'], + ) + self.assertTupleEqual( + actual_coordinate_variables, + (['/Soil_Moisture_Retrieval_Data_AM/latitude'], []), + ) + with self.subTest('No coordinate variables for the requested variables'): + # should return empty lists + actual_coordinate_variables = get_coordinate_variables( + self.test_varinfo, + ['/Soil_Moisture_Retrieval_Data_AM/no_coordinate_variable'], + ) + self.assertTupleEqual(actual_coordinate_variables, ([], [])) + with self.subTest('Missing coordinate variables'): + # should return empty lists + missing_coordinate_variables = get_coordinate_variables( + self.test_varinfo, + ['/Soil_Moisture_Retrieval_Data_AM/variable_with_missing_coordinates'], + ) + self.assertTupleEqual(missing_coordinate_variables, ([], [])) + with self.subTest('Fake coordinate variables'): + # should return empty lists + fake_coordinate_variables = get_coordinate_variables( + self.test_varinfo, + ['/Soil_Moisture_Retrieval_Data_AM/variable_with_fake_coordinates'], + ) + self.assertTupleEqual(fake_coordinate_variables, ([], [])) + + def test_interpolate_dim_values_from_sample_pts(self): + """Ensure that the dimension scale generated from the + provided dimension values are accurate for ascending and + descending scales + """ + + with self.subTest('valid ascending dim scale'): + dim_values_asc = np.array([2, 4]) + dim_indices_asc = np.array([0, 1]) + dim_size_asc = 12 + expected_dim_asc = np.array([2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24]) + dim_array_values = interpolate_dim_values_from_sample_pts( + dim_values_asc, dim_indices_asc, dim_size_asc + ) + self.assertTrue(np.array_equal(dim_array_values, expected_dim_asc)) + + with self.subTest('valid descending dim scale'): + dim_values_desc = np.array([100, 70]) + dim_indices_desc = np.array([2, 5]) + dim_size_desc = 10 + expected_dim_desc = np.array([120, 110, 100, 90, 80, 70, 60, 50, 40, 30]) + + dim_array_values = interpolate_dim_values_from_sample_pts( + dim_values_desc, dim_indices_desc, dim_size_desc + ) + self.assertTrue(np.array_equal(dim_array_values, expected_dim_desc)) + + with self.subTest('invalid dimension values'): + dim_values_invalid = np.array([2, 2]) + dim_indices_asc = np.array([0, 1]) + dim_size_asc = 12 + with self.assertRaises(InvalidCoordinateData) as context: + interpolate_dim_values_from_sample_pts( + dim_values_invalid, dim_indices_asc, dim_size_asc + ) + self.assertEqual( + context.exception.message, + 'No distinct valid coordinate points - ' 'dim_index=0, dim_value=2', + ) + + with self.subTest('invalid dimension indices'): + dim_values_desc = np.array([100, 70]) + dim_indices_invalid = np.array([5, 5]) + dim_size_desc = 10 + with self.assertRaises(InvalidCoordinateData) as context: + interpolate_dim_values_from_sample_pts( + dim_values_desc, dim_indices_invalid, dim_size_desc + ) + self.assertEqual( + context.exception.message, + 'No distinct valid coordinate points - ' 'dim_index=5, dim_value=100', + ) + + def test_get_2d_coordinate_array(self): + """Ensures that the expected lat/lon arrays are retrieved + for the coordinate variables + """ + expected_shape = (406, 964) + with self.subTest('Expected latitude array'): + with Dataset(self.nc4file, 'r') as prefetch_dataset: + lat_prefetch_arr = get_2d_coordinate_array( + prefetch_dataset, + self.latitude, + ) + self.assertTupleEqual(lat_prefetch_arr.shape, expected_shape) + np.testing.assert_array_equal( + lat_prefetch_arr, prefetch_dataset[self.latitude][:] + ) + with self.subTest('Expected longitude array'): + with Dataset(self.nc4file, 'r') as prefetch_dataset: + lon_prefetch_arr = get_2d_coordinate_array( + prefetch_dataset, self.longitude + ) + self.assertTupleEqual(lon_prefetch_arr.shape, expected_shape) + np.testing.assert_array_equal( + lon_prefetch_arr, prefetch_dataset[self.longitude][:] + ) + with self.subTest('Missing coordinate'): + with Dataset(self.nc4file, 'r') as prefetch_dataset: + with self.assertRaises(MissingCoordinateVariable) as context: + coord_arr = ( + get_2d_coordinate_array( + prefetch_dataset, + '/Soil_Moisture_Retrieval_Data_AM/longitude_centroid', + ), + ) + self.assertEqual( + context.exception.message, + 'Coordinate: "/Soil_Moisture_Retrieval_Data_AM/latitude_centroid" is ' + 'not present in coordinate prefetch file.', + ) + + def test_get_dimension_array_names(self): + """Ensure that the expected projected dimension name + is returned for the coordinate variables + """ + + expected_dimension_names = [ + '/Soil_Moisture_Retrieval_Data_AM/dim_y', + '/Soil_Moisture_Retrieval_Data_AM/dim_x', + ] + + with self.subTest( + 'Retrieves expected projected dimension names for a science variable' + ): + self.assertListEqual( + get_dimension_array_names(self.varinfo, self.latitude), + expected_dimension_names, + ) + + with self.subTest( + 'Retrieves expected dimension names for the longitude variable' + ): + self.assertEqual( + get_dimension_array_names(self.varinfo, self.longitude), + expected_dimension_names, + ) + + with self.subTest('Raises exception for missing coordinate variable'): + with self.assertRaises(MissingVariable) as context: + get_dimension_array_names( + self.varinfo, '/Soil_Moisture_Retrieval_Data_AM/random_variable' + ) + self.assertEqual( + context.exception.message, + '"/Soil_Moisture_Retrieval_Data_AM/random_variable" is ' + 'not present in source granule file.', + ) + + def test_get_dimension_array_names_from_coordinate_variables(self): + """Ensure that the expected projected dimension name + is returned for the coordinate variables + """ + + expected_override_dimensions_AM = [ + '/Soil_Moisture_Retrieval_Data_AM/dim_y', + '/Soil_Moisture_Retrieval_Data_AM/dim_x', + ] + expected_override_dimensions_PM = [ + '/Soil_Moisture_Retrieval_Data_PM/dim_y', + '/Soil_Moisture_Retrieval_Data_PM/dim_x', + ] + + with self.subTest( + 'Retrieves expected override dimensions for the science variable' + ): + self.assertListEqual( + get_dimension_array_names_from_coordinate_variables( + self.varinfo, '/Soil_Moisture_Retrieval_Data_AM/surface_flag' + ), + expected_override_dimensions_AM, + ) + + with self.subTest( + 'Retrieves expected override dimensions for the longitude variable' + ): + self.assertListEqual( + get_dimension_array_names_from_coordinate_variables( + self.varinfo, self.longitude + ), + expected_override_dimensions_AM, + ) + + with self.subTest( + 'Retrieves expected override dimensions for the latitude variable' + ): + self.assertListEqual( + get_dimension_array_names_from_coordinate_variables( + self.varinfo, self.latitude + ), + expected_override_dimensions_AM, + ) + + with self.subTest( + 'Retrieves expected override dimensions science variable with a different grid' + ): + self.assertListEqual( + get_dimension_array_names_from_coordinate_variables( + self.varinfo, '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm' + ), + expected_override_dimensions_PM, + ) + with self.subTest( + 'Retrieves empty dimensions list when science variable has no coordinates' + ): + self.assertListEqual( + get_dimension_array_names_from_coordinate_variables( + self.varinfo, '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm' + ), + expected_override_dimensions_PM, + ) + + def test_get_row_col_sizes_from_coordinates(self): + """Ensure that the correct row and column sizes are + returned for the requested coordinates + """ + + with self.subTest('Retrieves the expected row col sizes from the coordinates'): + expected_row_col_sizes = (5, 10) + self.assertEqual( + get_row_col_sizes_from_coordinates( + self.lat_arr, self.lon_arr, dim_order_is_y_x=True + ), + expected_row_col_sizes, + ) + with self.subTest('Retrieves the expected row col sizes for the dim array'): + self.assertEqual( + get_row_col_sizes_from_coordinates( + np.array([1, 2, 3, 4]), + np.array([5, 6, 7, 8, 9]), + dim_order_is_y_x=True, + ), + (4, 5), + ) + with self.subTest('Retrieves the expected row col sizes for the dim array'): + self.assertEqual( + get_row_col_sizes_from_coordinates( + np.array([1, 2, 3, 4]), + np.array([5, 6, 7, 8, 9]), + dim_order_is_y_x=False, + ), + (5, 4), + ) + with self.subTest( + 'Raises an exception when the lat and lon array shapes do not match' + ): + lat_mismatched_array = np.array([[1, 2, 3], [3, 4, 5]]) + lon_mismatched_array = np.array([[6, 7], [8, 9], [10, 11]]) + with self.assertRaises(IncompatibleCoordinateVariables) as context: + get_row_col_sizes_from_coordinates( + lat_mismatched_array, lon_mismatched_array, dim_order_is_y_x=True + ) + self.assertEqual( + context.exception.message, + f'Longitude coordinate shape: "{lon_mismatched_array.shape}"' + f'does not match the latitude coordinate shape: "{lat_mismatched_array.shape}"', + ) + with self.subTest( + 'Raises an exception when Both arrays are 1-D, but latitude has a zero size' + ): + lat_empty_size_array = np.array([]) + with self.assertRaises(IncompatibleCoordinateVariables) as context: + get_row_col_sizes_from_coordinates( + lat_empty_size_array, np.array([5, 6, 7, 8]), dim_order_is_y_x=True + ) + + with self.subTest( + 'Raises an exception when Both arrays are 1-D, but longitude has a zero size' + ): + lon_empty_size_array = np.array([]) + with self.assertRaises(IncompatibleCoordinateVariables) as context: + get_row_col_sizes_from_coordinates( + np.array([6, 7, 8, 9]), lon_empty_size_array, dim_order_is_y_x=True + ) + + with self.subTest( + 'Raises an exception when latitude array that is zero dimensional' + ): + lat_empty_ndim_array = np.array( + 0, + ) + with self.assertRaises(IncompatibleCoordinateVariables) as context: + get_row_col_sizes_from_coordinates( + lat_empty_ndim_array, np.array([1, 2, 3, 4]), dim_order_is_y_x=True + ) + + with self.subTest( + 'Raises an exception when longitude array that is zero dimensional' + ): + lon_empty_ndim_array = np.array( + 0, + ) + with self.assertRaises(IncompatibleCoordinateVariables) as context: + get_row_col_sizes_from_coordinates( + np.array([1, 2, 3, 4]), lon_empty_ndim_array, dim_order_is_y_x=True + ) + with self.subTest('when lat/lon arr is more than 2D'): + lat_3d_array = np.arange(24).reshape(2, 3, 4) + lon_3d_array = np.arange(24).reshape(2, 3, 4) + self.assertEqual( + get_row_col_sizes_from_coordinates( + lat_3d_array, lon_3d_array, dim_order_is_y_x=True + ), + (3, 4), + ) + + def test_get_valid_indices(self): + """Ensure that latitude and longitude values are correctly identified as + ascending or descending. + + """ + expected_valid_indices_lon_arr_over_range = np.array([0, 1, 2, 6, 7, 8, 9]) + + fill_array = np.array([-9999.0, -9999.0, -9999.0, -9999.0]) + + with self.subTest('valid indices for latitude with fill values'): + expected_valid_indices_lat_arr_with_fill = np.array( + [False, True, True, True, True] + ) + valid_indices_lat_arr = get_valid_indices( + self.lat_arr[:, 2], self.varinfo.get_variable(self.latitude) + ) + np.testing.assert_array_equal( + valid_indices_lat_arr, expected_valid_indices_lat_arr_with_fill + ) + with self.subTest('valid indices for longitude with fill values'): + expected_valid_indices_lon_arr_with_fill = np.array( + [True, True, True, False, False, False, True, True, True, True] + ) + valid_indices_lon_arr = get_valid_indices( + self.lon_arr[0, :], self.varinfo.get_variable(self.longitude) + ) + np.testing.assert_array_equal( + valid_indices_lon_arr, expected_valid_indices_lon_arr_with_fill + ) + with self.subTest('latitude values beyond valid range'): + expected_valid_indices_lat_arr_over_range = np.array( + [True, True, True, False, False] + ) + valid_indices_lat_arr = get_valid_indices( + self.lat_arr[:, 3], self.varinfo.get_variable(self.latitude) + ) + np.testing.assert_array_equal( + valid_indices_lat_arr, expected_valid_indices_lat_arr_over_range + ) + with self.subTest('longitude values beyond valid range'): + expected_valid_indices_lon_arr_over_range = np.array( + [True, True, True, False, False, False, True, True, True, True] + ) + valid_indices_lon_arr = get_valid_indices( + self.lon_arr[1, :], self.varinfo.get_variable(self.longitude) + ) + np.testing.assert_array_equal( + valid_indices_lon_arr, expected_valid_indices_lon_arr_over_range + ) + with self.subTest('all fill values - no valid indices'): + expected_valid_indices_fill_values = np.array([False, False, False, False]) + valid_indices_all_fill = get_valid_indices( + fill_array, self.varinfo.get_variable(self.longitude) + ) + np.testing.assert_array_equal( + valid_indices_all_fill, expected_valid_indices_fill_values + ) + + def test_get_variables_with_anonymous_dims(self): + """Ensure that variables with no dimensions are + retrieved for the requested science variable + + """ + + with self.subTest('Retrieves variables with no dimensions'): + requested_science_variables = { + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm', + } + variables_with_anonymous_dims = get_variables_with_anonymous_dims( + self.test_varinfo, requested_science_variables + ) + self.assertSetEqual( + variables_with_anonymous_dims, + requested_science_variables, + ) + with self.subTest('Does not retrieve variables with dimensions'): + variables_with_anonymous_dims = get_variables_with_anonymous_dims( + self.test_varinfo, {'/Soil_Moisture_Retrieval_Data_AM/variable_has_dim'} + ) + self.assertTrue(len(variables_with_anonymous_dims) == 0) + + with self.subTest( + 'Only retrieves variables with anonymous dimensions,' + 'when the request has both' + ): + requested_science_variables_with_dimensions = { + '/Soil_Moisture_Retrieval_Data_AM/variable_has_dim', + '/Soil_Moisture_Retrieval_Data_AM/variable_has_anonymous_dim', + } + variables_with_anonymous_dims = get_variables_with_anonymous_dims( + self.test_varinfo, requested_science_variables_with_dimensions + ) + self.assertSetEqual( + variables_with_anonymous_dims, + {'/Soil_Moisture_Retrieval_Data_AM/variable_has_anonymous_dim'}, + ) + with self.subTest( + 'retrieves variables with fake dimensions,' 'when the request has both' + ): + variables_with_fake_dims = get_variables_with_anonymous_dims( + self.test_varinfo, + {'/Soil_Moisture_Retrieval_Data_PM/variable_with_fake_dims'}, + ) + self.assertSetEqual( + variables_with_fake_dims, + {'/Soil_Moisture_Retrieval_Data_PM/variable_with_fake_dims'}, + ) + + def test_any_absent_dimension_variables(self): + """Ensure that variables with fake dimensions are + detected with a True return value + + """ + + with self.subTest('Returns true for variables with fake dimensions'): + variable_has_fake_dims = any_absent_dimension_variables( + self.test_varinfo, + '/Soil_Moisture_Retrieval_Data_PM/variable_with_fake_dims', + ) + self.assertTrue(variable_has_fake_dims) + with self.subTest('Returns false for variables with dimensions'): + variable_has_fake_dims = any_absent_dimension_variables( + self.test_varinfo, '/Soil_Moisture_Retrieval_Data_AM/variable_has_dim' + ) + self.assertFalse(variable_has_fake_dims) + + def test_get_max_spread_pts(self): + """Ensure that two valid sets of indices are returned by the function + with a masked dataset as input + + """ + + with self.subTest('Get two sets of valid indices for points from coordinates'): + valid_values = np.array( + [ + [True, True, True, True, False, False, True, True, True, True], + [True, True, True, False, False, False, True, True, True, True], + [True, True, True, False, True, False, True, True, True, True], + [True, True, True, False, False, False, True, True, True, True], + [True, True, False, False, False, False, True, True, True, True], + ] + ) + expected_indices = [[0, 0], [0, 9]] + actual_indices = get_max_spread_pts(~valid_values) + self.assertTrue(actual_indices[0] == expected_indices[0]) + self.assertTrue(actual_indices[1] == expected_indices[1]) + + with self.subTest('With just one valid index in the coordinates'): + valid_values = np.array( + [ + [False, False, False], + [False, True, False], + [False, False, False], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_max_spread_pts(~valid_values) + self.assertEqual( + context.exception.message, + 'Only one valid point in coordinate data', + ) + + with self.subTest('No valid points from coordinates'): + valid_values = np.array( + [ + [False, False, False], + [False, False, False], + [False, False, False], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_max_spread_pts(~valid_values) + self.assertEqual( + context.exception.message, + 'No valid coordinate data', + ) + + def test_get_valid_sample_pts(self): + """Ensure that two sets of valid indices are + returned by the method with a set of lat/lon coordinates as input + + """ + with self.subTest('Get two sets of valid indices from coordinates dataset'): + expected_grid_indices = ( + [[0, 0], [4, 0]], + [[0, 0], [0, 9]], + ) + actual_row_indices, actual_col_indices = get_valid_sample_pts( + self.lat_arr, + self.lon_arr, + self.varinfo.get_variable(self.latitude), + self.varinfo.get_variable(self.longitude), + ) + self.assertListEqual(actual_row_indices, expected_grid_indices[0]) + self.assertListEqual(actual_col_indices, expected_grid_indices[1]) + self.assertTupleEqual( + (actual_row_indices, actual_col_indices), expected_grid_indices + ) + with self.subTest('Only a single valid point in coordinates dataset'): + lat_arr = np.array( + [ + [-9999.0, -9999.0, 40.1, -9999.0, -9999.0], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + lon_arr = np.array( + [ + [-9999.0, -9999.0, 100.1, -9999.0, -9999.0], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_valid_sample_pts( + lat_arr, + lon_arr, + self.varinfo.get_variable(self.latitude), + self.varinfo.get_variable(self.longitude), + ) + self.assertEqual( + context.exception.message, + 'No valid coordinate data', + ) + with self.subTest('valid points in one row in coordinates dataset'): + lat_arr = np.array( + [ + [40.1, 40.1, 40.1, 40.1, 40.1], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + lon_arr = np.array( + [ + [-179.0, -10.0, 100.1, 130.0, 179.0], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_valid_sample_pts( + lat_arr, + lon_arr, + self.varinfo.get_variable(self.latitude), + self.varinfo.get_variable(self.longitude), + ) + self.assertEqual( + context.exception.message, + 'No valid coordinate data', + ) + with self.subTest('valid points in one column in coordinates dataset'): + lat_arr = np.array( + [ + [-9999.0, -9999.0, 40.1, -9999.0, -9999.0], + [-9999.0, -9999.0, -50.0, -9999.0, -9999.0], + ] + ) + lon_arr = np.array( + [ + [-9999.0, -9999.0, 100.1, -9999.0, -9999.0], + [-9999.0, -9999.0, 100.1, -9999.0, -9999.0], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_valid_sample_pts( + lat_arr, + lon_arr, + self.varinfo.get_variable(self.latitude), + self.varinfo.get_variable(self.longitude), + ) + self.assertEqual( + context.exception.message, + 'No valid coordinate data', + ) + with self.subTest('no valid points in coordinates dataset'): + lat_arr = np.array( + [ + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + lon_arr = np.array( + [ + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + [-9999.0, -9999.0, -9999.0, -9999.0, -9999.0], + ] + ) + with self.assertRaises(InvalidCoordinateData) as context: + get_valid_sample_pts( + lat_arr, + lon_arr, + self.varinfo.get_variable(self.latitude), + self.varinfo.get_variable(self.longitude), + ) + self.assertEqual( + context.exception.message, + 'No valid coordinate data', + ) + + def test_get_dimension_order_and_dim_values(self): + """Ensure that the correct dimension order index + is returned with a geo array of lat/lon values + """ + crs = CRS.from_cf( + { + 'false_easting': 0.0, + 'false_northing': 0.0, + 'longitude_of_central_meridian': 0.0, + 'standard_parallel': 30.0, + 'grid_mapping_name': 'lambert_cylindrical_equal_area', + } + ) + with self.subTest('Get y_x order when projected_dim is changing across row'): + row_indices = [[0, 0], [4, 0]] + expected_dim_values = [7341677.255608977, -7338157.219843731] + y_x_order, dim_values = get_dimension_order_and_dim_values( + self.lat_arr, self.lon_arr, row_indices, crs, is_row=True + ) + self.assertEqual(y_x_order, True) + self.assertListEqual(dim_values, expected_dim_values) + with self.subTest('Get y_x order when projected_dim is changing across column'): + col_indices = [[0, 0], [0, 9]] + expected_dim_values = [-17299990.048985746, 17213152.396759935] + y_x_order, dim_values = get_dimension_order_and_dim_values( + self.lat_arr, self.lon_arr, col_indices, crs, is_row=False + ) + self.assertEqual(y_x_order, True) + self.assertListEqual(dim_values, expected_dim_values) + with self.subTest('Get x_y order when projected_dim is changing across row'): + row_indices = [[0, 0], [4, 0]] + expected_dim_values = [-17299990.048985746, 17213152.396759935] + y_x_order, dim_values = get_dimension_order_and_dim_values( + self.lat_arr_reversed, + self.lon_arr_reversed, + row_indices, + crs, + is_row=True, + ) + self.assertEqual(y_x_order, False) + self.assertListEqual(dim_values, expected_dim_values) + with self.subTest('Get x_y order when projected_dim is changing across column'): + col_indices = [[0, 0], [0, 9]] + expected_dim_values = [7341677.255608977, -7341677.255608977] + y_x_order, dim_values = get_dimension_order_and_dim_values( + self.lat_arr_reversed, + self.lon_arr_reversed, + col_indices, + crs, + is_row=False, + ) + self.assertEqual(y_x_order, False) + self.assertListEqual(dim_values, expected_dim_values) + with self.subTest('Get y_x order when projected_dims are not varying'): + lat_arr = np.array( + [ + [89.1, 89.1, 89.1], + [89.1, 89.1, 89.1], + [89.1, 89.1, 89.1], + [89.1, 89.1, 89.1], + [89.1, 89.1, 89.1], + ] + ) + lon_arr = np.array( + [ + [-178.1, -178.1, -178.1], + [-178.1, -178.1, -178.1], + [-178.1, -178.1, -178.1], + [-178.1, -178.1, -178.1], + [-178.1, -178.1, -178.1], + ] + ) + row_indices = [[0, 0], [4, 0]] + with self.assertRaises(InvalidCoordinateData) as context: + get_dimension_order_and_dim_values( + lat_arr, lon_arr, row_indices, crs, is_row=True + ) + self.assertEqual( + context.exception.message, + 'lat/lon values are constant', + ) + + def test_create_dimension_arrays_from_coordinates( + self, + ): + """Ensure that the correct x and y dim arrays + are returned from a lat/lon prefetch dataset and + crs provided. + """ + smap_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + 'hoss/hoss_config.json', + ) + smap_file_path = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' + + latitude_coordinate = smap_varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/latitude' + ) + longitude_coordinate = smap_varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/longitude' + ) + projected_dimension_names_am = [ + '/Soil_Moisture_Retrieval_Data_AM/dim_y', + '/Soil_Moisture_Retrieval_Data_AM/dim_x', + ] + projected_dimension_names_pm = [ + '/Soil_Moisture_Retrieval_Data_PM/dim_y', + '/Soil_Moisture_Retrieval_Data_PM/dim_x', + ] + crs = CRS.from_cf( + { + 'false_easting': 0.0, + 'false_northing': 0.0, + 'longitude_of_central_meridian': 0.0, + 'standard_parallel': 30.0, + 'grid_mapping_name': 'lambert_cylindrical_equal_area', + } + ) + expected_xdim = np.array([-17349514.353068016, 17349514.353068016]) + expected_ydim = np.array([7296524.6913595535, -7296524.691359556]) + + with self.subTest('Projected x-y dim arrays from coordinate datasets'): + with Dataset(smap_file_path, 'r') as smap_prefetch: + x_y_dim_am = create_dimension_arrays_from_coordinates( + smap_prefetch, + latitude_coordinate, + longitude_coordinate, + crs, + projected_dimension_names_am, + ) + x_y_dim_pm = create_dimension_arrays_from_coordinates( + smap_prefetch, + latitude_coordinate, + longitude_coordinate, + crs, + projected_dimension_names_pm, + ) + + self.assertListEqual( + list(x_y_dim_am.keys()), projected_dimension_names_am + ) + self.assertListEqual( + list(x_y_dim_pm.keys()), projected_dimension_names_pm + ) + self.assertEqual( + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_y'][0], + expected_ydim[0], + ) + self.assertEqual( + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_y'][-1], + expected_ydim[-1], + ) + self.assertEqual( + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_x'][0], + expected_xdim[0], + ) + self.assertEqual( + x_y_dim_am['/Soil_Moisture_Retrieval_Data_AM/dim_x'][-1], + expected_xdim[-1], + ) + self.assertEqual( + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_y'][0], + expected_ydim[0], + ) + self.assertEqual( + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_y'][-1], + expected_ydim[-1], + ) + self.assertEqual( + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_x'][0], + expected_xdim[0], + ) + self.assertEqual( + x_y_dim_pm['/Soil_Moisture_Retrieval_Data_PM/dim_x'][-1], + expected_xdim[-1], + ) + with self.subTest('Invalid data in coordinate datasets'): + prefetch = { + '/Soil_Moisture_Retrieval_Data_AM/latitude': np.array( + [ + [89.3, 89.3, -9999, 89.3, 89.3], + [-9999, -9999, -60.2, -60.2, -60.2], + [-88.1, -9999, -88.1, -88.1, -88.1], + ] + ), + '/Soil_Moisture_Retrieval_Data_AM/longitude': np.array( + [ + [-9999, -9999, -9999, -9999, 178.4], + [-179.3, -9999, -9999, -9999, -9999], + [-179.3, -9999, -9999, -9999, -9999], + ] + ), + } + with self.assertRaises(InvalidCoordinateData) as context: + create_dimension_arrays_from_coordinates( + prefetch, + latitude_coordinate, + longitude_coordinate, + crs, + projected_dimension_names_am, + ) + self.assertEqual( + context.exception.message, + 'lat/lon values are constant', + ) + with self.subTest('Cannot determine x-y order in coordinate datasets'): + prefetch = { + '/Soil_Moisture_Retrieval_Data_AM/latitude': np.array( + [ + [89.3, 89.3, -9999, 89.3, 89.3], + [-9999, -9999, 89.3, 89.3, 89.3], + [89.3, 89.3, 89.3, 89.3, 89.3], + ] + ), + '/Soil_Moisture_Retrieval_Data_AM/longitude': np.array( + [ + [-9999, -9999, -9999, -9999, 178.4], + [-179.3, -9999, -9999, -9999, -9999], + [-179.3, -9999, -9999, -9999, -9999], + ] + ), + } + with self.assertRaises(InvalidCoordinateData) as context: + create_dimension_arrays_from_coordinates( + prefetch, + latitude_coordinate, + longitude_coordinate, + crs, + projected_dimension_names_am, + ) + self.assertEqual( + context.exception.message, + 'lat/lon values are constant', + ) diff --git a/tests/unit/test_dimension_utilities.py b/tests/unit/test_dimension_utilities.py index 5ad7a21..448b493 100644 --- a/tests/unit/test_dimension_utilities.py +++ b/tests/unit/test_dimension_utilities.py @@ -14,8 +14,8 @@ from varinfo import VarInfoFromDmr from hoss.dimension_utilities import ( - add_bounds_variables, add_index_range, + check_add_artificial_bounds, get_bounds_array, get_dimension_bounds, get_dimension_extents, @@ -23,15 +23,20 @@ get_dimension_indices_from_bounds, get_dimension_indices_from_values, get_fill_slice, + get_prefetch_variables, + get_range_strings, get_requested_index_ranges, is_almost_in, is_dimension_ascending, is_index_subset, needs_bounds, - prefetch_dimension_variables, write_bounds, ) -from hoss.exceptions import InvalidNamedDimension, InvalidRequestedRange +from hoss.exceptions import ( + InvalidIndexSubsetRequest, + InvalidNamedDimension, + InvalidRequestedRange, +) class TestDimensionUtilities(TestCase): @@ -313,6 +318,45 @@ def test_add_index_range(self): '/sst_dtime[][12:34][]', ) + def test_get_range_strings(self): + """Ensure the correct combinations of range_strings are added as + suffixes to the input variable based upon that variable's dimensions. + If a dimension range has the lower index > upper index, that + indicates the bounding box crosses the edge of the grid. In this + instance, the full range of the variable should be retrieved. + + """ + with self.subTest('all dimensions found in index_ranges'): + variable_list = ['/Grid/lon', '/Grid/lat'] + index_ranges = {'/Grid/lat': (600, 699), '/Grid/lon': (2200, 2299)} + expected_range_strings = range_strings = ['[2200:2299]', '[600:699]'] + self.assertEqual( + get_range_strings(variable_list, index_ranges), expected_range_strings + ) + + with self.subTest('only some dimensions found in index range'): + variable_list = ['/Grid/time', '/Grid/lon', '/Grid/lat'] + index_ranges = {'/Grid/lat': (600, 699), '/Grid/lon': (2200, 2299)} + expected_range_strings = ['[]', '[2200:2299]', '[600:699]'] + self.assertEqual( + get_range_strings(variable_list, index_ranges), + expected_range_strings, + ) + + with self.subTest('No variables found in index ranges'): + variable_list = ['/Grid/time', '/Grid/lon', '/Grid/lat'] + self.assertEqual(get_range_strings(variable_list, {}), ['[]', '[]', '[]']) + with self.subTest( + 'when dimension range lower index is greater than upper index' + ): + variable_list = ['/Grid/time', '/Grid/lon', '/Grid/lat'] + index_ranges = {'/Grid/lat': (699, 600), '/Grid/lon': (2200, 2299)} + expected_range_strings = ['[]', '[2200:2299]', '[]'] + self.assertEqual( + get_range_strings(variable_list, index_ranges), + expected_range_strings, + ) + def test_get_fill_slice(self): """Ensure that a slice object is correctly formed for a requested dimension. @@ -326,10 +370,10 @@ def test_get_fill_slice(self): with self.subTest('A filled dimension returns slice(start, stop).'): self.assertEqual(get_fill_slice('/longitude', fill_ranges), slice(16, 200)) - @patch('hoss.dimension_utilities.add_bounds_variables') + @patch('hoss.dimension_utilities.check_add_artificial_bounds') @patch('hoss.dimension_utilities.get_opendap_nc4') - def test_prefetch_dimension_variables( - self, mock_get_opendap_nc4, mock_add_bounds_variables + def test_get_prefetch_variables( + self, mock_get_opendap_nc4, mock_check_add_artificial_bounds ): """Ensure that when a list of required variables is specified, a request to OPeNDAP will be sent requesting only those that are @@ -343,13 +387,13 @@ def test_prefetch_dimension_variables( mock_get_opendap_nc4.return_value = prefetch_path access_token = 'access' - output_dir = 'tests/output' + output_dir = self.temp_dir url = 'https://url_to_opendap_granule' required_variables = {'/latitude', '/longitude', '/time', '/wind_speed'} required_dimensions = {'/latitude', '/longitude', '/time'} self.assertEqual( - prefetch_dimension_variables( + get_prefetch_variables( url, self.varinfo, required_variables, @@ -364,14 +408,140 @@ def test_prefetch_dimension_variables( mock_get_opendap_nc4.assert_called_once_with( url, required_dimensions, output_dir, self.logger, access_token, self.config ) - - mock_add_bounds_variables.assert_called_once_with( + mock_check_add_artificial_bounds.assert_called_once_with( prefetch_path, required_dimensions, self.varinfo, self.logger ) + @patch('hoss.dimension_utilities.check_add_artificial_bounds') + @patch('hoss.dimension_utilities.get_opendap_nc4') + def test_get_prefetch_variables_with_anonymous_dimensions( + self, + mock_get_opendap_nc4, + mock_check_add_artificial_bounds, + ): + """Ensure that when a list of required variables is specified, + and the required dimension variables are not present, + checks and retrieves coordinate variables which is used + in the opendap prefetch request. + + """ + prefetch_path = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' + mock_get_opendap_nc4.return_value = prefetch_path + access_token = 'access' + output_dir = self.temp_dir + url = 'https://url_to_opendap_granule' + prefetch_variables = { + '/Soil_Moisture_Retrieval_Data_AM/latitude', + '/Soil_Moisture_Retrieval_Data_AM/longitude', + } + requested_variables = { + '/Soil_Moisture_Retrieval_Data_AM/albedo', + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + } + varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + config_file='hoss/hoss_config.json', + ) + + self.assertEqual( + get_prefetch_variables( + url, + varinfo, + requested_variables, + output_dir, + self.logger, + access_token, + self.config, + ), + prefetch_path, + ) + mock_get_opendap_nc4.assert_called_once_with( + url, prefetch_variables, output_dir, self.logger, access_token, self.config + ) + mock_check_add_artificial_bounds.assert_called_once_with( + prefetch_path, prefetch_variables, varinfo, self.logger + ) + + @patch('hoss.dimension_utilities.get_coordinate_variables') + @patch('hoss.dimension_utilities.check_add_artificial_bounds') + @patch('hoss.dimension_utilities.get_opendap_nc4') + def test_get_prefetch_variables_with_no_anonymous_dimensions( + self, + mock_get_opendap_nc4, + mock_check_add_artificial_bounds, + mock_get_coordinate_variables, + ): + """Ensure that when a list of required variables is specified, + If dimension variables are not present and two coordinate + variables are also not present, the opendap prefetch request + will not include any dimension variables. + """ + prefetch_path = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' + mock_get_opendap_nc4.return_value = prefetch_path + access_token = 'access' + output_dir = self.temp_dir + url = 'https://url_to_opendap_granule' + + requested_variables = { + '/Soil_Moisture_Retrieval_Data_AM/albedo', + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + } + varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + config_file='hoss/hoss_config.json', + ) + with self.subTest('No coordinate variables'): + mock_get_coordinate_variables.return_value = ([], []) + with self.assertRaises(InvalidIndexSubsetRequest): + get_prefetch_variables( + url, + varinfo, + requested_variables, + output_dir, + self.logger, + access_token, + self.config, + ) + + mock_get_coordinate_variables.assert_called_once_with( + varinfo, + requested_variables, + ) + mock_get_opendap_nc4.assert_not_called() + mock_check_add_artificial_bounds.assert_not_called() + + mock_get_coordinate_variables.reset_mock() + mock_get_opendap_nc4.reset_mock() + mock_check_add_artificial_bounds.reset_mock() + + with self.subTest('Only one coordinate variable'): + mock_get_coordinate_variables.return_value = ( + ['/Soil_Moisture_Retrieval_Data_AM/latitude'], + [], + ) + with self.assertRaises(InvalidIndexSubsetRequest): + get_prefetch_variables( + url, + varinfo, + requested_variables, + output_dir, + self.logger, + access_token, + self.config, + ) + + mock_get_coordinate_variables.assert_called_once_with( + varinfo, + requested_variables, + ) + mock_get_opendap_nc4.assert_not_called() + mock_check_add_artificial_bounds.assert_not_called() + @patch('hoss.dimension_utilities.needs_bounds') @patch('hoss.dimension_utilities.write_bounds') - def test_add_bounds_variables(self, mock_write_bounds, mock_needs_bounds): + def test_check_add_artificial_bounds(self, mock_write_bounds, mock_needs_bounds): """Ensure that `write_bounds` is called when it's needed, and that it's not called when it's not needed. @@ -389,7 +559,7 @@ def test_add_bounds_variables(self, mock_write_bounds, mock_needs_bounds): with self.subTest('Bounds need to be written'): mock_needs_bounds.return_value = True - add_bounds_variables( + check_add_artificial_bounds( prefetch_dataset_name, required_dimensions, varinfo_prefetch, @@ -402,7 +572,7 @@ def test_add_bounds_variables(self, mock_write_bounds, mock_needs_bounds): with self.subTest('Bounds should not be written'): mock_needs_bounds.return_value = False - add_bounds_variables( + check_add_artificial_bounds( prefetch_dataset_name, required_dimensions, varinfo_prefetch, @@ -569,7 +739,7 @@ def test_prefetch_dimensions_with_bounds(self, mock_get_opendap_nc4): } self.assertEqual( - prefetch_dimension_variables( + get_prefetch_variables( url, self.varinfo_with_bounds, required_variables, diff --git a/tests/unit/test_projection_utilities.py b/tests/unit/test_projection_utilities.py index 0da8f66..164eb9e 100644 --- a/tests/unit/test_projection_utilities.py +++ b/tests/unit/test_projection_utilities.py @@ -194,6 +194,28 @@ def test_get_variable_crs(self): 'present in granule .dmr file.', ) + with self.subTest( + 'attributes for missing grid_mapping retrieved from earthdata-varinfo configuration file' + ): + smap_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + 'hoss/hoss_config.json', + ) + expected_crs = CRS.from_cf( + { + 'false_easting': 0.0, + 'false_northing': 0.0, + 'longitude_of_central_meridian': 0.0, + 'standard_parallel': 30.0, + 'grid_mapping_name': 'lambert_cylindrical_equal_area', + } + ) + actual_crs = get_variable_crs( + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', smap_varinfo + ) + self.assertEqual(actual_crs, expected_crs) + def test_get_projected_x_y_extents(self): """Ensure that the expected values for the x and y dimension extents are recovered for a known projected grid and requested input. diff --git a/tests/unit/test_spatial.py b/tests/unit/test_spatial.py index e5ee6d3..bc299b9 100644 --- a/tests/unit/test_spatial.py +++ b/tests/unit/test_spatial.py @@ -17,6 +17,7 @@ get_longitude_in_grid, get_projected_x_y_index_ranges, get_spatial_index_ranges, + get_x_y_index_ranges_from_coordinates, ) @@ -56,6 +57,44 @@ def test_get_spatial_index_ranges_projected(self): {'/x': (37, 56), '/y': (7, 26)}, ) + def test_get_spatial_index_ranges_projected_from_coordinates(self): + """Ensure that correct index ranges can be calculated for a SMAP L3 + granule. This granule has variables that do no have dimensions, but + they have coordinate attributes. + + """ + harmony_message = Message({'subset': {'bbox': [2, 54, 42, 72]}}) + smap_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + 'hoss/hoss_config.json', + ) + prefetch_path = 'tests/data/SC_SPL3SMP_009_prefetch.nc4' + required_variables = { + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm', + '/Soil_Moisture_Retrieval_Data_AM/latitude', + '/Soil_Moisture_Retrieval_Data_AM/longitude', + '/Soil_Moisture_Retrieval_Data_PM/latitude_pm', + '/Soil_Moisture_Retrieval_Data_PM/longitude_pm', + } + expected_index_ranges = { + '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 594), + '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), + '/Soil_Moisture_Retrieval_Data_PM/dim_x': (487, 594), + '/Soil_Moisture_Retrieval_Data_PM/dim_y': (9, 38), + } + + self.assertDictEqual( + get_spatial_index_ranges( + required_variables, + smap_varinfo, + prefetch_path, + harmony_message, + ), + expected_index_ranges, + ) + def test_get_spatial_index_ranges_geographic(self): """Ensure that correct index ranges can be calculated for: @@ -182,6 +221,126 @@ def test_get_spatial_index_ranges_geographic(self): {'/latitude': (5, 44), '/longitude': (160, 199)}, ) + @patch('hoss.spatial.get_dimension_index_range') + @patch('hoss.spatial.get_projected_x_y_extents') + def test_get_x_y_index_ranges_from_coordinates( + self, + mock_get_x_y_extents, + mock_get_dimension_index_range, + ): + """Ensure that x and y index ranges are only requested only when there are + no projected dimensions and when there are coordinate datasets, + and the values have not already been calculated. + + The example used in this test is for the SMAP SPL3SMP collection, + (SMAP L3 Radiometer Global Daily 36 km EASE-Grid Soil Moisture) + which has a Equal-Area Scalable Earth Grid (EASE-Grid 2.0) CRS for + a projected grid which is lambert_cylindrical_equal_area projection + + """ + smap_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + 'hoss/hoss_config.json', + ) + smap_file_path = 'tests/data/SC_SPL3SMP_008_prefetch.nc4' + expected_index_ranges = { + '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 595), + '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), + } + bbox = BBox(2, 54, 42, 72) + + latitude_coordinate = smap_varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/latitude' + ) + longitude_coordinate = smap_varinfo.get_variable( + '/Soil_Moisture_Retrieval_Data_AM/longitude' + ) + + crs = CRS.from_cf( + { + 'false_easting': 0.0, + 'false_northing': 0.0, + 'longitude_of_central_meridian': 0.0, + 'standard_parallel': 30.0, + 'grid_mapping_name': 'lambert_cylindrical_equal_area', + } + ) + + x_y_extents = { + 'x_min': 192972.56050179302, + 'x_max': 4052423.7705376535, + 'y_min': 5930779.396449475, + 'y_max': 6979878.9118312765, + } + + mock_get_x_y_extents.return_value = x_y_extents + + # When ranges are derived, they are first calculated for x, then y: + mock_get_dimension_index_range.side_effect = [(487, 595), (9, 38)] + + with self.subTest( + 'Projected grid from coordinates gets expected dimension ranges' + ): + with Dataset(smap_file_path, 'r') as smap_prefetch: + self.assertDictEqual( + get_x_y_index_ranges_from_coordinates( + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + smap_varinfo, + smap_prefetch, + latitude_coordinate, + longitude_coordinate, + {}, + bounding_box=bbox, + shape_file_path=None, + ), + expected_index_ranges, + ) + + mock_get_x_y_extents.assert_called_once_with( + ANY, ANY, crs, shape_file=None, bounding_box=bbox + ) + + self.assertEqual(mock_get_dimension_index_range.call_count, 2) + mock_get_dimension_index_range.assert_has_calls( + [ + call( + ANY, + x_y_extents['x_min'], + x_y_extents['x_max'], + bounds_values=None, + ), + call( + ANY, + x_y_extents['y_min'], + x_y_extents['y_max'], + bounds_values=None, + ), + ] + ) + + mock_get_x_y_extents.reset_mock() + mock_get_dimension_index_range.reset_mock() + + with self.subTest('Function does not rederive known index ranges'): + with Dataset(smap_file_path, 'r') as smap_prefetch: + self.assertDictEqual( + get_x_y_index_ranges_from_coordinates( + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + smap_varinfo, + smap_prefetch, + latitude_coordinate, + longitude_coordinate, + expected_index_ranges, + bounding_box=bbox, + shape_file_path=None, + ), + {}, + ) + + mock_get_x_y_extents.assert_not_called() + mock_get_dimension_index_range.assert_not_called() + @patch('hoss.spatial.get_dimension_index_range') @patch('hoss.spatial.get_projected_x_y_extents') def test_get_projected_x_y_index_ranges( diff --git a/tests/unit/test_subset.py b/tests/unit/test_subset.py index abbb5f1..f2a20f4 100644 --- a/tests/unit/test_subset.py +++ b/tests/unit/test_subset.py @@ -62,7 +62,7 @@ def tearDown(self): @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_granule_not_geo( self, @@ -126,7 +126,7 @@ def test_subset_granule_not_geo( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_granule_geo( self, @@ -216,7 +216,7 @@ def test_subset_granule_geo( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_non_geo_no_variables( self, @@ -290,7 +290,7 @@ def test_subset_non_geo_no_variables( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_geo_no_variables( self, @@ -404,7 +404,7 @@ def test_subset_geo_no_variables( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_non_variable_dimensions( self, @@ -537,7 +537,7 @@ def test_subset_non_variable_dimensions( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_bounds_reference( self, @@ -638,7 +638,7 @@ def test_subset_bounds_reference( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_temporal( self, @@ -742,7 +742,7 @@ def test_subset_temporal( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_geo_temporal( self, @@ -860,7 +860,7 @@ def test_subset_geo_temporal( @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') @patch('hoss.subset.get_request_shape_file') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_granule_shape( self, @@ -971,7 +971,7 @@ def test_subset_granule_shape( @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') @patch('hoss.subset.get_request_shape_file') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_granule_shape_and_bbox( self, @@ -1083,7 +1083,7 @@ def test_subset_granule_shape_and_bbox( @patch('hoss.subset.get_requested_index_ranges') @patch('hoss.subset.get_temporal_index_ranges') @patch('hoss.subset.get_spatial_index_ranges') - @patch('hoss.subset.prefetch_dimension_variables') + @patch('hoss.subset.get_prefetch_variables') @patch('hoss.subset.get_varinfo') def test_subset_granule_geo_named( self, @@ -1454,3 +1454,121 @@ def test_fill_variable(self, mock_get_fill_slice): ] ) mock_get_fill_slice.reset_mock() + + @patch('hoss.subset.get_opendap_nc4') + @patch('hoss.subset.get_spatial_index_ranges') + @patch('hoss.subset.get_prefetch_variables') + @patch('hoss.subset.get_varinfo') + def test_subset_granule_with_no_dimensions( + self, + mock_get_varinfo, + mock_get_prefetch_variables, + mock_get_spatial_index_ranges, + mock_get_opendap_nc4, + ): + """Ensure a request to extract both a variable and spatial subset for + a granule without dimensions but with valid coordinate attributes + without error. Because a bounding box is specified in this request, + the prefetch functionality and the HOSS spatial_index + functionality in `hoss.spatial.py` should be called. + + """ + harmony_message = Message( + {'accessToken': self.access_token, 'subset': {'bbox': [2, 54, 42, 72]}} + ) + harmony_source = Source( + { + 'collection': 'C1268452378-EEDTEST', + 'shortName': 'SPL3SMP', + 'variables': [ + { + 'id': 'V1255903615-EEDTEST', + 'name': 'surface_flag', + 'fullPath': '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + }, + { + 'id': 'V1238395077-EEDTEST', + 'name': 'surface_flag_pm', + 'fullPath': '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm', + }, + ], + } + ) + granule_url = 'https://harmony.earthdata.nasa.gov/bucket/spl3smp' + collection_short_name = 'SPL3SMP' + smap_varinfo = VarInfoFromDmr( + 'tests/data/SC_SPL3SMP_008.dmr', + 'SPL3SMP', + 'hoss/hoss_config.json', + ) + prefetch_path = 'tests/data/SC_SPL3SMP_009_prefetch.nc4' + subset_output_path = 'SC_SPL3SMP.009_296012210.nc4' + required_variables = { + '/Soil_Moisture_Retrieval_Data_AM/surface_flag', + '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm', + '/Soil_Moisture_Retrieval_Data_AM/latitude', + '/Soil_Moisture_Retrieval_Data_AM/longitude', + '/Soil_Moisture_Retrieval_Data_PM/latitude_pm', + '/Soil_Moisture_Retrieval_Data_PM/longitude_pm', + } + + variables_with_ranges = { + '/Soil_Moisture_Retrieval_Data_AM/longitude[9:38][487:595]', + '/Soil_Moisture_Retrieval_Data_PM/longitude_pm[9:38][487:595]', + '/Soil_Moisture_Retrieval_Data_AM/latitude[9:38][487:595]', + '/Soil_Moisture_Retrieval_Data_AM/surface_flag[9:38][487:595]', + '/Soil_Moisture_Retrieval_Data_PM/surface_flag_pm[9:38][487:595]', + '/Soil_Moisture_Retrieval_Data_PM/latitude_pm[9:38][487:595]', + } + expected_index_ranges = { + '/Soil_Moisture_Retrieval_Data_AM/dim_x': (487, 595), + '/Soil_Moisture_Retrieval_Data_AM/dim_y': (9, 38), + '/Soil_Moisture_Retrieval_Data_PM/dim_x': (487, 595), + '/Soil_Moisture_Retrieval_Data_PM/dim_y': (9, 38), + } + + mock_get_varinfo.return_value = smap_varinfo + mock_get_prefetch_variables.return_value = prefetch_path + mock_get_spatial_index_ranges.return_value = expected_index_ranges + mock_get_opendap_nc4.return_value = subset_output_path + + output_path = subset_granule( + granule_url, + harmony_source, + self.output_dir, + harmony_message, + self.logger, + self.config, + ) + + self.assertEqual(output_path, subset_output_path) + mock_get_varinfo.assert_called_once_with( + granule_url, + self.output_dir, + self.logger, + collection_short_name, + self.access_token, + self.config, + ) + + mock_get_prefetch_variables.assert_called_once_with( + granule_url, + smap_varinfo, + required_variables, + self.output_dir, + self.logger, + self.access_token, + self.config, + ) + mock_get_spatial_index_ranges.assert_called_once_with( + required_variables, smap_varinfo, prefetch_path, harmony_message, None + ) + + mock_get_opendap_nc4.assert_called_once_with( + granule_url, + variables_with_ranges, + self.output_dir, + self.logger, + self.access_token, + self.config, + )