From 5dd3ec0156c33dbf28b6d131d3fad4ed8c3acfb5 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 30 Apr 2024 14:14:41 +1000 Subject: [PATCH 01/12] Remove unused variable. --- datacube_ows/product_ranges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacube_ows/product_ranges.py b/datacube_ows/product_ranges.py index 1ec5afeec..39f050902 100644 --- a/datacube_ows/product_ranges.py +++ b/datacube_ows/product_ranges.py @@ -322,7 +322,6 @@ def add_ranges(dc: datacube.Datacube, product_names: list[str], merge_only: bool ows_multiproducts: list[OWSMultiProductLayer] = [] errors = False for pname in product_names: - dc_product = None ows_product = get_config().product_index.get(pname) if not ows_product: ows_product = get_config().native_product_index.get(pname) @@ -386,6 +385,7 @@ def add_ranges(dc: datacube.Datacube, product_names: list[str], merge_only: bool print("Done.") return errors + def get_ranges(dc: datacube.Datacube, product: OWSNamedLayer, path: str | None = None) -> dict[str, Any] | None: cfg = product.global_cfg From 2d14eb5a05eac5487055f1fe6845605e32d8bb03 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 30 Apr 2024 15:59:36 +1000 Subject: [PATCH 02/12] Start making config responsible for managing ODC access. --- datacube_ows/config_utils.py | 30 ++++---- datacube_ows/ows_configuration.py | 114 +++++++++++++++++------------- datacube_ows/product_ranges.py | 14 ++-- datacube_ows/styles/base.py | 6 +- datacube_ows/styles/component.py | 4 +- datacube_ows/wcs1.py | 6 +- datacube_ows/wcs1_utils.py | 56 +++++++-------- tests/conftest.py | 3 +- tests/test_cfg_bandidx.py | 17 ++--- tests/test_cfg_layer.py | 12 ++-- tests/test_styles.py | 2 +- 11 files changed, 140 insertions(+), 124 deletions(-) diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 5cf06705f..76d7767c8 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -228,7 +228,7 @@ def __setattr__(self, name: str, val: Any) -> None: super().__setattr__(name, val) # Validate against database and prepare for use. - def make_ready(self, dc: Datacube, *args, **kwargs) -> None: + def make_ready(self, *args, **kwargs) -> None: """ Perform second phase initialisation with a database connection. @@ -660,15 +660,15 @@ def __init__(self, cfg: CFG_DICT, product_cfg: "datacube_ows.ows_configuration.O """ super().__init__(cfg, **kwargs) cfg = cast(CFG_DICT, self._raw_cfg) - self.product = product_cfg - pq_names = self.product.parse_pq_names(cfg) + self.layer = product_cfg + pq_names = self.layer.parse_pq_names(cfg) self.pq_names = cast(list[str], pq_names["pq_names"]) self.pq_low_res_names = pq_names["pq_low_res_names"] self.main_products = pq_names["main_products"] self.pq_band = str(cfg["band"]) self.canonical_band_name = self.pq_band # Update for aliasing on make_ready if "fuse_func" in cfg: - self.pq_fuse_func: Optional[FunctionWrapper] = FunctionWrapper(self.product, cast(CFG_DICT, cfg["fuse_func"])) + self.pq_fuse_func: Optional[FunctionWrapper] = FunctionWrapper(self.layer, cast(CFG_DICT, cfg["fuse_func"])) else: self.pq_fuse_func = None self.pq_ignore_time = bool(cfg.get("ignore_time", False)) @@ -679,7 +679,7 @@ def __init__(self, cfg: CFG_DICT, product_cfg: "datacube_ows.ows_configuration.O self.declare_unready("info_mask") # pylint: disable=attribute-defined-outside-init - def make_ready(self, dc: Datacube, *args, **kwargs) -> None: + def make_ready(self, *args: Any, **kwargs: Any) -> None: """ Second round (db-aware) intialisation. @@ -691,21 +691,21 @@ def make_ready(self, dc: Datacube, *args, **kwargs) -> None: self.pq_low_res_products: list[Product] = [] for pqn in self.pq_names: if pqn is not None: - pq_product = dc.index.products.get_by_name(pqn) + pq_product = self.layer.dc.index.products.get_by_name(pqn) if pq_product is None: - raise ConfigException(f"Could not find flags product {pqn} for layer {self.product.name} in datacube") + raise ConfigException(f"Could not find flags product {pqn} for layer {self.layer.name} in datacube") self.pq_products.append(pq_product) for pqn in self.pq_low_res_names: if pqn is not None: - pq_product = dc.index.products.get_by_name(pqn) + pq_product = self.layer.dc.index.products.get_by_name(pqn) if pq_product is None: - raise ConfigException(f"Could not find flags low_res product {pqn} for layer {self.product.name} in datacube") + raise ConfigException(f"Could not find flags low_res product {pqn} for layer {self.layer.name} in datacube") self.pq_low_res_products.append(pq_product) # Resolve band alias if necessary. if self.main_products: try: - self.canonical_band_name = self.product.band_idx.band(self.pq_band) + self.canonical_band_name = self.layer.band_idx.band(self.pq_band) except ConfigException: pass @@ -717,9 +717,9 @@ def make_ready(self, dc: Datacube, *args, **kwargs) -> None: meas = product.lookup_measurements([str(self.canonical_band_name)])[str(self.canonical_band_name)] except KeyError: raise ConfigException( - f"Band {self.pq_band} does not exist in product {product.name} - cannot be used as a flag band for layer {self.product.name}.") + f"Band {self.pq_band} does not exist in product {product.name} - cannot be used as a flag band for layer {self.layer.name}.") if "flags_definition" not in meas: - raise ConfigException(f"Band {self.pq_band} in product {product.name} has no flags_definition in ODC - cannot be used as a flag band for layer {self.product.name}.") + raise ConfigException(f"Band {self.pq_band} in product {product.name} has no flags_definition in ODC - cannot be used as a flag band for layer {self.layer.name}.") # pyre-ignore[16] self.flags_def: dict[str, dict[str, RAW_CFG]] = meas["flags_definition"] for bitname in self.ignore_info_flags: @@ -728,7 +728,7 @@ def make_ready(self, dc: Datacube, *args, **kwargs) -> None: continue flag = 1 << bit self.info_mask &= ~flag - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) FlagBand = OWSFlagBand | OWSFlagBandStandalone @@ -789,7 +789,7 @@ def add_flag_band(self, fb: FlagBand) -> None: self.declare_unready("low_res_products") # pylint: disable=attribute-defined-outside-init - def make_ready(self, dc: Datacube, *args, **kwargs) -> None: + def make_ready(self, *args, **kwargs) -> None: """ Second round (db-aware) intialisation. @@ -804,7 +804,7 @@ def make_ready(self, dc: Datacube, *args, **kwargs) -> None: break if self.main_product: self.bands = set(self.layer.band_idx.band(b) for b in self.bands) - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) @classmethod def build_list_from_masks(cls, masks: Iterable["datacube_ows.styles.base.StyleMask"], diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 989c6a1f5..6645f659e 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -27,7 +27,8 @@ from babel.messages.pofile import read_po from datacube import Datacube from datacube.api.query import GroupBy -from datacube.model import Measurement +from datacube.model import Measurement, Product +from datacube.cfg import ODCConfig, ODCEnvironment from odc.geo import CRS from odc.geo.geobox import GeoBox from ows import Version @@ -40,7 +41,7 @@ OWSMetadataConfig, cfg_expand, get_file_loc, import_python_obj, load_json_obj) -from datacube_ows.cube_pool import ODCInitException, cube, get_cube +from datacube_ows.cube_pool import ODCInitException from datacube_ows.ogc_utils import create_geobox from datacube_ows.resource_limits import (OWSResourceManagementRules, parse_cache_age) @@ -86,8 +87,8 @@ def __init__(self, layer: "OWSNamedLayer", band_cfg: CFG_DICT): band_cfg = {} super().__init__(band_cfg) self.band_cfg = cast(dict[str, list[str]], band_cfg) - self.product = layer - self.product_name = layer.name + self.layer = layer + self.layer_name = layer.name self.parse_metadata(band_cfg) self._idx: dict[str, str] = {} self.add_aliases(self.band_cfg) @@ -96,22 +97,22 @@ def __init__(self, layer: "OWSNamedLayer", band_cfg: CFG_DICT): self.declare_unready("_dtypes") def global_config(self) -> "OWSConfig": - return self.product.global_config() + return self.layer.global_config() def get_obj_label(self) -> str: - return self.product.get_obj_label() + ".bands" + return self.layer.get_obj_label() + ".bands" def add_aliases(self, cfg: dict[str, list[str]]) -> None: for b, aliases in cfg.items(): if b in self._idx: - raise ConfigException(f"Duplicate band name/alias: {b} in layer {self.product_name}") + raise ConfigException(f"Duplicate band name/alias: {b} in layer {self.layer_name}") self._idx[b] = b for a in aliases: if a != b and a in self._idx: - raise ConfigException(f"Duplicate band name/alias: {a} in layer {self.product_name}") + raise ConfigException(f"Duplicate band name/alias: {a} in layer {self.layer_name}") self._idx[a] = b - def make_ready(self, dc: Datacube, *args, **kwargs) -> None: + def make_ready(self, *args: Any, **kwargs: Any) -> None: def floatify_nans(inp: float | int | str) -> float | int: if isinstance(inp, str) and inp == "nan": return float(inp) @@ -125,9 +126,9 @@ def floatify_nans(inp: float | int | str) -> float | int: self._nodata_vals: dict[str, int | float] = {} self._dtypes: dict[str, numpy.dtype] = {} first_product = True - for product in self.product.products: + for product in self.layer.products: if first_product and default_to_all: - native_bands = dc.list_measurements().loc[product.name] + native_bands = self.layer.dc.list_measurements().loc[product.name] for b in native_bands.index: self.band_cfg[b] = [b] self.add_aliases(self.band_cfg) @@ -145,19 +146,19 @@ def floatify_nans(inp: float | int | str) -> float | int: if ((numpy.isnan(nodata) and not numpy.isnan(floatify_nans(prod_measurements[k].nodata))) or (not numpy.isnan(nodata) and prod_measurements[k].nodata != nodata)): raise ConfigException( - f"Nodata value mismatch between products for band {k} in multiproduct layer {self.product.name}") + f"Nodata value mismatch between products for band {k} in multiproduct layer {self.layer_name}") if prod_measurements[k].dtype != self._dtypes[k]: raise ConfigException( - f"Data type mismatch between products for band {k} in multiproduct layer {self.product.name}") + f"Data type mismatch between products for band {k} in multiproduct layer {self.layer_name}") except KeyError as e: - raise ConfigException(f"Product {product.name} in layer {self.product.name} is missing band {e}") + raise ConfigException(f"Product {product.name} in layer {self.layer_name} is missing band {e}") first_product = False - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) def band(self, name_alias: str) -> str: if name_alias in self._idx: return self._idx[name_alias] - raise ConfigException(f"Unknown band name/alias: {name_alias} in layer {self.product.name}") + raise ConfigException(f"Unknown band name/alias: {name_alias} in layer {self.layer_name}") def locale_band(self, name_alias: str) -> str: try: @@ -167,7 +168,7 @@ def locale_band(self, name_alias: str) -> str: for b in self.band_cfg.keys(): if name_alias == self.band_label(b): return b - raise ConfigException(f"Unknown band: {name_alias} in layer {self.product.name}") + raise ConfigException(f"Unknown band: {name_alias} in layer {self.layer_name}") def band_label(self, name_alias) -> str: canonical_name = self.band(name_alias) @@ -321,17 +322,17 @@ def unready_layer_count(self) -> int: def layer_count(self) -> int: return sum([l.layer_count() for l in self.child_layers]) - def make_ready(self, dc: Datacube, *args, **kwargs) -> None: + def make_ready(self, *args, **kwargs) -> None: still_unready = [] for lyr in self.unready_layers: try: - lyr.make_ready(dc, *args, **kwargs) + lyr.make_ready(*args, **kwargs) self.child_layers.append(lyr) except ConfigException as e: _LOG.error("Could not load layer %s: %s", lyr.title, str(e)) still_unready.append(lyr) self.unready_layers = still_unready - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) class TimeRes(Enum): @@ -412,6 +413,10 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold self.name = name cfg = cast(CFG_DICT, self._raw_cfg) self.hide = False + self.local_env: ODCEnvironment | None = None + local_env = cast(str | None, cfg.get("env")) + if local_env: + self.local_env = ODCConfig.get_environment(env=local_env) try: self.parse_product_names(cfg) if len(self.low_res_product_names) not in (0, len(self.product_names)): @@ -564,47 +569,55 @@ def time_axis_representation(self) -> str: return "" # pylint: disable=attribute-defined-outside-init - def make_ready(self, dc: Datacube, *args, **kwargs): - self.products = [] - self.low_res_products = [] + def make_ready(self, *args: Any, **kwargs: Any) -> None: + if self.local_env: + try: + self.dc: Datacube = Datacube(env=self.local_env, app=self.global_cfg.odc_app) + except Exception as e: + _LOG.error("ODC initialisation failed: %s", str(e)) + raise (ODCInitException(e)) + else: + self.dc = self.global_cfg.dc + self.products: list[Product] = [] + self.low_res_products: list[Product] = [] for i, prod_name in enumerate(self.product_names): if self.low_res_product_names: low_res_prod_name = self.low_res_product_names[i] else: low_res_prod_name = None - product = dc.index.products.get_by_name(prod_name) + product = self.dc.index.products.get_by_name(prod_name) if not product: raise ConfigException(f"Could not find product {prod_name} in datacube for layer {self.name}") self.products.append(product) if low_res_prod_name: - product = dc.index.products.get_by_name(low_res_prod_name) + product = self.dc.index.products.get_by_name(low_res_prod_name) if not product: raise ConfigException(f"Could not find product {low_res_prod_name} in datacube for layer {self.name}") self.low_res_products.append(product) self.product = self.products[0] self.definition = self.product.definition - self.force_range_update(dc) - self.band_idx.make_ready(dc) - self.resource_limits.make_ready(dc) + self.force_range_update() + self.band_idx.make_ready() + self.resource_limits.make_ready() self.all_flag_band_names: set[str] = set() for fb in self.flag_bands.values(): - fb.make_ready(dc) + fb.make_ready() if fb.pq_band in self.all_flag_band_names: raise ConfigException(f"Duplicate flag band name: {fb.pq_band}") self.all_flag_band_names.add(fb.pq_band) - self.ready_image_processing(dc) + self.ready_image_processing() self.ready_native_specs() if self.global_cfg.wcs: - self.ready_wcs(dc) + self.ready_wcs() for style in self.styles: - style.make_ready(dc, *args, **kwargs) + style.make_ready(*args, **kwargs) for fpb in self.allflag_productbands: - fpb.make_ready(dc, *args, **kwargs) + fpb.make_ready(*args, **kwargs) if not self.multi_product: self.global_cfg.native_product_index[self.product_name] = self if not self.hide: - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) # pylint: disable=attribute-defined-outside-init def parse_image_processing(self, cfg: CFG_DICT): @@ -633,7 +646,7 @@ def parse_image_processing(self, cfg: CFG_DICT): self.fuse_func = None # pylint: disable=attribute-defined-outside-init - def ready_image_processing(self, dc: Datacube) -> None: + def ready_image_processing(self) -> None: self.always_fetch_bands = list([self.band_idx.band(b) for b in cast(list[str], self.raw_afb)]) # pylint: disable=attribute-defined-outside-init @@ -803,7 +816,7 @@ def ready_native_specs(self): self.name, repr(self.cfg_native_resolution), self.resolution_x, self.resolution_y) # pylint: disable=attribute-defined-outside-init - def ready_wcs(self, dc: Datacube): + def ready_wcs(self): if self.global_cfg.wcs and self.wcs: # Prepare Rectified Grids @@ -878,16 +891,11 @@ def parse_product_names(self, cfg: CFG_DICT): def parse_pq_names(self, cfg: CFG_DICT): raise NotImplementedError() - def force_range_update(self, ext_dc: Datacube | None = None) -> None: - if ext_dc: - dc: Datacube | None = ext_dc - else: - dc = get_cube() - assert dc is not None + def force_range_update(self) -> None: self.hide = False try: from datacube_ows.product_ranges import get_ranges - self._ranges = get_ranges(dc, self) + self._ranges = get_ranges(self) if self._ranges is None: raise Exception("Null product range") self.bboxes = self.extract_bboxes() @@ -1240,7 +1248,12 @@ def __init__(self, refresh=False, cfg: CFG_DICT | None = None, self.initialised = True #pylint: disable=attribute-defined-outside-init - def make_ready(self, dc: Datacube, *args, **kwargs): + def make_ready(self, *args: Any, **kwargs: Any) -> None: + try: + self.dc: Datacube = Datacube(env=self.default_env, app=self.odc_app) + except Exception as e: + _LOG.error("ODC initialisation failed: %s", str(e)) + raise (ODCInitException(e)) if self.msg_file_name: try: with open(self.msg_file_name, "rb") as fp: @@ -1250,8 +1263,8 @@ def make_ready(self, dc: Datacube, *args, **kwargs): else: self.set_msg_src(None) self.native_product_index: dict[str, OWSNamedLayer] = {} - self.root_layer_folder.make_ready(dc, *args, **kwargs) - super().make_ready(dc, *args, **kwargs) + self.root_layer_folder.make_ready(*args, **kwargs) + super().make_ready(*args, **kwargs) def export_metadata(self) -> Catalog: if self.catalog is None: @@ -1283,12 +1296,15 @@ def export_metadata(self) -> Catalog: "folder.ows_root_hidden.title", "folder.ows_root_hidden.abstract", "folder.ows_root_hidden.local_keywords", - ]: + ]: continue self.catalog.add(id=k, string=v, auto_comments=[v]) return self.catalog def parse_global(self, cfg: CFG_DICT, ignore_msgfile: bool): + default_env = cast(str, cfg.get("env")) + self.odc_app = cast(str, cfg.get("odc_app", "ows")) + self.default_env = ODCConfig.get_environment(default_env) self._response_headers = cast(dict[str, str], cfg.get("response_headers", {})) services = cast(dict[str, bool], cfg.get("services", {})) self.wms = services.get("wms", True) @@ -1482,9 +1498,7 @@ def get_config(refresh=False, called_from_update_ranges=False) -> OWSConfig: cfg = OWSConfig(refresh=refresh, called_from_update_ranges=called_from_update_ranges) if not cfg.ready: try: - with cube() as dc: - assert dc is not None # For type checker - cfg.make_ready(dc) + cfg.make_ready() except ODCInitException: pass return cfg diff --git a/datacube_ows/product_ranges.py b/datacube_ows/product_ranges.py index 39f050902..8e87376de 100644 --- a/datacube_ows/product_ranges.py +++ b/datacube_ows/product_ranges.py @@ -386,21 +386,21 @@ def add_ranges(dc: datacube.Datacube, product_names: list[str], merge_only: bool return errors -def get_ranges(dc: datacube.Datacube, product: OWSNamedLayer, +def get_ranges(layer: OWSNamedLayer, path: str | None = None) -> dict[str, Any] | None: - cfg = product.global_cfg - conn = get_sqlconn(dc) - if product.multi_product: + cfg = layer.global_cfg + conn = get_sqlconn(layer.dc) + if layer.multi_product: if path is not None: raise Exception("Combining subproducts and multiproducts is not yet supported") results = conn.execute(text(""" SELECT * FROM wms.multiproduct_ranges WHERE wms_product_name=:pname"""), - {"pname": product.name} + {"pname": layer.name} ) else: - prod_id = product.product.id + prod_id = layer.product.id if path is not None: results = conn.execute(text(""" SELECT * @@ -420,7 +420,7 @@ def get_ranges(dc: datacube.Datacube, product: OWSNamedLayer, ) for result in results: conn.close() - if product.time_resolution.is_subday(): + if layer.time_resolution.is_subday(): dt_parser: Callable[[str], datetime | date] = lambda dts: datetime.fromisoformat(dts) else: dt_parser = lambda dts: datetime.strptime(dts, "%Y-%m-%d").date() diff --git a/datacube_ows/styles/base.py b/datacube_ows/styles/base.py index 32fde1dcc..a372a5143 100644 --- a/datacube_ows/styles/base.py +++ b/datacube_ows/styles/base.py @@ -221,7 +221,7 @@ def get_obj_label(self) -> str: return f"style.{self.product.name}.{self.name}" # pylint: disable=attribute-defined-outside-init - def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: + def make_ready(self, *args, **kwargs) -> None: """ Second-phase (db aware) initialisation @@ -262,7 +262,7 @@ def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: raise ConfigException(f"Same flag band name {band} appears in different PQ product (sets)") self.flag_bands.add(band) for fp in self.flag_products: - fp.make_ready(dc) + fp.make_ready() if not self.stand_alone: # TODO: Should be able to remove this pyre-ignore after ows_configuration is typed. # pyre-ignore[16] @@ -270,7 +270,7 @@ def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: if band not in self.needed_bands: self.needed_bands.add(band) self.flag_bands.add(band) - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) def odc_needed_bands(self) -> Iterable[datacube.model.Measurement]: # pyre-ignore[16] diff --git a/datacube_ows/styles/component.py b/datacube_ows/styles/component.py index 5be3f2f13..b9efb0697 100644 --- a/datacube_ows/styles/component.py +++ b/datacube_ows/styles/component.py @@ -88,7 +88,7 @@ def __init__(self, product: "OWSNamedLayer", } # pylint: disable=attribute-defined-outside-init - def make_ready(self, dc, *args, **kwargs) -> None: + def make_ready(self, *args: Any, **kwargs: Any) -> None: """ Second-phase (db aware) initialisation @@ -102,7 +102,7 @@ def make_ready(self, dc, *args, **kwargs) -> None: self.rgb_components[band] = component else: self.rgb_components[band] = self.dealias_components(component) - super().make_ready(dc, *args, **kwargs) + super().make_ready(*args, **kwargs) self.raw_rgb_components = {} def dealias_components(self, comp_in: LINEAR_COMP_DICT | None) -> LINEAR_COMP_DICT | None: diff --git a/datacube_ows/wcs1.py b/datacube_ows/wcs1.py index f8eb9b224..ade17eaa0 100644 --- a/datacube_ows/wcs1.py +++ b/datacube_ows/wcs1.py @@ -117,14 +117,14 @@ def get_coverage(args): return json_response(qprof.profile()) headers = { "Content-Type": req.format.mime, - 'content-disposition': 'attachment; filename=%s.%s' % (req.product_name, req.format.extension) + 'content-disposition': 'attachment; filename=%s.%s' % (req.layer_name, req.format.extension) } - headers.update(req.product.resource_limits.wcs_cache_rules.cache_headers(n_datasets)) + headers.update(req.layer.resource_limits.wcs_cache_rules.cache_headers(n_datasets)) return ( req.format.renderer(req.version)(req, data), 200, cfg.response_headers({ "Content-Type": req.format.mime, - 'content-disposition': 'attachment; filename=%s.%s' % (req.product_name, req.format.extension) + 'content-disposition': 'attachment; filename=%s.%s' % (req.layer_name, req.format.extension) }) ) diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index 9dea3d13c..7aea8aa74 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -37,10 +37,10 @@ def __init__(self, args): WCS1Exception.MISSING_PARAMETER_VALUE, locator="COVERAGE parameter", valid_keys=list(cfg.product_index)) - self.product_name = args["coverage"] - self.product = cfg.product_index.get(self.product_name) - if not self.product or not self.product.wcs: - raise WCS1Exception("Invalid coverage: %s" % self.product_name, + self.layer_name = args["coverage"] + self.layer = cfg.product_index.get(self.layer_name) + if not self.layer or not self.layer.wcs: + raise WCS1Exception("Invalid coverage: %s" % self.layer_name, WCS1Exception.COVERAGE_NOT_DEFINED, locator="COVERAGE parameter", valid_keys=list(cfg.product_index)) @@ -139,7 +139,7 @@ def __init__(self, args): # CEOS treats no supplied time argument as all time. # I'm really not sure what the right thing to do is, but QGIS wants us to do SOMETHING - use configured # default. - self.times = [self.product.default_time] + self.times = [self.layer.default_time] else: # TODO: the min/max/res format option? # It's a bit underspeced. I'm not sure what the "res" would look like. @@ -150,12 +150,12 @@ def __init__(self, args): continue try: time = parse(t).date() - if time not in self.product.ranges["time_set"]: + if time not in self.layer.ranges["time_set"]: raise WCS1Exception( - "Time value '%s' not a valid date for coverage %s" % (t, self.product_name), + "Time value '%s' not a valid date for coverage %s" % (t, self.layer_name), WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys=[d.strftime('%Y-%m-%d') for d in self.product.ranges["time_set"]] + valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] ) self.times.append(time) except ValueError: @@ -163,7 +163,7 @@ def __init__(self, args): "Time value '%s' not a valid ISO-8601 date" % t, WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys=[d.strftime('%Y-%m-%d') for d in self.product.ranges["time_set"]] + valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] ) self.times.sort() @@ -172,7 +172,7 @@ def __init__(self, args): "No valid ISO-8601 dates", WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys = [d.strftime('%Y-%m-%d') for d in self.product.ranges["time_set"]] + valid_keys = [d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] ) elif len(self.times) > 1 and not self.format.multi_time: raise WCS1Exception( @@ -191,28 +191,28 @@ def __init__(self, args): if not b: continue try: - self.bands.append(self.product.band_idx.locale_band(b)) + self.bands.append(self.layer.band_idx.locale_band(b)) except ConfigException: raise WCS1Exception(f"Invalid measurement: {b}", WCS1Exception.INVALID_PARAMETER_VALUE, locator="MEASUREMENTS parameter", - valid_keys=self.product.band_idx.band_labels()) + valid_keys=self.layer.band_idx.band_labels()) if not bands: raise WCS1Exception("No measurements supplied", WCS1Exception.INVALID_PARAMETER_VALUE, locator="MEASUREMENTS parameter", - valid_keys = self.product.band_idx.band_labels()) + valid_keys = self.layer.band_idx.band_labels()) elif "styles" in args and args["styles"]: # Use style bands. # Non-standard protocol extension. # # As we have correlated WCS and WMS service implementations, # we can accept a style from WMS, and return the bands used for it. - self.bands = get_bands_from_styles(args["styles"], self.product) + self.bands = get_bands_from_styles(args["styles"], self.layer) if not self.bands: - self.bands = self.product.band_idx.band_labels() + self.bands = self.layer.band_idx.band_labels() else: - self.bands = self.product.band_idx.band_labels() + self.bands = self.layer.band_idx.band_labels() # Argument: EXCEPTIONS (optional - defaults to XML) if "exceptions" in args and args["exceptions"] != "application/vnd.ogc.se_xml": @@ -311,7 +311,7 @@ def get_coverage_data(req, qprof): with cube() as dc: if not dc: raise WCS1Exception("Database connectivity failure") - stacker = DataStacker(req.product, + stacker = DataStacker(req.layer, req.geobox, req.times, bands=req.bands) @@ -321,12 +321,12 @@ def get_coverage_data(req, qprof): qprof["n_datasets"] = n_datasets try: - req.product.resource_limits.check_wcs(n_datasets, - req.geobox.height, req.geobox.width, - sum(req.product.band_idx.dtype_size(b) for b in req.bands), - len(req.times)) + req.layer.resource_limits.check_wcs(n_datasets, + req.geobox.height, req.geobox.width, + sum(req.layer.band_idx.dtype_size(b) for b in req.bands), + len(req.times)) except ResourceLimited as e: - if e.wcs_hard or not req.product.low_res_product_names: + if e.wcs_hard or not req.layer.low_res_product_names: raise WCS1Exception( f"This request processes too much data to be served in a reasonable amount of time. ({e}) " + "Please reduce the bounds of your request and try again.") @@ -350,7 +350,7 @@ def get_coverage_data(req, qprof): y_range[1], num=req.height ) - if req.product.time_resolution.is_subday(): + if req.layer.time_resolution.is_subday(): timevals = [ numpy.datetime64(dt.astimezone(pytz.utc).isoformat(), "ns") for dt in req.times @@ -361,7 +361,7 @@ def get_coverage_data(req, qprof): nparrays = { band: (("time", yname, xname), numpy.full((len(req.times), len(yvals), len(xvals)), - req.product.band_idx.nodata_val(band)) + req.layer.band_idx.nodata_val(band)) ) for band in req.bands } @@ -369,7 +369,7 @@ def get_coverage_data(req, qprof): nparrays = { band: (("time", xname, yname), numpy.full((len(req.times), len(xvals), len(yvals)), - req.product.band_idx.nodata_val(band)) + req.layer.band_idx.nodata_val(band)) ) for band in req.bands } @@ -399,7 +399,7 @@ def get_coverage_data(req, qprof): qprof.end_event("load-data") # Clean extent flag band from output - sanitised_bands = [req.product.band_idx.locale_band(b) for b in req.bands] + sanitised_bands = [req.layer.band_idx.locale_band(b) for b in req.bands] for k, v in output.data_vars.items(): if k not in sanitised_bands: output = output.drop_vars([k]) @@ -434,7 +434,7 @@ def get_tiff(req, data): yname = cfg.published_CRSs[req.response_crsid]["vertical_coord"] nodata = 0 for band in data.data_vars: - nodata = req.product.band_idx.nodata_val(band) + nodata = req.layer.band_idx.nodata_val(band) with MemoryFile() as memfile: # pylint: disable=protected-access, bad-continuation with memfile.open( @@ -451,7 +451,7 @@ def get_tiff(req, data): dtype=dtype) as dst: for idx, band in enumerate(data.data_vars, start=1): dst.write(data[band].values, idx) - dst.set_band_description(idx, req.product.band_idx.band_label(band)) + dst.set_band_description(idx, req.layer.band_idx.band_label(band)) if cfg.wcs_tiff_statistics: dst.update_tags(idx, STATISTICS_MINIMUM=data[band].values.min()) dst.update_tags(idx, STATISTICS_MAXIMUM=data[band].values.max()) diff --git a/tests/conftest.py b/tests/conftest.py index 65bbafc8d..be7c91e2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,7 +111,7 @@ def lookup_measurements(ls): @pytest.fixture -def minimal_global_cfg(): +def minimal_global_cfg(minimal_dc): global_cfg = MagicMock() global_cfg.keywords = {"global"} global_cfg.product_index = {} @@ -119,6 +119,7 @@ def minimal_global_cfg(): global_cfg.contact_org = None global_cfg.contact_position = None global_cfg.abstract = "Global Abstract" + global_cfg.dc = minimal_dc global_cfg.authorities = { "auth0": "http://test.url/auth0", "auth1": "http://test.url/auth1", diff --git a/tests/test_cfg_bandidx.py b/tests/test_cfg_bandidx.py index 5b4e324e7..ff0acb527 100644 --- a/tests/test_cfg_bandidx.py +++ b/tests/test_cfg_bandidx.py @@ -13,10 +13,11 @@ @pytest.fixture -def minimal_prod(): +def minimal_prod(minimal_dc): glob = MagicMock() glob.internationalised = False product = MagicMock() + product.dc = minimal_dc product.name = "foo" product.product_name = "foo" product.get_obj_label.return_value = "layer.foo" @@ -44,7 +45,7 @@ def lookup_measurements(ls): def test_bidx_p_minimal(minimal_prod): bidx = BandIndex(minimal_prod, None) - assert bidx.product_name == "foo" + assert bidx.layer_name == "foo" assert bidx.band_cfg == {} assert bidx._idx == {} assert not bidx.ready @@ -121,14 +122,14 @@ def test_bidx_p_label(minimal_prod): assert "splat" in str(excinfo.value) -def test_bidx_makeready(minimal_prod, minimal_dc): +def test_bidx_makeready(minimal_prod): bidx = BandIndex(minimal_prod, { "band1": [], "band2": ["alias2"], "band3": ["alias3", "band3"], "band4": ["band4", "alias4"] }) - bidx.make_ready(minimal_dc) + bidx.make_ready() assert bidx.ready assert bidx.band("band1") == "band1" assert bidx.band("alias2") == "band2" @@ -136,10 +137,10 @@ def test_bidx_makeready(minimal_prod, minimal_dc): assert bidx.band("alias4") == "band4" -def test_bidx_makeready_default(minimal_prod, minimal_dc): +def test_bidx_makeready_default(minimal_prod): import numpy as np bidx = BandIndex(minimal_prod, {}) - bidx.make_ready(minimal_dc) + bidx.make_ready() assert bidx.ready assert "band1" in bidx.band_cfg assert "band1" in bidx.measurements @@ -153,7 +154,7 @@ def test_bidx_makeready_default(minimal_prod, minimal_dc): assert np.isnan(bidx.nodata_val("band1")) -def test_bidx_makeready_invalid_band(minimal_prod, minimal_dc): +def test_bidx_makeready_invalid_band(minimal_prod): bidx = BandIndex(minimal_prod, { "band1": ["band1", "valid"], "bandx": ["invalid"] @@ -161,6 +162,6 @@ def test_bidx_makeready_invalid_band(minimal_prod, minimal_dc): assert bidx.band("valid") == "band1" assert bidx.band("invalid") == "bandx" with pytest.raises(ConfigException) as excinfo: - bidx.make_ready(minimal_dc) + bidx.make_ready() assert "is missing band" in str(excinfo.value) assert "bandx" in str(excinfo.value) diff --git a/tests/test_cfg_layer.py b/tests/test_cfg_layer.py index 8d4a981be..f79abd4e3 100644 --- a/tests/test_cfg_layer.py +++ b/tests/test_cfg_layer.py @@ -169,14 +169,14 @@ def test_make_ready_catch_errors(minimal_global_cfg, minimal_dc): assert lyr.ready -def test_minimal_named_layer(minimal_layer_cfg, minimal_global_cfg, minimal_dc, mock_range): +def test_minimal_named_layer(minimal_layer_cfg, minimal_global_cfg, mock_range): lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) assert lyr.name == "a_layer" assert not lyr.ready with patch("datacube_ows.product_ranges.get_ranges") as get_rng: get_rng.return_value = mock_range - lyr.make_ready(minimal_dc) + lyr.make_ready() assert lyr.ready assert not lyr.hide assert lyr.default_time == mock_range["times"][-1] @@ -185,12 +185,12 @@ def test_minimal_named_layer(minimal_layer_cfg, minimal_global_cfg, minimal_dc, assert lyr.mosaic_date_func is None -def test_duplicate_named_layer(minimal_layer_cfg, minimal_global_cfg, minimal_dc, mock_range): +def test_duplicate_named_layer(minimal_layer_cfg, minimal_global_cfg, mock_range): lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) with patch("datacube_ows.product_ranges.get_ranges") as get_rng: get_rng.return_value = mock_range - lyr.make_ready(minimal_dc) + lyr.make_ready() with pytest.raises(ConfigException) as e: lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) @@ -198,11 +198,11 @@ def test_duplicate_named_layer(minimal_layer_cfg, minimal_global_cfg, minimal_dc assert "Duplicate layer name" in str(e.value) -def test_lowres_named_layer(minimal_layer_cfg, minimal_global_cfg, minimal_dc): +def test_lowres_named_layer(minimal_layer_cfg, minimal_global_cfg): minimal_layer_cfg["low_res_product_name"] = "smol_foo" lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) - lyr.make_ready(minimal_dc) + lyr.make_ready() assert len(lyr.low_res_products) == 1 diff --git a/tests/test_styles.py b/tests/test_styles.py index d5da21f61..f78a2244d 100644 --- a/tests/test_styles.py +++ b/tests/test_styles.py @@ -63,7 +63,7 @@ def __init__(self, pq_name, band): product_layer.always_fetch_bands = ["red", "green", "blue"] product_layer.band_idx = BandIndex.__new__(BandIndex) product_layer.band_idx._unready_attributes = [] - product_layer.band_idx.product = product_layer + product_layer.band_idx.layer = product_layer product_layer.band_idx.band_cfg = { "red": ["crimson", "foo", ], "green": [], From e90f197ff45ffc509ce4b884844944083fe830fe Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 1 May 2024 09:46:40 +1000 Subject: [PATCH 03/12] Remove cube_pool. --- datacube_ows/config_utils.py | 7 + datacube_ows/cube_pool.py | 109 ------- datacube_ows/data.py | 230 +++++++------- datacube_ows/feature_info.py | 236 +++++++------- datacube_ows/ogc.py | 21 +- datacube_ows/ows_configuration.py | 16 +- datacube_ows/wcs1_utils.py | 190 ++++++------ datacube_ows/wcs2_utils.py | 462 ++++++++++++++-------------- datacube_ows/wms_utils.py | 36 +-- integration_tests/test_cube_pool.py | 17 - integration_tests/test_layers.py | 10 +- integration_tests/test_mv_index.py | 86 +++--- integration_tests/utils.py | 30 +- 13 files changed, 647 insertions(+), 803 deletions(-) delete mode 100644 datacube_ows/cube_pool.py delete mode 100644 integration_tests/test_cube_pool.py diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 76d7767c8..cf0603812 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -157,6 +157,13 @@ class OWSConfigNotReady(ConfigException): """ +class ODCInitException(ConfigException): + """ + Exception raised when a Datacube index could not be created + """ + + + class OWSConfigEntry: """ Base class for all configuration objects diff --git a/datacube_ows/cube_pool.py b/datacube_ows/cube_pool.py deleted file mode 100644 index 40043ac25..000000000 --- a/datacube_ows/cube_pool.py +++ /dev/null @@ -1,109 +0,0 @@ -# This file is part of datacube-ows, part of the Open Data Cube project. -# See https://opendatacube.org for more information. -# -# Copyright (c) 2017-2024 OWS Contributors -# SPDX-License-Identifier: Apache-2.0 - -import logging -from contextlib import contextmanager -from threading import Lock -from typing import Generator - -from datacube import Datacube - -_LOG: logging.Logger = logging.getLogger(__name__) - - -class ODCInitException(Exception): - def __init__(self, e: Exception): - super().__init__(str(e)) - self.cause = e - - def __str__(self): - return "ODC initialisation failed:" + str(self.cause) - - -# CubePool class -class CubePool: - """ - A Cube pool is a thread-safe resource pool for managing Datacube objects (which map to database connections). - """ - # _instances, global mapping of CubePools by app name - _instances: dict[str, "CubePool"] = {} - - _cubes_lock_: bool = False - - _instance: Datacube | None = None - - def __new__(cls, app: str) -> "CubePool": - """ - Construction of CubePools is managed. Constructing a cubepool for an app string that already has a cubepool - constructed, returns the existing cubepool, not a new one. - """ - if app not in cls._instances: - cls._instances[app] = super(CubePool, cls).__new__(cls) - return cls._instances[app] - - def __init__(self, app: str) -> None: - """ - Obtain the cube pool for the nominated app string, or create one if one does not exist yet. - - :param app: The app string used to construct any Datacube objects created by the pool. - """ - self.app: str = app - if not self._cubes_lock_: - self._cubes_lock: Lock = Lock() - self._cubes_lock_ = True - - def get_cube(self) -> Datacube | None: - """ - Return a Datacube object. Either generating a new Datacube, or recycling an unassigned one already in the pool. - - :return: a Datacube object (or None on error). - """ - self._cubes_lock.acquire() - try: - if self._instance is None: - self._instance = self._new_cube() - # pylint: disable=broad-except - except Exception as e: - _LOG.error("ODC initialisation failed: %s", str(e)) - raise(ODCInitException(e)) - finally: - self._cubes_lock.release() - return self._instance - - def _new_cube(self) -> Datacube: - return Datacube(app=self.app) - - -# Lowlevel CubePool API -def get_cube(app: str = "ows") -> Datacube | None: - """ - Obtain a Datacube object from the appropriate pool - - :param app: The app pool to use - defaults to "ows". - :return: a Datacube object (or None) in case of database error. - :raises: ODCInitException - """ - return CubePool(app=app).get_cube() - - -# High Level Cube Pool API -@contextmanager -def cube(app: str = "ows") -> Generator[Datacube | None, None, None]: - """ - Context manager for using a Datacube object from a pool. - - E.g. - - with cube() as dc: - # Do stuff that needs a datacube object here. - data = dc.load(.....) - - - :param app: The pool to obtain the app from - defaults to "ows". - :return: A Datacube context manager. - :raises: ODCInitException - """ - yield get_cube(app) diff --git a/datacube_ows/data.py b/datacube_ows/data.py index 9e03686a9..e06ffc169 100644 --- a/datacube_ows/data.py +++ b/datacube_ows/data.py @@ -17,7 +17,6 @@ from rasterio.features import rasterize from rasterio.io import MemoryFile -from datacube_ows.cube_pool import cube from datacube_ows.http_utils import FlaskResponse, json_response, png_response from datacube_ows.loading import DataStacker, ProductBandQuery from datacube_ows.mv_index import MVSelectOpts @@ -98,133 +97,130 @@ def get_map(args: dict[str, str]) -> FlaskResponse: raise WMSException("Style %s does not support GetMap requests with %d dates" % (params.style.name, n_dates), WMSException.INVALID_DIMENSION_VALUE, locator="Time parameter") qprof["n_dates"] = n_dates - with cube() as dc: + try: + # Tiling. + stacker = DataStacker(params.layer, params.geobox, params.times, params.resampling, style=params.style) + qprof["zoom_factor"] = params.zf + qprof.start_event("count-datasets") + n_datasets = stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.COUNT) + qprof.end_event("count-datasets") + qprof["n_datasets"] = n_datasets + qprof["zoom_level_base"] = params.resources.base_zoom_level + qprof["zoom_level_adjusted"] = params.resources.load_adjusted_zoom_level try: - if not dc: - raise WMSException("Database connectivity failure") - # Tiling. - stacker = DataStacker(params.product, params.geobox, params.times, params.resampling, style=params.style) - qprof["zoom_factor"] = params.zf - qprof.start_event("count-datasets") - n_datasets = stacker.datasets(dc.index, mode=MVSelectOpts.COUNT) - qprof.end_event("count-datasets") - qprof["n_datasets"] = n_datasets - qprof["zoom_level_base"] = params.resources.base_zoom_level - qprof["zoom_level_adjusted"] = params.resources.load_adjusted_zoom_level - try: - params.product.resource_limits.check_wms(n_datasets, params.zf, params.resources) - except ResourceLimited as e: - stacker.resource_limited = True - qprof["resource_limited"] = str(e) - if qprof.active: - q_ds_dict = cast(dict[ProductBandQuery, xarray.DataArray], - stacker.datasets(dc.index, mode=MVSelectOpts.DATASETS)) - qprof["datasets"] = [] - for q, dsxr in q_ds_dict.items(): - query_res: dict[str, Any] = {} - query_res["query"] = str(q) - query_res["datasets"] = [ - [ - f"{ds.id} ({ds.type.name})" - for ds in tdss - ] - for tdss in dsxr.values + params.layer.resource_limits.check_wms(n_datasets, params.zf, params.resources) + except ResourceLimited as e: + stacker.resource_limited = True + qprof["resource_limited"] = str(e) + if qprof.active: + q_ds_dict = cast(dict[ProductBandQuery, xarray.DataArray], + stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.DATASETS)) + qprof["datasets"] = [] + for q, dsxr in q_ds_dict.items(): + query_res: dict[str, Any] = {} + query_res["query"] = str(q) + query_res["datasets"] = [ + [ + f"{ds.id} ({ds.type.name})" + for ds in tdss ] - qprof["datasets"].append(query_res) - if stacker.resource_limited and not params.product.low_res_product_names: - qprof.start_event("extent-in-query") - extent = cast(geom.Geometry | None, stacker.datasets(dc.index, mode=MVSelectOpts.EXTENT)) - qprof.end_event("extent-in-query") - if extent is None: - qprof["write_action"] = "No extent: Write Empty" - raise EmptyResponse() - else: - qprof["write_action"] = "Polygon" - qprof.start_event("write") - body = _write_polygon( - params.geobox, - extent, - params.product.resource_limits.zoom_fill, - params.product) - qprof.end_event("write") - elif n_datasets == 0: - qprof["write_action"] = "No datasets: Write Empty" + for tdss in dsxr.values + ] + qprof["datasets"].append(query_res) + if stacker.resource_limited and not params.layer.low_res_product_names: + qprof.start_event("extent-in-query") + extent = cast(geom.Geometry | None, stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.EXTENT)) + qprof.end_event("extent-in-query") + if extent is None: + qprof["write_action"] = "No extent: Write Empty" raise EmptyResponse() else: - if stacker.resource_limited: - qprof.start_event("count-summary-datasets") - qprof["n_summary_datasets"] = stacker.datasets(dc.index, mode=MVSelectOpts.COUNT) - qprof.end_event("count-summary-datasets") - qprof.start_event("fetch-datasets") - datasets = cast(dict[ProductBandQuery, xarray.DataArray], stacker.datasets(dc.index)) - for flagband, dss in datasets.items(): - if not dss.any(): - _LOG.warning("Flag band %s returned no data", str(flagband)) - if len(dss.time) != n_dates and flagband.main: - qprof["write_action"] = f"{n_dates} requested, only {len(dss.time)} found - returning empty image" - raise EmptyResponse() - qprof.end_event("fetch-datasets") - _LOG.debug("load start %s %s", datetime.now().time(), args["requestid"]) - qprof.start_event("load-data") - data = stacker.data(datasets) - qprof.end_event("load-data") - if not data: - qprof["write_action"] = "No Data: Write Empty" + qprof["write_action"] = "Polygon" + qprof.start_event("write") + body = _write_polygon( + params.geobox, + extent, + params.layer.resource_limits.zoom_fill, + params.layer) + qprof.end_event("write") + elif n_datasets == 0: + qprof["write_action"] = "No datasets: Write Empty" + raise EmptyResponse() + else: + if stacker.resource_limited: + qprof.start_event("count-summary-datasets") + qprof["n_summary_datasets"] = stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.COUNT) + qprof.end_event("count-summary-datasets") + qprof.start_event("fetch-datasets") + datasets = cast(dict[ProductBandQuery, xarray.DataArray], stacker.datasets(params.layer.dc.index)) + for flagband, dss in datasets.items(): + if not dss.any(): + _LOG.warning("Flag band %s returned no data", str(flagband)) + if len(dss.time) != n_dates and flagband.main: + qprof["write_action"] = f"{n_dates} requested, only {len(dss.time)} found - returning empty image" raise EmptyResponse() - _LOG.debug("load stop %s %s", datetime.now().time(), args["requestid"]) - qprof.start_event("build-masks") - td_masks = [] - for npdt in data.time.values: - td = data.sel(time=npdt) - td_ext_mask_man: numpy.ndarray | None = None - td_ext_mask: xarray.DataArray | None = None - band = "" - for band in params.style.needed_bands: - if band not in params.style.flag_bands: - if params.product.data_manual_merge: - if td_ext_mask_man is None: - td_ext_mask_man = ~numpy.isnan(td[band]) - else: - td_ext_mask_man &= ~numpy.isnan(td[band]) + qprof.end_event("fetch-datasets") + _LOG.debug("load start %s %s", datetime.now().time(), args["requestid"]) + qprof.start_event("load-data") + data = stacker.data(datasets) + qprof.end_event("load-data") + if not data: + qprof["write_action"] = "No Data: Write Empty" + raise EmptyResponse() + _LOG.debug("load stop %s %s", datetime.now().time(), args["requestid"]) + qprof.start_event("build-masks") + td_masks = [] + for npdt in data.time.values: + td = data.sel(time=npdt) + td_ext_mask_man: numpy.ndarray | None = None + td_ext_mask: xarray.DataArray | None = None + band = "" + for band in params.style.needed_bands: + if band not in params.style.flag_bands: + if params.layer.data_manual_merge: + if td_ext_mask_man is None: + td_ext_mask_man = ~numpy.isnan(td[band]) else: - for f in params.product.extent_mask_func: - if td_ext_mask is None: - td_ext_mask = f(td, band) - else: - td_ext_mask &= f(td, band) - if params.product.data_manual_merge: - td_ext_mask = xarray.DataArray(td_ext_mask_man) - if td_ext_mask is None: - td_ext_mask = xarray.DataArray( - ~numpy.zeros( - td[band].values.shape, - dtype=numpy.bool_ - ), - td[band].coords - ) - td_masks.append(td_ext_mask) - extent_mask = xarray.concat(td_masks, dim=data.time) - qprof.end_event("build-masks") - qprof["write_action"] = "Write Data" - if mdh and mdh.preserve_user_date_order: - sorter = user_date_sorter( - params.product, - data.time.values, - params.geobox.geographic_extent, - params.times) - data = data.sortby(sorter) - extent_mask = extent_mask.sortby(sorter) + td_ext_mask_man &= ~numpy.isnan(td[band]) + else: + for f in params.layer.extent_mask_func: + if td_ext_mask is None: + td_ext_mask = f(td, band) + else: + td_ext_mask &= f(td, band) + if params.layer.data_manual_merge: + td_ext_mask = xarray.DataArray(td_ext_mask_man) + if td_ext_mask is None: + td_ext_mask = xarray.DataArray( + ~numpy.zeros( + td[band].values.shape, + dtype=numpy.bool_ + ), + td[band].coords + ) + td_masks.append(td_ext_mask) + extent_mask = xarray.concat(td_masks, dim=data.time) + qprof.end_event("build-masks") + qprof["write_action"] = "Write Data" + if mdh and mdh.preserve_user_date_order: + sorter = user_date_sorter( + params.layer, + data.time.values, + params.geobox.geographic_extent, + params.times) + data = data.sortby(sorter) + extent_mask = extent_mask.sortby(sorter) - body = _write_png(data, params.style, extent_mask, qprof) - except EmptyResponse: - qprof.start_event("write") - body = _write_empty(params.geobox) - qprof.end_event("write") + body = _write_png(data, params.style, extent_mask, qprof) + except EmptyResponse: + qprof.start_event("write") + body = _write_empty(params.geobox) + qprof.end_event("write") if params.ows_stats: return json_response(qprof.profile()) else: - return png_response(body, extra_headers=params.product.resource_limits.wms_cache_rules.cache_headers(n_datasets)) + return png_response(body, extra_headers=params.layer.resource_limits.wms_cache_rules.cache_headers(n_datasets)) @log_call diff --git a/datacube_ows/feature_info.py b/datacube_ows/feature_info.py index 1fa961a87..a2cddd3f8 100644 --- a/datacube_ows/feature_info.py +++ b/datacube_ows/feature_info.py @@ -19,11 +19,9 @@ from pandas import Timestamp from datacube_ows.config_utils import CFG_DICT, RAW_CFG, ConfigException -from datacube_ows.cube_pool import cube from datacube_ows.http_utils import (FlaskResponse, html_json_response, json_response) from datacube_ows.loading import DataStacker, ProductBandQuery -from datacube_ows.ogc_exceptions import WMSException from datacube_ows.ows_configuration import OWSNamedLayer, get_config from datacube_ows.styles import StyleDef from datacube_ows.time_utils import dataset_center_time, tz_for_geometry @@ -155,128 +153,124 @@ def feature_info(args: dict[str, str]) -> FlaskResponse: geo_point_geobox = GeoBox.from_geopolygon( geo_point, params.geobox.resolution, crs=params.geobox.crs) tz = tz_for_geometry(geo_point_geobox.geographic_extent) - stacker = DataStacker(params.product, geo_point_geobox, params.times) + stacker = DataStacker(params.layer, geo_point_geobox, params.times) # --- Begin code section requiring datacube. cfg = get_config() - with cube() as dc: - if not dc: - raise WMSException("Database connectivity failure") - all_time_datasets = cast(xarray.DataArray, stacker.datasets(dc.index, all_time=True, point=geo_point)) - - # Taking the data as a single point so our indexes into the data should be 0,0 - h_coord = cast(str, cfg.published_CRSs[params.crsid]["horizontal_coord"]) - v_coord = cast(str, cfg.published_CRSs[params.crsid]["vertical_coord"]) - s3_bucket = cfg.s3_bucket - s3_url = cfg.s3_url - isel_kwargs = { - h_coord: 0, - v_coord: 0 - } - if any(all_time_datasets): - # Group datasets by time, load only datasets that match the idx_date - global_info_written = False - feature_json["data"] = [] - fi_date_index: dict[datetime, RAW_CFG] = {} - time_datasets = cast( - dict[ProductBandQuery, xarray.DataArray], - stacker.datasets(dc.index, all_flag_bands=True, point=geo_point) - ) - data = stacker.data(time_datasets, skip_corrections=True) - if data is not None: - for dt in data.time.values: - td = data.sel(time=dt) - # Global data that should apply to all dates, but needs some data to extract - if not global_info_written: - global_info_written = True - # Non-geographic coordinate systems need to be projected onto a geographic - # coordinate system. Why not use EPSG:4326? - # Extract coordinates in CRS - data_x = getattr(td, h_coord) - data_y = getattr(td, v_coord) - - x = data_x[isel_kwargs[h_coord]].item() - y = data_y[isel_kwargs[v_coord]].item() - pt = geom.point(x, y, params.crs) - - # Project to EPSG:4326 - crs_geo = geom.CRS("EPSG:4326") - ptg = pt.to_crs(crs_geo) - - # Capture lat/long coordinates - feature_json["lon"], feature_json["lat"] = ptg.coords[0] - - date_info: CFG_DICT = {} - - ds: Dataset | None = None - for pbq, dss in time_datasets.items(): - if pbq.main: - ds = dss.sel(time=dt).values.tolist()[0] - break - assert ds is not None - if params.product.multi_product: - if "platform" in ds.metadata_doc: - date_info["source_product"] = "%s (%s)" % (ds.type.name, ds.metadata_doc["platform"]["code"]) - else: - date_info["source_product"] = ds.type.name - - # Extract data pixel - pixel_ds: xarray.Dataset = td.isel(**isel_kwargs) # type: ignore[arg-type] - - # Get accurate timestamp from dataset - assert ds.time is not None # For type checker - if params.product.time_resolution.is_summary(): - date_info["time"] = ds.time.begin.strftime("%Y-%m-%d") - else: - date_info["time"] = dataset_center_time(ds).strftime("%Y-%m-%d %H:%M:%S %Z") - # Collect raw band values for pixel and derived bands from styles - date_info["bands"] = cast(RAW_CFG, _make_band_dict(params.product, pixel_ds)) - derived_band_dict = cast(RAW_CFG, _make_derived_band_dict(pixel_ds, params.product.style_index)) - if derived_band_dict: - date_info["band_derived"] = derived_band_dict - # Add any custom-defined fields. - for k, f in params.product.feature_info_custom_includes.items(): - date_info[k] = f(date_info["bands"]) - - cast(list[RAW_CFG], feature_json["data"]).append(date_info) - fi_date_index[dt] = cast(dict[str, list[RAW_CFG]], feature_json)["data"][-1] - feature_json["data_available_for_dates"] = [] - pt_native = None - for d in all_time_datasets.coords["time"].values: - dt_datasets = all_time_datasets.sel(time=d) - for ds in dt_datasets.values.item(): - assert ds is not None # For type checker - if pt_native is None: - pt_native = geo_point.to_crs(ds.crs) - elif pt_native.crs != ds.crs: - pt_native = geo_point.to_crs(ds.crs) - if ds.extent and ds.extent.contains(pt_native): - # tolist() converts a numpy datetime64 to a python datatime - dt = Timestamp(stacker.group_by.group_by_func(ds)).to_pydatetime() - if params.product.time_resolution.is_subday(): - cast(list[RAW_CFG], feature_json["data_available_for_dates"]).append(dt.isoformat()) - else: - cast(list[RAW_CFG], feature_json["data_available_for_dates"]).append(dt.strftime("%Y-%m-%d")) + all_time_datasets = cast(xarray.DataArray, stacker.datasets(params.layer.dc.index, all_time=True, point=geo_point)) + + # Taking the data as a single point so our indexes into the data should be 0,0 + h_coord = cast(str, cfg.published_CRSs[params.crsid]["horizontal_coord"]) + v_coord = cast(str, cfg.published_CRSs[params.crsid]["vertical_coord"]) + s3_bucket = cfg.s3_bucket + s3_url = cfg.s3_url + isel_kwargs = { + h_coord: 0, + v_coord: 0 + } + if any(all_time_datasets): + # Group datasets by time, load only datasets that match the idx_date + global_info_written = False + feature_json["data"] = [] + fi_date_index: dict[datetime, RAW_CFG] = {} + time_datasets = cast( + dict[ProductBandQuery, xarray.DataArray], + stacker.datasets(params.layer.dc.index, all_flag_bands=True, point=geo_point) + ) + data = stacker.data(time_datasets, skip_corrections=True) + if data is not None: + for dt in data.time.values: + td = data.sel(time=dt) + # Global data that should apply to all dates, but needs some data to extract + if not global_info_written: + global_info_written = True + # Non-geographic coordinate systems need to be projected onto a geographic + # coordinate system. Why not use EPSG:4326? + # Extract coordinates in CRS + data_x = getattr(td, h_coord) + data_y = getattr(td, v_coord) + + x = data_x[isel_kwargs[h_coord]].item() + y = data_y[isel_kwargs[v_coord]].item() + pt = geom.point(x, y, params.crs) + + # Project to EPSG:4326 + crs_geo = geom.CRS("EPSG:4326") + ptg = pt.to_crs(crs_geo) + + # Capture lat/long coordinates + feature_json["lon"], feature_json["lat"] = ptg.coords[0] + + date_info: CFG_DICT = {} + + ds: Dataset | None = None + for pbq, dss in time_datasets.items(): + if pbq.main: + ds = dss.sel(time=dt).values.tolist()[0] break - if time_datasets: - feature_json["data_links"] = cast( - RAW_CFG, - sorted(get_s3_browser_uris(time_datasets, pt_native, s3_url, s3_bucket))) - else: - feature_json["data_links"] = [] - if params.product.feature_info_include_utc_dates: - unsorted_dates: list[str] = [] - for tds in all_time_datasets: - for ds in tds.values.item(): - assert ds is not None and ds.time is not None # for type checker - if params.product.time_resolution.is_solar(): - unsorted_dates.append(ds.center_time.strftime("%Y-%m-%d")) - elif params.product.time_resolution.is_subday(): - unsorted_dates.append(ds.time.begin.isoformat()) - else: - unsorted_dates.append(ds.time.begin.strftime("%Y-%m-%d")) - feature_json["data_available_for_utc_dates"] = sorted( - d.center_time.strftime("%Y-%m-%d") for d in all_time_datasets) - # --- End code section requiring datacube. + assert ds is not None + if params.layer.multi_product: + if "platform" in ds.metadata_doc: + date_info["source_product"] = "%s (%s)" % (ds.type.name, ds.metadata_doc["platform"]["code"]) + else: + date_info["source_product"] = ds.type.name + + # Extract data pixel + pixel_ds: xarray.Dataset = td.isel(**isel_kwargs) # type: ignore[arg-type] + + # Get accurate timestamp from dataset + assert ds.time is not None # For type checker + if params.layer.time_resolution.is_summary(): + date_info["time"] = ds.time.begin.strftime("%Y-%m-%d") + else: + date_info["time"] = dataset_center_time(ds).strftime("%Y-%m-%d %H:%M:%S %Z") + # Collect raw band values for pixel and derived bands from styles + date_info["bands"] = cast(RAW_CFG, _make_band_dict(params.layer, pixel_ds)) + derived_band_dict = cast(RAW_CFG, _make_derived_band_dict(pixel_ds, params.layer.style_index)) + if derived_band_dict: + date_info["band_derived"] = derived_band_dict + # Add any custom-defined fields. + for k, f in params.layer.feature_info_custom_includes.items(): + date_info[k] = f(date_info["bands"]) + + cast(list[RAW_CFG], feature_json["data"]).append(date_info) + fi_date_index[dt] = cast(dict[str, list[RAW_CFG]], feature_json)["data"][-1] + feature_json["data_available_for_dates"] = [] + pt_native = None + for d in all_time_datasets.coords["time"].values: + dt_datasets = all_time_datasets.sel(time=d) + for ds in dt_datasets.values.item(): + assert ds is not None # For type checker + if pt_native is None: + pt_native = geo_point.to_crs(ds.crs) + elif pt_native.crs != ds.crs: + pt_native = geo_point.to_crs(ds.crs) + if ds.extent and ds.extent.contains(pt_native): + # tolist() converts a numpy datetime64 to a python datatime + dt = Timestamp(stacker.group_by.group_by_func(ds)).to_pydatetime() + if params.layer.time_resolution.is_subday(): + cast(list[RAW_CFG], feature_json["data_available_for_dates"]).append(dt.isoformat()) + else: + cast(list[RAW_CFG], feature_json["data_available_for_dates"]).append(dt.strftime("%Y-%m-%d")) + break + if time_datasets: + feature_json["data_links"] = cast( + RAW_CFG, + sorted(get_s3_browser_uris(time_datasets, pt_native, s3_url, s3_bucket))) + else: + feature_json["data_links"] = [] + if params.layer.feature_info_include_utc_dates: + unsorted_dates: list[str] = [] + for tds in all_time_datasets: + for ds in tds.values.item(): + assert ds is not None and ds.time is not None # for type checker + if params.layer.time_resolution.is_solar(): + unsorted_dates.append(ds.center_time.strftime("%Y-%m-%d")) + elif params.layer.time_resolution.is_subday(): + unsorted_dates.append(ds.time.begin.isoformat()) + else: + unsorted_dates.append(ds.time.begin.strftime("%Y-%m-%d")) + feature_json["data_available_for_utc_dates"] = sorted( + d.center_time.strftime("%Y-%m-%d") for d in all_time_datasets) result: CFG_DICT = { "type": "FeatureCollection", diff --git a/datacube_ows/ogc.py b/datacube_ows/ogc.py index f1447762a..f8cdf50ef 100644 --- a/datacube_ows/ogc.py +++ b/datacube_ows/ogc.py @@ -12,7 +12,6 @@ from sqlalchemy import text from datacube_ows import __version__ -from datacube_ows.cube_pool import cube from datacube_ows.http_utils import (capture_headers, get_service_base_url, lower_get_args, resp_headers) from datacube_ows.legend_generator import create_legend_for_style @@ -179,17 +178,15 @@ def ogc_wcs_impl(): @metrics.summary('ows_heartbeat_pings', "Ping durations", labels={"status": lambda r: r.status}) def ping(): db_ok = False - with cube() as dc: - if dc: - # pylint: disable=protected-access - with dc.index._db.give_me_a_connection() as conn: - results = conn.execute(text(""" - SELECT * - FROM wms.product_ranges - LIMIT 1""") - ) - for r in results: - db_ok = True + cfg = get_config() + with cfg.dc.index._db.give_me_a_connection() as conn: + results = conn.execute(text(""" + SELECT * + FROM wms.product_ranges + LIMIT 1""") + ) + for r in results: + db_ok = True if db_ok: return (render_template("ping.html", status="Up"), 200, resp_headers({"Content-Type": "text/html"})) else: diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 6645f659e..784704118 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -36,12 +36,11 @@ from datacube_ows.config_utils import (CFG_DICT, RAW_CFG, ConfigException, FlagProductBands, FunctionWrapper, - OWSConfigEntry, OWSEntryNotFound, - OWSExtensibleConfigEntry, OWSFlagBand, - OWSMetadataConfig, cfg_expand, - get_file_loc, import_python_obj, - load_json_obj) -from datacube_ows.cube_pool import ODCInitException + ODCInitException, OWSConfigEntry, + OWSEntryNotFound, OWSExtensibleConfigEntry, + OWSFlagBand, OWSMetadataConfig, + cfg_expand, get_file_loc, + import_python_obj, load_json_obj) from datacube_ows.ogc_utils import create_geobox from datacube_ows.resource_limits import (OWSResourceManagementRules, parse_cache_age) @@ -575,7 +574,7 @@ def make_ready(self, *args: Any, **kwargs: Any) -> None: self.dc: Datacube = Datacube(env=self.local_env, app=self.global_cfg.odc_app) except Exception as e: _LOG.error("ODC initialisation failed: %s", str(e)) - raise (ODCInitException(e)) + raise ODCInitException(e) else: self.dc = self.global_cfg.dc self.products: list[Product] = [] @@ -1253,7 +1252,7 @@ def make_ready(self, *args: Any, **kwargs: Any) -> None: self.dc: Datacube = Datacube(env=self.default_env, app=self.odc_app) except Exception as e: _LOG.error("ODC initialisation failed: %s", str(e)) - raise (ODCInitException(e)) + raise ODCInitException(e) if self.msg_file_name: try: with open(self.msg_file_name, "rb") as fp: @@ -1493,7 +1492,6 @@ def response_headers(self, d: dict[str, str]) -> dict[str, str]: hdrs.update(d) return hdrs - def get_config(refresh=False, called_from_update_ranges=False) -> OWSConfig: cfg = OWSConfig(refresh=refresh, called_from_update_ranges=called_from_update_ranges) if not cfg.ready: diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index 7aea8aa74..4da96ab2d 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -15,7 +15,6 @@ from rasterio import MemoryFile from datacube_ows.config_utils import ConfigException -from datacube_ows.cube_pool import cube from datacube_ows.loading import DataStacker from datacube_ows.mv_index import MVSelectOpts from datacube_ows.ogc_exceptions import WCS1Exception @@ -308,103 +307,100 @@ def __init__(self, args): def get_coverage_data(req, qprof): # pylint: disable=too-many-locals, protected-access - with cube() as dc: - if not dc: - raise WCS1Exception("Database connectivity failure") - stacker = DataStacker(req.layer, - req.geobox, - req.times, - bands=req.bands) - qprof.start_event("count-datasets") - n_datasets = stacker.datasets(dc.index, mode=MVSelectOpts.COUNT) - qprof.end_event("count-datasets") - qprof["n_datasets"] = n_datasets - - try: - req.layer.resource_limits.check_wcs(n_datasets, - req.geobox.height, req.geobox.width, - sum(req.layer.band_idx.dtype_size(b) for b in req.bands), - len(req.times)) - except ResourceLimited as e: - if e.wcs_hard or not req.layer.low_res_product_names: - raise WCS1Exception( - f"This request processes too much data to be served in a reasonable amount of time. ({e}) " - + "Please reduce the bounds of your request and try again.") - stacker.resource_limited = True - qprof["resource_limited"] = str(e) - if n_datasets == 0: - # Return an empty coverage file with full metadata? - qprof.start_event("build_empty_dataset") - cfg = get_config() - x_range = (req.minx, req.maxx) - y_range = (req.miny, req.maxy) - xname = cfg.published_CRSs[req.response_crsid]["horizontal_coord"] - yname = cfg.published_CRSs[req.response_crsid]["vertical_coord"] - xvals = numpy.linspace( - x_range[0], - x_range[1], - num=req.width - ) - yvals = numpy.linspace( - y_range[0], - y_range[1], - num=req.height - ) - if req.layer.time_resolution.is_subday(): - timevals = [ - numpy.datetime64(dt.astimezone(pytz.utc).isoformat(), "ns") - for dt in req.times - ] - else: - timevals = req.times - if cfg.published_CRSs[req.request_crsid]["vertical_coord_first"]: - nparrays = { - band: (("time", yname, xname), - numpy.full((len(req.times), len(yvals), len(xvals)), - req.layer.band_idx.nodata_val(band)) - ) - for band in req.bands - } - else: - nparrays = { - band: (("time", xname, yname), - numpy.full((len(req.times), len(xvals), len(yvals)), - req.layer.band_idx.nodata_val(band)) - ) - for band in req.bands - } - data = xarray.Dataset( - nparrays, - coords={ - "time": timevals, - xname: xvals, - yname: yvals, - } - ).astype("int16") - qprof.start_event("end_empty_dataset") - qprof["write_action"] = "Write Empty" - - return n_datasets, data - - qprof.start_event("fetch-datasets") - datasets = stacker.datasets(index=dc.index) - qprof.end_event("fetch-datasets") - if qprof.active: - qprof["datasets"] = { - str(q): [str(i) for i in ids] - for q, ids in stacker.datasets(dc.index, mode=MVSelectOpts.IDS).items() + stacker = DataStacker(req.layer, + req.geobox, + req.times, + bands=req.bands) + qprof.start_event("count-datasets") + n_datasets = stacker.datasets(req.layer.dc.index, mode=MVSelectOpts.COUNT) + qprof.end_event("count-datasets") + qprof["n_datasets"] = n_datasets + + try: + req.layer.resource_limits.check_wcs(n_datasets, + req.geobox.height, req.geobox.width, + sum(req.layer.band_idx.dtype_size(b) for b in req.bands), + len(req.times)) + except ResourceLimited as e: + if e.wcs_hard or not req.layer.low_res_product_names: + raise WCS1Exception( + f"This request processes too much data to be served in a reasonable amount of time. ({e}) " + + "Please reduce the bounds of your request and try again.") + stacker.resource_limited = True + qprof["resource_limited"] = str(e) + if n_datasets == 0: + # Return an empty coverage file with full metadata? + qprof.start_event("build_empty_dataset") + cfg = get_config() + x_range = (req.minx, req.maxx) + y_range = (req.miny, req.maxy) + xname = cfg.published_CRSs[req.response_crsid]["horizontal_coord"] + yname = cfg.published_CRSs[req.response_crsid]["vertical_coord"] + xvals = numpy.linspace( + x_range[0], + x_range[1], + num=req.width + ) + yvals = numpy.linspace( + y_range[0], + y_range[1], + num=req.height + ) + if req.layer.time_resolution.is_subday(): + timevals = [ + numpy.datetime64(dt.astimezone(pytz.utc).isoformat(), "ns") + for dt in req.times + ] + else: + timevals = req.times + if cfg.published_CRSs[req.request_crsid]["vertical_coord_first"]: + nparrays = { + band: (("time", yname, xname), + numpy.full((len(req.times), len(yvals), len(xvals)), + req.layer.band_idx.nodata_val(band)) + ) + for band in req.bands + } + else: + nparrays = { + band: (("time", xname, yname), + numpy.full((len(req.times), len(xvals), len(yvals)), + req.layer.band_idx.nodata_val(band)) + ) + for band in req.bands + } + data = xarray.Dataset( + nparrays, + coords={ + "time": timevals, + xname: xvals, + yname: yvals, } - qprof.start_event("load-data") - output = stacker.data(datasets, skip_corrections=True) - qprof.end_event("load-data") - - # Clean extent flag band from output - sanitised_bands = [req.layer.band_idx.locale_band(b) for b in req.bands] - for k, v in output.data_vars.items(): - if k not in sanitised_bands: - output = output.drop_vars([k]) - qprof["write_action"] = "Write Data" - return n_datasets, output + ).astype("int16") + qprof.start_event("end_empty_dataset") + qprof["write_action"] = "Write Empty" + + return n_datasets, data + + qprof.start_event("fetch-datasets") + datasets = stacker.datasets(index=req.layer.dc.index) + qprof.end_event("fetch-datasets") + if qprof.active: + qprof["datasets"] = { + str(q): [str(i) for i in ids] + for q, ids in stacker.datasets(req.layer.dc.index, mode=MVSelectOpts.IDS).items() + } + qprof.start_event("load-data") + output = stacker.data(datasets, skip_corrections=True) + qprof.end_event("load-data") + + # Clean extent flag band from output + sanitised_bands = [req.layer.band_idx.locale_band(b) for b in req.bands] + for k, v in output.data_vars.items(): + if k not in sanitised_bands: + output = output.drop_vars([k]) + qprof["write_action"] = "Write Data" + return n_datasets, output def get_tiff(req, data): diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index 5ad659607..d137805d6 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -12,7 +12,6 @@ from ows.wcs.v20 import ScaleAxis, ScaleExtent, ScaleSize, Slice, Trim from rasterio import MemoryFile -from datacube_ows.cube_pool import cube from datacube_ows.loading import DataStacker from datacube_ows.mv_index import MVSelectOpts from datacube_ows.ogc_exceptions import WCS2Exception @@ -59,250 +58,245 @@ def get_coverage_data(request, styles, qprof): locator="COVERAGE parameter", valid_keys=list(cfg.product_index)) - with cube() as dc: - if not dc: - raise WCS2Exception("Database connectivity failure") - # - # CRS handling - # - - native_crs = layer.native_CRS - subsetting_crs = uniform_crs(cfg, request.subsetting_crs or native_crs) - output_crs = uniform_crs(cfg, request.output_crs or subsetting_crs) - - if subsetting_crs not in cfg.published_CRSs: - raise WCS2Exception("Invalid subsettingCrs: %s" % subsetting_crs, - WCS2Exception.SUBSETTING_CRS_NOT_SUPPORTED, - locator=subsetting_crs, - valid_keys=list(cfg.published_CRSs)) - - output_crs = uniform_crs(cfg, request.output_crs or subsetting_crs or native_crs) - - if output_crs not in cfg.published_CRSs: - raise WCS2Exception("Invalid outputCrs: %s" % output_crs, - WCS2Exception.OUTPUT_CRS_NOT_SUPPORTED, - locator=output_crs, - valid_keys=list(cfg.published_CRSs)) - - # - # Subsetting/Scaling - # - - scaler = WCSScaler(layer, subsetting_crs) - times = layer.ranges["times"] - - subsets = request.subsets - - if len(subsets) != len(set(subset.dimension.lower() for subset in subsets)): - dimensions = [subset.dimension.lower() for subset in subsets] - duplicate_dimensions = [ - item - for item, count in collections.Counter(dimensions).items() - if count > 1 - ] - - raise WCS2Exception("Duplicate dimension%s: %s" % ( - 's' if len(duplicate_dimensions) > 1 else '', - ', '.join(duplicate_dimensions) - ), - WCS2Exception.INVALID_SUBSETTING, - locator=','.join(duplicate_dimensions) - ) + # + # CRS handling + # - for subset in subsets: - dimension = subset.dimension.lower() - if dimension == 'time': - if isinstance(subset, Trim): - if "," in subset.high: - raise WCS2Exception( - "Subsets can only contain 2 elements - the lower and upper bounds. For arbitrary date lists, use WCS1", - WCS2Exception.INVALID_SUBSETTING, - locator="time") - if layer.time_resolution.is_subday(): - low = parse(subset.low) if subset.low is not None else None - low = default_to_utc(low) - high = parse(subset.high) if subset.high is not None else None - high = default_to_utc(high) - else: - low = parse(subset.low).date() if subset.low is not None else None - high = parse(subset.high).date() if subset.high is not None else None - if low is not None: - times = [ - time for time in times - if time >= low - ] - if high is not None: - times = [ - time for time in times - if time <= high - ] - elif isinstance(subset, Slice): - point = parse(subset.point).date() - times = [point] - else: - try: - if isinstance(subset, Trim): - scaler.trim(dimension, subset.low, subset.high) - elif isinstance(subset, Slice): - scaler.slice(dimension, subset.point) - except WCSScalerUnknownDimension: - raise WCS2Exception('Invalid subsetting axis %s' % subset.dimension, - WCS2Exception.INVALID_AXIS_LABEL, - locator=subset.dimension) - - # - # Transform spatial extent to native CRS. - # - scaler.to_crs(output_crs) - - # - # Scaling - # - - scales = request.scales - if len(scales) != len(set(subset.axis.lower() for subset in scales)): - axes = [subset.axis.lower() for subset in scales] - duplicate_axes = [ - item - for item, count in collections.Counter(axes).items() - if count > 1 - ] - raise WCS2Exception('Duplicate scales for ax%ss: %s' % ( - 'i' if len(duplicate_axes) == 1 else 'e', - ', '.join(duplicate_axes) - ), - WCS2Exception.INVALID_SCALE_FACTOR, - locator=','.join(duplicate_axes) - ) + native_crs = layer.native_CRS + subsetting_crs = uniform_crs(cfg, request.subsetting_crs or native_crs) + if subsetting_crs not in cfg.published_CRSs: + raise WCS2Exception("Invalid subsettingCrs: %s" % subsetting_crs, + WCS2Exception.SUBSETTING_CRS_NOT_SUPPORTED, + locator=subsetting_crs, + valid_keys=list(cfg.published_CRSs)) - for scale in scales: - axis = scale.axis.lower() + output_crs = uniform_crs(cfg, request.output_crs or subsetting_crs or native_crs) - if axis in ('time', 'k'): - raise WCS2Exception('Cannot scale axis %s' % scale.axis, - WCS2Exception.INVALID_SCALE_FACTOR, - locator=scale.axis - ) - else: - if isinstance(scale, ScaleAxis): - scaler.scale_axis(axis, scale.factor) - elif isinstance(scale, ScaleSize): - scaler.scale_size(axis, scale.size) - elif isinstance(scale, ScaleExtent): - scaler.scale_extent(axis, scale.low, scale.high) - - # - # Rangesubset - # - - band_labels = layer.band_idx.band_labels() - if request.range_subset: - bands = [] - for range_subset in request.range_subset: - if isinstance(range_subset, str): - if range_subset not in band_labels: - raise WCS2Exception('No such field %s' % range_subset, - WCS2Exception.NO_SUCH_FIELD, - locator=range_subset, - valid_keys=band_labels - ) - bands.append(range_subset) + if output_crs not in cfg.published_CRSs: + raise WCS2Exception("Invalid outputCrs: %s" % output_crs, + WCS2Exception.OUTPUT_CRS_NOT_SUPPORTED, + locator=output_crs, + valid_keys=list(cfg.published_CRSs)) + + # + # Subsetting/Scaling + # + + scaler = WCSScaler(layer, subsetting_crs) + times = layer.ranges["times"] + + subsets = request.subsets + + if len(subsets) != len(set(subset.dimension.lower() for subset in subsets)): + dimensions = [subset.dimension.lower() for subset in subsets] + duplicate_dimensions = [ + item + for item, count in collections.Counter(dimensions).items() + if count > 1 + ] + + raise WCS2Exception("Duplicate dimension%s: %s" % ( + 's' if len(duplicate_dimensions) > 1 else '', + ', '.join(duplicate_dimensions) + ), + WCS2Exception.INVALID_SUBSETTING, + locator=','.join(duplicate_dimensions) + ) + + for subset in subsets: + dimension = subset.dimension.lower() + if dimension == 'time': + if isinstance(subset, Trim): + if "," in subset.high: + raise WCS2Exception( + "Subsets can only contain 2 elements - the lower and upper bounds. For arbitrary date lists, use WCS1", + WCS2Exception.INVALID_SUBSETTING, + locator="time") + if layer.time_resolution.is_subday(): + low = parse(subset.low) if subset.low is not None else None + low = default_to_utc(low) + high = parse(subset.high) if subset.high is not None else None + high = default_to_utc(high) else: - if range_subset.start not in band_labels: - raise WCS2Exception('No such field %s' % range_subset.start, - WCS2Exception.ILLEGAL_FIELD_SEQUENCE, - locator=range_subset.start, - valid_keys = band_labels) - if range_subset.end not in band_labels: - raise WCS2Exception('No such field %s' % range_subset.end, - WCS2Exception.ILLEGAL_FIELD_SEQUENCE, - locator=range_subset.end, - valid_keys = band_labels) - - start = band_labels.index(range_subset.start) - end = band_labels.index(range_subset.end) - bands.extend(band_labels[start:(end + 1) if end > start else (end - 1)]) - # Uncomment to restore original styles parameter hack. - # - # elif styles: - # bands = get_bands_from_styles(styles, layer, version=2) - # if not bands: - # bands = band_labels + low = parse(subset.low).date() if subset.low is not None else None + high = parse(subset.high).date() if subset.high is not None else None + if low is not None: + times = [ + time for time in times + if time >= low + ] + if high is not None: + times = [ + time for time in times + if time <= high + ] + elif isinstance(subset, Slice): + point = parse(subset.point).date() + times = [point] else: - bands = band_labels + try: + if isinstance(subset, Trim): + scaler.trim(dimension, subset.low, subset.high) + elif isinstance(subset, Slice): + scaler.slice(dimension, subset.point) + except WCSScalerUnknownDimension: + raise WCS2Exception('Invalid subsetting axis %s' % subset.dimension, + WCS2Exception.INVALID_AXIS_LABEL, + locator=subset.dimension) - # - # Format handling - # + # + # Transform spatial extent to native CRS. + # + scaler.to_crs(output_crs) + + # + # Scaling + # - if not request.format: - fmt = cfg.wcs_formats_by_name[layer.native_format] + scales = request.scales + if len(scales) != len(set(subset.axis.lower() for subset in scales)): + axes = [subset.axis.lower() for subset in scales] + duplicate_axes = [ + item + for item, count in collections.Counter(axes).items() + if count > 1 + ] + raise WCS2Exception('Duplicate scales for ax%ss: %s' % ( + 'i' if len(duplicate_axes) == 1 else 'e', + ', '.join(duplicate_axes) + ), + WCS2Exception.INVALID_SCALE_FACTOR, + locator=','.join(duplicate_axes) + ) + + for scale in scales: + axis = scale.axis.lower() + + if axis in ('time', 'k'): + raise WCS2Exception('Cannot scale axis %s' % scale.axis, + WCS2Exception.INVALID_SCALE_FACTOR, + locator=scale.axis + ) else: - try: - fmt = cfg.wcs_formats_by_mime[request.format] - except KeyError: - raise WCS2Exception("Unsupported format: %s" % request.format, - WCS2Exception.INVALID_PARAMETER_VALUE, - locator="FORMAT", - valid_keys=list(cfg.wcs_formats_by_mime)) - - if len(times) > 1 and not fmt.multi_time: - raise WCS2Exception( - "Format does not support multi-time datasets - " - "either constrain the time dimension or choose a different format", - WCS2Exception.INVALID_SUBSETTING, - locator="FORMAT or SUBSET" + if isinstance(scale, ScaleAxis): + scaler.scale_axis(axis, scale.factor) + elif isinstance(scale, ScaleSize): + scaler.scale_size(axis, scale.size) + elif isinstance(scale, ScaleExtent): + scaler.scale_extent(axis, scale.low, scale.high) + + # + # Rangesubset + # + + band_labels = layer.band_idx.band_labels() + if request.range_subset: + bands = [] + for range_subset in request.range_subset: + if isinstance(range_subset, str): + if range_subset not in band_labels: + raise WCS2Exception('No such field %s' % range_subset, + WCS2Exception.NO_SUCH_FIELD, + locator=range_subset, + valid_keys=band_labels ) - affine = scaler.affine() - geobox = GeoBox((scaler.size.y, scaler.size.x), - affine, cfg.crs(output_crs)) - - stacker = DataStacker(layer, - geobox, - times, - bands=bands) - qprof.end_event("setup") - qprof.start_event("count-datasets") - n_datasets = stacker.datasets(dc.index, mode=MVSelectOpts.COUNT) - qprof.end_event("count-datasets") - qprof["n_datasets"] = n_datasets + bands.append(range_subset) + else: + if range_subset.start not in band_labels: + raise WCS2Exception('No such field %s' % range_subset.start, + WCS2Exception.ILLEGAL_FIELD_SEQUENCE, + locator=range_subset.start, + valid_keys = band_labels) + if range_subset.end not in band_labels: + raise WCS2Exception('No such field %s' % range_subset.end, + WCS2Exception.ILLEGAL_FIELD_SEQUENCE, + locator=range_subset.end, + valid_keys = band_labels) + + start = band_labels.index(range_subset.start) + end = band_labels.index(range_subset.end) + bands.extend(band_labels[start:(end + 1) if end > start else (end - 1)]) + # Uncomment to restore original styles parameter hack. + # + # elif styles: + # bands = get_bands_from_styles(styles, layer, version=2) + # if not bands: + # bands = band_labels + else: + bands = band_labels + # + # Format handling + # + + if not request.format: + fmt = cfg.wcs_formats_by_name[layer.native_format] + else: try: - layer.resource_limits.check_wcs(n_datasets, - geobox.height, geobox.width, - sum(layer.band_idx.dtype_size(b) for b in bands), - len(times) - ) - except ResourceLimited as e: - if e.wcs_hard or not layer.low_res_product_names: - raise WCS2Exception( - f"This request processes too much data to be served in a reasonable amount of time. ({e}) " - + "Please reduce the bounds of your request and try again.") - stacker.resource_limited = True - qprof["resource_limited"] = str(e) - - if n_datasets == 0: - raise WCS2Exception("The requested spatio-temporal subsets return no data.", - WCS2Exception.INVALID_SUBSETTING, - http_response=404) - - qprof.start_event("fetch-datasets") - datasets = stacker.datasets(dc.index) - qprof.end_event("fetch-datasets") - if qprof.active: - qprof["datasets"] = { - str(q): [str(i) for i in ids] - for q, ids in stacker.datasets(dc.index, mode=MVSelectOpts.IDS).items() - } - qprof.start_event("load-data") - output = stacker.data(datasets, skip_corrections=True) - qprof.end_event("load-data") - - # Clean extent flag band from output - raw_bands = [layer.band_idx.locale_band(b) for b in bands] - for k, v in output.data_vars.items(): - if k not in raw_bands: - output = output.drop_vars([k]) + fmt = cfg.wcs_formats_by_mime[request.format] + except KeyError: + raise WCS2Exception("Unsupported format: %s" % request.format, + WCS2Exception.INVALID_PARAMETER_VALUE, + locator="FORMAT", + valid_keys=list(cfg.wcs_formats_by_mime)) + + if len(times) > 1 and not fmt.multi_time: + raise WCS2Exception( + "Format does not support multi-time datasets - " + "either constrain the time dimension or choose a different format", + WCS2Exception.INVALID_SUBSETTING, + locator="FORMAT or SUBSET" + ) + affine = scaler.affine() + geobox = GeoBox((scaler.size.y, scaler.size.x), + affine, cfg.crs(output_crs)) + + stacker = DataStacker(layer, + geobox, + times, + bands=bands) + qprof.end_event("setup") + qprof.start_event("count-datasets") + n_datasets = stacker.datasets(layer.dc.index, mode=MVSelectOpts.COUNT) + qprof.end_event("count-datasets") + qprof["n_datasets"] = n_datasets + + try: + layer.resource_limits.check_wcs(n_datasets, + geobox.height, geobox.width, + sum(layer.band_idx.dtype_size(b) for b in bands), + len(times) + ) + except ResourceLimited as e: + if e.wcs_hard or not layer.low_res_product_names: + raise WCS2Exception( + f"This request processes too much data to be served in a reasonable amount of time. ({e}) " + + "Please reduce the bounds of your request and try again.") + stacker.resource_limited = True + qprof["resource_limited"] = str(e) + + if n_datasets == 0: + raise WCS2Exception("The requested spatio-temporal subsets return no data.", + WCS2Exception.INVALID_SUBSETTING, + http_response=404) + + qprof.start_event("fetch-datasets") + datasets = stacker.datasets(layer.dc.index) + qprof.end_event("fetch-datasets") + if qprof.active: + qprof["datasets"] = { + str(q): [str(i) for i in ids] + for q, ids in stacker.datasets(layer.dc.index, mode=MVSelectOpts.IDS).items() + } + qprof.start_event("load-data") + output = stacker.data(datasets, skip_corrections=True) + qprof.end_event("load-data") + + # Clean extent flag band from output + raw_bands = [layer.band_idx.locale_band(b) for b in bands] + for k, v in output.data_vars.items(): + if k not in raw_bands: + output = output.drop_vars([k]) # # TODO: configurable diff --git a/datacube_ows/wms_utils.py b/datacube_ows/wms_utils.py index 7c004d74c..c5612a0fc 100644 --- a/datacube_ows/wms_utils.py +++ b/datacube_ows/wms_utils.py @@ -128,21 +128,21 @@ def img_coords_to_geopoint(geobox, i, j): geobox.crs) -def get_product_from_arg(args, argname="layers") -> OWSNamedLayer: +def get_layer_from_arg(args, argname="layers") -> OWSNamedLayer: layers = args.get(argname, "").split(",") if len(layers) != 1: raise WMSException("Multi-layer requests not supported") - layer = layers[0] - layer_chunks = layer.split("__") - layer = layer_chunks[0] + lyr = layers[0] + layer_chunks = lyr.split("__") + lyr = layer_chunks[0] cfg = get_config() - product = cfg.product_index.get(layer) - if not product: - raise WMSException("Layer %s is not defined" % layer, + layer = cfg.product_index.get(lyr) + if not layer: + raise WMSException("Layer %s is not defined" % lyr, WMSException.LAYER_NOT_DEFINED, locator="Layer parameter", valid_keys=list(cfg.product_index)) - return product + return layer def get_arg(args, argname, verbose_name, lower=False, @@ -308,21 +308,21 @@ def __init__(self, args): permitted_values=list(self.cfg.published_CRSs)) self.crs = self.cfg.crs(self.crsid) # Layers - self.product = self.get_product(args) + self.layer = self.get_layer(args) self.geometry = _get_polygon(args, self.crs) # BBox, height and width parameters self.geobox = _get_geobox(args, self.crs) # Time parameter - self.times = get_times(args, self.product) + self.times = get_times(args, self.layer) self.method_specific_init(args) def method_specific_init(self, args): pass - def get_product(self, args) -> OWSNamedLayer: - return get_product_from_arg(args) + def get_layer(self, args) -> OWSNamedLayer: + return get_layer_from_arg(args) def single_style_from_args(product, args, required=True): @@ -386,7 +386,7 @@ def single_style_from_args(product, args, required=True): class GetLegendGraphicParameters(): def __init__(self, args): - self.product = get_product_from_arg(args, 'layer') + self.product = get_layer_from_arg(args, 'layer') # Validate Format parameter self.format = get_arg(args, "format", "image format", @@ -407,7 +407,7 @@ def method_specific_init(self, args): lower=True, permitted_values=["image/png"]) - self.style = single_style_from_args(self.product, args) + self.style = single_style_from_args(self.layer, args) cfg = get_config() if self.geobox.width > cfg.wms_max_width: raise WMSException(f"Width {self.geobox.width} exceeds supported maximum {self.cfg.wms_max_width}.", @@ -425,8 +425,8 @@ def method_specific_init(self, args): self.resampling = Resampling.nearest self.resources = RequestScale( - native_crs=geom.CRS(self.product.native_CRS), - native_resolution=(self.product.resolution_x, self.product.resolution_y), + native_crs=geom.CRS(self.layer.native_CRS), + native_resolution=(self.layer.resolution_x, self.layer.resolution_y), geobox=self.geobox, n_dates=len(self.times), request_bands=self.style.odc_needed_bands(), @@ -434,8 +434,8 @@ def method_specific_init(self, args): class GetFeatureInfoParameters(GetParameters): - def get_product(self, args): - return get_product_from_arg(args, "query_layers") + def get_layer(self, args): + return get_layer_from_arg(args, "query_layers") def method_specific_init(self, args): # Validate Formata parameter diff --git a/integration_tests/test_cube_pool.py b/integration_tests/test_cube_pool.py deleted file mode 100644 index 96252b4be..000000000 --- a/integration_tests/test_cube_pool.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file is part of datacube-ows, part of the Open Data Cube project. -# See https://opendatacube.org for more information. -# -# Copyright (c) 2017-2024 OWS Contributors -# SPDX-License-Identifier: Apache-2.0 - -from datacube import Datacube - -from datacube_ows.cube_pool import get_cube - - -def test_basic_cube_pool(): - dc_1 = get_cube(app="test") - dc_2 = get_cube(app="test") - assert dc_1 == dc_2 - dc_unalloc = Datacube(app="test") - assert dc_1 != dc_unalloc diff --git a/integration_tests/test_layers.py b/integration_tests/test_layers.py index fd55ddc65..c3d47f55d 100644 --- a/integration_tests/test_layers.py +++ b/integration_tests/test_layers.py @@ -6,7 +6,6 @@ import os -from datacube_ows.cube_pool import cube from datacube_ows.ows_configuration import OWSConfig, get_config, read_config src_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -44,8 +43,7 @@ def test_missing_metadata_file(monkeypatch): raw_cfg["global"]["translations_directory"] = None raw_cfg["global"]["languages"] = ["en"] cfg = OWSConfig(refresh=True, cfg=raw_cfg) - with cube() as dc: - cfg.make_ready(dc) + cfg.make_ready() assert "Over-ridden" not in cfg.title assert "aardvark" not in cfg.title @@ -70,8 +68,7 @@ def test_metadata_file_ignore(monkeypatch): raw_cfg = read_config() raw_cfg["global"]["message_file"] = "integration_tests/cfg/message.po" cfg = OWSConfig(refresh=True, cfg=raw_cfg, ignore_msgfile=True) - with cube() as dc: - cfg.make_ready(dc) + cfg.make_ready() assert "Over-ridden" not in cfg.title assert "aardvark" not in cfg.title @@ -90,8 +87,7 @@ def test_metadata_read(monkeypatch, product_name): raw_cfg = read_config() raw_cfg["global"]["message_file"] = "integration_tests/cfg/message.po" cfg = OWSConfig(refresh=True, cfg=raw_cfg) - with cube() as dc: - cfg.make_ready(dc) + cfg.make_ready() assert "Over-ridden" in cfg.title assert "aardvark" in cfg.title diff --git a/integration_tests/test_mv_index.py b/integration_tests/test_mv_index.py index d7568b9f4..970e56d19 100644 --- a/integration_tests/test_mv_index.py +++ b/integration_tests/test_mv_index.py @@ -7,7 +7,6 @@ import pytest from odc.geo.geom import box -from datacube_ows.cube_pool import cube from datacube_ows.mv_index import MVSelectOpts, mv_search from datacube_ows.ows_configuration import get_config from datacube_ows.time_utils import local_solar_date_range @@ -16,25 +15,24 @@ def test_full_layer(): cfg = get_config() lyr = list(cfg.product_index.values())[0] - with cube() as dc: - sel = mv_search(dc.index, MVSelectOpts.COUNT, products=lyr.products) - assert sel > 0 + sel = mv_search(lyr.dc.index, MVSelectOpts.COUNT, products=lyr.products) + assert sel > 0 def test_select_all(): cfg = get_config() lyr = list(cfg.product_index.values())[0] - with cube() as dc: - rows = mv_search(dc.index, MVSelectOpts.ALL, products=lyr.products) - for row in rows: - assert len(row) > 1 + rows = mv_search(lyr.dc.index, MVSelectOpts.ALL, products=lyr.products) + for row in rows: + assert len(row) > 1 def test_no_products(): - with cube() as dc: - with pytest.raises(Exception) as e: - sel = mv_search(dc.index, MVSelectOpts.COUNT) - assert "Must filter by product/layer" in str(e.value) + cfg = get_config() + lyr = list(cfg.product_index.values())[0] + with pytest.raises(Exception) as e: + sel = mv_search(lyr.dc.index, MVSelectOpts.COUNT) + assert "Must filter by product/layer" in str(e.value) def test_bad_set_opt(): @@ -64,31 +62,28 @@ def test_time_search(): ) time_rng = local_solar_date_range(MockGeobox(geom), time) - with cube() as dc: - sel = mv_search( - dc.index, MVSelectOpts.COUNT, times=[time_rng], products=lyr.products - ) - assert sel > 0 + sel = mv_search( + lyr.dc.index, MVSelectOpts.COUNT, times=[time_rng], products=lyr.products + ) + assert sel > 0 def test_count(): cfg = get_config() lyr = list(cfg.product_index.values())[0] - with cube() as dc: - count = mv_search(dc.index, MVSelectOpts.COUNT, products=lyr.products) - ids = mv_search(dc.index, MVSelectOpts.IDS, products=lyr.products) - assert len(ids) == count + count = mv_search(lyr.dc.index, MVSelectOpts.COUNT, products=lyr.products) + ids = mv_search(lyr.dc.index, MVSelectOpts.IDS, products=lyr.products) + assert len(ids) == count def test_datasets(): cfg = get_config() lyr = list(cfg.product_index.values())[0] - with cube() as dc: - dss = mv_search(dc.index, MVSelectOpts.DATASETS, products=lyr.products) - ids = mv_search(dc.index, MVSelectOpts.IDS, products=lyr.products) - assert len(ids) == len(dss) - for ds in dss: - assert ds.id in ids + dss = mv_search(lyr.dc.index, MVSelectOpts.DATASETS, products=lyr.products) + ids = mv_search(lyr.dc.index, MVSelectOpts.IDS, products=lyr.products) + assert len(ids) == len(dss) + for ds in dss: + assert ds.id in ids def test_extent_and_spatial(): @@ -112,22 +107,21 @@ def test_extent_and_spatial(): small_bbox[0], small_bbox[1], small_bbox[2], small_bbox[3], "EPSG:4326" ) - with cube() as dc: - all_ext = mv_search( - dc.index, MVSelectOpts.EXTENT, geom=layer_ext_geom, products=lyr.products - ) - small_ext = mv_search( - dc.index, MVSelectOpts.EXTENT, geom=small_geom, products=lyr.products - ) - assert layer_ext_geom.contains(all_ext) - assert small_geom.contains(small_ext) - assert all_ext.contains(small_ext) - assert small_ext.area < all_ext.area - - all_count = mv_search( - dc.index, MVSelectOpts.COUNT, geom=layer_ext_geom, products=lyr.products - ) - small_count = mv_search( - dc.index, MVSelectOpts.COUNT, geom=small_geom, products=lyr.products - ) - assert small_count <= all_count + all_ext = mv_search( + lyr.dc.index, MVSelectOpts.EXTENT, geom=layer_ext_geom, products=lyr.products + ) + small_ext = mv_search( + lyr.dc.index, MVSelectOpts.EXTENT, geom=small_geom, products=lyr.products + ) + assert layer_ext_geom.contains(all_ext) + assert small_geom.contains(small_ext) + assert all_ext.contains(small_ext) + assert small_ext.area < all_ext.area + + all_count = mv_search( + lyr.dc.index, MVSelectOpts.COUNT, geom=layer_ext_geom, products=lyr.products + ) + small_count = mv_search( + lyr.dc.index, MVSelectOpts.COUNT, geom=small_geom, products=lyr.products + ) + assert small_count <= all_count diff --git a/integration_tests/utils.py b/integration_tests/utils.py index f337b0396..a1fd7d7bc 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -9,7 +9,6 @@ from odc.geo.geom import BoundingBox, Geometry, point from shapely.ops import triangulate, unary_union -from datacube_ows.cube_pool import cube from datacube_ows.mv_index import MVSelectOpts, mv_search @@ -371,21 +370,20 @@ def subsets( ): ext_times = time.slice(self.layer.ranges["times"]) search_times = [self.layer.search_times(t) for t in ext_times] - with cube() as dc: - if space.needs_full_extent() and not self.full_extent: - self.full_extent = mv_search( - dc.index, products=self.layer.products, sel=MVSelectOpts.EXTENT - ) - if space.needs_time_extent(): - time_extent = mv_search( - dc.index, - products=self.layer.products, - sel=MVSelectOpts.EXTENT, - times=search_times, - ) - else: - time_extent = None + if space.needs_full_extent() and not self.full_extent: + self.full_extent = mv_search( + self.layer.dc.index, products=self.layer.products, sel=MVSelectOpts.EXTENT + ) + if space.needs_time_extent(): + time_extent = mv_search( + self.layer.dc.index, + products=self.layer.products, + sel=MVSelectOpts.EXTENT, + times=search_times, + ) + else: + time_extent = None - extent = space.subset(time_extent, self.full_extent) + extent = space.subset(time_extent, self.full_extent) return extent, ext_times From 261aceff2db0153629ca6a01f2de59f5459a94bf Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 11:26:08 +1000 Subject: [PATCH 04/12] Decide where to stop for this PR and documentation. --- datacube_ows/ows_configuration.py | 32 ++++++++++++++++++------------- docs/cfg_global.rst | 12 ++++++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 784704118..bb866a709 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -414,8 +414,12 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold self.hide = False self.local_env: ODCEnvironment | None = None local_env = cast(str | None, cfg.get("env")) - if local_env: - self.local_env = ODCConfig.get_environment(env=local_env) + # TODO: MULTIDB_SUPPORT + # After refactoring the range tables, Uncomment this code for multi-database support + # (Don't forget to add to documentation) + # + # if local_env: + # self.local_env = ODCConfig.get_environment(env=local_env) try: self.parse_product_names(cfg) if len(self.low_res_product_names) not in (0, len(self.product_names)): @@ -508,7 +512,6 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold self.declare_unready("default_time") self.declare_unready("_ranges") self.declare_unready("bboxes") - # TODO: sub-ranges self.band_idx: BandIndex = BandIndex(self, cast(CFG_DICT, cfg.get("bands"))) self.cfg_native_resolution = cfg.get("native_resolution") self.cfg_native_crs = cfg.get("native_crs") @@ -569,14 +572,18 @@ def time_axis_representation(self) -> str: # pylint: disable=attribute-defined-outside-init def make_ready(self, *args: Any, **kwargs: Any) -> None: - if self.local_env: - try: - self.dc: Datacube = Datacube(env=self.local_env, app=self.global_cfg.odc_app) - except Exception as e: - _LOG.error("ODC initialisation failed: %s", str(e)) - raise ODCInitException(e) - else: - self.dc = self.global_cfg.dc + # TODO: MULTIDB_SUPPORT + # After refactoring the range tables, Uncomment this code for multi-database support + # (Don't forget to add to documentation) + # + # if self.local_env: + # try: + # self.dc: Datacube = Datacube(env=self.local_env, app=self.global_cfg.odc_app) + # except Exception as e: + # _LOG.error("ODC initialisation failed: %s", str(e)) + # raise ODCInitException(e) + # else: + self.dc = self.global_cfg.dc self.products: list[Product] = [] self.low_res_products: list[Product] = [] for i, prod_name in enumerate(self.product_names): @@ -1301,9 +1308,8 @@ def export_metadata(self) -> Catalog: return self.catalog def parse_global(self, cfg: CFG_DICT, ignore_msgfile: bool): - default_env = cast(str, cfg.get("env")) + self.default_env = cast(str, cfg.get("env")) self.odc_app = cast(str, cfg.get("odc_app", "ows")) - self.default_env = ODCConfig.get_environment(default_env) self._response_headers = cast(dict[str, str], cfg.get("response_headers", {})) services = cast(dict[str, bool], cfg.get("services", {})) self.wms = services.get("wms", True) diff --git a/docs/cfg_global.rst b/docs/cfg_global.rst index fb2ef6d5e..e895d128c 100644 --- a/docs/cfg_global.rst +++ b/docs/cfg_global.rst @@ -14,6 +14,18 @@ to all services and all layers/coverages. The Global section is always required and contains the following entries: +Open Data Cube Environment +========================== + +Normally OWS defers to the ODC-1.9 configuration environment (`default`, with legacy support for `datacube`). + +You can override this with the ``env`` global config entry:: + + "env": "dev" + +Future releases will allow this to be over-ridden at the layer level, allowing one OWS instance to serve data +from multiple indexes. + Metadata Separation and Internationalisation ============================================ From 2db136c83d0a983dc485296d28a72a3a58a06516 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 11:37:27 +1000 Subject: [PATCH 05/12] Lintage and disable isort. --- .pre-commit-config.yaml | 4 ---- datacube_ows/config_utils.py | 1 - 2 files changed, 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29e018513..c9c7ce9be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,10 +5,6 @@ repos: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace -- repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - repo: https://github.com/PyCQA/flake8 rev: 7.0.0 hooks: diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index cf0603812..0b41fbdde 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -14,7 +14,6 @@ import fsspec from babel.messages import Catalog, Message -from datacube import Datacube from datacube.model import Product from datacube.utils.masking import make_mask from flask_babel import gettext as _ From 3acdf274a7603c5fbce8b824d0c5c550edbf4d51 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 11:59:34 +1000 Subject: [PATCH 06/12] Minor reformat. --- datacube_ows/ows_configuration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index bb866a709..f2e83684f 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -414,6 +414,7 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold self.hide = False self.local_env: ODCEnvironment | None = None local_env = cast(str | None, cfg.get("env")) + self.local_env = ODCConfig.get_environment(env=local_env) # TODO: MULTIDB_SUPPORT # After refactoring the range tables, Uncomment this code for multi-database support # (Don't forget to add to documentation) @@ -598,7 +599,9 @@ def make_ready(self, *args: Any, **kwargs: Any) -> None: if low_res_prod_name: product = self.dc.index.products.get_by_name(low_res_prod_name) if not product: - raise ConfigException(f"Could not find product {low_res_prod_name} in datacube for layer {self.name}") + raise ConfigException( + f"Could not find product {low_res_prod_name} in datacube for layer {self.name}" + ) self.low_res_products.append(product) self.product = self.products[0] self.definition = self.product.definition From ce1eb0d222676184c28b6736c268e2ff4ca1a9c6 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 12:18:02 +1000 Subject: [PATCH 07/12] Fix global config unit tests by monkeypatching Datacube construction. --- tests/test_cfg_global.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_cfg_global.py b/tests/test_cfg_global.py index f6cd21b4f..6030a602b 100644 --- a/tests/test_cfg_global.py +++ b/tests/test_cfg_global.py @@ -10,10 +10,13 @@ from datacube_ows.ows_configuration import ContactInfo, OWSConfig -def test_minimal_global(minimal_global_raw_cfg, minimal_dc): +def test_minimal_global(minimal_global_raw_cfg, minimal_dc, monkeypatch): + def fake_dc(*args, **kwargs): + return minimal_dc + monkeypatch.setattr("datacube.Datacube", fake_dc) OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) - cfg.make_ready(minimal_dc) + cfg.make_ready() assert cfg.ready assert cfg.initialised assert not cfg.wcs_tiff_statistics @@ -45,7 +48,10 @@ def test_wcs_only(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): assert cfg.wcs_tiff_statistics assert cfg.default_geographic_CRS == "urn:ogc:def:crs:OGC:1.3:CRS84" -def test_geog_crs(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): +def test_geog_crs(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc, monkeypatch): + def fake_dc(*args, **kwargs): + return minimal_dc + monkeypatch.setattr("datacube.Datacube", fake_dc) OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -199,7 +205,10 @@ def test_tiff_stats(minimal_global_raw_cfg, wcs_global_cfg): cfg = OWSConfig(cfg=minimal_global_raw_cfg) assert not cfg.wcs_tiff_statistics -def test_crs_lookup_fail(minimal_global_raw_cfg, minimal_dc): +def test_crs_lookup_fail(monkeypatch, minimal_global_raw_cfg, minimal_dc): + def fake_dc(*args, **kwargs): + return minimal_dc + monkeypatch.setattr("datacube.Datacube", fake_dc) OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) with pytest.raises(ConfigException) as excinfo: From 07b36688a4e661422952f7c74eed82bcec3c0313 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 12:40:30 +1000 Subject: [PATCH 08/12] Monkey patch BEFORE importing the thing you are monkeypatching, that's important. --- tests/test_cfg_global.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/test_cfg_global.py b/tests/test_cfg_global.py index 6030a602b..380fb19a0 100644 --- a/tests/test_cfg_global.py +++ b/tests/test_cfg_global.py @@ -6,14 +6,16 @@ import pytest +import datacube from datacube_ows.config_utils import ConfigException -from datacube_ows.ows_configuration import ContactInfo, OWSConfig -def test_minimal_global(minimal_global_raw_cfg, minimal_dc, monkeypatch): +def test_minimal_global(monkeypatch, minimal_global_raw_cfg, minimal_dc): def fake_dc(*args, **kwargs): return minimal_dc - monkeypatch.setattr("datacube.Datacube", fake_dc) + monkeypatch.setattr(datacube, "Datacube", fake_dc) + assert datacube.Datacube("foo") == minimal_dc + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) cfg.make_ready() @@ -24,6 +26,7 @@ def fake_dc(*args, **kwargs): def test_global_no_title(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None del minimal_global_raw_cfg["global"]["title"] with pytest.raises(ConfigException) as excinfo: @@ -31,7 +34,11 @@ def test_global_no_title(minimal_global_raw_cfg): assert "Entity global has no title" in str(excinfo.value) -def test_wcs_only(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): +def test_wcs_only(monkeypatch, minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): + def fake_dc(*args, **kwargs): + return minimal_dc + monkeypatch.setattr(datacube, "Datacube", fake_dc) + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -40,7 +47,7 @@ def test_wcs_only(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): } minimal_global_raw_cfg["wcs"] = wcs_global_cfg cfg = OWSConfig(cfg=minimal_global_raw_cfg) - cfg.make_ready(minimal_dc) + cfg.make_ready() assert cfg.ready assert cfg.wcs assert not cfg.wms @@ -52,6 +59,7 @@ def test_geog_crs(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc, monkeypatc def fake_dc(*args, **kwargs): return minimal_dc monkeypatch.setattr("datacube.Datacube", fake_dc) + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -88,6 +96,7 @@ def fake_dc(*args, **kwargs): def test_contact_details_parse(minimal_global_cfg): + from datacube_ows.ows_configuration import ContactInfo addr1 = ContactInfo.parse({}, minimal_global_cfg) assert addr1 is None addr2 = ContactInfo.parse({"address": {}}, minimal_global_cfg) @@ -97,6 +106,7 @@ def test_contact_details_parse(minimal_global_cfg): def test_wcs_no_native_format(minimal_global_raw_cfg, wcs_global_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -113,6 +123,7 @@ def test_wcs_no_native_format(minimal_global_raw_cfg, wcs_global_cfg): def test_no_services(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wms": False, @@ -124,6 +135,7 @@ def test_no_services(minimal_global_raw_cfg): def test_no_published_crss(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig del minimal_global_raw_cfg["global"]["published_CRSs"] with pytest.raises(ConfigException) as e: cfg = OWSConfig(cfg=minimal_global_raw_cfg) @@ -132,6 +144,7 @@ def test_no_published_crss(minimal_global_raw_cfg): def test_bad_geographic_crs(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["published_CRSs"]["EPSG:7777"] = { "geographic": True, @@ -157,6 +170,7 @@ def test_bad_geographic_crs(minimal_global_raw_cfg): def test_bad_crs_alias(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["published_CRSs"]["EPSG:7777"] = { "alias": "EPSG:6666", @@ -166,6 +180,7 @@ def test_bad_crs_alias(minimal_global_raw_cfg): def test_no_wcs(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} with pytest.raises(ConfigException) as excinfo: @@ -175,6 +190,7 @@ def test_no_wcs(minimal_global_raw_cfg): def test_no_wcs_formats(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = { @@ -186,6 +202,7 @@ def test_no_wcs_formats(minimal_global_raw_cfg): def test_bad_wcs_format(minimal_global_raw_cfg, wcs_global_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = wcs_global_cfg @@ -198,6 +215,7 @@ def test_bad_wcs_format(minimal_global_raw_cfg, wcs_global_cfg): def test_tiff_stats(minimal_global_raw_cfg, wcs_global_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = wcs_global_cfg @@ -209,6 +227,7 @@ def test_crs_lookup_fail(monkeypatch, minimal_global_raw_cfg, minimal_dc): def fake_dc(*args, **kwargs): return minimal_dc monkeypatch.setattr("datacube.Datacube", fake_dc) + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) with pytest.raises(ConfigException) as excinfo: @@ -217,7 +236,8 @@ def fake_dc(*args, **kwargs): assert "is not published" in str(excinfo.value) -def test_no_langs(minimal_global_raw_cfg, minimal_dc): +def test_no_langs(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = [] with pytest.raises(ConfigException) as excinfo: @@ -226,6 +246,7 @@ def test_no_langs(minimal_global_raw_cfg, minimal_dc): def test_two_langs(minimal_global_raw_cfg, minimal_dc): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = ["fr", "en"] cfg = OWSConfig(cfg=minimal_global_raw_cfg) @@ -236,6 +257,7 @@ def test_two_langs(minimal_global_raw_cfg, minimal_dc): def test_internationalised(minimal_global_raw_cfg, minimal_dc): + from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = ["fr", "en"] @@ -244,7 +266,8 @@ def test_internationalised(minimal_global_raw_cfg, minimal_dc): assert cfg.global_config().internationalised -def test_bad_integers_in_wms_section(minimal_global_raw_cfg, minimal_dc): +def test_bad_integers_in_wms_section(minimal_global_raw_cfg): + from datacube_ows.ows_configuration import OWSConfig minimal_global_raw_cfg["wms"] = {} minimal_global_raw_cfg["wms"]["max_width"] = "very big" with pytest.raises(ConfigException) as e: From 8eacec25239e0cda22974b374ed96d4be4e5a815 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 13:10:53 +1000 Subject: [PATCH 09/12] Monkeypatching the right module is even more important. --- tests/test_cfg_global.py | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/tests/test_cfg_global.py b/tests/test_cfg_global.py index 380fb19a0..631fbb34d 100644 --- a/tests/test_cfg_global.py +++ b/tests/test_cfg_global.py @@ -6,16 +6,14 @@ import pytest -import datacube from datacube_ows.config_utils import ConfigException +from datacube_ows.ows_configuration import OWSConfig, ContactInfo def test_minimal_global(monkeypatch, minimal_global_raw_cfg, minimal_dc): def fake_dc(*args, **kwargs): return minimal_dc - monkeypatch.setattr(datacube, "Datacube", fake_dc) - assert datacube.Datacube("foo") == minimal_dc - from datacube_ows.ows_configuration import OWSConfig + monkeypatch.setattr("datacube_ows.ows_configuration.Datacube", fake_dc) OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) cfg.make_ready() @@ -26,7 +24,6 @@ def fake_dc(*args, **kwargs): def test_global_no_title(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None del minimal_global_raw_cfg["global"]["title"] with pytest.raises(ConfigException) as excinfo: @@ -37,8 +34,7 @@ def test_global_no_title(minimal_global_raw_cfg): def test_wcs_only(monkeypatch, minimal_global_raw_cfg, wcs_global_cfg, minimal_dc): def fake_dc(*args, **kwargs): return minimal_dc - monkeypatch.setattr(datacube, "Datacube", fake_dc) - from datacube_ows.ows_configuration import OWSConfig + monkeypatch.setattr("datacube_ows.ows_configuration.Datacube", fake_dc) OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -58,8 +54,7 @@ def fake_dc(*args, **kwargs): def test_geog_crs(minimal_global_raw_cfg, wcs_global_cfg, minimal_dc, monkeypatch): def fake_dc(*args, **kwargs): return minimal_dc - monkeypatch.setattr("datacube.Datacube", fake_dc) - from datacube_ows.ows_configuration import OWSConfig + monkeypatch.setattr("datacube_ows.ows_configuration.Datacube", fake_dc) OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -96,7 +91,6 @@ def fake_dc(*args, **kwargs): def test_contact_details_parse(minimal_global_cfg): - from datacube_ows.ows_configuration import ContactInfo addr1 = ContactInfo.parse({}, minimal_global_cfg) assert addr1 is None addr2 = ContactInfo.parse({"address": {}}, minimal_global_cfg) @@ -106,7 +100,6 @@ def test_contact_details_parse(minimal_global_cfg): def test_wcs_no_native_format(minimal_global_raw_cfg, wcs_global_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wcs": True, @@ -123,7 +116,6 @@ def test_wcs_no_native_format(minimal_global_raw_cfg, wcs_global_cfg): def test_no_services(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = { "wms": False, @@ -135,7 +127,6 @@ def test_no_services(minimal_global_raw_cfg): def test_no_published_crss(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig del minimal_global_raw_cfg["global"]["published_CRSs"] with pytest.raises(ConfigException) as e: cfg = OWSConfig(cfg=minimal_global_raw_cfg) @@ -144,7 +135,6 @@ def test_no_published_crss(minimal_global_raw_cfg): def test_bad_geographic_crs(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["published_CRSs"]["EPSG:7777"] = { "geographic": True, @@ -170,7 +160,6 @@ def test_bad_geographic_crs(minimal_global_raw_cfg): def test_bad_crs_alias(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["published_CRSs"]["EPSG:7777"] = { "alias": "EPSG:6666", @@ -180,7 +169,6 @@ def test_bad_crs_alias(minimal_global_raw_cfg): def test_no_wcs(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} with pytest.raises(ConfigException) as excinfo: @@ -190,7 +178,6 @@ def test_no_wcs(minimal_global_raw_cfg): def test_no_wcs_formats(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = { @@ -202,7 +189,6 @@ def test_no_wcs_formats(minimal_global_raw_cfg): def test_bad_wcs_format(minimal_global_raw_cfg, wcs_global_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = wcs_global_cfg @@ -215,7 +201,6 @@ def test_bad_wcs_format(minimal_global_raw_cfg, wcs_global_cfg): def test_tiff_stats(minimal_global_raw_cfg, wcs_global_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["services"] = {"wcs": True} minimal_global_raw_cfg["wcs"] = wcs_global_cfg @@ -226,8 +211,7 @@ def test_tiff_stats(minimal_global_raw_cfg, wcs_global_cfg): def test_crs_lookup_fail(monkeypatch, minimal_global_raw_cfg, minimal_dc): def fake_dc(*args, **kwargs): return minimal_dc - monkeypatch.setattr("datacube.Datacube", fake_dc) - from datacube_ows.ows_configuration import OWSConfig + monkeypatch.setattr("datacube_ows.ows_configuration.Datacube", fake_dc) OWSConfig._instance = None cfg = OWSConfig(cfg=minimal_global_raw_cfg) with pytest.raises(ConfigException) as excinfo: @@ -237,7 +221,6 @@ def fake_dc(*args, **kwargs): def test_no_langs(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = [] with pytest.raises(ConfigException) as excinfo: @@ -246,7 +229,6 @@ def test_no_langs(minimal_global_raw_cfg): def test_two_langs(minimal_global_raw_cfg, minimal_dc): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = ["fr", "en"] cfg = OWSConfig(cfg=minimal_global_raw_cfg) @@ -257,7 +239,6 @@ def test_two_langs(minimal_global_raw_cfg, minimal_dc): def test_internationalised(minimal_global_raw_cfg, minimal_dc): - from datacube_ows.ows_configuration import OWSConfig OWSConfig._instance = None minimal_global_raw_cfg["global"]["supported_languages"] = ["fr", "en"] @@ -267,7 +248,6 @@ def test_internationalised(minimal_global_raw_cfg, minimal_dc): def test_bad_integers_in_wms_section(minimal_global_raw_cfg): - from datacube_ows.ows_configuration import OWSConfig minimal_global_raw_cfg["wms"] = {} minimal_global_raw_cfg["wms"]["max_width"] = "very big" with pytest.raises(ConfigException) as e: From c8baec939877e0af2a9fd82423031cc067c86722 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 13:12:54 +1000 Subject: [PATCH 10/12] Tweak trigger for linting GHA. --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6fee828a5..3ca69a746 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,6 +18,7 @@ on: push: branches: - 'master' + - 'develop-1.9' paths: - '**' - '.github/workflows/lint.yml' From 3cc088872fdfd1398aa4a71574cac3f7b7f5e8f4 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 14:43:47 +1000 Subject: [PATCH 11/12] Fix syntax error in linting GHA. --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3ca69a746..c5ab7dd80 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -52,7 +52,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ["3.10"] + python-version: "3.10" - run: python -m pip install flake8 - name: flake8 cleanup imported but unused uses: liskin/gh-problem-matcher-wrap@v3 @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: "3.10" name: MyPy steps: - name: checkout git From 722bc6eb20c53854288e33977da3d8101c94c4f0 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 2 May 2024 14:46:29 +1000 Subject: [PATCH 12/12] Fix syntax error in linting GHA. --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c5ab7dd80..28d48e0ee 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: "3.10" + python-version: ["3.10"] name: MyPy steps: - name: checkout git