diff --git a/.env_simple b/.env_simple index bdbacd3c0..ab1e83e50 100644 --- a/.env_simple +++ b/.env_simple @@ -4,12 +4,17 @@ ################ # ODC DB Config # ############## -ODC_DEFAULT_DB_URL=postgresql://opendatacubeusername:opendatacubepassword@postgres:5432/opendatacube -# Needed for docker db image. -DB_PORT=5432 -DB_USERNAME=opendatacubeusername -DB_PASSWORD=opendatacubepassword -DB_DATABASE=opendatacube +ODC_DEFAULT_DB_URL=postgresql://opendatacubeusername:opendatacubepassword@postgres:5432/odc_postgres +ODC_OWSPOSTGIS_DB_URL=postgresql://opendatacubeusername:opendatacubepassword@postgres:5432/odc_postgis + +# Needed for docker db image and db readiness probe. +POSTGRES_PORT=5432 +POSTGRES_HOSTNAME=postgres +POSTGRES_USER=opendatacubeusername +SERVER_DB_USERNAME=opendatacubeusername +POSTGRES_PASSWORD=opendatacubepassword +POSTGRES_DB="odc_postgres,odc_postgis" +READY_PROBE_DB=odc_postgis ################# # OWS CFG Config diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 28d48e0ee..353213bbc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,7 +40,6 @@ jobs: - name: Install dependencies and run pylint run: | pip install .[test,dev] - pip install pylint pylint -j 2 --reports no datacube_ows --disable=C,R,W,E1136 flake8: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9c7ce9be..a7f9c71ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: # hooks: # - id: bandit - repo: https://github.com/PyCQA/pylint - rev: v3.1.0 + rev: v3.2.3 hooks: - id: pylint args: ["--disable=C,R,W,E1136"] diff --git a/README.rst b/README.rst index f906ded41..7434b5d58 100644 --- a/README.rst +++ b/README.rst @@ -185,6 +185,10 @@ The following instructions are for installing on a clean Linux system. export DATACUBE_OWS_CFG=datacube_ows.ows_cfg_example.ows_cfg datacube-ows-update --write-role ubuntu --schema + # If you are not using the `default` ODC environment, you can specify the environment to create the schema in: + + datacube-ows-update -E myenv --write-role ubuntu --schema + * Create a configuration file for your service, and all data products you wish to publish in it. @@ -199,7 +203,9 @@ The following instructions are for installing on a clean Linux system. * When additional datasets are added to the datacube, the following steps will need to be run:: + # Update the materialised views (postgis index driver only - can be skipped for the postgis index driver): datacube-ows-update --views + # Update the range tables (both index drivers) datacube-ows-update * If you are accessing data on AWS S3 and running `datacube_ows` on Ubuntu you may encounter errors with ``GetMap`` diff --git a/check-code-all.sh b/check-code-all.sh index 040dd2f14..1c0fe4be2 100755 --- a/check-code-all.sh +++ b/check-code-all.sh @@ -5,30 +5,45 @@ set -ex # ensure db is ready sh ./docker/ows/wait-for-db -# Initialise ODC schema +# Initialise ODC schemas datacube system init +datacube -E owspostgis system init +datacube -E owspostgis spindex create 3577 +datacube -E owspostgis system init # Add extended metadata types datacube metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_landsat_ard.odc-type.yaml datacube metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml +datacube -E owspostgis metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_landsat_ard.odc-type.yaml +datacube -E owspostgis metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml + # Test products datacube product add ./integration_tests/metadata/s2_l2a_prod.yaml datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2am_ard_3.odc-product.yaml datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2bm_ard_3.odc-product.yaml datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/land_and_vegetation/c3_fc/ga_ls_fc_3.odc-product.yaml +datacube -E owspostgis product add ./integration_tests/metadata/s2_l2a_prod.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2am_ard_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2bm_ard_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/land_and_vegetation/c3_fc/ga_ls_fc_3.odc-product.yaml + # add flag masking products -datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/sea_ocean_coast/geodata_coast_100k/geodata_coast_100k.odc-product.yaml +datacube product add ./integration_tests/metadata/product_geodata_coast_100k.yaml datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/inland_water/c3_wo/ga_ls_wo_3.odc-product.yaml +datacube -E owspostgis product add ./integration_tests/metadata/product_geodata_coast_100k.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/inland_water/c3_wo/ga_ls_wo_3.odc-product.yaml + # Geomedian for summary product testing datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/geomedian-au/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/geomedian-au/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml -# S2 datasets from us-west-2 (might not work) +# S2 datasets from us-west-2 and eo3ified geodata_coast MDL=./integration_tests/metadata python ${MDL}/metadata_importer.py < FlaskResponse: stacker = DataStacker(params.layer, params.geobox, params.times, params.resampling, style=params.style) qprof["zoom_factor"] = params.zf qprof.start_event("count-datasets") - n_datasets = stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.COUNT) + n_datasets = stacker.n_datasets() qprof.end_event("count-datasets") qprof["n_datasets"] = n_datasets qprof["zoom_level_base"] = params.resources.base_zoom_level @@ -113,8 +112,7 @@ def get_map(args: dict[str, str]) -> FlaskResponse: stacker.resource_limited = True qprof["resource_limited"] = str(e) if qprof.active: - q_ds_dict = cast(dict[ProductBandQuery, xarray.DataArray], - stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.DATASETS)) + q_ds_dict = stacker.datasets() qprof["datasets"] = [] for q, dsxr in q_ds_dict.items(): query_res: dict[str, Any] = {} @@ -129,7 +127,7 @@ def get_map(args: dict[str, str]) -> FlaskResponse: qprof["datasets"].append(query_res) if stacker.resource_limited and not params.layer.low_res_product_names: qprof.start_event("extent-in-query") - extent = cast(geom.Geometry | None, stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.EXTENT)) + extent = stacker.extent(crs=params.crs) qprof.end_event("extent-in-query") if extent is None: qprof["write_action"] = "No extent: Write Empty" @@ -149,10 +147,10 @@ def get_map(args: dict[str, str]) -> FlaskResponse: else: if stacker.resource_limited: qprof.start_event("count-summary-datasets") - qprof["n_summary_datasets"] = stacker.datasets(params.layer.dc.index, mode=MVSelectOpts.COUNT) + qprof["n_summary_datasets"] = stacker.n_datasets() qprof.end_event("count-summary-datasets") qprof.start_event("fetch-datasets") - datasets = cast(dict[ProductBandQuery, xarray.DataArray], stacker.datasets(params.layer.dc.index)) + datasets = stacker.datasets() for flagband, dss in datasets.items(): if not dss.any(): _LOG.warning("Flag band %s returned no data", str(flagband)) diff --git a/datacube_ows/feature_info.py b/datacube_ows/feature_info.py index 6f2b4fbb9..af0ed73c0 100644 --- a/datacube_ows/feature_info.py +++ b/datacube_ows/feature_info.py @@ -158,7 +158,7 @@ def feature_info(args: dict[str, str]) -> FlaskResponse: stacker = DataStacker(params.layer, geo_point_geobox, params.times) # --- Begin code section requiring datacube. cfg = get_config() - all_time_datasets = cast(xarray.DataArray, stacker.datasets(params.layer.dc.index, all_time=True, point=geo_point)) + all_time_datasets = stacker.datasets_all_time(point=geo_point) # Taking the data as a single point so our indexes into the data should be 0,0 h_coord = cast(str, cfg.published_CRSs[params.crsid]["horizontal_coord"]) @@ -174,10 +174,7 @@ def feature_info(args: dict[str, str]) -> FlaskResponse: global_info_written = False feature_json["data"] = [] fi_date_index: dict[datetime, RAW_CFG] = {} - time_datasets = cast( - dict[ProductBandQuery, xarray.DataArray], - stacker.datasets(params.layer.dc.index, all_flag_bands=True, point=geo_point) - ) + time_datasets = stacker.datasets(all_flag_bands=True, point=geo_point) data = stacker.data(time_datasets, skip_corrections=True) if data is not None: for dt in data.time.values: diff --git a/datacube_ows/index/api.py b/datacube_ows/index/api.py index 1a196ab71..06ec1b896 100644 --- a/datacube_ows/index/api.py +++ b/datacube_ows/index/api.py @@ -13,7 +13,7 @@ from datacube import Datacube from datacube.index.abstract import AbstractIndex from datacube.model import Product, Dataset -from odc.geo import Geometry, CRS +from odc.geo.geom import Geometry, CRS, polygon from datacube_ows.config_utils import CFG_DICT, ConfigException @@ -128,6 +128,7 @@ def extent(self, products: Iterable[Product] | None = None, crs: CRS | None = None ) -> Geometry | None: + geom = self._prep_geom(layer, geom) if crs is None: crs = CRS("epsg:4326") ext: Geometry | None = None @@ -148,6 +149,33 @@ def extent(self, return ext.to_crs(crs) return ext + def _prep_geom(self, layer: "OWSNamedLayer", any_geom: Geometry | None) -> Geometry | None: + # Prepare a Geometry for geospatial search + # Perhaps Core can be updated so this is not needed? + if any_geom is None: + # None? Leave as None + return None + if any_geom.geom_type == "Point": + # Point? Expand to a polygon covering a single native pixel. + any_geom = any_geom.to_crs(layer.native_CRS) + x, y = any_geom.coords[0] + delta_x, delta_y = layer.cfg_native_resolution + return polygon( + ( + (x, y), + (x + delta_x, y), + (x + delta_x, y + delta_y), + (x, y + delta_y), + (x, y), + ), + crs=layer.native_CRS + ) + elif any_geom.geom_type in ("MultiPoint", "LineString", "MultiLineString"): + # Not a point, but not a polygon or multipolygon? Expand to polygon by taking convex hull + return any_geom.convex_hull + else: + # Return polygons and multipolygons as is. + return any_geom class OWSAbstractIndexDriver(ABC): @classmethod diff --git a/datacube_ows/index/postgis/api.py b/datacube_ows/index/postgis/api.py new file mode 100644 index 000000000..3e260d302 --- /dev/null +++ b/datacube_ows/index/postgis/api.py @@ -0,0 +1,157 @@ +# This file is part of datacube-ows, part of the Open Data Cube project. +# See https://opendatacube.org for more information. +# +# Copyright (c) 2017-2024 OWS Contributors +# SPDX-License-Identifier: Apache-2.0 + +import click +import datetime + +from threading import Lock +from typing import Any, Iterable, Type +from uuid import UUID + +from odc.geo import Geometry, CRS +from datacube import Datacube +from datacube.model import Product, Dataset, Range + +from datacube_ows.ows_configuration import OWSNamedLayer +from datacube_ows.index.api import OWSAbstractIndex, OWSAbstractIndexDriver, LayerSignature, LayerExtent, TimeSearchTerm +from datacube_ows.index.sql import run_sql +from .product_ranges import create_range_entry as create_range_entry_impl, get_ranges as get_ranges_impl +from ...utils import default_to_utc + + +class OWSPostgisIndex(OWSAbstractIndex): + name: str = "postgis" + + # method to delete obsolete schemas etc. + def cleanup_schema(self, dc: Datacube): + # No obsolete schema for postgis databases to clean up. + pass + + # Schema creation method + def create_schema(self, dc: Datacube): + click.echo("Creating/updating schema and tables...") + self._run_sql(dc, "ows_schema/create") + + # Permission management method + def grant_perms(self, dc: Datacube, role: str, read_only: bool = False): + if read_only: + self._run_sql(dc, "ows_schema/grants/read_only", role=role) + else: + self._run_sql(dc, "ows_schema/grants/read_write", role=role) + + # Spatiotemporal index update method (e.g. refresh materialised views) + def update_geotemporal_index(self, dc: Datacube): + # Native ODC geotemporal index used in postgis driver. + pass + + def create_range_entry(self, layer: OWSNamedLayer, cache: dict[LayerSignature, list[str]]) -> None: + create_range_entry_impl(layer, cache) + + def get_ranges(self, layer: OWSNamedLayer) -> LayerExtent | None: + return get_ranges_impl(layer) + + def _query(self, + layer: OWSNamedLayer, + times: Iterable[TimeSearchTerm] | None = None, + geom: Geometry | None = None, + products: Iterable[Product] | None = None + ) -> dict[str, Any]: + query: dict[str, Any] = {} + if geom: + query["geopolygon"] = self._prep_geom(layer, geom) + if products is not None: + query["product"] = [p.name for p in products] + if times is not None: + + def normalise_to_dtr(unnorm: datetime.datetime | datetime.date) -> tuple[datetime.datetime, datetime.datetime]: + if isinstance(unnorm, datetime.datetime): + st: datetime.datetime = default_to_utc(unnorm) + tmax = st + datetime.timedelta(seconds=1) + elif isinstance(t, datetime.date): + st = datetime.datetime(unnorm.year, unnorm.month, unnorm.day, tzinfo=datetime.timezone.utc) + tmax = st + datetime.timedelta(days=1) + else: + raise ValueError("Not a datetime object") + return st, tmax + + time_args = [] + for t in times: + if isinstance(t, (datetime.date, datetime.datetime)): + start, tmax = normalise_to_dtr(t) + time_args.append(Range(start, tmax)) + else: + st, et = t + st, _ = normalise_to_dtr(st) + et, _ = normalise_to_dtr(et) + time_args.append(Range(st, et)) + if len(time_args) > 1: + raise ValueError("Huh?") + query["time"] = time_args[0] + return query + + def ds_search(self, + layer: OWSNamedLayer, + times: Iterable[TimeSearchTerm] | None = None, + geom: Geometry | None = None, + products: Iterable[Product] | None = None + ) -> Iterable[Dataset]: + return layer.dc.index.datasets.search(**self._query(layer, times, geom, products)) + + def dsid_search(self, + layer: OWSNamedLayer, + times: Iterable[TimeSearchTerm] | None = None, + geom: Geometry | None = None, + products: Iterable[Product] | None = None + ) -> Iterable[UUID]: + for ds in layer.dc.index.datasets.search_returning(field_names=["id"], + **self._query(layer, times, geom, products)): + yield ds.id # type: ignore[attr-defined] + + def count(self, + layer: OWSNamedLayer, + times: Iterable[TimeSearchTerm] | None = None, + geom: Geometry | None = None, + products: Iterable[Product] | None = None + ) -> int: + return layer.dc.index.datasets.count(**self._query(layer, times, geom, products)) + + def extent(self, + layer: OWSNamedLayer, + times: Iterable[TimeSearchTerm] | None = None, + geom: Geometry | None = None, + products: Iterable[Product] | None = None, + crs: CRS | None = None + ) -> Geometry | None: + if crs is None: + crs = CRS("epsg:4326") + return layer.dc.index.datasets.spatial_extent( + self.dsid_search(layer, times=times, geom=geom, products=products), + crs=crs + ) + + def _run_sql(self, dc: Datacube, path: str, **params: str) -> bool: + return run_sql(dc, self.name, path, **params) + + +pgisdriverlock = Lock() + + +class OWSPostgisIndexDriver(OWSAbstractIndexDriver): + _driver = None + @classmethod + def ows_index_class(cls) -> Type[OWSAbstractIndex]: + return OWSPostgisIndex + + @classmethod + def ows_index(cls) -> OWSAbstractIndex: + with pgisdriverlock: + if cls._driver is None: + cls._driver = OWSPostgisIndex() + return cls._driver + + +def ows_index_driver_init(): + return OWSPostgisIndexDriver() diff --git a/datacube_ows/index/postgis/product_ranges.py b/datacube_ows/index/postgis/product_ranges.py new file mode 100644 index 000000000..7e43654cb --- /dev/null +++ b/datacube_ows/index/postgis/product_ranges.py @@ -0,0 +1,252 @@ +# This file is part of datacube-ows, part of the Open Data Cube project. +# See https://opendatacube.org for more information. +# +# Copyright (c) 2017-2024 OWS Contributors +# SPDX-License-Identifier: Apache-2.0 + +import logging +import math +from datetime import date, datetime, timezone +from typing import Callable +import click + +import datacube +import odc.geo +import sqlalchemy.exc +from psycopg2.extras import Json +from sqlalchemy import text + +from odc.geo.geom import CRS + +from datacube_ows.ows_configuration import OWSNamedLayer +from datacube_ows.utils import get_sqlconn +from datacube_ows.index.api import CoordRange, LayerSignature, LayerExtent + +_LOG = logging.getLogger(__name__) + + +def jsonise_bbox(bbox: odc.geo.geom.BoundingBox) -> dict[str, float]: + return { + "top": bbox.top, + "bottom": bbox.bottom, + "left": bbox.left, + "right": bbox.right, + } + + +def create_range_entry(layer: OWSNamedLayer, cache: dict[LayerSignature, list[str]]) -> None: + meta = LayerSignature(time_res=layer.time_resolution.value, + products=tuple(layer.product_names), + env=layer.local_env._name, + datasets=layer.dc.index.datasets.count(product=layer.product_names)) + + click.echo(f"Postgis Updating range for layer {layer.name}") + click.echo(f"(signature: {meta.as_json()!r})") + conn = get_sqlconn(layer.dc) + txn = conn.begin() + if meta in cache: + template = cache[meta][0] + click.echo(f"Layer {template} has same signature - reusing") + cache[meta].append(layer.name) + try: + conn.execute(text(""" + INSERT INTO ows.layer_ranges + (layer, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated) + SELECT :layer_id, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated + FROM ows.layer_ranges lr2 + WHERE lr2.layer = :template_id"""), + { + "layer_id": layer.name, + "template_id": template + }) + except sqlalchemy.exc.IntegrityError: + conn.execute(text(""" + UPDATE ows.layer_ranges lr1 + SET lat_min = lr2.lat_min, + lat_max = lr2.lat_max, + lon_min = lr2.lon_min, + lon_max = lr2.lon_max, + dates = lr2.dates, + bboxes = lr2.bboxes, + meta = lr2.meta, + last_updated = lr2.last_updated + FROM ows.layer_ranges lr2 + WHERE lr1.layer = :layer_id + AND lr2.layer = :template_id"""), + { + "layer_id": layer.name, + "template_id": template + }) + else: + # insert empty row if one does not already exist + conn.execute(text(""" + INSERT INTO ows.layer_ranges + (layer, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated) + VALUES + (:p_layer, 0, 0, 0, 0, :empty, :empty, :meta, :now) + ON CONFLICT (layer) DO NOTHING + """), + { + "p_layer": layer.name, "empty": Json(""), + "meta": Json(meta.as_json()), "now": datetime.now(tz=timezone.utc) + }) + + prodids = [p.id for p in layer.products] + + # Set default timezone + conn.execute(text("""set timezone to 'Etc/UTC'""")) + + # Loop over dates + dates = set() # Should get to here! + if layer.time_resolution.is_solar(): + results = conn.execute(text( + """ + select lower(dt.search_val), upper(dt.search_val), + lower(lon.search_val), upper(lon.search_val) + from odc.dataset ds, odc.dataset_search_datetime dt, odc.dataset_search_num lon + where ds.product_ref = ANY(:prodids) + AND ds.id = dt.dataset_ref + AND ds.id = lon.dataset_ref + AND dt.search_key = :time + AND lon.search_key = :lon + """), + {"prodids": prodids, "time": "time", "lon": "lon"}) + for result in results: + dt1, dt2, ll, lu = result + lon = (ll + lu) / 2 + dt = dt1 + (dt2 - dt1) / 2 + dt = dt.astimezone(timezone.utc) + + solar_day = datacube.api.query._convert_to_solar_time(dt, lon).date() + dates.add(solar_day) + else: + results = conn.execute(text( + """ + select + array_agg(dt.search_val) + from odc.dataset_search_datetime dt, + odc.dataset ds + WHERE ds.product_ref = ANY(:prodids) + AND ds.id = dt.dataset_ref + AND dt.search_key = 'time' + """), + {"prodids": prodids} + ) + for result in results: + for dat_ran in result[0]: + dates.add(dat_ran.lower) + + if layer.time_resolution.is_subday(): + date_formatter = lambda d: d.isoformat() + else: + date_formatter = lambda d: d.strftime("%Y-%m-%d") + + dates = sorted(dates) + conn.execute(text(""" + UPDATE ows.layer_ranges + SET dates = :dates + WHERE layer= :layer_id + """), + { + "dates": Json(list(map(date_formatter, dates))), + "layer_id": layer.name + } + ) + # calculate bounding boxes + # Get extent polygon from materialised views + + base_crs = CRS(layer.native_CRS) + if base_crs not in layer.dc.index.spatial_indexes(): + click.echo(f"Native CRS for layer {layer.name} ({layer.native_CRS}) does not have a spatial index. " + "Using epsg:4326 for extent calculations.") + base_crs = CRS("EPSG:4326") + + base_extent = None + for product in layer.products: + prod_extent = layer.dc.index.products.spatial_extent(product, base_crs) + if base_extent is None: + base_extent = prod_extent + else: + base_extent = base_extent | prod_extent + assert base_extent is not None + all_bboxes = bbox_projections(base_extent, layer.global_cfg.crses) + + conn.execute(text(""" + UPDATE ows.layer_ranges + SET bboxes = :bbox, + lat_min = :lat_min, + lat_max = :lat_max, + lon_min = :lon_min, + lon_max = :lon_max + WHERE layer = :layer_id + """), { + "bbox": Json(all_bboxes), + "layer_id": layer.name, + "lat_min": all_bboxes['EPSG:4326']['bottom'], + "lat_max": all_bboxes['EPSG:4326']['top'], + "lon_min": all_bboxes['EPSG:4326']['left'], + "lon_max": all_bboxes['EPSG:4326']['right'] + }) + + cache[meta] = [layer.name] + + txn.commit() + conn.close() + + +def bbox_projections(starting_box: odc.geo.Geometry, crses: dict[str, odc.geo.CRS]) -> dict[str, dict[str, float]]: + result = {} + for crsid, crs in crses.items(): + if crs.valid_region is not None: + test_box = starting_box.to_crs("epsg:4326") + clipped_crs_region = (test_box & crs.valid_region) + if clipped_crs_region.wkt == 'POLYGON EMPTY': + continue + clipped_crs_bbox = clipped_crs_region.to_crs(crs).boundingbox + else: + clipped_crs_bbox = None + if clipped_crs_bbox is not None: + result[crsid] = jsonise_bbox(clipped_crs_bbox) + else: + projbbox = starting_box.to_crs(crs).boundingbox + result[crsid] = sanitise_bbox(projbbox) + return result + + +def sanitise_bbox(bbox: odc.geo.geom.BoundingBox) -> dict[str, float]: + def sanitise_coordinate(coord: float, fallback: float) -> float: + return coord if math.isfinite(coord) else fallback + return { + "top": sanitise_coordinate(bbox.top, float("9.999999999e99")), + "bottom": sanitise_coordinate(bbox.bottom, float("-9.999999999e99")), + "left": sanitise_coordinate(bbox.left, float("-9.999999999e99")), + "right": sanitise_coordinate(bbox.right, float("9.999999999e99")), + } + + +def get_ranges(layer: OWSNamedLayer) -> LayerExtent | None: + cfg = layer.global_cfg + conn = get_sqlconn(layer.dc) + results = conn.execute(text(""" + SELECT * + FROM ows.layer_ranges + WHERE layer=:pname"""), + {"pname": layer.name} + ) + + for result in results: + conn.close() + if layer.time_resolution.is_subday(): + dt_parser: Callable[[str], datetime | date] = lambda dts: datetime.fromisoformat(dts) + else: + dt_parser = lambda dts: datetime.strptime(dts, "%Y-%m-%d").date() + times = [dt_parser(d) for d in result.dates if d is not None] + if not times: + return None + return LayerExtent( + lat=CoordRange(min=float(result.lat_min), max=float(result.lat_max)), + lon=CoordRange(min=float(result.lon_min), max=float(result.lon_max)), + times=times, + bboxes=result.bboxes + ) + return None diff --git a/datacube_ows/index/postgres/api.py b/datacube_ows/index/postgres/api.py index 1dff0ef0a..65584f2e3 100644 --- a/datacube_ows/index/postgres/api.py +++ b/datacube_ows/index/postgres/api.py @@ -18,7 +18,7 @@ from datacube_ows.index.api import OWSAbstractIndex, OWSAbstractIndexDriver, LayerSignature, LayerExtent, TimeSearchTerm from .product_ranges import create_range_entry as create_range_entry_impl, get_ranges as get_ranges_impl from .mv_index import MVSelectOpts, mv_search -from .sql import run_sql +from datacube_ows.index.sql import run_sql class OWSPostgresIndex(OWSAbstractIndex): @@ -26,29 +26,29 @@ class OWSPostgresIndex(OWSAbstractIndex): # method to delete obsolete schemas etc. def cleanup_schema(self, dc: Datacube): - run_sql(dc, "ows_schema/cleanup") + self._run_sql(dc, "ows_schema/cleanup") # Schema creation method def create_schema(self, dc: Datacube): click.echo("Creating/updating schema and tables...") - run_sql(dc, "ows_schema/create") + self._run_sql(dc, "ows_schema/create") click.echo("Creating/updating materialised views...") - run_sql(dc, "extent_views/create") + self._run_sql(dc, "extent_views/create") click.echo("Setting ownership of materialised views...") - run_sql(dc, "extent_views/grants/refresh_owner") + self._run_sql(dc, "extent_views/grants/refresh_owner") # Permission management method def grant_perms(self, dc: Datacube, role: str, read_only: bool = False): if read_only: - run_sql(dc, "ows_schema/grants/read_only", role=role) - run_sql(dc, "extent_views/grants/read_only", role=role) + self._run_sql(dc, "ows_schema/grants/read_only", role=role) + self._run_sql(dc, "extent_views/grants/read_only", role=role) else: - run_sql(dc, "ows_schema/grants/read_write", role=role) - run_sql(dc, "extent_views/grants/write_refresh", role=role) + self._run_sql(dc, "ows_schema/grants/read_write", role=role) + self._run_sql(dc, "extent_views/grants/write_refresh", role=role) # Spatiotemporal index update method (e.g. refresh materialised views) def update_geotemporal_index(self, dc: Datacube): - run_sql(dc, "extent_views/refresh") + self._run_sql(dc, "extent_views/refresh") def create_range_entry(self, layer: OWSNamedLayer, cache: dict[LayerSignature, list[str]]) -> None: create_range_entry_impl(layer, cache) @@ -97,6 +97,9 @@ def extent(self, else: return extent.to_crs(crs) + def _run_sql(self, dc: Datacube, path: str, **params: str) -> bool: + return run_sql(dc, self.name, path, **params) + pgdriverlock = Lock() diff --git a/datacube_ows/index/postgres/product_ranges.py b/datacube_ows/index/postgres/product_ranges.py index 9b8425fc0..bac6306e9 100644 --- a/datacube_ows/index/postgres/product_ranges.py +++ b/datacube_ows/index/postgres/product_ranges.py @@ -6,8 +6,9 @@ import logging import math +import click from datetime import date, datetime, timezone -from typing import cast, Callable, Iterable +from typing import cast, Callable import datacube import odc.geo @@ -17,24 +18,14 @@ from odc.geo.geom import Geometry -from datacube_ows.ows_configuration import OWSConfig, OWSNamedLayer, get_config -from datacube_ows.mv_index import MVSelectOpts, mv_search +from datacube_ows.ows_configuration import OWSNamedLayer +from datacube_ows.index.postgres.mv_index import MVSelectOpts, mv_search from datacube_ows.utils import get_sqlconn from datacube_ows.index.api import CoordRange, LayerSignature, LayerExtent _LOG = logging.getLogger(__name__) -def get_crsids(cfg: OWSConfig | None = None) -> Iterable[str]: - if not cfg: - cfg = get_config() - return cfg.internal_CRSs.keys() - - -def get_crses(cfg: OWSConfig | None = None) -> dict[str, odc.geo.CRS]: - return {crsid: odc.geo.CRS(crsid) for crsid in get_crsids(cfg)} - - def jsonise_bbox(bbox: odc.geo.geom.BoundingBox) -> dict[str, float]: if isinstance(bbox, dict): return bbox @@ -53,12 +44,13 @@ def create_range_entry(layer: OWSNamedLayer, cache: dict[LayerSignature, list[st env=layer.local_env._name, datasets=layer.dc.index.datasets.count(product=layer.product_names)) - print(f"Updating range for layer {layer.name}") + click.echo(f"Postgres Updating range for layer {layer.name}") + click.echo(f"(signature: {meta.as_json()!r})") conn = get_sqlconn(layer.dc) txn = conn.begin() if meta in cache: template = cache[meta][0] - print(f"Layer {template} has same signature - reusing") + click.echo(f"Layer {template} has same signature - reusing") cache[meta].append(layer.name) try: conn.execute(text(""" diff --git a/datacube_ows/index/postgres/sql.py b/datacube_ows/index/sql.py similarity index 82% rename from datacube_ows/index/postgres/sql.py rename to datacube_ows/index/sql.py index 258713278..88f099756 100644 --- a/datacube_ows/index/postgres/sql.py +++ b/datacube_ows/index/sql.py @@ -25,13 +25,13 @@ def get_sqlconn(dc: Datacube) -> sqlalchemy.Connection: return dc.index._db._engine.connect() # type: ignore[attr-defined] -def run_sql(dc: Datacube, path: str, **params: str) -> bool: - if not importlib.resources.files("datacube_ows").joinpath(f"sql/postgres/{path}").is_dir(): +def run_sql(dc: Datacube, driver_name: str, path: str, **params: str) -> bool: + if not importlib.resources.files("datacube_ows").joinpath(f"sql/{driver_name}/{path}").is_dir(): print("Cannot find SQL resource directory - check your datacube-ows installation") return False files = sorted( - importlib.resources.files("datacube_ows").joinpath(f"sql/postgres/{path}").iterdir() # type: ignore[type-var] + importlib.resources.files("datacube_ows").joinpath(f"sql/{driver_name}/{path}").iterdir() # type: ignore[type-var] ) filename_req_pattern = re.compile(r"\d+[_a-zA-Z0-9]+_requires_(?P[_a-zA-Z0-9]+)\.sql") @@ -50,36 +50,39 @@ def run_sql(dc: Datacube, path: str, **params: str) -> bool: reqs = req_match.group("reqs").split("_") else: reqs = [] - ref = importlib.resources.files("datacube_ows").joinpath(f"sql/postgres/{path}/{f}") + if reqs: + try: + kwargs = {v: params[v] for v in reqs} + except KeyError as e: + click.echo(f"Required parameter {e} for file {f} not supplied - skipping") + all_ok = False + continue + else: + kwargs = {} + ref = importlib.resources.files("datacube_ows").joinpath(f"sql/{driver_name}/{path}/{f}") with ref.open("rb") as fp: sql = "" first = True for line in fp: sline = str(line, "utf-8") if first and sline.startswith("--"): - click.echo(f" - Running {sline[2:]}") + if reqs: + click.echo(f" - Running {sline[2:].format(**kwargs)}") + else: + click.echo(f" - Running {sline[2:]}") else: sql = sql + "\n" + sline first = False if reqs: - try: - kwargs = {v: params[v] for v in reqs} - except KeyError as e: - click.echo(f"Required parameter {e} for file {f} not supplied - skipping") - all_ok = False - continue sql = sql.format(**kwargs) try: result = conn.execute(sqlalchemy.text(sql)) - click.echo(f" ... succeeded(?) with {result!r} rowcount {result.rowcount}") - if result.returns_rows: - for r in result: - click.echo(f" ... succeeded(?) with {r!r}") + click.echo(f" ... succeeded(?) with rowcount {result.rowcount}") except sqlalchemy.exc.ProgrammingError as e: if isinstance(e.orig, psycopg2.errors.InsufficientPrivilege): click.echo( - f"Insufficient Privileges (user {dc.index.environment.db_username}). Schema altering actions should be run by a role with admin privileges" + f"Insufficient Privileges (user {dc.index.environment.db_username}). Schema altering actions should be run by a role with admin privileges" ) raise AbortRun() from None elif isinstance(e.orig, psycopg2.errors.DuplicateObject): diff --git a/datacube_ows/loading.py b/datacube_ows/loading.py index f6addba94..ee7c67892 100644 --- a/datacube_ows/loading.py +++ b/datacube_ows/loading.py @@ -14,11 +14,9 @@ import numpy import xarray from odc.geo.geobox import GeoBox -from odc.geo.geom import Geometry +from odc.geo.geom import Geometry, CRS from odc.geo.warp import Resampling -from sqlalchemy.engine import Row -from datacube_ows.mv_index import MVSelectOpts, mv_search from datacube_ows.ogc_exceptions import WMSException from datacube_ows.ows_configuration import OWSNamedLayer from datacube_ows.startup_utils import CredentialManager @@ -161,36 +159,77 @@ def __init__(self, def needed_bands(self) -> list[str]: return self._needed_bands - @log_call - def n_datasets(self, - index: datacube.index.Index, - all_time: bool = False, - point: Geometry | None = None) -> int: - return cast(int, self.datasets(index, - all_time=all_time, point=point, - mode=MVSelectOpts.COUNT)) + def n_datasets(self) -> int: + if self.style: + # we have a style - lets go with that. + queries = ProductBandQuery.style_queries(self.style) + else: + # Just take needed bands. + queries = [ProductBandQuery.simple_layer_query(self._layer, self.needed_bands())] + geom = self._geobox.extent + for query in queries: + if query.ignore_time: + qry_times = None + else: + qry_times = self._times + return self._layer.ows_index().count(self._layer, times=qry_times, geom=geom, products=query.products) + return 0 - def datasets(self, index: datacube.index.Index, - all_flag_bands: bool = False, - all_time: bool = False, - point: Geometry | None = None, - mode: MVSelectOpts = MVSelectOpts.DATASETS) -> (int - | Iterable[Row] - | Iterable[UUID] - | xarray.DataArray - | Geometry - | None - | dict[ProductBandQuery, PerPBQReturnType]): - if mode == MVSelectOpts.EXTENT or all_time: - # Not returning datasets - use main product only - queries = [ - ProductBandQuery.simple_layer_query( + def extent(self, crs: CRS | None = None) -> Geometry | None: + query = ProductBandQuery.simple_layer_query( + self._layer, + self.needed_bands(), + self.resource_limited + ) + geom = self._geobox.extent + if query.ignore_time: + times = None + else: + times = self._times + return self._layer.ows_index().extent(self._layer, times=times, geom=geom, products=query.products, crs=crs) + + def dsids(self) -> dict[ProductBandQuery, Iterable[UUID]]: + if self.style: + # we have a style - lets go with that. + queries = ProductBandQuery.style_queries(self.style) + else: + # Just take needed bands. + queries = [ProductBandQuery.simple_layer_query(self._layer, self.needed_bands())] + results: list[tuple[ProductBandQuery, Iterable[UUID]]] = [] + for query in queries: + if query.ignore_time: + qry_times = None + else: + qry_times = self._times + result = self._layer.ows_index().dsid_search(self._layer, times=qry_times, geom=self._geobox.extent, + products=query.products) + results.append((query, result)) + return OrderedDict(results) + + def datasets_all_time(self, point: Geometry | None = None) -> xarray.DataArray: + query = ProductBandQuery.simple_layer_query( self._layer, self.needed_bands(), self.resource_limited) + if point: + geom = point + else: + geom = self._geobox.extent + result = self._layer.ows_index().ds_search( + layer=self._layer, + geom=geom, + products=query.products) + grpd_result = datacube.Datacube.group_datasets( + cast(Iterable[datacube.model.Dataset], result), + self.group_by + ) + return grpd_result - ] - elif self.style: + def datasets(self, + all_flag_bands: bool = False, + point: Geometry | None = None, + ) -> dict[ProductBandQuery, xarray.DataArray]: + if self.style: # we have a style - lets go with that. queries = ProductBandQuery.style_queries(self.style) elif all_flag_bands: @@ -203,40 +242,22 @@ def datasets(self, index: datacube.index.Index, geom = point else: geom = self._geobox.extent - if all_time: - times = None - else: - times = self._times - results: list[tuple[ProductBandQuery, PerPBQReturnType]] = [] + results: list[tuple[ProductBandQuery, xarray.DataArray]] = [] for query in queries: if query.ignore_time: qry_times = None else: - qry_times = times - result = mv_search(index, - sel=mode, + qry_times = self._times + result = self._layer.ows_index().ds_search( + layer=self._layer, times=qry_times, geom=geom, products=query.products) - if mode == MVSelectOpts.DATASETS: - grpd_result = datacube.Datacube.group_datasets( - cast(Iterable[datacube.model.Dataset], result), - self.group_by - ) - if all_time: - return grpd_result - results.append((query, grpd_result)) - elif mode == MVSelectOpts.IDS: - result_ids = cast(Iterable[UUID], result) - if all_time: - return result_ids - results.append((query, result_ids)) - elif mode == MVSelectOpts.ALL: - return cast(Iterable[Row], result) - elif mode == MVSelectOpts.COUNT: - return cast(int, result) - else: # MVSelectOpts.EXTENT - return cast(Geometry | None, result) + grpd_result = datacube.Datacube.group_datasets( + cast(Iterable[datacube.model.Dataset], result), + self.group_by + ) + results.append((query, grpd_result)) return OrderedDict(results) def create_nodata_filled_flag_bands(self, data: xarray.Dataset, pbq: ProductBandQuery) -> xarray.Dataset: diff --git a/datacube_ows/mv_index.py b/datacube_ows/mv_index.py deleted file mode 100644 index 7668ca22d..000000000 --- a/datacube_ows/mv_index.py +++ /dev/null @@ -1,180 +0,0 @@ -# This file is part of datacube-ows, part of the Open Data Cube project. -# See https://opendatacube.org for more information. -# -# Copyright (c) 2017-2024 OWS Contributors -# SPDX-License-Identifier: Apache-2.0 - -import datetime -import json -from enum import Enum -from types import UnionType -from typing import Iterable, Type, cast -from uuid import UUID as UUID_ - -import pytz -from datacube.index import Index -from datacube.model import Dataset, Product -from geoalchemy2 import Geometry -from odc.geo.geom import Geometry as ODCGeom -from psycopg2.extras import DateTimeTZRange -from sqlalchemy import (SMALLINT, Column, MetaData, Table, and_, or_, select, - text) -from sqlalchemy.dialects.postgresql import TSTZRANGE, UUID -from sqlalchemy.engine import Row -from sqlalchemy.engine.base import Engine -from sqlalchemy.sql.elements import ClauseElement -from sqlalchemy.sql.functions import count, func - -from datacube_ows.utils import default_to_utc - - -def get_sqlalc_engine(index: Index) -> Engine: - # pylint: disable=protected-access - return index._db._engine # type: ignore[attr-defined] - - -def get_st_view(meta: MetaData) -> Table: - return Table('space_time_view', meta, - Column('id', UUID()), - Column('dataset_type_ref', SMALLINT()), - Column('spatial_extent', Geometry(from_text='ST_GeomFromGeoJSON', name='geometry')), - Column('temporal_extent', TSTZRANGE()), - schema="ows") - - -_meta = MetaData() -st_view = get_st_view(_meta) - - -class MVSelectOpts(Enum): - """ - Enum for mv_search_datasets sel parameter. - - ALL: return all columns, select *, as result set - IDS: return list of database_ids only. - DATASETS: return list of ODC dataset objects - COUNT: return a count of matching datasets - EXTENT: return full extent of query result as a Geometry - """ - ALL = 0 - IDS = 1 - COUNT = 2 - EXTENT = 3 - DATASETS = 4 - - def sel(self, stv: Table) -> list[ClauseElement]: - if self == self.ALL: - return [stv] - if self == self.IDS or self == self.DATASETS: - return [stv.c.id] - if self == self.COUNT: - return [cast(ClauseElement, count(stv.c.id))] - if self == self.EXTENT: - return [text("ST_AsGeoJSON(ST_Union(spatial_extent))")] - raise AssertionError("Invalid selection option") - - -selection_return_types: dict[MVSelectOpts, Type | UnionType] = { - MVSelectOpts.ALL: Iterable[Row], - MVSelectOpts.IDS: Iterable[UUID_], - MVSelectOpts.DATASETS: Iterable[Dataset], - MVSelectOpts.COUNT: int, - MVSelectOpts.EXTENT: ODCGeom | None, -} - - -SelectOut = Iterable[Row] | Iterable[UUID_] | Iterable[Dataset] | int | ODCGeom | None -DateOrDateTime = datetime.datetime | datetime.date -TimeSearchTerm = tuple[datetime.datetime, datetime.datetime] | tuple[datetime.date, datetime.date] | DateOrDateTime - - -def mv_search(index: Index, - sel: MVSelectOpts = MVSelectOpts.IDS, - times: Iterable[TimeSearchTerm] | None = None, - geom: ODCGeom | None = None, - products: Iterable[Product] | None = None) -> SelectOut: - """ - Perform a dataset query via the space_time_view - - :param products: An iterable of combinable products to search - :param index: A datacube index (required) - - :param sel: Selection mode - a MVSelectOpts enum. Defaults to IDS. - :param times: A list of pairs of datetimes (with time zone) - :param geom: A odc.geo.geom.Geometry object - - :return: See MVSelectOpts doc - """ - engine = get_sqlalc_engine(index) - stv = st_view - if products is None: - raise Exception("Must filter by product/layer") - prod_ids = [p.id for p in products] - - s = select(*sel.sel(stv)).where(stv.c.dataset_type_ref.in_(prod_ids)) # type: ignore[call-overload] - if times is not None: - or_clauses = [] - for t in times: - if isinstance(t, datetime.datetime): - st: datetime.datetime = datetime.datetime(t.year, t.month, t.day, t.hour, t.minute, t.second) - st = default_to_utc(t) - if not st.tzinfo: - st = st.replace(tzinfo=pytz.utc) - tmax = st + datetime.timedelta(seconds=1) - or_clauses.append( - and_( - func.lower(stv.c.temporal_extent) >= t, - func.lower(stv.c.temporal_extent) < tmax, - ) - ) - elif isinstance(t, datetime.date): - st = datetime.datetime(t.year, t.month, t.day, tzinfo=pytz.utc) - tmax = st + datetime.timedelta(days=1) - or_clauses.append( - and_( - func.lower(stv.c.temporal_extent) >= st, - func.lower(stv.c.temporal_extent) < tmax, - ) - ) - else: - or_clauses.append( - stv.c.temporal_extent.op("&&")(DateTimeTZRange(*t)) - ) - s = s.where(or_(*or_clauses)) - orig_crs = None - if geom is not None: - orig_crs = geom.crs - if str(geom.crs) != "EPSG:4326": - geom = geom.to_crs("EPSG:4326") - geom_js = json.dumps(geom.json) - s = s.where(stv.c.spatial_extent.intersects(geom_js)) - # print(s) # Print SQL Statement - with engine.connect() as conn: - if sel == MVSelectOpts.ALL: - return conn.execute(s) - elif sel == MVSelectOpts.IDS: - return [r[0] for r in conn.execute(s)] - elif sel in (MVSelectOpts.COUNT, MVSelectOpts.EXTENT): - for r in conn.execute(s): - if sel == MVSelectOpts.COUNT: - return cast(int, r[0]) - else: # MVSelectOpts.EXTENT - geojson = r[0] - if geojson is None: - return None - uniongeom = ODCGeom(json.loads(geojson), crs="EPSG:4326") - if geom: - intersect = uniongeom.intersection(geom) - if intersect.wkt == 'POLYGON EMPTY': - return None - if orig_crs and orig_crs != "EPSG:4326": - intersect = intersect.to_crs(orig_crs) - else: - intersect = uniongeom - return intersect - elif sel == MVSelectOpts.DATASETS: - ids = [r[0] for r in conn.execute(s)] - return index.datasets.bulk_get(ids) - else: - raise Exception("Invalid Selection Option") - raise Exception("Unreachable code reached") diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 19913597f..de393b350 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -1,4 +1,3 @@ -# This file is part of datacube-ows, part of the Open Data Cube project. # See https://opendatacube.org for more information. # # Copyright (c) 2017-2024 OWS Contributors @@ -255,6 +254,10 @@ def __init__(self, cfg: CFG_DICT, object_label: str, parent_layer: Optional["OWS self._cached_local_env: ODCEnvironment | None = None self._cached_dc: Datacube | None = None self.parse_metadata(cfg) + # Do we have a local ODC environment override? + local_env = cfg.get("env") + if local_env is not None: + self._local_env = ODCConfig.get_environment(env=str(local_env)) # Inherit or override attribution if "attribution" in cfg: self.attribution = AttributionCfg.parse( # type: ignore[assignment] diff --git a/datacube_ows/sql/postgis/ows_schema/create/001_create_schema.sql b/datacube_ows/sql/postgis/ows_schema/create/001_create_schema.sql new file mode 100644 index 000000000..1862c31ec --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/create/001_create_schema.sql @@ -0,0 +1,3 @@ +-- Creating/replacing ows schema + +create schema if not exists ows; diff --git a/datacube_ows/sql/postgis/ows_schema/create/002_create_product_rng.sql b/datacube_ows/sql/postgis/ows_schema/create/002_create_product_rng.sql new file mode 100644 index 000000000..bc947bacf --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/create/002_create_product_rng.sql @@ -0,0 +1,17 @@ +-- Creating/replacing product ranges table + +create table if not exists ows.layer_ranges ( + layer varchar(255) not null primary key, + + lat_min decimal not null, + lat_max decimal not null, + lon_min decimal not null, + lon_max decimal not null, + + dates jsonb not null, + + bboxes jsonb not null, + + meta jsonb not null, + last_updated timestamp not null +); diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_only/001_grant_usage_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_only/001_grant_usage_requires_role.sql new file mode 100644 index 000000000..598fee7a2 --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_only/001_grant_usage_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting usage on schema + +GRANT USAGE ON SCHEMA ows TO {role} diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql new file mode 100644 index 000000000..dc1bc3ee0 --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting select on layer ranges table to {role} + +GRANT SELECT ON ows.layer_ranges TO {role}; diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_only/003_grant_odc_user_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_only/003_grant_odc_user_requires_role.sql new file mode 100644 index 000000000..1af4c9729 --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_only/003_grant_odc_user_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting odc_user role to {role} + +GRANT odc_user to {role}; diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_write/001_grant_usage_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_write/001_grant_usage_requires_role.sql new file mode 100644 index 000000000..598fee7a2 --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_write/001_grant_usage_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting usage on schema + +GRANT USAGE ON SCHEMA ows TO {role} diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql new file mode 100644 index 000000000..6d7e3f30a --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting update/insert/delete on all tables in schema + +GRANT update, select, insert, delete ON ALL TABLES IN SCHEMA ows TO {role} diff --git a/datacube_ows/sql/postgis/ows_schema/grants/read_write/003_grant_odc_user_requires_role.sql b/datacube_ows/sql/postgis/ows_schema/grants/read_write/003_grant_odc_user_requires_role.sql new file mode 100644 index 000000000..1af4c9729 --- /dev/null +++ b/datacube_ows/sql/postgis/ows_schema/grants/read_write/003_grant_odc_user_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting odc_user role to {role} + +GRANT odc_user to {role}; diff --git a/datacube_ows/sql/postgres/ows_schema/grants/read_write/003_grant_agdc_user_requires_role.sql b/datacube_ows/sql/postgres/ows_schema/grants/read_write/003_grant_agdc_user_requires_role.sql new file mode 100644 index 000000000..5783f1b9e --- /dev/null +++ b/datacube_ows/sql/postgres/ows_schema/grants/read_write/003_grant_agdc_user_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting agdc_user role to {role} + +GRANT agdc_user to {role}; diff --git a/datacube_ows/styles/component.py b/datacube_ows/styles/component.py index b9efb0697..564242ffa 100644 --- a/datacube_ows/styles/component.py +++ b/datacube_ows/styles/component.py @@ -165,7 +165,7 @@ def transform_single_date_data(self, data: Dataset) -> Dataset: else: imgband_data = imgband_component if imgband_data is None: - null_np = np.zeros(tuple(data.sizes.values()), 'uint8') + null_np = np.zeros(tuple(data.sizes.values()), 'float64') imgband_data = DataArray(null_np, data.coords, tuple(data.sizes.keys())) if imgband != "alpha": imgband_data = self.compress_band(imgband, imgband_data) diff --git a/datacube_ows/update_ranges_impl.py b/datacube_ows/update_ranges_impl.py index e5059074c..5f5318956 100755 --- a/datacube_ows/update_ranges_impl.py +++ b/datacube_ows/update_ranges_impl.py @@ -31,7 +31,7 @@ help="(Only valid with --schema) Role(s) to grant both read and write/update database permissions to") @click.option("--cleanup", is_flag=True, default=False, help="Cleanup up any datacube-ows 1.8.x tables/views") -@click.option("-e", "--env", default=None, +@click.option("-E", "--env", default=None, help="(Only valid with --schema or --read-role or --write-role or --cleanup) environment to write to.") @click.option("--version", is_flag=True, default=False, help="Print version string and exit") @@ -127,10 +127,14 @@ def main(layers: list[str], app = cfg.odc_app + "-update" errors: bool = False if schema or read_role or write_role or cleanup or views: - if cfg.default_env and env is None: - dc = Datacube(env=cfg.default_env, app=app) - else: - dc = Datacube(env=env, app=app) + try: + if cfg.default_env and env is None: + dc = Datacube(env=cfg.default_env, app=app) + else: + dc = Datacube(env=env, app=app) + except: + click.echo(f"Unable to connect to the {env or cfg.default_env} database.") + sys.exit(1) click.echo(f"Applying database schema updates to the {dc.index.environment.db_database} database:...") try: @@ -167,7 +171,7 @@ def main(layers: list[str], click.echo("") click.echo(" Try running with the --schema options first.") sys.exit(1) - elif isinstance(e.orig, psycopg2.errors.NotNullViloation): + elif isinstance(e.orig, psycopg2.errors.NotNullViolation): click.echo("ERROR: OWS materialised views are most likely missing a newly indexed product") click.echo("") click.echo(" Try running with the --viewes options first.") diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index 2e0dc3d1a..93722529a 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -16,7 +16,6 @@ from datacube_ows.config_utils import ConfigException from datacube_ows.loading import DataStacker -from datacube_ows.mv_index import MVSelectOpts from datacube_ows.ogc_exceptions import WCS1Exception from datacube_ows.ows_configuration import get_config from datacube_ows.resource_limits import ResourceLimited @@ -312,7 +311,7 @@ def get_coverage_data(req, qprof): req.times, bands=req.bands) qprof.start_event("count-datasets") - n_datasets = stacker.datasets(req.layer.dc.index, mode=MVSelectOpts.COUNT) + n_datasets = stacker.n_datasets() qprof.end_event("count-datasets") qprof["n_datasets"] = n_datasets @@ -385,12 +384,12 @@ def get_coverage_data(req, qprof): return n_datasets, data qprof.start_event("fetch-datasets") - datasets = stacker.datasets(index=req.layer.dc.index) + datasets = stacker.datasets() qprof.end_event("fetch-datasets") if qprof.active: qprof["datasets"] = { str(q): [str(i) for i in ids] - for q, ids in stacker.datasets(req.layer.dc.index, mode=MVSelectOpts.IDS).items() + for q, ids in stacker.dsids().items() } qprof.start_event("load-data") output = stacker.data(datasets, skip_corrections=True) diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index 482a10f49..7911c6389 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -13,7 +13,6 @@ from rasterio import MemoryFile from datacube_ows.loading import DataStacker -from datacube_ows.mv_index import MVSelectOpts from datacube_ows.ogc_exceptions import WCS2Exception from datacube_ows.ows_configuration import get_config from datacube_ows.resource_limits import ResourceLimited @@ -257,7 +256,7 @@ def get_coverage_data(request, styles, qprof): bands=bands) qprof.end_event("setup") qprof.start_event("count-datasets") - n_datasets = stacker.datasets(layer.dc.index, mode=MVSelectOpts.COUNT) + n_datasets = stacker.n_datasets() qprof.end_event("count-datasets") qprof["n_datasets"] = n_datasets @@ -281,12 +280,12 @@ def get_coverage_data(request, styles, qprof): http_response=404) qprof.start_event("fetch-datasets") - datasets = stacker.datasets(layer.dc.index) + datasets = stacker.datasets() qprof.end_event("fetch-datasets") if qprof.active: qprof["datasets"] = { str(q): [str(i) for i in ids] - for q, ids in stacker.datasets(layer.dc.index, mode=MVSelectOpts.IDS).items() + for q, ids in stacker.dsids().items() } qprof.start_event("load-data") output = stacker.data(datasets, skip_corrections=True) diff --git a/docker-compose.db.yaml b/docker-compose.db.yaml index 58f17007e..7e722e5de 100644 --- a/docker-compose.db.yaml +++ b/docker-compose.db.yaml @@ -5,15 +5,13 @@ services: # db build: docker/database/ environment: - - POSTGRES_DB=${DB_DATABASE} - - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_USER=${DB_USERNAME} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_USER=${POSTGRES_USER} ports: - - "${DB_PORT}:5432" + - "${POSTGRES_PORT}:${POSTGRES_PORT}" restart: always # Overwrite ows so it can talk to docker db ows: ports: - 8000:8000 - environment: - DB_PORT: 5432 diff --git a/docker-compose.yaml b/docker-compose.yaml index fdfb0e901..087e4d013 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,14 +21,15 @@ services: # Hard coded for now. ODC_ENVIRONMENT: default ODC_DEFAULT_INDEX_DRIVER: postgres + ODC_OWSPOSTGIS_INDEX_DRIVER: postgis # Please switch to single entry url configuration for postgres url ODC_DEFAULT_DB_URL: ${ODC_DEFAULT_DB_URL} - # Overridden by preferred URL entry above. - DB_HOSTNAME: ${DB_HOSTNAME} - DB_PORT: ${DB_PORT} - DB_USERNAME: ${DB_USERNAME} - DB_PASSWORD: ${DB_PASSWORD} - DB_DATABASE: ${DB_DATABASE} + ODC_OWSPOSTGIS_DB_URL: ${ODC_OWSPOSTGIS_DB_URL} + # for wait-for-db check + READY_PROBE_DB: ${READY_PROBE_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} + SERVER_DB_USERNAME: ${SERVER_DB_USERNAME} # Path from the PYTHONPATH to the config object (default PYTHONPATH is /env) PYTHONPATH: ${PYTHONPATH} DATACUBE_OWS_CFG: ${DATACUBE_OWS_CFG} diff --git a/docker/ows/wait-for-db b/docker/ows/wait-for-db index 732e49707..f06c6a122 100755 --- a/docker/ows/wait-for-db +++ b/docker/ows/wait-for-db @@ -3,8 +3,8 @@ RETRIES=10 # Wait until Database is ready -until pg_isready --dbname=$DB_DATABASE --host=$DB_HOSTNAME --port=$DB_PORT --username=$DB_USERNAME || [ $RETRIES -eq 0 ]; do - echo "Waiting for $DB_HOSTNAME server, $((RETRIES-=1)) remaining attempts..." +until pg_isready --dbname=$READY_PROBE_DB --host=$POSTGRES_HOSTNAME --username=$POSTGRES_USER || [ $RETRIES -eq 0 ]; do + echo "Waiting for $READY_PROBE_DB databases, $((RETRIES-=1)) remaining attempts..." sleep 2 done diff --git a/docs/database.rst b/docs/database.rst index 030ce43cc..d430bb50d 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -14,20 +14,24 @@ System Architecture Diagram --------------------------- .. figure:: diagrams/ows_diagram1.9.png - :target: /_images/ows_diagram.png + :target: /_images/ows_diagram1.9.png OWS Architecture Diagram, including Database structure. Open Data Cube Native Entities ------------------------------ -The core of the Datacube-OWS application database is the ``agdc`` schema see: +The core of the Datacube-OWS application database is found in either +the ``agdc`` schema (for the legacy ``postgres`` index driver), or +the ``odc`` schema (for the new ``postgis`` index driver). +See `Open Data Cube `_. + This schema is created and maintained with the ``datacube`` command. OWS only needs read access to this schema. -Materialised Views over ODC Indexes ------------------------------------ +Materialised Views over ODC Indexes (Postgres driver only) +---------------------------------------------------------- The materialised views provide a dataset level extent index using `PostGIS `_ datatypes. @@ -37,13 +41,15 @@ step to populate the Layer Extent Cache, as described below, and for doing dataset queries for GetMap, GetFeatureInfo and GetCoverage requests. -This layer will eventually become folded up into functionality in core, but -in the meantime (and while using the legacy ``postgres`` index driver), -it must be maintained separately using the ``datacube-ows-update`` (``update_ranges)``) -command. +The materialised views must be manually updated whenever new data +is added to the underlying ODC index, as described below. + +With the new postgis index driver, the functionality provided by the +materialised views is available directly from the ODC index, and so +no materialised views are required. -Range Tables (Layer Extent Cache) ----------------------------------- +Ranges Table (Layer Extent Cache) +--------------------------------- Range tables serve as a cache of full-layer spatio-temporal extents for generating GetCapabilities documents efficiently. @@ -53,7 +59,7 @@ Creating/Maintaining the OWS Schema Creating or updating an OWS schema is performed with following options to ``datacube-ows-update``. -Note that the options in this schema requires database superuser privileges. +Note that the options in this section requires database superuser/admin privileges. =================================== Creating or Updating the OWS Schema @@ -64,6 +70,13 @@ updates to the form required by the installed version of ``datacube-ows``:: datacube-ows-update --schema +The ``--schema`` option creates/updates the OWS schema in the database defined by the ``default`` +ODC environment only. If you are using multiple ODC environments, or are simply not using the ``default`` +environment, you will need to pass the target environment name with the ``-E`` option. E.g. to +create an OWS schema in the ``myenv`` ODC environment:: + + datacube-ows-update -E myenv --schema + ========================================== Cleaning up an old datacube-ows 1.8 schema ========================================== @@ -80,19 +93,21 @@ You can combine the ``--schema`` and ``--cleanup`` options in the one invocation datacube-ows-update --schema --cleanup -The new schema is always created before dropping the new one, regardless of the order you specify the options. +The new schema is always created before dropping the old one, regardless of the order you specify the options. + +The ``--cleanup`` option targets the ``default`` ODC environment database unless a target environment is supplied +with the ``-E`` option. ====================================== Granting permissions to database roles ====================================== The ``datacube-ows`` web application requires permissions to read from the various tables and views in the ``ows`` -schema. These can permissions can be granted to a database role with the ``--read-role `` argument:: +schema. These can permissions (including read-access to the ODC tables can be granted to a database role with +the ``--read-role `` argument:: datacube-ows-update --read-role role1 -The role used by ``datacube-ows`` also needs read-access to the ODC tables. These should managed with the ``datacube users`` -CLI tool - refer to the datacube-core documentation. You do not need to use --read-role and --write-role on the same user - granting write permissions automatically grants read permissions as well. @@ -108,6 +123,10 @@ multiple roles:: datacube-ows-update --schema --cleanup --read-role role1 --read-role role2 --read-role role3 --write-role admin +The ``--read-role`` and ``--write-role`` options are executed against the ODC environment database identified +by the ``-E`` option (Default is ``default``):: + + datacube-ows-update -E myenv --read-role role1 --read-role role2 --read-role role3 --write-role admin Updating/Maintaining OWS data ----------------------------- @@ -125,7 +144,7 @@ manually refreshed, with the ``--views`` flag. ``datacube-ows-update --views`` -A lot of the speed of OWS comes from pushing +A lot of the speed of OWS with the ``postgres`` index driver comes from pushing expensive database calculations down into these materialised views, and refreshing them is slow and computationally expensive. Large, constantly updating databases will unavoidably have @@ -144,18 +163,31 @@ In a production environment you should not be refreshing views much more than 2 or 3 times a day unless your database is small (e.g. less than a few thousand datasets). +If working with multiple ODC environments/databases, you can specify which +environment to refresh the materialised views in with the ``-E`` option. +(The default is to use the ``default`` environment.) + +Materialised views are required for ``postgres`` index driver environments only. +In environments using the ``postgis`` index driver, the `--views` option +does nothing and may be skipped. + ===================== Updating range tables ===================== -The range table is updated from the materialised views by simply calling: +The range tables are updated from the materialised views by simply calling: datacube-ows-update Note that this operation is very fast and computationally light compared to refreshing the materialised views. -In a production environment, this should be run after refreshing the materialised views, as described above (after -waiting a couple of minutes for the final refresh to complete). +Range tables are updated in all ODC environments referenced by the active ODC configuration file. +The ``-E`` flag is therefore not valid for use with this calling mode. + +In a ``postgres`` driver production environment, this should be run after refreshing the materialised views, +as described above (after waiting a couple of minutes for the final refresh to complete). + +In a ``postgis`` driver production environment, this is the only required regular maintenance task. =========================================== Updating range tables for individual layers @@ -169,3 +201,6 @@ Specific layers can be updated using: This will need to be done after adding a new layer to the OWS configuration, or after changing the time resolution or the ODC product(s) of an existing layer. + +The target OWS database for each layer is determined from OWS configuration, so the ``-E`` flag is invalid +with this calling mode. diff --git a/docs/diagrams/ows_diagram1.9.png b/docs/diagrams/ows_diagram1.9.png index 93c0a77d0..6fa7855ba 100644 Binary files a/docs/diagrams/ows_diagram1.9.png and b/docs/diagrams/ows_diagram1.9.png differ diff --git a/docs/environment_variables.rst b/docs/environment_variables.rst index b95a72177..c5b663dd6 100644 --- a/docs/environment_variables.rst +++ b/docs/environment_variables.rst @@ -22,17 +22,38 @@ environment variable. The format of postgres connection URL is:: postgresql://:@:/ +If you are using an ODC environment other than ``default`` or are using multiple ODC environments, +you can specify the url for other environments in the same fashion, e.g. for environment ``myenv`` +use ``$ODC_MYENV_DB_URL``. + +If you want to use a ``postgis`` based ODC index, you should also specify the index driver by +setting e.g. ``$ODC_MYENV_INDEX_DRIVER`` to ``postgis``. + Other valid methods for configuring an OpenDatacube instance (e.g. a ``.datacube.conf`` file) should also work. Note that OWS currently only works with legacy/postgres index driver. Postgis support is hopefully coming soon. +The old `$DB_HOSTNAME`, `$DB_DATABASE` etc. environment variables are now STRONGLY DEPRECATED as they +only work in a single-index environment. + An ODC environment other than ``default`` can be used by setting the ``env`` option in the global OWS configuration. -Note that ``docker-compose`` arrangement used for integration testing on github also redundantly requires -the ``$DB_USERNAME``, ``$DB_PASSWORD``, ``$DB_DATABSE`` and ``$DB_PORT`` environment variables to set up -the generic docker postgres container. If you are connecting to an existing database, these variables -are not required. +For Running Integration Tests +----------------------------- + +The integration tests need to be able to call a running a OWS server connected the test database +and running the version of the OWS codebase being tested. + +SERVER_URL: + The URL of the test server. Defaults to ``http://localhost:5000`` + +SERVER_DB_USERNAME: + This is the database username used by the test server to connect to the test database. Defaults to + the same database username being used by the integration tests themselves. + +Note that ``docker-compose`` arrangement used for integration testing on github + Configuring AWS Access ---------------------- @@ -111,11 +132,13 @@ Docker and Docker-compose ------------------------- The provided ``Dockerfile`` and ``docker-compose.yaml`` read additional -environment variables at build time. Please refer to the `README `_ +environment variables at build time. +Please refer to the `README `_ for further details. -environment variables exclusive for docker-compose +Environment variables exclusive for docker-compose -------------------------------------------------- + OWS_CFG_DIR: path to a folder containing ows config files anywhere on the local machine @@ -124,3 +147,16 @@ OWS_CFG_MOUNT_DIR: PYTHONPATH: PYTHONPATH to ows config file + +POSTGRES_DB: +POSTGRES_USER: +POSTGRES_PASSWORD: + The db superuser name and password for the postgis database container. + If multiple databases are required, use a comma-separated list of database names + +POSTGRES_HOSTNAME: + The name of the database server/container. + +READY_PROBE_DB: + The (single) database to use for the startup database readiness probe. Should be set to one of the + values in ``$POSTGRES_DB`` diff --git a/docs/installation.rst b/docs/installation.rst index 617f62ec9..c5fb9b3af 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -51,10 +51,10 @@ of the config file, and edit it to reflect your requirements. $ DATACUBE_OWS_CFG=ows_local_cfg.ows_cfg * We currently recommend using pip with pre-built binary packages. Create a - new python 3.6 or 3.7 virtualenv and run pip install against the supplied + new python 3.10+ virtualenv and run pip install against the supplied requirements.txt:: - pip install --pre -r requirements.txt + pip install -e .[all] To install datacube-ows, run: @@ -129,7 +129,7 @@ E.g. to set up a new database: $ docker exec -it datacube-ows_ows_1 bash ows_1$ datacube system init - ows_1$ datacube-ows-update --schema --role ubuntu + ows_1$ datacube-ows-update --schema --write-role ubuntu ows_1$ datacube-ows-update diff --git a/docs/usage.rst b/docs/usage.rst index 54ea90888..14655458f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -92,7 +92,7 @@ Update extents of a new product or to update a product in Datacube to make it ea .. code-block:: console - $ datacube-ows-update --views --blocking + $ datacube-ows-update --views $ datacube-ows-update alos_palsar_mosaic Deploy the Digital Earth Africa OWS config available `here `_ diff --git a/integration_tests/cfg/ows_test_cfg.py b/integration_tests/cfg/ows_test_cfg.py index a1066f7bf..f5029ff5d 100644 --- a/integration_tests/cfg/ows_test_cfg.py +++ b/integration_tests/cfg/ows_test_cfg.py @@ -648,6 +648,11 @@ "horizontal_coord": "x", "vertical_coord": "y", }, + "EPSG:6933": { # Africa - invalid for our test data + "geographic": False, + "horizontal_coord": "x", + "vertical_coord": "y", + }, }, # If True the new EXPERIMENTAL materialised views are used for spatio-temporal extents. # If False (the default), the old "update_ranges" tables (and native ODC search methods) are used. @@ -731,9 +736,9 @@ # is also a coverage, that may be requested in WCS DescribeCoverage or WCS GetCoverage requests. "layers": [ { - "title": "s2", - "abstract": "Images from the sentinel 2 satellite", - "keywords": ["sentinel2"], + "title": "Postgres Data", + "abstract": "Data from the postgres test database", + "keywords": ["postgres"], "attribution": { # Attribution must contain at least one of ("title", "url" and "logo") # A human readable title for the attribution - e.g. the name of the attributed organisation @@ -752,200 +757,456 @@ "format": "image/png", }, }, - "label": "sentinel2", + "label": "postgres", "layers": [ { - "title": "Surface reflectance (Sentinel-2)", - "name": "s2_l2a", - "abstract": """layer s2_l2a""", - "product_name": "s2_l2a", - "bands": bands_sentinel, - "dynamic": True, - "resource_limits": reslim_continental, - "time_resolution": "subday", - "image_processing": { - "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", - "always_fetch_bands": [], - "manual_merge": False, # True - "apply_solar_corrections": False, + "title": "s2", + "abstract": "Images from the sentinel 2 satellite", + "keywords": ["sentinel2"], + "attribution": { + # Attribution must contain at least one of ("title", "url" and "logo") + # A human readable title for the attribution - e.g. the name of the attributed organisation + "title": "Open Data Cube - OWS", + # The associated - e.g. URL for the attributed organisation + "url": "https://www.opendatacube.org/", + # Logo image - e.g. for the attributed organisation + "logo": { + # Image width in pixels (optional) + "width": 268, + # Image height in pixels (optional) + "height": 68, + # URL for the logo image. (required if logo specified) + "url": "https://user-images.githubusercontent.com/4548530/112120795-b215b880-8c12-11eb-8bfa-1033961fb1ba.png", + # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.) + "format": "image/png", + }, }, - "flags": [ + "label": "sentinel2", + "layers": [ { - "band": "SCL", - "product": "s2_l2a", - "ignore_time": False, - "ignore_info_flags": [], - # This band comes from main product, so cannot set flags manual_merge independently - # "manual_merge": True, + "title": "Surface reflectance (Sentinel-2)", + "name": "s2_l2a", + "abstract": """layer s2_l2a""", + "product_name": "s2_l2a", + "bands": bands_sentinel, + "dynamic": True, + "resource_limits": reslim_continental, + "time_resolution": "subday", + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, # True + "apply_solar_corrections": False, + }, + "flags": [ + { + "band": "SCL", + "product": "s2_l2a", + "ignore_time": False, + "ignore_info_flags": [], + # This band comes from main product, so cannot set flags manual_merge independently + # "manual_merge": True, + }, + ], + "native_crs": "EPSG:3857", + "native_resolution": [30.0, -30.0], + "styling": { + "default_style": "simple_rgb", + "styles": styles_s2_list, + }, }, - ], - "native_crs": "EPSG:3857", - "native_resolution": [30.0, -30.0], - "styling": { - "default_style": "simple_rgb", - "styles": styles_s2_list, - }, + { + "inherits": { + "layer": "s2_l2a", + }, + "title": "s2_l2a Clone", + "abstract": "Imagery from the s2_l2a Clone", + "name": "s2_l2a_clone", + "low_res_product_name": "s2_l2a", + "image_processing": { + "extent_mask_func": [], + "manual_merge": True, + "apply_solar_corrections": True, + }, + "resource_limits": { + "wcs": { + "max_image_size": 2000 * 2000 * 3 * 2, + } + }, + "patch_url_function": f"{cfgbase}utils.trivial_identity", + }, + ] }, { - "inherits": { - "layer": "s2_l2a", - }, - "title": "s2_l2a Clone", - "abstract": "Imagery from the s2_l2a Clone", - "name": "s2_l2a_clone", - "low_res_product_name": "s2_l2a", - "image_processing": { - "extent_mask_func": [], - "manual_merge": True, - "apply_solar_corrections": True, - }, - "resource_limits": { - "wcs": { - "max_image_size": 2000 * 2000 * 3 * 2, + "title": "DEA Config Samples", + "abstract": "", + "layers": [ + { + "title": "DEA Surface Reflectance (Sentinel-2)", + "name": "s2_ard_granule_nbar_t", + "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) + This product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. + These corrections have been applied to all satellite imagery in the Sentinel-2 archive. This is undertaken to allow comparison of imagery acquired at different times, in different seasons and in different geographic locations. + These products also indicate where the imagery has been affected by cloud or cloud shadow, contains missing data or has been affected in other ways. The Surface Reflectance products are useful as a fundamental starting point for any further analysis, and underpin all other optical derived Digital Earth Australia products. + This is a definitive archive of daily Sentinel-2 data. This is processed using correct ancillary data to provide a more accurate product than the Near Real Time. + The Surface Reflectance product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. These corrections have been applied to all satellite imagery in the Sentinel-2 archive. + The Normalised Difference Chlorophyll Index (NDCI) is based on the method of Mishra & Mishra 2012, and adapted to bands on the Sentinel-2A & B sensors. + The index indicates levels of chlorophyll-a (chl-a) concentrations in complex turbid productive waters such as those encountered in many inland water bodies. The index has not been validated in Australian waters, and there are a range of environmental conditions that may have an effect on the accuracy of the derived index values in this test implementation, including: + - Influence on the remote sensing signal from nearby land and/or atmospheric effects + - Optically shallow water + - Cloud cover + Mishra, S., Mishra, D.R., 2012. Normalized difference chlorophyll index: A novel model for remote estimation of chlorophyll-a concentration in turbid productive waters. Remote Sensing of Environment, Remote Sensing of Urban Environments 117, 394–406. https://doi.org/10.1016/j.rse.2011.10.016 + For more information see http://pid.geoscience.gov.au/dataset/ga/129684 + https://cmi.ga.gov.au/data-products/dea/190/dea-surface-reflectance-nbart-sentinel-2-msi + For service status information, see https://status.dea.ga.gov.au + """, + "multi_product": True, + "product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "low_res_product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "bands": bands_sentinel2_ard_nbart, + "resource_limits": reslim_for_sentinel2, + "native_crs": "EPSG:3577", + "native_resolution": [10.0, -10.0], + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, + }, + "flags": [ + { + "band": "fmask_alias", + "products": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "products": ["geodata_coast_100k", "geodata_coast_100k"], + "ignore_time": True, + "ignore_info_flags": [] + }, + ], + "time_axis": { + "time_interval": 1 + }, + "styling": {"default_style": "ndci", "styles": styles_s2_ga_list}, + }, + { + "inherits": { + "layer": "s2_ard_granule_nbar_t", + }, + "title": "DEA Surface Reflectance Mosaic (Sentinel-2)", + "name": "s2_ard_latest_mosaic", + "multi_product": True, + "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) + + Latest imagery mosaic with no time dimension. + """, + "mosaic_date_func": { + "function": "datacube_ows.time_utils.rolling_window_ndays", + "pass_layer_cfg": True, + "kwargs": { + "ndays": 6, + } + } + }, + { + "title": "DEA Fractional Cover (Landsat)", + "name": "ga_ls_fc_3", + "abstract": """Geoscience Australia Landsat Fractional Cover Collection 3 + Fractional Cover (FC), developed by the Joint Remote Sensing Research Program, is a measurement that splits the landscape into three parts, or fractions: + green (leaves, grass, and growing crops) + brown (branches, dry grass or hay, and dead leaf litter) + bare ground (soil or rock) + DEA uses Fractional Cover to characterise every 30 m square of Australia for any point in time from 1987 to today. + https://cmi.ga.gov.au/data-products/dea/629/dea-fractional-cover-landsat-c3 + For service status information, see https://status.dea.ga.gov.au""", + "product_name": "ga_ls_fc_3", + "bands": bands_fc_3, + "resource_limits": reslim_for_sentinel2, + "dynamic": True, + "native_crs": "EPSG:3577", + "native_resolution": [25, -25], + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, + }, + "flags": [ + # flags is now a list of flag band definitions - NOT a dictionary with identifiers + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [], + }, + { + "band": "water", + "product": "ga_ls_wo_3", + "ignore_time": False, + "ignore_info_flags": [], + "fuse_func": "datacube_ows.wms_utils.wofls_fuser", + }, + ], + "styling": { + "default_style": "fc_rgb_unmasked", + "styles": [style_fc_c3_rgb_unmasked], + }, } - }, - "patch_url_function": f"{cfgbase}utils.trivial_identity", + ] }, - ] - }, - { - "title": "DEA Config Samples", - "abstract": "", - "layers": [ { - "title": "DEA Surface Reflectance (Sentinel-2)", - "name": "s2_ard_granule_nbar_t", - "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) - This product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. - These corrections have been applied to all satellite imagery in the Sentinel-2 archive. This is undertaken to allow comparison of imagery acquired at different times, in different seasons and in different geographic locations. - These products also indicate where the imagery has been affected by cloud or cloud shadow, contains missing data or has been affected in other ways. The Surface Reflectance products are useful as a fundamental starting point for any further analysis, and underpin all other optical derived Digital Earth Australia products. - This is a definitive archive of daily Sentinel-2 data. This is processed using correct ancillary data to provide a more accurate product than the Near Real Time. - The Surface Reflectance product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. These corrections have been applied to all satellite imagery in the Sentinel-2 archive. - The Normalised Difference Chlorophyll Index (NDCI) is based on the method of Mishra & Mishra 2012, and adapted to bands on the Sentinel-2A & B sensors. - The index indicates levels of chlorophyll-a (chl-a) concentrations in complex turbid productive waters such as those encountered in many inland water bodies. The index has not been validated in Australian waters, and there are a range of environmental conditions that may have an effect on the accuracy of the derived index values in this test implementation, including: - - Influence on the remote sensing signal from nearby land and/or atmospheric effects - - Optically shallow water - - Cloud cover - Mishra, S., Mishra, D.R., 2012. Normalized difference chlorophyll index: A novel model for remote estimation of chlorophyll-a concentration in turbid productive waters. Remote Sensing of Environment, Remote Sensing of Urban Environments 117, 394–406. https://doi.org/10.1016/j.rse.2011.10.016 - For more information see http://pid.geoscience.gov.au/dataset/ga/129684 - https://cmi.ga.gov.au/data-products/dea/190/dea-surface-reflectance-nbart-sentinel-2-msi - For service status information, see https://status.dea.ga.gov.au - """, - "multi_product": True, - "product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], - "low_res_product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], - "bands": bands_sentinel2_ard_nbart, + "title": "Landsat-8 Geomedian", + "name": "ls8_geomedian", + "abstract": """DEA Landsat-8 Geomedian""", + "product_name": "ga_ls8c_nbart_gm_cyear_3", + "bands": bands_c3_ls, "resource_limits": reslim_for_sentinel2, + "dynamic": False, "native_crs": "EPSG:3577", - "native_resolution": [10.0, -10.0], + "time_resolution": "summary", + "native_resolution": [25, -25], "image_processing": { "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", "always_fetch_bands": [], "manual_merge": False, }, - "flags": [ + "styling": { + "default_style": "simple_rgb", + "styles": styles_ls_list, + }, + } + ] ####### End of "postgres" layers + }, + { + "title": "Postgis Data", + "abstract": "Data from the postgis test database", + "keywords": ["postgis"], + "attribution": { + # Attribution must contain at least one of ("title", "url" and "logo") + # A human readable title for the attribution - e.g. the name of the attributed organisation + "title": "Open Data Cube - OWS", + # The associated - e.g. URL for the attributed organisation + "url": "https://www.opendatacube.org/", + # Logo image - e.g. for the attributed organisation + "logo": { + # Image width in pixels (optional) + "width": 268, + # Image height in pixels (optional) + "height": 68, + # URL for the logo image. (required if logo specified) + "url": "https://user-images.githubusercontent.com/4548530/112120795-b215b880-8c12-11eb-8bfa-1033961fb1ba.png", + # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.) + "format": "image/png", + }, + }, + "label": "postgis", + "layers": [ + { + "title": "s2 (postgis)", + "abstract": "Images from the sentinel 2 satellite (postgis db)", + "keywords": ["sentinel2"], + "label": "sentinel2_pgis", + "layers": [ { - "band": "fmask_alias", - "products": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], - "ignore_time": False, - "ignore_info_flags": [] + "title": "Surface reflectance (Sentinel-2) (postgis db)", + "name": "s2_l2a_postgis", + "abstract": """layer s2_l2a (postgis db)""", + "product_name": "s2_l2a", + "bands": bands_sentinel, + "env": "owspostgis", + "dynamic": True, + "resource_limits": reslim_continental, + "time_resolution": "subday", + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, # True + "apply_solar_corrections": False, + }, + "flags": [ + { + "band": "SCL", + "product": "s2_l2a", + "ignore_time": False, + "ignore_info_flags": [], + # This band comes from main product, so cannot set flags manual_merge independently + # "manual_merge": True, + }, + ], + "native_crs": "EPSG:3857", + "native_resolution": [30.0, -30.0], + "styling": { + "default_style": "simple_rgb", + "styles": styles_s2_list, + }, }, { - "band": "land", - "products": ["geodata_coast_100k", "geodata_coast_100k"], - "ignore_time": True, - "ignore_info_flags": [] + "inherits": { + "layer": "s2_l2a_postgis", + }, + "title": "s2_l2a Clone (postgis db)", + "abstract": "Imagery from the s2_l2a Clone (postgis db)", + "name": "s2_l2a_clone_postgis", + "env": "owspostgis", + "low_res_product_name": "s2_l2a", + "image_processing": { + "extent_mask_func": [], + "manual_merge": True, + "apply_solar_corrections": True, + }, + "resource_limits": { + "wcs": { + "max_image_size": 2000 * 2000 * 3 * 2, + } + }, + "patch_url_function": f"{cfgbase}utils.trivial_identity", }, - ], - "time_axis": { - "time_interval": 1 - }, - "styling": {"default_style": "ndci", "styles": styles_s2_ga_list}, + ] }, { - "inherits": { - "layer": "s2_ard_granule_nbar_t", - }, - "title": "DEA Surface Reflectance Mosaic (Sentinel-2)", - "name": "s2_ard_latest_mosaic", - "multi_product": True, - "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) - -Latest imagery mosaic with no time dimension. - """, - "mosaic_date_func": { - "function": "datacube_ows.time_utils.rolling_window_ndays", - "pass_layer_cfg": True, - "kwargs": { - "ndays": 6, + "title": "DEA Config Samples", + "abstract": "", + "layers": [ + { + "title": "DEA Surface Reflectance (Sentinel-2) (postgis db)", + "name": "s2_ard_granule_nbar_t_postgis", + "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) + This product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. + These corrections have been applied to all satellite imagery in the Sentinel-2 archive. This is undertaken to allow comparison of imagery acquired at different times, in different seasons and in different geographic locations. + These products also indicate where the imagery has been affected by cloud or cloud shadow, contains missing data or has been affected in other ways. The Surface Reflectance products are useful as a fundamental starting point for any further analysis, and underpin all other optical derived Digital Earth Australia products. + This is a definitive archive of daily Sentinel-2 data. This is processed using correct ancillary data to provide a more accurate product than the Near Real Time. + The Surface Reflectance product has been corrected to account for variations caused by atmospheric properties, sun position and sensor view angle at time of image capture. These corrections have been applied to all satellite imagery in the Sentinel-2 archive. + The Normalised Difference Chlorophyll Index (NDCI) is based on the method of Mishra & Mishra 2012, and adapted to bands on the Sentinel-2A & B sensors. + The index indicates levels of chlorophyll-a (chl-a) concentrations in complex turbid productive waters such as those encountered in many inland water bodies. The index has not been validated in Australian waters, and there are a range of environmental conditions that may have an effect on the accuracy of the derived index values in this test implementation, including: + - Influence on the remote sensing signal from nearby land and/or atmospheric effects + - Optically shallow water + - Cloud cover + Mishra, S., Mishra, D.R., 2012. Normalized difference chlorophyll index: A novel model for remote estimation of chlorophyll-a concentration in turbid productive waters. Remote Sensing of Environment, Remote Sensing of Urban Environments 117, 394–406. https://doi.org/10.1016/j.rse.2011.10.016 + For more information see http://pid.geoscience.gov.au/dataset/ga/129684 + https://cmi.ga.gov.au/data-products/dea/190/dea-surface-reflectance-nbart-sentinel-2-msi + For service status information, see https://status.dea.ga.gov.au (postgis db) + """, + "multi_product": True, + "product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "low_res_product_names": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "bands": bands_sentinel2_ard_nbart, + "env": "owspostgis", + "resource_limits": reslim_for_sentinel2, + "native_crs": "EPSG:3577", + "native_resolution": [10.0, -10.0], + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, + }, + "flags": [ + { + "band": "fmask_alias", + "products": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], + "ignore_time": False, + "ignore_info_flags": [] + }, + { + "band": "land", + "products": ["geodata_coast_100k", "geodata_coast_100k"], + "ignore_time": True, + "ignore_info_flags": [] + }, + ], + "time_axis": { + "time_interval": 1 + }, + "styling": {"default_style": "ndci", "styles": styles_s2_ga_list}, + }, + { + "inherits": { + "layer": "s2_ard_granule_nbar_t_postgis", + }, + "title": "DEA Surface Reflectance Mosaic (Sentinel-2) (postgis)", + "name": "s2_ard_latest_mosaic_postgis", + "multi_product": True, + "abstract": """Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + Terrain Illumination Correction (Sentinel-2 MSI) + + Latest imagery mosaic with no time dimension. (postgis db) + """, + "mosaic_date_func": { + "function": "datacube_ows.time_utils.rolling_window_ndays", + "pass_layer_cfg": True, + "kwargs": { + "ndays": 6, + } + } + }, + { + "title": "DEA Fractional Cover (Landsat) (postgis db)", + "name": "ga_ls_fc_3_postgis", + "abstract": """Geoscience Australia Landsat Fractional Cover Collection 3 + Fractional Cover (FC), developed by the Joint Remote Sensing Research Program, is a measurement that splits the landscape into three parts, or fractions: + green (leaves, grass, and growing crops) + brown (branches, dry grass or hay, and dead leaf litter) + bare ground (soil or rock) + DEA uses Fractional Cover to characterise every 30 m square of Australia for any point in time from 1987 to today. + https://cmi.ga.gov.au/data-products/dea/629/dea-fractional-cover-landsat-c3 + For service status information, see https://status.dea.ga.gov.au (postgis db)""", + "product_name": "ga_ls_fc_3", + "bands": bands_fc_3, + "resource_limits": reslim_for_sentinel2, + "env": "owspostgis", + "dynamic": True, + "native_crs": "EPSG:3577", + "native_resolution": [25, -25], + "image_processing": { + "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", + "always_fetch_bands": [], + "manual_merge": False, + }, + "flags": [ + # flags is now a list of flag band definitions - NOT a dictionary with identifiers + { + "band": "land", + "product": "geodata_coast_100k", + "ignore_time": True, + "ignore_info_flags": [], + }, + { + "band": "water", + "product": "ga_ls_wo_3", + "ignore_time": False, + "ignore_info_flags": [], + "fuse_func": "datacube_ows.wms_utils.wofls_fuser", + }, + ], + "styling": { + "default_style": "fc_rgb_unmasked", + "styles": [style_fc_c3_rgb_unmasked], + }, } - } + ] }, { - "title": "DEA Fractional Cover (Landsat)", - "name": "ga_ls_fc_3", - "abstract": """Geoscience Australia Landsat Fractional Cover Collection 3 - Fractional Cover (FC), developed by the Joint Remote Sensing Research Program, is a measurement that splits the landscape into three parts, or fractions: - green (leaves, grass, and growing crops) - brown (branches, dry grass or hay, and dead leaf litter) - bare ground (soil or rock) - DEA uses Fractional Cover to characterise every 30 m square of Australia for any point in time from 1987 to today. - https://cmi.ga.gov.au/data-products/dea/629/dea-fractional-cover-landsat-c3 - For service status information, see https://status.dea.ga.gov.au""", - "product_name": "ga_ls_fc_3", - "bands": bands_fc_3, + "title": "Landsat-8 Geomedian (postgis db)", + "name": "ls8_geomedian_postgis", + "abstract": """DEA Landsat-8 Geomedian (postgis db)""", + "product_name": "ga_ls8c_nbart_gm_cyear_3", + "bands": bands_c3_ls, + "env": "owspostgis", "resource_limits": reslim_for_sentinel2, - "dynamic": True, + "dynamic": False, "native_crs": "EPSG:3577", + "time_resolution": "summary", "native_resolution": [25, -25], "image_processing": { "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", "always_fetch_bands": [], "manual_merge": False, }, - "flags": [ - # flags is now a list of flag band definitions - NOT a dictionary with identifiers - { - "band": "land", - "product": "geodata_coast_100k", - "ignore_time": True, - "ignore_info_flags": [], - }, - { - "band": "water", - "product": "ga_ls_wo_3", - "ignore_time": False, - "ignore_info_flags": [], - "fuse_func": "datacube_ows.wms_utils.wofls_fuser", - }, - ], "styling": { - "default_style": "fc_rgb_unmasked", - "styles": [style_fc_c3_rgb_unmasked], + "default_style": "simple_rgb", + "styles": styles_ls_list, }, } - ] + ] ####### End of "postgis" layers }, - { - "title": "Landsat-8 Geomedian", - "name": "ls8_geomedian", - "abstract": """DEA Landsat-8 Geomedian""", - "product_name": "ga_ls8c_nbart_gm_cyear_3", - "bands": bands_c3_ls, - "resource_limits": reslim_for_sentinel2, - "dynamic": False, - "native_crs": "EPSG:3577", - "time_resolution": "summary", - "native_resolution": [25, -25], - "image_processing": { - "extent_mask_func": "datacube_ows.ogc_utils.mask_by_val", - "always_fetch_bands": [], - "manual_merge": False, - }, - "styling": { - "default_style": "simple_rgb", - "styles": styles_ls_list, - }, - } - ], ##### End of "layers" list. + ] ##### End of "layers" list. } #### End of test configuration object diff --git a/integration_tests/cfg/test_translations/de/LC_MESSAGES/ows_cfg.po b/integration_tests/cfg/test_translations/de/LC_MESSAGES/ows_cfg.po new file mode 100644 index 000000000..2689c7196 --- /dev/null +++ b/integration_tests/cfg/test_translations/de/LC_MESSAGES/ows_cfg.po @@ -0,0 +1,31 @@ +# German translations for PROJECT. +# Copyright (C) 2024 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2024-07-03 17:29+1000\n" +"PO-Revision-Date: 2024-07-03 17:29+1000\n" +"Last-Translator: FULL NAME \n" +"Language: de\n" +"Language-Team: de \n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.15.0\n" + +msgid "global.title" +msgstr "Over-ridden: aardvark" + +msgid "folder.sentinel2.abstract" +msgstr "Over-ridden: bunny-rabbit" + +msgid "layer.s2_l2a.title" +msgstr "Over-ridden: chook" + +msgid "style.s2_l2a.simple_rgb.title" +msgstr "Over-ridden: donkey" diff --git a/integration_tests/cfg/test_translations/en/LC_MESSAGES/ows_cfg.po b/integration_tests/cfg/test_translations/en/LC_MESSAGES/ows_cfg.po new file mode 100644 index 000000000..1091369fe --- /dev/null +++ b/integration_tests/cfg/test_translations/en/LC_MESSAGES/ows_cfg.po @@ -0,0 +1,31 @@ +# English translations for PROJECT. +# Copyright (C) 2024 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2024-07-03 17:29+1000\n" +"PO-Revision-Date: 2024-07-03 17:29+1000\n" +"Last-Translator: FULL NAME \n" +"Language: en\n" +"Language-Team: en \n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.15.0\n" + +msgid "global.title" +msgstr "Over-ridden: aardvark" + +msgid "folder.sentinel2.abstract" +msgstr "Over-ridden: bunny-rabbit" + +msgid "layer.s2_l2a.title" +msgstr "Over-ridden: chook" + +msgid "style.s2_l2a.simple_rgb.title" +msgstr "Over-ridden: donkey" diff --git a/integration_tests/cfg/translations/de/LC_MESSAGES/ows_cfg.po b/integration_tests/cfg/translations/de/LC_MESSAGES/ows_cfg.po index 4677bba88..8d40721c4 100644 --- a/integration_tests/cfg/translations/de/LC_MESSAGES/ows_cfg.po +++ b/integration_tests/cfg/translations/de/LC_MESSAGES/ows_cfg.po @@ -7,7 +7,7 @@ msgstr "" "Project-Id-Version: Open web-services for the Open Data Cube " "2022-03-24T23:29:57.407805\n" "Report-Msgid-Bugs-To: test@example.com\n" -"POT-Creation-Date: 2022-08-26 13:23+1000\n" +"POT-Creation-Date: 2024-07-03 17:29+1000\n" "PO-Revision-Date: 2022-03-24 23:33+0000\n" "Last-Translator: FULL NAME \n" "Language: de\n" @@ -16,7 +16,7 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.10.3\n" +"Generated-By: Babel 2.15.0\n" msgid "global.title" msgstr "This is the German translation of the title" diff --git a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.mo b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.mo index 5c23f0271..56adeb3ed 100644 Binary files a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.mo and b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.mo differ diff --git a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po index a1201b33b..70e821693 100644 --- a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po +++ b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: Over-ridden: aardvark 2022-03-24T23:29:57.407805\n" "Report-Msgid-Bugs-To: test@example.com\n" -"POT-Creation-Date: 2024-05-22 10:33+1000\n" +"POT-Creation-Date: 2024-07-03 17:29+1000\n" "PO-Revision-Date: 2022-03-24 23:33+0000\n" "Last-Translator: FULL NAME \n" "Language: en\n" diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 6fdf3982d..ac18b457d 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -96,11 +96,19 @@ def product_name(): @pytest.fixture -def role_name(): +def write_role_name(): odc_env = ODCConfig.get_environment() return odc_env.db_username +@pytest.fixture +def read_role_name(write_role_name): + if read_role_name := os.environ.get("SERVER_DB_USERNAME"): + return read_role_name + else: + return write_role_name + + @pytest.fixture def multiproduct_name(): return "s2_ard_granule_nbar_t" diff --git a/integration_tests/metadata/COAST_100K_15_-40.yaml b/integration_tests/metadata/COAST_100K_15_-40.yaml new file mode 100644 index 000000000..d13bb3075 --- /dev/null +++ b/integration_tests/metadata/COAST_100K_15_-40.yaml @@ -0,0 +1,41 @@ +$schema: https://schemas.opendatacube.org/dataset +crs: epsg:3577 +extent: + lat: + begin: -35.72031832673588 + end: -34.70891190784338 + lon: + begin: 148.63868967606328 + end: 149.73417611149185 +grids: + default: + shape: + - 4000 + - 4000 + transform: + - 25.0 + - 0 + - 1500000.0 + - 0 + - -25.0 + - -3900000.0 + - 0 + - 0 + - 1 +id: 2921b863-9c09-4d5b-a561-5b8d5ddf24bc +label: COAST_100L_15_-40 +lineage: + source_datasets: {} +measurements: + land: + path: COAST_100K_15_-40.tif +product: + name: geodata_coast_100k +properties: + created: '2018-12-03T04:28:16.827902' + datetime: '2004-01-01T00:00:00' + odc:file_format: GeoTIFF + odc:region_code: 15_-40 + eo:instrument: unknown + eo:platform: unknown + proj:epsg: 3577 diff --git a/integration_tests/metadata/COAST_100K_8_-21.yaml b/integration_tests/metadata/COAST_100K_8_-21.yaml new file mode 100644 index 000000000..88ff130d8 --- /dev/null +++ b/integration_tests/metadata/COAST_100K_8_-21.yaml @@ -0,0 +1,41 @@ +$schema: https://schemas.opendatacube.org/dataset +crs: epsg:3577 +extent: + lat: + begin: -19.38776011021664 + end: -18.43108812058793 + lon: + begin: 139.58869840733868 + end: 140.59834925204385 +grids: + default: + shape: + - 4000 + - 4000 + transform: + - 25.0 + - 0 + - 800000.0 + - 0 + - -25.0 + - -2000000.0 + - 0 + - 0 + - 1 +id: 701478bc-2625-4743-99bf-10865d3bb2da +label: COAST_100K_8_-21 +lineage: + source_datasets: {} +measurements: + land: + path: COAST_100K_8_-21.tif +product: + name: geodata_coast_100k +properties: + created: '2018-12-03T04:31:43.146830' + datetime: '2004-01-01T00:00:00' + odc:file_format: GeoTIFF + odc:region_code: 8_-21 + eo:instrument: unknown + eo:platform: unknown + proj:epsg: 3577 diff --git a/integration_tests/metadata/metadata_importer.py b/integration_tests/metadata/metadata_importer.py index ec88df60f..7f8c3a128 100644 --- a/integration_tests/metadata/metadata_importer.py +++ b/integration_tests/metadata/metadata_importer.py @@ -7,8 +7,10 @@ from datacube.index.hl import Doc2Dataset dc = Datacube() +dc_pgis = Datacube(env="owspostgis") -doc2ds = Doc2Dataset(dc.index, products=["s2_l2a"], skip_lineage=True, verify_lineage=False) +doc2ds = Doc2Dataset(dc.index, products=["s2_l2a", "geodata_coast_100k"], skip_lineage=True, verify_lineage=False) +doc2ds_pgis = Doc2Dataset(dc_pgis.index, products=["s2_l2a", "geodata_coast_100k"], skip_lineage=True, verify_lineage=False) for line in fileinput.input(): filename, uri = line.split() @@ -20,7 +22,14 @@ del doc["extent"] ds, err = doc2ds(doc, uri) if ds: - dc.index.datasets.add(ds) + dc.index.datasets.add(ds, with_lineage=False) else: - print("Dataset add failed:", err) + print("Dataset add (postgres) failed:", err) + exit(1) + + ds, err = doc2ds_pgis(doc, uri) + if ds: + dc_pgis.index.datasets.add(ds, with_lineage=False) + else: + print("Dataset add (postgis) failed:", err) exit(1) diff --git a/integration_tests/metadata/product_geodata_coast_100k.yaml b/integration_tests/metadata/product_geodata_coast_100k.yaml new file mode 100644 index 000000000..69e7585cf --- /dev/null +++ b/integration_tests/metadata/product_geodata_coast_100k.yaml @@ -0,0 +1,37 @@ +name: geodata_coast_100k +description: Coastline data for Australia +metadata_type: eo3 +license: CC-BY-4.0 +metadata: + product: + name: geodata_coast_100k +measurements: +- name: land + dtype: uint8 + flags_definition: + land_type: + bits: + - 0 + - 1 + description: Sea, Mainland or Island + values: + 0: sea + 1: island + 2: mainland + sea: + bits: + - 0 + - 1 + description: Sea + values: + 0: true + nodata: 0 + units: '1' +load: + crs: 'EPSG:3577' + resolution: + y: -25 + x: 25 + align: + y: 0 + x: 0 diff --git a/integration_tests/test_mv_index.py b/integration_tests/test_mv_index.py index 234f719a1..ea158d004 100644 --- a/integration_tests/test_mv_index.py +++ b/integration_tests/test_mv_index.py @@ -7,7 +7,7 @@ import pytest from odc.geo.geom import box -from datacube_ows.mv_index import MVSelectOpts, mv_search +from datacube_ows.index.postgres.mv_index import MVSelectOpts, mv_search from datacube_ows.ows_configuration import get_config from datacube_ows.time_utils import local_solar_date_range diff --git a/integration_tests/test_update_ranges.py b/integration_tests/test_update_ranges.py index 6834fb56e..b333ffae0 100644 --- a/integration_tests/test_update_ranges.py +++ b/integration_tests/test_update_ranges.py @@ -16,18 +16,38 @@ def test_update_ranges_schema_without_roles(runner): assert "Insufficient Privileges" not in result.output assert "Cannot find SQL resource" not in result.output assert result.exit_code == 0 + result = runner.invoke(main, ["-E", "owspostgis", "--schema"]) + assert "appear to be missing" not in result.output + assert "Insufficient Privileges" not in result.output + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 -def test_update_ranges_schema_with_roles(runner, role_name): - result = runner.invoke(main, ["--schema", "--read-role", role_name, "--write-role", role_name]) +def test_update_ranges_schema_with_roles(runner, read_role_name, write_role_name): + result = runner.invoke(main, ["--schema", "--read-role", read_role_name, "--write-role", write_role_name]) assert "appear to be missing" not in result.output assert "Insufficient Privileges" not in result.output assert "Cannot find SQL resource" not in result.output assert result.exit_code == 0 + result = runner.invoke(main, ["-E", "owspostgis", + "--schema", "--read-role", read_role_name, "--write-role", write_role_name]) + assert "appear to be missing" not in result.output + assert "Insufficient Privileges" not in result.output + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 + result = runner.invoke(main, ["-E", "nonononodontcallanenviornmentthis", + "--schema", "--read-role", read_role_name, "--write-role", write_role_name]) + assert "Unable to connect to the nonono" in result.output + assert result.exit_code == 1 -def test_update_ranges_roles_only(runner, role_name): - result = runner.invoke(main, ["--read-role", role_name, "--write-role", role_name]) +def test_update_ranges_roles_only(runner, read_role_name, write_role_name): + result = runner.invoke(main, ["--read-role", read_role_name, "--write-role", write_role_name]) + assert "appear to be missing" not in result.output + assert "Insufficient Privileges" not in result.output + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 + result = runner.invoke(main, ["-E", "owspostgis", "--read-role", read_role_name, "--write-role", write_role_name]) assert "appear to be missing" not in result.output assert "Insufficient Privileges" not in result.output assert "Cannot find SQL resource" not in result.output diff --git a/integration_tests/test_version.py b/integration_tests/test_version.py index d6e98a75b..2d437d999 100644 --- a/integration_tests/test_version.py +++ b/integration_tests/test_version.py @@ -11,7 +11,7 @@ from datacube_ows.update_ranges_impl import main -def test_updates_ranges_schema(runner, role_name): +def test_updates_ranges_version(runner): result = runner.invoke(main, ["--version"]) assert __version__ in result.output assert result.exit_code == 0 diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index ad7a03a67..4a845b6fd 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -1194,20 +1194,24 @@ def test_wcs20_getcoverage_multidate_netcdf(ows_server): # Ensure that we have at least some layers available contents = list(wcs.contents) - assert len(contents) == 6 - layer = cfg.layer_index[contents[0]] - extent = ODCExtent(layer) - subsets = extent.wcs2_subsets( - ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO, crs="EPSG:4326" - ) - resp = wcs.getCoverage( - identifier=[contents[0]], - format="application/x-netcdf", - subsets=subsets, - subsettingcrs="EPSG:4326", - scalesize="x(400),y(300)", - ) - assert resp + assert len(contents) == 12 + for i in (0, 11): + layer = cfg.layer_index[contents[i]] + extent = ODCExtent(layer) + subsets = extent.wcs2_subsets( + ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO, crs="EPSG:4326" + ) + if len(subsets[2]) < 2: + continue + resp = wcs.getCoverage( + identifier=[layer.name], + format="application/x-netcdf", + subsets=subsets, + subsettingcrs="EPSG:4326", + scalesize="x(400),y(300)", + timeout=90 + ) + assert resp def test_wcs21_server(ows_server): # N.B. At time of writing owslib does not support WCS 2.1, so we have to make requests manually. diff --git a/integration_tests/utils.py b/integration_tests/utils.py index 107b1a6c3..c91bdede7 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -9,8 +9,6 @@ from odc.geo.geom import BoundingBox, Geometry, point from shapely.ops import triangulate, unary_union -from datacube_ows.mv_index import MVSelectOpts, mv_search - class WCS20Extent: def __init__(self, desc_cov): @@ -227,8 +225,8 @@ def subset(self, time_extent, full_extent): hslice_bbox = BoundingBox( left=bbox.left, right=bbox.right, - top=offset_y + 0.02 * height, - bottom=offset_y - 0.02 * height, + top=offset_y + 0.01 * height, + bottom=offset_y - 0.01 * height, ) hslice_geom = geom_from_bbox(hslice_bbox) hslice_geom = hslice_geom.intersection(time_extent) @@ -236,7 +234,7 @@ def subset(self, time_extent, full_extent): hslice_bbox = BoundingBox( left=bbox.left, right=bbox.right, - top=bbox.bottom + 0.02 * height, + top=bbox.bottom + 0.01 * height, bottom=bbox.bottom, ) hslice_geom = geom_from_bbox(hslice_bbox) @@ -255,8 +253,8 @@ def subset(self, time_extent, full_extent): elif self == self.OFFSET_SUBSET_FOR_TIMES: offset_x = centre_x + 0.25 * height vslice_bbox = BoundingBox( - left=offset_x - 0.02 * width, - right=offset_x + 0.02 * width, + left=offset_x - 0.01 * width, + right=offset_x + 0.01 * width, top=slice_bbox.top, bottom=slice_bbox.bottom, ) @@ -371,14 +369,11 @@ def subsets( ext_times = time.slice(self.layer.ranges.times) search_times = [self.layer.search_times(t) for t in ext_times] if space.needs_full_extent() and not self.full_extent: - self.full_extent = mv_search( - self.layer.dc.index, products=self.layer.products, sel=MVSelectOpts.EXTENT - ) + self.full_extent = self.layer.ows_index().extent(layer=self.layer, products=self.layer.products) if space.needs_time_extent(): - time_extent = mv_search( - self.layer.dc.index, + time_extent = self.layer.ows_index().extent( + layer=self.layer, products=self.layer.products, - sel=MVSelectOpts.EXTENT, times=search_times, ) else: diff --git a/ows_cfg_report.json b/ows_cfg_report.json index 8298b8820..082481178 100644 --- a/ows_cfg_report.json +++ b/ows_cfg_report.json @@ -1 +1,165 @@ -{"total_layers_count": 6, "layers": [{"layer": "s2_l2a","product": ["s2_l2a"], "styles_count": 8, "styles_list": ["simple_rgb", "style_ls_simple_rgb_clone", "infra_red", "rgb_ndvi", "blue", "ndvi", "ndvi_expr", "ndvi_delta"]}, {"layer": "s2_l2a_clone", "product": ["s2_l2a"], "styles_count": 8, "styles_list": ["simple_rgb", "style_ls_simple_rgb_clone", "infra_red", "rgb_ndvi", "blue", "ndvi", "ndvi_expr", "ndvi_delta"]}, {"layer": "s2_ard_granule_nbar_t", "product": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], "styles_count": 2, "styles_list": ["ndci", "mndwi"]}, {"layer": "s2_ard_latest_mosaic", "product": ["ga_s2am_ard_3", "ga_s2bm_ard_3"], "styles_count": 2, "styles_list": ["ndci", "mndwi"]}, {"layer": "ga_ls_fc_3", "product": ["ga_ls_fc_3"], "styles_count": 1, "styles_list": ["fc_rgb_unmasked"]},{"layer": "ls8_geomedian", "product": ["ga_ls8c_nbart_gm_cyear_3"], "styles_count": 3, "styles_list": ["simple_rgb","infra_red","ndvi"]}]} +{ + "total_layers_count": 12, + "layers": [ + { + "layer": "s2_l2a", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_l2a_clone", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_ard_granule_nbar_t", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "s2_ard_latest_mosaic", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "ga_ls_fc_3", + "product": [ + "ga_ls_fc_3" + ], + "styles_count": 1, + "styles_list": [ + "fc_rgb_unmasked" + ] + }, + { + "layer": "ls8_geomedian", + "product": [ + "ga_ls8c_nbart_gm_cyear_3" + ], + "styles_count": 3, + "styles_list": [ + "simple_rgb", + "infra_red", + "ndvi" + ] + }, + { + "layer": "s2_l2a_postgis", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_l2a_clone_postgis", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_ard_granule_nbar_t_postgis", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "s2_ard_latest_mosaic_postgis", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "ga_ls_fc_3_postgis", + "product": [ + "ga_ls_fc_3" + ], + "styles_count": 1, + "styles_list": [ + "fc_rgb_unmasked" + ] + }, + { + "layer": "ls8_geomedian_postgis", + "product": [ + "ga_ls8c_nbart_gm_cyear_3" + ], + "styles_count": 3, + "styles_list": [ + "simple_rgb", + "infra_red", + "ndvi" + ] + } + ] +} diff --git a/setup.py b/setup.py index 73954af56..510e40673 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from setuptools import find_packages, setup install_requirements = [ - 'datacube[performance,s3]>=1.9.0-rc4', + 'datacube[performance,s3]>=1.9.0-rc9', 'flask', 'requests', 'affine', @@ -111,6 +111,7 @@ ], "datacube_ows.plugins.index": [ 'postgres = datacube_ows.index.postgres.api:ows_index_driver_init', + 'postgis = datacube_ows.index.postgis.api:ows_index_driver_init', ] }, python_requires=">=3.10.0", diff --git a/tests/test_driver_cache.py b/tests/test_driver_cache.py index 055f6ffab..09cf036d7 100644 --- a/tests/test_driver_cache.py +++ b/tests/test_driver_cache.py @@ -7,6 +7,9 @@ def test_index_driver_cache(): from datacube_ows.index.driver import ows_index_drivers + a = 2 + a = a + 1 assert "postgres" in ows_index_drivers() + assert "postgis" in ows_index_drivers() from datacube_ows.index.driver import ows_index_driver_by_name assert ows_index_driver_by_name("postgres") is not None diff --git a/tests/test_mv_selopts.py b/tests/test_mv_selopts.py index 68b80fb56..e9b910a53 100644 --- a/tests/test_mv_selopts.py +++ b/tests/test_mv_selopts.py @@ -4,7 +4,7 @@ # Copyright (c) 2017-2024 OWS Contributors # SPDX-License-Identifier: Apache-2.0 -from datacube_ows.mv_index import MVSelectOpts +from datacube_ows.index.postgres.mv_index import MVSelectOpts def test_all(): diff --git a/tests/test_update_ranges.py b/tests/test_update_ranges.py index 3e0a93514..0150d5ed3 100644 --- a/tests/test_update_ranges.py +++ b/tests/test_update_ranges.py @@ -10,7 +10,7 @@ import pytest from click.testing import CliRunner from datacube_ows.update_ranges_impl import main -from datacube_ows.index.postgres.sql import run_sql +from datacube_ows.index.sql import run_sql @pytest.fixture @@ -67,8 +67,8 @@ def test_update_ranges_misuse_cases(runner, role_name, layer_name): def test_run_sql(minimal_dc): - assert not run_sql(minimal_dc, "no_such_directory") + assert not run_sql(minimal_dc, "postgres", "no_such_directory") - assert not run_sql(minimal_dc, "templates") + assert not run_sql(minimal_dc, "postgres", "templates") - assert not run_sql(minimal_dc, "ows_schema/grants/read_only") + assert not run_sql(minimal_dc, "postgres", "ows_schema/grants/read_only") diff --git a/wordlist.txt b/wordlist.txt index 9f842b0ca..9b2a0bc80 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -77,6 +77,7 @@ cfg ci cli cloudfront +codebase codecov cogs COGs @@ -259,6 +260,7 @@ multiproc multiproduct mv mydb +myenv mypassword mysecretpassword myuser