From d1ac06516675fddda3f3ce383b6d25b774750188 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Fri, 10 May 2024 20:36:35 +1000 Subject: [PATCH] Postgres ranges rebuild (#1017) * Implement new schema and management - nothing is using it yet! * Start getting update process onto new schema - incomplete work in progress. Includes some product->layer name refactoring. * Writing to the new layer-based schema, with batch caching. * Reading from the new layer range table. More product->layer renaming. * Passing mypy, failing tests. * Passing unit tests, server intialising. Integration tests still failing. * Passing integration tests. * make datacube/env handling more generic (one step closer to multi-db) and passing mypy. * Passing all tests. * Add new tests and fix broken tests. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * lintage. * lintage. * Don't rely on DEA Explorer * Update db to postgres 16 and use DB_URL * Revert main docker-compose.yaml * Need port as well. * Fix nodb test fixture for GH * Opps - used non-raw github link. * Fix ows-update call in GHA test prep script. * Update documentation. * Fix spelling or add (non-)words to wordlist. * Various fixes/cleanups found on self-review. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Make no_db in test_no_db_routes a proper fixture. * Documentation edits * Some cleanup in wms_utils.py * Some cleanup in update_ranges_impl.py * Make access in initialiser more consistent. * Provide better examples of role granting in scripts and documentation. * Fix inconsistent indentation. * Typo --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .env_simple | 3 +- CONTRIBUTING.rst | 23 +- README.rst | 22 +- check-code-all.sh | 4 +- datacube_ows/cfg_parser_impl.py | 4 +- datacube_ows/loading.py | 36 +- datacube_ows/mv_index.py | 10 +- datacube_ows/ogc.py | 21 +- datacube_ows/ows_configuration.py | 121 ++-- datacube_ows/product_ranges.py | 548 +++++++----------- .../create/001_postgis_extension.sql | 2 +- ...eate_view_owner_role_ignore_duplicates.sql | 3 + .../create/010_create_new_time_view.sql | 2 +- ..._raw.sql => 011_create_new_space_view.sql} | 2 +- .../create/012_create_new_spacetime_view.sql | 4 +- .../create/020_create_index_1.sql | 2 +- .../create/021_create_index_2.sql | 2 +- .../create/022_create_index_3.sql | 2 +- .../create/023_create_index_4.sql | 2 +- .../create/030_rename_old_space_time_view.sql | 2 +- .../create/031_rename_new_space_time_view.sql | 2 +- .../create/032_drop_old_space_time_view.sql | 3 + .../create/040_drop_old_space_time_view.sql | 3 - .../create/040_drop_old_space_view.sql | 3 + .../create/041_drop_old_time_view.sql | 2 +- .../create/042_drop_old_space_view.sql | 3 - ...view.sql => 042_rename_new_space_view.sql} | 2 +- ..._view.sql => 043_rename_new_time_view.sql} | 2 +- ...ame_index_1.sql => 050_rename_index_1.sql} | 2 +- ...ame_index_2.sql => 051_rename_index_2.sql} | 2 +- ...ame_index_3.sql => 052_rename_index_3.sql} | 2 +- ...ame_index_4.sql => 053_rename_index_4.sql} | 2 +- .../sql/extent_views/create/060_grant.sql | 3 - .../001_grant_read_requires_role.sql | 3 + .../refresh_owner/001_set_owner_time_view.sql | 3 + .../002_set_owner_space_view.sql | 3 + .../003_set_owner_spacetime_view.sql | 3 + .../001_grant_refresh_requires_role.sql | 3 + .../extent_views/refresh/002_refresh_time.sql | 2 +- .../refresh/003_refresh_space.sql | 2 +- .../refresh/004_refresh_spacetime.sql | 2 +- .../cleanup/001_drop_space_time_view.sql | 3 + .../ows_schema/cleanup/002_drop_time_view.sql | 3 + .../cleanup/003_drop_space_view.sql | 3 + .../cleanup/010_drop_subproduct_range.sql | 3 + .../cleanup/011_drop_multiproduct_range.sql | 3 + .../cleanup/012_drop_product_range.sql | 3 + .../ows_schema/create/001_create_schema.sql | 3 + .../create/002_create_product_rng.sql | 10 +- .../001_grant_usage_requires_role.sql | 3 + .../001_grant_usage_requires_role.sql | 3 + .../002_grant_writetables_requires_role.sql | 3 + datacube_ows/sql/use_space_time.sql | 11 - .../wms_schema/create/001_create_schema.sql | 3 - .../create/003_create_subproduct_rng.sql | 16 - .../create/004_create_multiproduct_rng.sql | 14 - .../create/005_grant_requires_role.sql | 3 - datacube_ows/styles/base.py | 2 +- datacube_ows/templates/wcs_capabilities.xml | 2 +- datacube_ows/templates/wms_capabilities.xml | 2 +- datacube_ows/templates/wmts_capabilities.xml | 2 +- datacube_ows/update_ranges_impl.py | 223 ++++--- datacube_ows/wcs1.py | 4 +- datacube_ows/wcs1_utils.py | 14 +- datacube_ows/wcs2.py | 28 +- datacube_ows/wcs2_utils.py | 6 +- datacube_ows/wms_utils.py | 43 +- docker/database/Dockerfile | 2 +- docs/database.rst | 124 ++-- docs/diagrams/db-relationship-diagram.svg | 196 ------- docs/environment_variables.rst | 14 +- integration_tests/test_layers.py | 2 +- integration_tests/test_mv_index.py | 18 +- integration_tests/test_update_ranges.py | 42 +- integration_tests/test_wcs_server.py | 76 +-- integration_tests/test_wms_server.py | 3 +- integration_tests/utils.py | 2 +- tests/cfg/minimal_cfg.py | 1 + tests/conftest.py | 28 +- tests/test_cfg_global.py | 4 +- tests/test_cfg_layer.py | 36 +- tests/test_cfg_wcs.py | 6 +- tests/test_no_db_routes.py | 38 +- tests/test_styles.py | 2 +- tests/test_update_ranges.py | 73 +++ tests/test_wms_utils.py | 28 +- wordlist.txt | 5 + 87 files changed, 963 insertions(+), 1017 deletions(-) create mode 100644 datacube_ows/sql/extent_views/create/003_create_view_owner_role_ignore_duplicates.sql rename datacube_ows/sql/extent_views/create/{011_create_new_space_view_raw.sql => 011_create_new_space_view.sql} (97%) create mode 100644 datacube_ows/sql/extent_views/create/032_drop_old_space_time_view.sql delete mode 100644 datacube_ows/sql/extent_views/create/040_drop_old_space_time_view.sql create mode 100644 datacube_ows/sql/extent_views/create/040_drop_old_space_view.sql delete mode 100644 datacube_ows/sql/extent_views/create/042_drop_old_space_view.sql rename datacube_ows/sql/extent_views/create/{050_rename_new_space_view.sql => 042_rename_new_space_view.sql} (53%) rename datacube_ows/sql/extent_views/create/{051_rename_new_time_view.sql => 043_rename_new_time_view.sql} (53%) rename datacube_ows/sql/extent_views/create/{052_rename_index_1.sql => 050_rename_index_1.sql} (64%) rename datacube_ows/sql/extent_views/create/{053_rename_index_2.sql => 051_rename_index_2.sql} (64%) rename datacube_ows/sql/extent_views/create/{054_rename_index_3.sql => 052_rename_index_3.sql} (65%) rename datacube_ows/sql/extent_views/create/{055_rename_index_4.sql => 053_rename_index_4.sql} (65%) delete mode 100644 datacube_ows/sql/extent_views/create/060_grant.sql create mode 100644 datacube_ows/sql/extent_views/grants/read_only/001_grant_read_requires_role.sql create mode 100644 datacube_ows/sql/extent_views/grants/refresh_owner/001_set_owner_time_view.sql create mode 100644 datacube_ows/sql/extent_views/grants/refresh_owner/002_set_owner_space_view.sql create mode 100644 datacube_ows/sql/extent_views/grants/refresh_owner/003_set_owner_spacetime_view.sql create mode 100644 datacube_ows/sql/extent_views/grants/write_refresh/001_grant_refresh_requires_role.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/001_drop_space_time_view.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/002_drop_time_view.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/003_drop_space_view.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/010_drop_subproduct_range.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/011_drop_multiproduct_range.sql create mode 100644 datacube_ows/sql/ows_schema/cleanup/012_drop_product_range.sql create mode 100644 datacube_ows/sql/ows_schema/create/001_create_schema.sql rename datacube_ows/sql/{wms_schema => ows_schema}/create/002_create_product_rng.sql (51%) create mode 100644 datacube_ows/sql/ows_schema/grants/read_only/001_grant_usage_requires_role.sql create mode 100644 datacube_ows/sql/ows_schema/grants/read_write/001_grant_usage_requires_role.sql create mode 100644 datacube_ows/sql/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql delete mode 100644 datacube_ows/sql/use_space_time.sql delete mode 100644 datacube_ows/sql/wms_schema/create/001_create_schema.sql delete mode 100644 datacube_ows/sql/wms_schema/create/003_create_subproduct_rng.sql delete mode 100644 datacube_ows/sql/wms_schema/create/004_create_multiproduct_rng.sql delete mode 100644 datacube_ows/sql/wms_schema/create/005_grant_requires_role.sql delete mode 100644 docs/diagrams/db-relationship-diagram.svg create mode 100644 tests/test_update_ranges.py diff --git a/.env_simple b/.env_simple index fa2e5ea7a..bdbacd3c0 100644 --- a/.env_simple +++ b/.env_simple @@ -4,7 +4,8 @@ ################ # ODC DB Config # ############## -DB_HOSTNAME=postgres +ODC_DEFAULT_DB_URL=postgresql://opendatacubeusername:opendatacubepassword@postgres:5432/opendatacube +# Needed for docker db image. DB_PORT=5432 DB_USERNAME=opendatacubeusername DB_PASSWORD=opendatacubepassword diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d1de5dfa7..f4d992c37 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -95,7 +95,7 @@ Before you submit a pull request, check that it meets these guidelines: 1. The pull request should include tests (and should pass them - and all pre-existing tests!) 2. If the pull request adds or modifies functionality, the docs should be updated. -3. The pull request should work for Python 3.7+. Check the results of +3. The pull request should work for Python 3.10+. Check the results of the github actions and make sure that your PR passes all checks and does not decrease test coverage. @@ -143,8 +143,9 @@ indexing and create db dump # now go to ows container docker exec -it datacube-ows_ows_1 bash - datacube-ows-update --schema --role - datacube-ows-update --views + # Run this a database superuser role + datacube-ows-update --schema --read-role --write-role + # Run this as the user above datacube-ows-update exit @@ -178,22 +179,6 @@ manually modify translation for `de` for `assert` test to pass, then create `ows docker cp datacube-ows_ows_1:/tmp/translations datacube-ows/integrations/cfg/ -Generating database relationship diagram ----------------------------------------- - -.. code-block:: console - - docker run -it --rm -v "$PWD:/output" --network="host" schemaspy/schemaspy:snapshot -u $DB_USERNAME -host localhost -port $DB_PORT -db $DB_DATABASE -t pgsql11 -schemas wms -norows -noviews -pfp -imageformat svg - -Merge relationship diagram and orphan diagram - -.. code-block:: console - - python3 svg_stack.py --direction=h --margin=100 ../wms/diagrams/summary/relationships.real.large.svg ../wms/diagrams/orphans/orphans.svg > ows.merged.large.svg - - cp svg_stack/ows.merged.large.svg ../datacube-ows/docs/diagrams/db-relationship-diagram.svg - - Links ----- diff --git a/README.rst b/README.rst index bf221034f..f906ded41 100644 --- a/README.rst +++ b/README.rst @@ -3,13 +3,13 @@ datacube-ows ============ .. image:: https://github.com/opendatacube/datacube-ows/workflows/Linting/badge.svg - :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3ALinting + :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3ACode%20Linting .. image:: https://github.com/opendatacube/datacube-ows/workflows/Tests/badge.svg :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3ATests .. image:: https://github.com/opendatacube/datacube-ows/workflows/Docker/badge.svg - :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3ADocker + :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3ADockerfile%20Linting .. image:: https://github.com/opendatacube/datacube-ows/workflows/Scan/badge.svg :target: https://github.com/opendatacube/datacube-ows/actions?query=workflow%3A%22Scan%22 @@ -17,9 +17,6 @@ datacube-ows .. image:: https://codecov.io/gh/opendatacube/datacube-ows/branch/master/graph/badge.svg :target: https://codecov.io/gh/opendatacube/datacube-ows -.. image:: https://img.shields.io/pypi/v/datacube?label=datacube - :alt: PyPI - Datacube Open Web Services -------------------------- @@ -41,7 +38,7 @@ Features System Architecture ------------------- -.. image:: docs/diagrams/ows_diagram.png +.. image:: docs/diagrams/ows_diagram1.9.png :width: 700 Community @@ -141,14 +138,14 @@ To run the standard Docker image, create a docker volume containing your ows con -e AWS_DEFAULT_REGION=ap-southeast-2 \ # AWS Default Region (supply even if NOT accessing files on S3! See Issue #151) -e SENTRY_DSN=https://key@sentry.local/projid \ # Key for Sentry logging (optional) \ # Database connection URL: postgresql://:@:/ - -e ODC_DEFAULT_DB_URL=postgresql://cube:DataCube@172.17.0.1:5432/datacube \ + -e ODC_DEFAULT_DB_URL=postgresql://myuser:mypassword@172.17.0.1:5432/mydb \ -e PYTHONPATH=/code # The default PATH is under env, change this to target /code -p 8080:8000 \ # Publish the gunicorn port (8000) on the Docker \ # container at port 8008 on the host machine. --mount source=test_cfg,target=/code/datacube_ows/config \ # Mount the docker volume where the config lives name_of_built_container -The image is based on the standard ODC container. +The image is based on the standard ODC container and an external database Installation with Conda ------------ @@ -157,7 +154,7 @@ The following instructions are for installing on a clean Linux system. * Create a conda python 3.8 and activate conda environment:: - conda create -n ows -c conda-forge python=3.8 datacube pre_commit postgis + conda create -n ows -c conda-forge python=3.10 datacube pre_commit postgis conda activate ows * install the latest release using pip install:: @@ -186,7 +183,7 @@ The following instructions are for installing on a clean Linux system. # to create schema, tables and materialised views used by datacube-ows. export DATACUBE_OWS_CFG=datacube_ows.ows_cfg_example.ows_cfg - datacube-ows-update --role ubuntu --schema + datacube-ows-update --write-role ubuntu --schema * Create a configuration file for your service, and all data products you wish to publish in @@ -253,8 +250,9 @@ Local Postgres database | xargs -n1 -I {} datacube dataset add s3://deafrica-data/{} 5. Write an ows config file to identify the products you want available in ows, see example here: https://github.com/opendatacube/datacube-ows/blob/master/datacube_ows/ows_cfg_example.py -6. Run `datacube-ows-update --schema --role ` to create ows specific tables -7. Run `datacube-ows-update` to generate ows extents. +6. Run ``datacube-ows-update --schema --read-role --write-role `` as a database + superuser role to create ows specific tables and views +7. Run ``datacube-ows-update`` as ``db_write_role`` to populate ows extent tables. Apache2 mod_wsgi ---------------- diff --git a/check-code-all.sh b/check-code-all.sh index 8ac70aa07..da11d7c1a 100755 --- a/check-code-all.sh +++ b/check-code-all.sh @@ -21,7 +21,7 @@ datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-c # Geomedian for summary product testing -datacube product add https://explorer-aws.dea.ga.gov.au/products/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/geomedian-au/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml # S2 multiproduct datasets datacube dataset add https://dea-public-data.s3.ap-southeast-2.amazonaws.com/baseline/ga_s2bm_ard_3/52/LGM/2017/07/19/20170719T030622/ga_s2bm_ard_3-2-1_52LGM_2017-07-19_final.odc-metadata.yaml --ignore-lineage @@ -44,7 +44,7 @@ datacube dataset add https://dea-public-data.s3.ap-southeast-2.amazonaws.com/der datacube dataset add https://dea-public-data.s3.ap-southeast-2.amazonaws.com/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x17/y37/2021--P1Y/ga_ls8c_nbart_gm_cyear_3_x17y37_2021--P1Y_final.odc-metadata.yaml --ignore-lineage # create material view for ranges extents -datacube-ows-update --schema --role $DB_USERNAME +datacube-ows-update --schema --write-role $DB_USERNAME datacube-ows-update # run test diff --git a/datacube_ows/cfg_parser_impl.py b/datacube_ows/cfg_parser_impl.py index 502931299..02834b229 100755 --- a/datacube_ows/cfg_parser_impl.py +++ b/datacube_ows/cfg_parser_impl.py @@ -117,12 +117,12 @@ def parse_path(path: str | None, parse_only: bool, folders: bool, styles: bool, click.echo() click.echo("Layers and Styles") click.echo("=================") - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): click.echo(f"{lyr.name} [{','.join(lyr.product_names)}]") print_styles(lyr) click.echo() if input_file or output_file: - layers_report(cfg.product_index, input_file, output_file) + layers_report(cfg.layer_index, input_file, output_file) return True diff --git a/datacube_ows/loading.py b/datacube_ows/loading.py index 330037694..f6addba94 100644 --- a/datacube_ows/loading.py +++ b/datacube_ows/loading.py @@ -125,37 +125,37 @@ def simple_layer_query(cls, layer: OWSNamedLayer, class DataStacker: @log_call def __init__(self, - product: OWSNamedLayer, + layer: OWSNamedLayer, geobox: GeoBox, times: list[datetime.datetime], resampling: Resampling | None = None, style: StyleDef | None = None, bands: list[str] | None = None): - self._product = product - self.cfg = product.global_cfg + self._layer = layer + self.cfg = layer.global_cfg self._geobox = geobox self._resampling = resampling if resampling is not None else "nearest" self.style = style if style: self._needed_bands = list(style.needed_bands) elif bands: - self._needed_bands = [self._product.band_idx.locale_band(b) for b in bands] + self._needed_bands = [self._layer.band_idx.locale_band(b) for b in bands] else: - self._needed_bands = list(self._product.band_idx.measurements.keys()) + self._needed_bands = list(self._layer.band_idx.measurements.keys()) - for band in self._product.always_fetch_bands: + for band in self._layer.always_fetch_bands: if band not in self._needed_bands: self._needed_bands.append(band) self.raw_times = times - if product.mosaic_date_func: - self._times = [product.mosaic_date_func(product.ranges["times"])] + if self._layer.mosaic_date_func: + self._times = [self._layer.mosaic_date_func(layer.ranges.times)] else: self._times = [ - self._product.search_times( + self._layer.search_times( t, self._geobox) for t in times ] - self.group_by = self._product.dataset_groupby() + self.group_by = self._layer.dataset_groupby() self.resource_limited = False def needed_bands(self) -> list[str]: @@ -185,7 +185,7 @@ def datasets(self, index: datacube.index.Index, # Not returning datasets - use main product only queries = [ ProductBandQuery.simple_layer_query( - self._product, + self._layer, self.needed_bands(), self.resource_limited) @@ -194,10 +194,10 @@ def datasets(self, index: datacube.index.Index, # we have a style - lets go with that. queries = ProductBandQuery.style_queries(self.style) elif all_flag_bands: - queries = ProductBandQuery.full_layer_queries(self._product, self.needed_bands()) + queries = ProductBandQuery.full_layer_queries(self._layer, self.needed_bands()) else: # Just take needed bands. - queries = [ProductBandQuery.simple_layer_query(self._product, self.needed_bands())] + queries = [ProductBandQuery.simple_layer_query(self._layer, self.needed_bands())] if point: geom = point @@ -338,14 +338,14 @@ def manual_data_stack(self, d = self.read_data_for_single_dataset(ds, measurements, self._geobox, fuse_func=fuse_func) extent_mask = None for band in non_flag_bands: - for f in self._product.extent_mask_func: + for f in self._layer.extent_mask_func: if extent_mask is None: extent_mask = f(d, band) else: extent_mask &= f(d, band) if extent_mask is not None: d = d.where(extent_mask) - if self._product.solar_correction and not skip_corrections: + if self._layer.solar_correction and not skip_corrections: for band in non_flag_bands: d[band] = solar_correct_data(d[band], ds) if merged is None: @@ -383,7 +383,7 @@ def read_data(self, measurements=measurements, fuse_func=fuse_func, skip_broken_datasets=skip_broken, - patch_url=self._product.patch_url, + patch_url=self._layer.patch_url, resampling=resampling) except Exception as e: _LOG.error("Error (%s) in load_data: %s", e.__class__.__name__, str(e)) @@ -399,7 +399,7 @@ def read_data_for_single_dataset(self, resampling: Resampling = "nearest", fuse_func: datacube.api.core.FuserFunction | None = None) -> xarray.Dataset: datasets = [dataset] - dc_datasets = datacube.Datacube.group_datasets(datasets, self._product.time_resolution.dataset_groupby()) + dc_datasets = datacube.Datacube.group_datasets(datasets, self._layer.time_resolution.dataset_groupby()) CredentialManager.check_cred() try: return datacube.Datacube.load_data( @@ -408,7 +408,7 @@ def read_data_for_single_dataset(self, measurements=measurements, fuse_func=fuse_func, skip_broken_datasets=skip_broken, - patch_url=self._product.patch_url, + patch_url=self._layer.patch_url, resampling=resampling) except Exception as e: _LOG.error("Error (%s) in load_data: %s", e.__class__.__name__, str(e)) diff --git a/datacube_ows/mv_index.py b/datacube_ows/mv_index.py index 1dc1572ad..7668ca22d 100644 --- a/datacube_ows/mv_index.py +++ b/datacube_ows/mv_index.py @@ -35,11 +35,11 @@ def get_sqlalc_engine(index: Index) -> Engine: def get_st_view(meta: MetaData) -> Table: return Table('space_time_view', meta, - Column('id', UUID()), - Column('dataset_type_ref', SMALLINT()), - Column('spatial_extent', Geometry(from_text='ST_GeomFromGeoJSON', name='geometry')), - Column('temporal_extent', TSTZRANGE()) - ) + Column('id', UUID()), + Column('dataset_type_ref', SMALLINT()), + Column('spatial_extent', Geometry(from_text='ST_GeomFromGeoJSON', name='geometry')), + Column('temporal_extent', TSTZRANGE()), + schema="ows") _meta = MetaData() diff --git a/datacube_ows/ogc.py b/datacube_ows/ogc.py index f8cdf50ef..23a61bd0a 100644 --- a/datacube_ows/ogc.py +++ b/datacube_ows/ogc.py @@ -179,14 +179,17 @@ def ogc_wcs_impl(): def ping(): db_ok = False cfg = get_config() - with cfg.dc.index._db.give_me_a_connection() as conn: - results = conn.execute(text(""" - SELECT * - FROM wms.product_ranges - LIMIT 1""") - ) - for r in results: - db_ok = True + try: + with cfg.dc.index._db.give_me_a_connection() as conn: + results = conn.execute(text(""" + SELECT * + FROM ows.layer_ranges + LIMIT 1""") + ) + for r in results: + db_ok = True + except Exception: + pass if db_ok: return (render_template("ping.html", status="Up"), 200, resp_headers({"Content-Type": "text/html"})) else: @@ -202,7 +205,7 @@ def ping(): def legend(layer, style, dates=None): # pylint: disable=redefined-outer-name cfg = get_config() - product = cfg.product_index.get(layer) + product = cfg.layer_index.get(layer) if not product: return ("Unknown Layer", 404, resp_headers({"Content-Type": "text/plain"})) if dates is None: diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index f2e83684f..26618f379 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -50,6 +50,10 @@ from datacube_ows.utils import (group_by_begin_datetime, group_by_mosaic, group_by_solar) +TYPE_CHECKING = False +if TYPE_CHECKING: + from datacube_ows.product_ranges import LayerExtent + _LOG = logging.getLogger(__name__) @@ -247,7 +251,8 @@ def __init__(self, cfg: CFG_DICT, object_label: str, parent_layer: Optional["OWS self.object_label = object_label self.global_cfg: "OWSConfig" = kwargs["global_cfg"] self.parent_layer = parent_layer - + self._cached_local_env: ODCEnvironment | None = None + self._cached_dc: Datacube | None = None self.parse_metadata(cfg) # Inherit or override attribution if "attribution" in cfg: @@ -260,6 +265,57 @@ def __init__(self, cfg: CFG_DICT, object_label: str, parent_layer: Optional["OWS else: self.attribution = self.global_cfg.attribution + @property + def local_env(self) -> ODCEnvironment: + # If we have cached inherited environment, use it. + if self._cached_local_env: + return self._cached_local_env + + # If we have our own custom environment, use it. + if hasattr(self, "_local_env") and self._local_env is not None: + return self._local_env + + # If we have a parent layer, then it's their problem. + if self.parent_layer: + # remember, so we don't have to ask our parent layer again. As a layer we must learn to adult. + self._cached_local_env = self.parent_layer.local_env + else: + # If we have no parent layer, then we have to ask the global government. + # and remember, so we don't have to deal with the global government again. + self._cached_local_env = self.global_cfg.default_env + + return self._cached_local_env + + @property + def dc(self) -> Datacube: + # If we have cached inherited datacube, use it. + if self._cached_dc: + return self._cached_dc + + # If we have our own custom dc, use it. + if hasattr(self, "_dc"): + # pylint: disable=access-member-before-definition + return self._dc # type: ignore[has-type] + + # If we have our own custom environment, try to make a Datacube from it: + if hasattr(self, "_local_env") and self._local_env is not None: + try: + self._dc = Datacube(env=self._local_env, app=self.global_cfg.odc_app) + return self._dc + except Exception as e: + raise ODCInitException(str(e)) from None + + # If we have a parent layer, then it's their problem. + if self.parent_layer: + # remember, so we don't have to ask our parent layer again. As a layer we must learn to adult. + self._cached_dc = self.parent_layer.dc + else: + # If we have no parent layer, then we have to ask the global government. + # and remember, so we don't have to deal with the global government again. + self._cached_dc = self.global_cfg.dc + + return self._cached_dc + def global_config(self) -> "OWSConfig": return self.global_cfg @@ -403,6 +459,7 @@ def dataset_groupby(self, product_names: list[str] | None = None, is_mosaic=Fals class OWSNamedLayer(OWSExtensibleConfigEntry, OWSLayer): INDEX_KEYS = ["layer"] named = True + multi_product: bool = False def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFolder | None = None, **kwargs): name = cast(str, cfg["name"]) @@ -412,15 +469,6 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold self.name = name cfg = cast(CFG_DICT, self._raw_cfg) self.hide = False - self.local_env: ODCEnvironment | None = None - local_env = cast(str | None, cfg.get("env")) - self.local_env = ODCConfig.get_environment(env=local_env) - # TODO: MULTIDB_SUPPORT - # After refactoring the range tables, Uncomment this code for multi-database support - # (Don't forget to add to documentation) - # - # if local_env: - # self.local_env = ODCConfig.get_environment(env=local_env) try: self.parse_product_names(cfg) if len(self.low_res_product_names) not in (0, len(self.product_names)): @@ -560,10 +608,10 @@ def __init__(self, cfg: CFG_DICT, global_cfg: "OWSConfig", parent_layer: OWSFold # else: # self.sub_product_extractor = None # And finally, add to the global product index. - existing = self.global_cfg.product_index.get(self.name) + existing = self.global_cfg.layer_index.get(self.name) if existing and existing != self: raise ConfigException(f"Duplicate layer name: {self.name}") - self.global_cfg.product_index[self.name] = self + self.global_cfg.layer_index[self.name] = self def time_axis_representation(self) -> str: if self.regular_time_axis: @@ -573,18 +621,6 @@ def time_axis_representation(self) -> str: # pylint: disable=attribute-defined-outside-init def make_ready(self, *args: Any, **kwargs: Any) -> None: - # TODO: MULTIDB_SUPPORT - # After refactoring the range tables, Uncomment this code for multi-database support - # (Don't forget to add to documentation) - # - # if self.local_env: - # try: - # self.dc: Datacube = Datacube(env=self.local_env, app=self.global_cfg.odc_app) - # except Exception as e: - # _LOG.error("ODC initialisation failed: %s", str(e)) - # raise ODCInitException(e) - # else: - self.dc = self.global_cfg.dc self.products: list[Product] = [] self.low_res_products: list[Product] = [] for i, prod_name in enumerate(self.product_names): @@ -909,10 +945,10 @@ def force_range_update(self) -> None: raise Exception("Null product range") self.bboxes = self.extract_bboxes() if self.default_time_rule == DEF_TIME_EARLIEST: - self.default_time = cast(datetime.datetime | datetime.date, self._ranges["start_time"]) + self.default_time = cast(datetime.datetime | datetime.date, self._ranges.start_time) elif isinstance(self.default_time_rule, datetime.date) and self.default_time_rule in cast(set[datetime.datetime | datetime.date], - self._ranges["time_set"]): + self._ranges.time_set): self.default_time = cast(datetime.datetime | datetime.date, self.default_time_rule) elif isinstance(self.default_time_rule, datetime.date): _LOG.warning("default_time for named_layer %s is explicit date (%s) that is " @@ -920,9 +956,9 @@ def force_range_update(self) -> None: self.name, self.default_time_rule.isoformat() ) - self.default_time = cast(datetime.datetime | datetime.date, self._ranges["end_time"]) + self.default_time = cast(datetime.datetime | datetime.date, self._ranges.end_time) else: - self.default_time = cast(datetime.datetime | datetime.date, self._ranges["end_time"]) + self.default_time = cast(datetime.datetime | datetime.date, self._ranges.end_time) # pylint: disable=broad-except except Exception as a: @@ -931,21 +967,23 @@ def force_range_update(self) -> None: self.hide = True self.bboxes = {} - def time_range(self, ranges: dict[str, Any] | None = None): + def time_range(self, + ranges: Optional["LayerExtent"] = None + ) -> tuple[datetime.datetime | datetime.date, datetime.datetime | datetime.date]: if ranges is None: ranges = self.ranges if self.regular_time_axis and self.time_axis_start: start = self.time_axis_start else: - start = ranges["times"][0] + start = ranges.start_time if self.regular_time_axis and self.time_axis_end: end = self.time_axis_end else: - end = ranges["times"][-1] + end = ranges.end_time return (start, end) @property - def ranges(self) -> dict[str, Any]: + def ranges(self) -> "LayerExtent": if self.dynamic: self.force_range_update() assert self._ranges is not None # For type checker @@ -955,7 +993,7 @@ def extract_bboxes(self) -> dict[str, Any]: if self._ranges is None: return {} bboxes = {} - for crs_id, bbox in cast(dict[str, dict[str, float]], self._ranges["bboxes"]).items(): + for crs_id, bbox in cast(dict[str, dict[str, float]], self._ranges.bboxes).items(): if crs_id in self.global_cfg.published_CRSs: # Assume we've already handled coordinate swapping for # Vertical-coord first CRSs. Top is top, left is left. @@ -973,7 +1011,7 @@ def layer_count(self) -> int: def search_times(self, t, geobox=None): if not geobox: - bbox = self.ranges["bboxes"][self.native_CRS] + bbox = self.ranges.bboxes[self.native_CRS] geobox = create_geobox( self.native_CRS, bbox["left"], bbox["bottom"], bbox["right"], bbox["top"], @@ -990,7 +1028,7 @@ def __str__(self): @classmethod def lookup_impl(cls, cfg: "OWSConfig", keyvals: dict[str, str], subs: CFG_DICT | None = None): try: - return cfg.product_index[keyvals["layer"]] + return cfg.layer_index[keyvals["layer"]] except KeyError: raise OWSEntryNotFound(f"Layer {keyvals['layer']} not found") @@ -1204,7 +1242,7 @@ def default_abstract(self) -> Optional[str]: @property def active_products(self) -> Iterable[OWSNamedLayer]: - return filter(lambda x: not x.hide, self.product_index.values()) + return filter(lambda x: not x.hide, self.layer_index.values()) @property def active_product_index(self) -> dict[str, OWSNamedLayer]: @@ -1255,6 +1293,9 @@ def __init__(self, refresh=False, cfg: CFG_DICT | None = None, ) self.catalog: Catalog | None = None self.initialised = True + self.declare_unready("dc") + self.declare_unready("crses") + self.declare_unready("native_product_index") #pylint: disable=attribute-defined-outside-init def make_ready(self, *args: Any, **kwargs: Any) -> None: @@ -1271,6 +1312,7 @@ def make_ready(self, *args: Any, **kwargs: Any) -> None: _LOG.warning("Message file %s does not exist - using metadata from config file", self.msg_file_name) else: self.set_msg_src(None) + self.crses = {s: self.crs(s) for s in self.published_CRSs} self.native_product_index: dict[str, OWSNamedLayer] = {} self.root_layer_folder.make_ready(*args, **kwargs) super().make_ready(*args, **kwargs) @@ -1311,8 +1353,9 @@ def export_metadata(self) -> Catalog: return self.catalog def parse_global(self, cfg: CFG_DICT, ignore_msgfile: bool): - self.default_env = cast(str, cfg.get("env")) - self.odc_app = cast(str, cfg.get("odc_app", "ows")) + default_env = cast(str, cfg.get("env")) + self.default_env = ODCConfig.get_environment(env=default_env) + self.odc_app = cast(str, cfg.get("odc_app", "datacube-ows")) self._response_headers = cast(dict[str, str], cfg.get("response_headers", {})) services = cast(dict[str, bool], cfg.get("services", {})) self.wms = services.get("wms", True) @@ -1461,7 +1504,7 @@ def parse_wmts(self, cfg: CFG_DICT): def parse_layers(self, cfg: list[CFG_DICT]): self.folder_index: dict[str, OWSFolder] = {} - self.product_index: dict[str, OWSNamedLayer] = {} + self.layer_index: dict[str, OWSNamedLayer] = {} self.declare_unready("native_product_index") self.root_layer_folder = OWSFolder(cast(CFG_DICT, { "title": "Root Folder (hidden)", diff --git a/datacube_ows/product_ranges.py b/datacube_ows/product_ranges.py index 8e87376de..26b74da74 100644 --- a/datacube_ows/product_ranges.py +++ b/datacube_ows/product_ranges.py @@ -3,23 +3,30 @@ # # Copyright (c) 2017-2024 OWS Contributors # SPDX-License-Identifier: Apache-2.0 - - +import dataclasses #pylint: skip-file +import logging import math +import click from datetime import date, datetime, timezone -from typing import Any, Callable, Iterable, cast +from typing import cast, Callable, Iterable, NamedTuple import datacube import odc.geo +import sqlalchemy.exc from psycopg2.extras import Json from sqlalchemy import text -from datacube_ows.ows_configuration import (OWSConfig, OWSMultiProductLayer, - OWSNamedLayer, TimeRes, get_config) +from odc.geo.geom import Geometry + +from datacube_ows.config_utils import CFG_DICT +from datacube_ows.ows_configuration import OWSConfig, OWSNamedLayer, get_config +from datacube_ows.mv_index import MVSelectOpts, mv_search from datacube_ows.utils import get_sqlconn +_LOG = logging.getLogger(__name__) + def get_crsids(cfg: OWSConfig | None = None) -> Iterable[str]: if not cfg: @@ -42,234 +49,169 @@ def jsonise_bbox(bbox: odc.geo.geom.BoundingBox) -> dict[str, float]: "right": bbox.right, } +@dataclasses.dataclass(frozen=True) +class LayerSignature: + time_res: str + products: tuple[str, ...] + env: str + datasets: int -def create_multiprod_range_entry(dc: datacube.Datacube, product: OWSMultiProductLayer, - crses: dict[str, odc.geo.CRS]) -> None: - print("Merging multiproduct ranges for %s (ODC products: %s)" % ( - product.name, - repr(product.product_names) - )) - conn = get_sqlconn(dc) - prodids = [p.id for p in product.products] - wms_name = product.name - - if all( - not datasets_exist(dc, p_name) - for p_name in product.product_names - ): - print("Could not find datasets for any product in multiproduct: ", product.name) - conn.close() - print("Done") - return + def as_json(self) -> dict[str, list[str] | str | int]: + return { + "time_res": self.time_res, + "products": list(self.products), + "env": self.env, + "datasets": self.datasets, + } + + +def create_range_entry(layer: OWSNamedLayer, cache: dict[LayerSignature, list[str]]) -> None: + meta = LayerSignature(time_res=layer.time_resolution.value, + products=tuple(layer.product_names), + env=layer.local_env._name, + datasets=layer.dc.index.datasets.count(product=layer.product_names)) + print(f"Updating range for layer {layer.name}") + conn = get_sqlconn(layer.dc) txn = conn.begin() - # Attempt to insert row - conn.execute(text(""" - INSERT INTO wms.multiproduct_ranges - (wms_product_name,lat_min,lat_max,lon_min,lon_max,dates,bboxes) + if meta in cache: + template = cache[meta][0] + print(f"Layer {template} has same signature - reusing") + cache[meta].append(layer.name) + try: + conn.execute(text(""" + INSERT INTO ows.layer_ranges + (layer, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated) + SELECT :layer_id, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated + FROM ows.layer_ranges lr2 + WHERE lr2.layer = :template_id"""), + { + "layer_id": layer.name, + "template_id": template + }) + except sqlalchemy.exc.IntegrityError: + conn.execute(text(""" + UPDATE ows.layer_ranges lr1 + SET lat_min = lr2.lat_min, + lat_max = lr2.lat_max, + lon_min = lr2.lon_min, + lon_max = lr2.lon_max, + dates = lr2.dates, + bboxes = lr2.bboxes, + meta = lr2.meta, + last_updated = lr2.last_updated + FROM ows.layer_ranges lr2 + WHERE lr1.layer = :layer_id + AND lr2.layer = :template_id"""), + { + "layer_id": layer.name, + "template_id": template + }) + else: + # insert empty row if one does not already exist + conn.execute(text(""" + INSERT INTO ows.layer_ranges + (layer, lat_min, lat_max, lon_min, lon_max, dates, bboxes, meta, last_updated) VALUES - (:p_id, 0, 0, 0, 0, :empty, :empty) - ON CONFLICT (wms_product_name) DO NOTHING - """), - {"p_id": wms_name, "empty": Json("")}) - - # Update extents - conn.execute(text(""" - UPDATE wms.multiproduct_ranges - SET lat_min = subq.lat_min, - lat_max = subq.lat_max, - lon_min = subq.lon_min, - lon_max = subq.lon_max - FROM ( - select min(lat_min) as lat_min, - max(lat_max) as lat_max, - min(lon_min) as lon_min, - max(lon_max) as lon_max - from wms.product_ranges - where id = ANY (:p_prodids) - ) as subq - WHERE wms_product_name = :p_id + (:p_layer, 0, 0, 0, 0, :empty, :empty, :meta, :now) + ON CONFLICT (layer) DO NOTHING """), - {"p_id": wms_name, "p_prodids": prodids}) - - # Create sorted list of dates - results = conn.execute(text( - """ - SELECT dates - FROM wms.product_ranges - WHERE id = ANY (:p_prodids) - """), {"p_prodids": prodids} - ) - dates = set() - for r in results: - for d in r[0]: - dates.add(d) - dates = sorted(dates) - conn.execute(text(""" - UPDATE wms.multiproduct_ranges + { + "p_layer": layer.name, "empty": Json(""), + "meta": Json(meta.as_json()), "now": datetime.now(tz=timezone.utc) + }) + + prodids = [p.id for p in layer.products] + # Update min/max lat/longs + conn.execute(text( + """ + UPDATE ows.layer_ranges lr + SET lat_min = st_ymin(subq.bbox), + lat_max = st_ymax(subq.bbox), + lon_min = st_xmin(subq.bbox), + lon_max = st_xmax(subq.bbox) + FROM ( + SELECT st_extent(stv.spatial_extent) as bbox + FROM ows.space_time_view stv + WHERE stv.dataset_type_ref = ANY(:prodids) + ) as subq + WHERE lr.layer = :layer_id + """), + {"layer_id": layer.name, "prodids": prodids}) + + # Set default timezone + conn.execute(text("""set timezone to 'Etc/UTC'""")) + + # Loop over dates + dates = set() # Should get to here! + if layer.time_resolution.is_solar(): + results = conn.execute(text( + """ + select + lower(temporal_extent), upper(temporal_extent), + ST_X(ST_Centroid(spatial_extent)) + from ows.space_time_view + WHERE dataset_type_ref = ANY(:prodids) + """), + {"prodids": prodids}) + for result in results: + dt1, dt2, lon = result + dt = dt1 + (dt2 - dt1) / 2 + dt = dt.astimezone(timezone.utc) + + solar_day = datacube.api.query._convert_to_solar_time(dt, lon).date() + dates.add(solar_day) + else: + results = conn.execute(text( + """ + select + array_agg(temporal_extent) + from ows.space_time_view + WHERE dataset_type_ref = ANY(:prodids) + """), + {"prodids": prodids} + ) + for result in results: + for dat_ran in result[0]: + dates.add(dat_ran.lower) + + if layer.time_resolution.is_subday(): + date_formatter = lambda d: d.isoformat() + else: + date_formatter = lambda d: d.strftime("%Y-%m-%d") + + dates = sorted(dates) + conn.execute(text(""" + UPDATE ows.layer_ranges SET dates = :dates - WHERE wms_product_name= :p_id - """), - { - "dates": Json(dates), - "p_id": wms_name - } - ) - - # calculate bounding boxes - results = list(conn.execute(text(""" - SELECT lat_min,lat_max,lon_min,lon_max - FROM wms.multiproduct_ranges - WHERE wms_product_name=:p_id + WHERE layer= :layer_id """), - {"p_id": wms_name})) + { + "dates": Json(list(map(date_formatter, dates))), + "layer_id": layer.name + } + ) + print("Dates written") - r = results[0] + # calculate bounding boxes + # Get extent polygon from materialised views - epsg4326 = odc.geo.CRS("EPSG:4326") - box = odc.geo.geom.box( - float(r[2]), - float(r[0]), - float(r[3]), - float(r[1]), - epsg4326) + extent_4386 = cast(Geometry, mv_search(layer.dc.index, MVSelectOpts.EXTENT, products=layer.products)) - cfg = get_config() - conn.execute(text(""" - UPDATE wms.multiproduct_ranges + all_bboxes = bbox_projections(extent_4386, layer.global_cfg.crses) + + conn.execute(text(""" + UPDATE ows.layer_ranges SET bboxes = :bbox - WHERE wms_product_name=:pname - """), - { - "bbox": Json({crsid: jsonise_bbox(box.to_crs(crs).boundingbox) for crsid, crs in get_crses(cfg).items()}), - "pname": wms_name - } - ) + WHERE layer = :layer_id + """), { + "bbox": Json(all_bboxes), + "layer_id": layer.name}) + + cache[meta] = [layer.name] txn.commit() conn.close() - return - - -def create_range_entry(dc: datacube.Datacube, product: datacube.model.Product, - crses: dict[str, odc.geo.CRS], time_resolution: TimeRes) -> None: - print("Updating range for ODC product %s..." % product.name) - # NB. product is an ODC product - conn = get_sqlconn(dc) - txn = conn.begin() - prodid = product.id - - # insert empty row if one does not already exist - conn.execute(text(""" - INSERT INTO wms.product_ranges - (id,lat_min,lat_max,lon_min,lon_max,dates,bboxes) - VALUES - (:p_id, 0, 0, 0, 0, :empty, :empty) - ON CONFLICT (id) DO NOTHING - """), - {"p_id": prodid, "empty": Json("")}) - - - # Update min/max lat/longs - conn.execute(text( - """ - UPDATE wms.product_ranges pr - SET lat_min = st_ymin(subq.bbox), - lat_max = st_ymax(subq.bbox), - lon_min = st_xmin(subq.bbox), - lon_max = st_xmax(subq.bbox) - FROM ( - SELECT st_extent(stv.spatial_extent) as bbox - FROM public.space_time_view stv - WHERE stv.dataset_type_ref = :p_id - ) as subq - WHERE pr.id = :p_id - """), - {"p_id": prodid}) - - # Set default timezone - conn.execute(text("""set timezone to 'Etc/UTC'""")) - - # Loop over dates - dates = set() - if time_resolution.is_solar(): - results = conn.execute(text( - """ - select - lower(temporal_extent), upper(temporal_extent), - ST_X(ST_Centroid(spatial_extent)) - from public.space_time_view - WHERE dataset_type_ref = :p_id - """), - {"p_id": prodid}) - for result in results: - dt1, dt2, lon = result - dt = dt1 + (dt2 - dt1) / 2 - dt = dt.astimezone(timezone.utc) - - solar_day = datacube.api.query._convert_to_solar_time(dt, lon).date() - dates.add(solar_day) - else: - results = conn.execute(text( - """ - select - array_agg(temporal_extent) - from public.space_time_view - WHERE dataset_type_ref = :p_id - """), - {"p_id": prodid} - ) - for result in results: - for dat_ran in result[0]: - dates.add(dat_ran.lower) - - if time_resolution.is_subday(): - date_formatter = lambda d: d.isoformat() - else: - date_formatter = lambda d: d.strftime("%Y-%m-%d") - - dates = sorted(dates) - conn.execute(text(""" - UPDATE wms.product_ranges - SET dates = :dates - WHERE id= :p_id - """), - { - "dates": Json(list(map(date_formatter, dates))), - "p_id": prodid - } - ) - - # calculate bounding boxes - lres = list(conn.execute(text(""" - SELECT lat_min,lat_max,lon_min,lon_max - FROM wms.product_ranges - WHERE id=:p_id - """), - {"p_id": prodid})) - - r = lres[0] - - epsg4326 = odc.geo.CRS("EPSG:4326") - box = odc.geo.geom.box( - float(r[2]), - float(r[0]), - float(r[3]), - float(r[1]), - epsg4326) - - all_bboxes = bbox_projections(box, crses) - - conn.execute(text(""" - UPDATE wms.product_ranges - SET bboxes = :bbox - WHERE id=:p_id - """), { - "bbox": Json(all_bboxes), - "p_id": product.id}) - - txn.commit() - conn.close() def bbox_projections(starting_box: odc.geo.Geometry, crses: dict[str, odc.geo.CRS]) -> dict[str, dict[str, float]]: @@ -302,122 +244,63 @@ def sanitise_coordinate(coord: float, fallback: float) -> float: def datasets_exist(dc: datacube.Datacube, product_name: str) -> bool: - conn = get_sqlconn(dc) + conn = get_sqlconn(dc) - results = conn.execute(text(""" - SELECT COUNT(*) - FROM agdc.dataset ds, agdc.dataset_type p - WHERE ds.archived IS NULL - AND ds.dataset_type_ref = p.id - AND p.name = :pname"""), - {"pname": product_name}) + results = conn.execute(text(""" + SELECT COUNT(*) + FROM agdc.dataset ds, agdc.dataset_type p + WHERE ds.archived IS NULL + AND ds.dataset_type_ref = p.id + AND p.name = :pname"""), {"pname": product_name}) - conn.close() + conn.close() - return list(results)[0][0] > 0 + return list(results)[0][0] > 0 -def add_ranges(dc: datacube.Datacube, product_names: list[str], merge_only: bool = False) -> bool: - odc_products: dict[str, dict[str, list[OWSNamedLayer]]] = {} # Maps OWS layer names to - ows_multiproducts: list[OWSMultiProductLayer] = [] +def add_ranges(cfg: OWSConfig, layer_names: list[str]) -> bool: + if not layer_names: + layer_names = list(cfg.layer_index.keys()) errors = False - for pname in product_names: - ows_product = get_config().product_index.get(pname) - if not ows_product: - ows_product = get_config().native_product_index.get(pname) - if ows_product: - for dc_pname in ows_product.product_names: - if dc_pname in odc_products: - odc_products[dc_pname]["ows"].append(ows_product) - else: - odc_products[dc_pname] = {"ows": [ows_product]} - print("OWS Layer %s maps to ODC Product(s): %s" % ( - ows_product.name, - repr(ows_product.product_names) - )) - if ows_product.multi_product: - ows_multiproducts.append(cast(OWSMultiProductLayer, ows_product)) - if not ows_product: - print("Could not find product", pname, "in OWS config") - dc_product = dc.index.products.get_by_name(pname) - if dc_product: - print("ODC Layer: %s" % pname) - if pname not in odc_products: - odc_products[pname] = {"ows": []} - else: - print("Unrecognised product name:", pname) - errors = True - continue - - if ows_multiproducts and merge_only: - print("Merge-only: Skipping range update of products:", repr(list(odc_products.keys()))) - else: - for pname, ows_prods in odc_products.items(): - dc_product = dc.index.products.get_by_name(pname) - if dc_product is None: - print("Could not find ODC product:", pname) - errors = True - elif datasets_exist(dc, dc_product.name): - print("Datasets exist for ODC product", - dc_product.name, - "(OWS layers", - ",".join(p.name for p in ows_prods["ows"]), - ")") - time_resolution = None - for ows_prod in ows_prods["ows"]: - if ows_prod: - new_tr = ows_prod.time_resolution - if time_resolution is not None and new_tr != time_resolution: - time_resolution = None - errors = True - print("Inconsistent time resolution for ODC product:", pname) - break - time_resolution = new_tr - if time_resolution is not None: - create_range_entry(dc, dc_product, get_crses(), time_resolution) - else: - print("Could not determine time_resolution for product: ", pname) - else: - print("Could not find any datasets for: ", pname) - for mp in ows_multiproducts: - create_multiprod_range_entry(dc, mp, get_crses()) + cache: dict[LayerSignature, list[str]] = {} + for name in layer_names: + if name not in cfg.layer_index: + click.echo(f"Layer '{name}' does not exist in the OWS configuration - skipping") + errors = True + continue + layer = cfg.layer_index[name] + create_range_entry(layer, cache) print("Done.") return errors -def get_ranges(layer: OWSNamedLayer, - path: str | None = None) -> dict[str, Any] | None: +class CoordRange(NamedTuple): + min: float + max: float + + +class LayerExtent: + def __init__(self, lat: CoordRange, lon: CoordRange, times: list[datetime | date], bboxes: CFG_DICT): + self.lat = lat + self.lon = lon + self.times = times + self.start_time = times[0] + self.end_time = times[-1] + self.time_set = set(times) + self.bboxes = bboxes + + +def get_ranges(layer: OWSNamedLayer) -> LayerExtent | None: cfg = layer.global_cfg conn = get_sqlconn(layer.dc) - if layer.multi_product: - if path is not None: - raise Exception("Combining subproducts and multiproducts is not yet supported") - results = conn.execute(text(""" - SELECT * - FROM wms.multiproduct_ranges - WHERE wms_product_name=:pname"""), - {"pname": layer.name} - ) - else: - prod_id = layer.product.id - if path is not None: - results = conn.execute(text(""" - SELECT * - FROM wms.sub_product_ranges - WHERE product_id=:pid and sub_product_id=:path"""), - { - "pid": prod_id, - "path": path - } - ) - else: - results = conn.execute(text(""" - SELECT * - FROM wms.product_ranges - WHERE id=:pid"""), - {"pid": prod_id} - ) + results = conn.execute(text(""" + SELECT * + FROM ows.layer_ranges + WHERE layer=:pname"""), + {"pname": layer.name} + ) + for result in results: conn.close() if layer.time_resolution.is_subday(): @@ -427,19 +310,10 @@ def get_ranges(layer: OWSNamedLayer, times = [dt_parser(d) for d in result.dates if d is not None] if not times: return None - return { - "lat": { - "min": float(result.lat_min), - "max": float(result.lat_max), - }, - "lon": { - "min": float(result.lon_min), - "max": float(result.lon_max), - }, - "times": times, - "start_time": times[0], - "end_time": times[-1], - "time_set": set(times), - "bboxes": cfg.alias_bboxes(result.bboxes) - } + return LayerExtent( + lat=CoordRange(min=float(result.lat_min), max=float(result.lat_max)), + lon=CoordRange(min=float(result.lon_min), max=float(result.lon_max)), + times=times, + bboxes=result.bboxes + ) return None diff --git a/datacube_ows/sql/extent_views/create/001_postgis_extension.sql b/datacube_ows/sql/extent_views/create/001_postgis_extension.sql index 0f841f156..148b64700 100644 --- a/datacube_ows/sql/extent_views/create/001_postgis_extension.sql +++ b/datacube_ows/sql/extent_views/create/001_postgis_extension.sql @@ -1,3 +1,3 @@ --- Installing Postgis extensions on public schema +-- Installing Postgis extensions create extension if not exists postgis diff --git a/datacube_ows/sql/extent_views/create/003_create_view_owner_role_ignore_duplicates.sql b/datacube_ows/sql/extent_views/create/003_create_view_owner_role_ignore_duplicates.sql new file mode 100644 index 000000000..66d443a65 --- /dev/null +++ b/datacube_ows/sql/extent_views/create/003_create_view_owner_role_ignore_duplicates.sql @@ -0,0 +1,3 @@ +-- Create database role to own the materialised views + +create role ows_view_owner; diff --git a/datacube_ows/sql/extent_views/create/010_create_new_time_view.sql b/datacube_ows/sql/extent_views/create/010_create_new_time_view.sql index 0182be3e4..054137631 100644 --- a/datacube_ows/sql/extent_views/create/010_create_new_time_view.sql +++ b/datacube_ows/sql/extent_views/create/010_create_new_time_view.sql @@ -2,7 +2,7 @@ -- Try all different locations for temporal extents and UNION them -CREATE MATERIALIZED VIEW IF NOT EXISTS time_view_new (dataset_type_ref, ID, temporal_extent) +CREATE MATERIALIZED VIEW IF NOT EXISTS ows.time_view_new (dataset_type_ref, ID, temporal_extent) AS with -- Crib metadata to use as for string matching various types diff --git a/datacube_ows/sql/extent_views/create/011_create_new_space_view_raw.sql b/datacube_ows/sql/extent_views/create/011_create_new_space_view.sql similarity index 97% rename from datacube_ows/sql/extent_views/create/011_create_new_space_view_raw.sql rename to datacube_ows/sql/extent_views/create/011_create_new_space_view.sql index e0546d07b..9f755e0bc 100644 --- a/datacube_ows/sql/extent_views/create/011_create_new_space_view_raw.sql +++ b/datacube_ows/sql/extent_views/create/011_create_new_space_view.sql @@ -2,7 +2,7 @@ -- Spatial extents per dataset (to be created as a column of the space-time table) -- Try all different locations for spatial extents and UNION them -CREATE MATERIALIZED VIEW IF NOT EXISTS space_view_new (ID, spatial_extent) +CREATE MATERIALIZED VIEW IF NOT EXISTS ows.space_view_new (ID, spatial_extent) AS with -- Crib metadata to use as for string matching various types diff --git a/datacube_ows/sql/extent_views/create/012_create_new_spacetime_view.sql b/datacube_ows/sql/extent_views/create/012_create_new_spacetime_view.sql index f3dfb8a75..21ba8699b 100644 --- a/datacube_ows/sql/extent_views/create/012_create_new_spacetime_view.sql +++ b/datacube_ows/sql/extent_views/create/012_create_new_spacetime_view.sql @@ -1,5 +1,5 @@ -- Creating NEW combined SPACE-TIME Materialised View -CREATE MATERIALIZED VIEW IF NOT EXISTS space_time_view_new (ID, dataset_type_ref, spatial_extent, temporal_extent) +CREATE MATERIALIZED VIEW IF NOT EXISTS ows.space_time_view_new (ID, dataset_type_ref, spatial_extent, temporal_extent) AS -select space_view_new.id, dataset_type_ref, spatial_extent, temporal_extent from space_view_new join time_view_new on space_view_new.id=time_view_new.id +select space_view_new.id, dataset_type_ref, spatial_extent, temporal_extent from ows.space_view_new join ows.time_view_new on space_view_new.id=time_view_new.id diff --git a/datacube_ows/sql/extent_views/create/020_create_index_1.sql b/datacube_ows/sql/extent_views/create/020_create_index_1.sql index 31abb257d..9d82878c9 100644 --- a/datacube_ows/sql/extent_views/create/020_create_index_1.sql +++ b/datacube_ows/sql/extent_views/create/020_create_index_1.sql @@ -1,5 +1,5 @@ -- Creating NEW Materialised View Index 1/4 CREATE INDEX space_time_view_geom_idx_new - ON space_time_view_new + ON ows.space_time_view_new USING GIST (spatial_extent) diff --git a/datacube_ows/sql/extent_views/create/021_create_index_2.sql b/datacube_ows/sql/extent_views/create/021_create_index_2.sql index e74bcc99b..9080073ea 100644 --- a/datacube_ows/sql/extent_views/create/021_create_index_2.sql +++ b/datacube_ows/sql/extent_views/create/021_create_index_2.sql @@ -1,5 +1,5 @@ -- Creating NEW Materialised View Index 2/4 CREATE INDEX space_time_view_time_idx_new - ON space_time_view_new + ON ows.space_time_view_new USING SPGIST (temporal_extent) diff --git a/datacube_ows/sql/extent_views/create/022_create_index_3.sql b/datacube_ows/sql/extent_views/create/022_create_index_3.sql index 2106a691b..88f7b8ce3 100644 --- a/datacube_ows/sql/extent_views/create/022_create_index_3.sql +++ b/datacube_ows/sql/extent_views/create/022_create_index_3.sql @@ -1,5 +1,5 @@ -- Creating NEW Materialised View Index 3/4 CREATE INDEX space_time_view_ds_idx_new - ON space_time_view_new + ON ows.space_time_view_new USING BTREE(dataset_type_ref) diff --git a/datacube_ows/sql/extent_views/create/023_create_index_4.sql b/datacube_ows/sql/extent_views/create/023_create_index_4.sql index bdeddad6e..25327b459 100644 --- a/datacube_ows/sql/extent_views/create/023_create_index_4.sql +++ b/datacube_ows/sql/extent_views/create/023_create_index_4.sql @@ -1,5 +1,5 @@ -- Creating NEW Materialised View Index 4/4 CREATE unique INDEX space_time_view_idx_new - ON space_time_view_new + ON ows.space_time_view_new USING BTREE(id) diff --git a/datacube_ows/sql/extent_views/create/030_rename_old_space_time_view.sql b/datacube_ows/sql/extent_views/create/030_rename_old_space_time_view.sql index af6170759..b5198e650 100644 --- a/datacube_ows/sql/extent_views/create/030_rename_old_space_time_view.sql +++ b/datacube_ows/sql/extent_views/create/030_rename_old_space_time_view.sql @@ -1,4 +1,4 @@ -- Renaming old spacetime view (OWS down) -ALTER MATERIALIZED VIEW IF EXISTS space_time_view +ALTER MATERIALIZED VIEW IF EXISTS ows.space_time_view RENAME TO space_time_view_old diff --git a/datacube_ows/sql/extent_views/create/031_rename_new_space_time_view.sql b/datacube_ows/sql/extent_views/create/031_rename_new_space_time_view.sql index a604f2ed3..82bbf5e3b 100644 --- a/datacube_ows/sql/extent_views/create/031_rename_new_space_time_view.sql +++ b/datacube_ows/sql/extent_views/create/031_rename_new_space_time_view.sql @@ -1,4 +1,4 @@ -- Renaming new view to space_time_view (OWS back up) -ALTER MATERIALIZED VIEW space_time_view_new +ALTER MATERIALIZED VIEW ows.space_time_view_new RENAME to space_time_view diff --git a/datacube_ows/sql/extent_views/create/032_drop_old_space_time_view.sql b/datacube_ows/sql/extent_views/create/032_drop_old_space_time_view.sql new file mode 100644 index 000000000..d442a3b69 --- /dev/null +++ b/datacube_ows/sql/extent_views/create/032_drop_old_space_time_view.sql @@ -0,0 +1,3 @@ +-- Dropping OLD spacetime view (and indexes) + +DROP MATERIALIZED VIEW IF EXISTS ows.space_time_view_old diff --git a/datacube_ows/sql/extent_views/create/040_drop_old_space_time_view.sql b/datacube_ows/sql/extent_views/create/040_drop_old_space_time_view.sql deleted file mode 100644 index 08fd0cae2..000000000 --- a/datacube_ows/sql/extent_views/create/040_drop_old_space_time_view.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Dropping OLD spacetime view (and indexes) - -DROP MATERIALIZED VIEW IF EXISTS space_time_view_old diff --git a/datacube_ows/sql/extent_views/create/040_drop_old_space_view.sql b/datacube_ows/sql/extent_views/create/040_drop_old_space_view.sql new file mode 100644 index 000000000..8d33ee253 --- /dev/null +++ b/datacube_ows/sql/extent_views/create/040_drop_old_space_view.sql @@ -0,0 +1,3 @@ +-- Dropping OLD space view + +DROP MATERIALIZED VIEW IF EXISTS ows.space_view diff --git a/datacube_ows/sql/extent_views/create/041_drop_old_time_view.sql b/datacube_ows/sql/extent_views/create/041_drop_old_time_view.sql index bb6b1fc7f..e55302b7c 100644 --- a/datacube_ows/sql/extent_views/create/041_drop_old_time_view.sql +++ b/datacube_ows/sql/extent_views/create/041_drop_old_time_view.sql @@ -1,3 +1,3 @@ -- Dropping OLD time view -DROP MATERIALIZED VIEW IF EXISTS time_view +DROP MATERIALIZED VIEW IF EXISTS ows.time_view diff --git a/datacube_ows/sql/extent_views/create/042_drop_old_space_view.sql b/datacube_ows/sql/extent_views/create/042_drop_old_space_view.sql deleted file mode 100644 index 0e1cf8525..000000000 --- a/datacube_ows/sql/extent_views/create/042_drop_old_space_view.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Dropping OLD space view - -DROP MATERIALIZED VIEW IF EXISTS space_view diff --git a/datacube_ows/sql/extent_views/create/050_rename_new_space_view.sql b/datacube_ows/sql/extent_views/create/042_rename_new_space_view.sql similarity index 53% rename from datacube_ows/sql/extent_views/create/050_rename_new_space_view.sql rename to datacube_ows/sql/extent_views/create/042_rename_new_space_view.sql index 1b7587caf..9786e7a1f 100644 --- a/datacube_ows/sql/extent_views/create/050_rename_new_space_view.sql +++ b/datacube_ows/sql/extent_views/create/042_rename_new_space_view.sql @@ -1,4 +1,4 @@ -- Renaming NEW space_view -ALTER MATERIALIZED VIEW space_view_new +ALTER MATERIALIZED VIEW ows.space_view_new RENAME to space_view diff --git a/datacube_ows/sql/extent_views/create/051_rename_new_time_view.sql b/datacube_ows/sql/extent_views/create/043_rename_new_time_view.sql similarity index 53% rename from datacube_ows/sql/extent_views/create/051_rename_new_time_view.sql rename to datacube_ows/sql/extent_views/create/043_rename_new_time_view.sql index 06a0ecae4..1fd281c6c 100644 --- a/datacube_ows/sql/extent_views/create/051_rename_new_time_view.sql +++ b/datacube_ows/sql/extent_views/create/043_rename_new_time_view.sql @@ -1,4 +1,4 @@ -- Renaming NEW time_view -ALTER MATERIALIZED VIEW time_view_new +ALTER MATERIALIZED VIEW ows.time_view_new RENAME TO time_view diff --git a/datacube_ows/sql/extent_views/create/052_rename_index_1.sql b/datacube_ows/sql/extent_views/create/050_rename_index_1.sql similarity index 64% rename from datacube_ows/sql/extent_views/create/052_rename_index_1.sql rename to datacube_ows/sql/extent_views/create/050_rename_index_1.sql index a29deab09..35945db5d 100644 --- a/datacube_ows/sql/extent_views/create/052_rename_index_1.sql +++ b/datacube_ows/sql/extent_views/create/050_rename_index_1.sql @@ -1,4 +1,4 @@ -- Renaming new Materialised View Index 1/4 -ALTER INDEX space_time_view_geom_idx_new +ALTER INDEX ows.space_time_view_geom_idx_new RENAME TO space_time_view_geom_idx diff --git a/datacube_ows/sql/extent_views/create/053_rename_index_2.sql b/datacube_ows/sql/extent_views/create/051_rename_index_2.sql similarity index 64% rename from datacube_ows/sql/extent_views/create/053_rename_index_2.sql rename to datacube_ows/sql/extent_views/create/051_rename_index_2.sql index d26c608f6..cee6dd4aa 100644 --- a/datacube_ows/sql/extent_views/create/053_rename_index_2.sql +++ b/datacube_ows/sql/extent_views/create/051_rename_index_2.sql @@ -1,4 +1,4 @@ -- Renaming new Materialised View Index 2/4 -ALTER INDEX space_time_view_time_idx_new +ALTER INDEX ows.space_time_view_time_idx_new RENAME TO space_time_view_time_idx diff --git a/datacube_ows/sql/extent_views/create/054_rename_index_3.sql b/datacube_ows/sql/extent_views/create/052_rename_index_3.sql similarity index 65% rename from datacube_ows/sql/extent_views/create/054_rename_index_3.sql rename to datacube_ows/sql/extent_views/create/052_rename_index_3.sql index 281f6de7e..fa459fa0f 100644 --- a/datacube_ows/sql/extent_views/create/054_rename_index_3.sql +++ b/datacube_ows/sql/extent_views/create/052_rename_index_3.sql @@ -1,4 +1,4 @@ -- Renaming new Materialised View Index 3/4 -ALTER INDEX space_time_view_ds_idx_new +ALTER INDEX ows.space_time_view_ds_idx_new RENAME TO space_time_view_ds_idx diff --git a/datacube_ows/sql/extent_views/create/055_rename_index_4.sql b/datacube_ows/sql/extent_views/create/053_rename_index_4.sql similarity index 65% rename from datacube_ows/sql/extent_views/create/055_rename_index_4.sql rename to datacube_ows/sql/extent_views/create/053_rename_index_4.sql index 4f5e84f76..366585d81 100644 --- a/datacube_ows/sql/extent_views/create/055_rename_index_4.sql +++ b/datacube_ows/sql/extent_views/create/053_rename_index_4.sql @@ -1,4 +1,4 @@ -- Renaming new Materialised View Index 4/4 -ALTER INDEX space_time_view_idx_new +ALTER INDEX ows.space_time_view_idx_new RENAME TO space_time_view_idx diff --git a/datacube_ows/sql/extent_views/create/060_grant.sql b/datacube_ows/sql/extent_views/create/060_grant.sql deleted file mode 100644 index dda7822ee..000000000 --- a/datacube_ows/sql/extent_views/create/060_grant.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Granting read permission to public - -GRANT SELECT ON space_time_view TO public; diff --git a/datacube_ows/sql/extent_views/grants/read_only/001_grant_read_requires_role.sql b/datacube_ows/sql/extent_views/grants/read_only/001_grant_read_requires_role.sql new file mode 100644 index 000000000..4b6a4349d --- /dev/null +++ b/datacube_ows/sql/extent_views/grants/read_only/001_grant_read_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting read permission to materialised view + +GRANT SELECT ON ows.space_time_view TO {role}; diff --git a/datacube_ows/sql/extent_views/grants/refresh_owner/001_set_owner_time_view.sql b/datacube_ows/sql/extent_views/grants/refresh_owner/001_set_owner_time_view.sql new file mode 100644 index 000000000..5d91390a1 --- /dev/null +++ b/datacube_ows/sql/extent_views/grants/refresh_owner/001_set_owner_time_view.sql @@ -0,0 +1,3 @@ +-- Set owner of time view. + +ALTER MATERIALIZED VIEW ows.time_view OWNER TO ows_view_owner; diff --git a/datacube_ows/sql/extent_views/grants/refresh_owner/002_set_owner_space_view.sql b/datacube_ows/sql/extent_views/grants/refresh_owner/002_set_owner_space_view.sql new file mode 100644 index 000000000..05e8804fc --- /dev/null +++ b/datacube_ows/sql/extent_views/grants/refresh_owner/002_set_owner_space_view.sql @@ -0,0 +1,3 @@ +-- Set owner of space view. + +ALTER MATERIALIZED VIEW ows.space_view OWNER TO ows_view_owner; diff --git a/datacube_ows/sql/extent_views/grants/refresh_owner/003_set_owner_spacetime_view.sql b/datacube_ows/sql/extent_views/grants/refresh_owner/003_set_owner_spacetime_view.sql new file mode 100644 index 000000000..01b005f0e --- /dev/null +++ b/datacube_ows/sql/extent_views/grants/refresh_owner/003_set_owner_spacetime_view.sql @@ -0,0 +1,3 @@ +-- Set owner of space-time view. + +ALTER MATERIALIZED VIEW ows.space_time_view OWNER TO ows_view_owner; diff --git a/datacube_ows/sql/extent_views/grants/write_refresh/001_grant_refresh_requires_role.sql b/datacube_ows/sql/extent_views/grants/write_refresh/001_grant_refresh_requires_role.sql new file mode 100644 index 000000000..b57968be5 --- /dev/null +++ b/datacube_ows/sql/extent_views/grants/write_refresh/001_grant_refresh_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting read permission to materialised view + +GRANT ows_view_owner TO {role}; diff --git a/datacube_ows/sql/extent_views/refresh/002_refresh_time.sql b/datacube_ows/sql/extent_views/refresh/002_refresh_time.sql index a4e8c43f4..202256068 100644 --- a/datacube_ows/sql/extent_views/refresh/002_refresh_time.sql +++ b/datacube_ows/sql/extent_views/refresh/002_refresh_time.sql @@ -1,3 +1,3 @@ -- Refreshing TIME materialized view (Blocking) -REFRESH MATERIALIZED VIEW time_view +REFRESH MATERIALIZED VIEW ows.time_view diff --git a/datacube_ows/sql/extent_views/refresh/003_refresh_space.sql b/datacube_ows/sql/extent_views/refresh/003_refresh_space.sql index 38f332cb2..263d91c4f 100644 --- a/datacube_ows/sql/extent_views/refresh/003_refresh_space.sql +++ b/datacube_ows/sql/extent_views/refresh/003_refresh_space.sql @@ -1,3 +1,3 @@ -- Refreshing SPACE materialized view (blocking) -REFRESH MATERIALIZED VIEW space_view +REFRESH MATERIALIZED VIEW ows.space_view diff --git a/datacube_ows/sql/extent_views/refresh/004_refresh_spacetime.sql b/datacube_ows/sql/extent_views/refresh/004_refresh_spacetime.sql index 343249028..78fcb1a1b 100644 --- a/datacube_ows/sql/extent_views/refresh/004_refresh_spacetime.sql +++ b/datacube_ows/sql/extent_views/refresh/004_refresh_spacetime.sql @@ -1,3 +1,3 @@ -- Refreshing combined SPACE-TIME materialized view (concurrently) -REFRESH MATERIALIZED VIEW CONCURRENTLY space_time_view +REFRESH MATERIALIZED VIEW CONCURRENTLY ows.space_time_view diff --git a/datacube_ows/sql/ows_schema/cleanup/001_drop_space_time_view.sql b/datacube_ows/sql/ows_schema/cleanup/001_drop_space_time_view.sql new file mode 100644 index 000000000..d8c30f84a --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/001_drop_space_time_view.sql @@ -0,0 +1,3 @@ +-- Dropping OLD spacetime view (and indexes) + +DROP MATERIALIZED VIEW IF EXISTS space_time_view; diff --git a/datacube_ows/sql/ows_schema/cleanup/002_drop_time_view.sql b/datacube_ows/sql/ows_schema/cleanup/002_drop_time_view.sql new file mode 100644 index 000000000..a91800236 --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/002_drop_time_view.sql @@ -0,0 +1,3 @@ +-- Dropping OLD time view + +DROP MATERIALIZED VIEW IF EXISTS time_view; diff --git a/datacube_ows/sql/ows_schema/cleanup/003_drop_space_view.sql b/datacube_ows/sql/ows_schema/cleanup/003_drop_space_view.sql new file mode 100644 index 000000000..4351b8043 --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/003_drop_space_view.sql @@ -0,0 +1,3 @@ +-- Dropping OLD space view + +DROP MATERIALIZED VIEW IF EXISTS space_view; diff --git a/datacube_ows/sql/ows_schema/cleanup/010_drop_subproduct_range.sql b/datacube_ows/sql/ows_schema/cleanup/010_drop_subproduct_range.sql new file mode 100644 index 000000000..df278ff95 --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/010_drop_subproduct_range.sql @@ -0,0 +1,3 @@ +-- Dropping OLD subproduct range table. + +DROP TABLE IF EXISTS wms.sub_product_ranges; diff --git a/datacube_ows/sql/ows_schema/cleanup/011_drop_multiproduct_range.sql b/datacube_ows/sql/ows_schema/cleanup/011_drop_multiproduct_range.sql new file mode 100644 index 000000000..f1f908662 --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/011_drop_multiproduct_range.sql @@ -0,0 +1,3 @@ +-- Dropping OLD multiproduct range table. + +DROP TABLE IF EXISTS wms.multiproduct_ranges; diff --git a/datacube_ows/sql/ows_schema/cleanup/012_drop_product_range.sql b/datacube_ows/sql/ows_schema/cleanup/012_drop_product_range.sql new file mode 100644 index 000000000..ccb9044b6 --- /dev/null +++ b/datacube_ows/sql/ows_schema/cleanup/012_drop_product_range.sql @@ -0,0 +1,3 @@ +-- Dropping OLD product range table. + +DROP TABLE IF EXISTS wms.product_ranges; diff --git a/datacube_ows/sql/ows_schema/create/001_create_schema.sql b/datacube_ows/sql/ows_schema/create/001_create_schema.sql new file mode 100644 index 000000000..1862c31ec --- /dev/null +++ b/datacube_ows/sql/ows_schema/create/001_create_schema.sql @@ -0,0 +1,3 @@ +-- Creating/replacing ows schema + +create schema if not exists ows; diff --git a/datacube_ows/sql/wms_schema/create/002_create_product_rng.sql b/datacube_ows/sql/ows_schema/create/002_create_product_rng.sql similarity index 51% rename from datacube_ows/sql/wms_schema/create/002_create_product_rng.sql rename to datacube_ows/sql/ows_schema/create/002_create_product_rng.sql index 7ade9b830..bc947bacf 100644 --- a/datacube_ows/sql/wms_schema/create/002_create_product_rng.sql +++ b/datacube_ows/sql/ows_schema/create/002_create_product_rng.sql @@ -1,7 +1,7 @@ -- Creating/replacing product ranges table -create table if not exists wms.product_ranges ( - id smallint not null primary key references agdc.dataset_type (id), +create table if not exists ows.layer_ranges ( + layer varchar(255) not null primary key, lat_min decimal not null, lat_max decimal not null, @@ -10,4 +10,8 @@ create table if not exists wms.product_ranges ( dates jsonb not null, - bboxes jsonb not null); + bboxes jsonb not null, + + meta jsonb not null, + last_updated timestamp not null +); diff --git a/datacube_ows/sql/ows_schema/grants/read_only/001_grant_usage_requires_role.sql b/datacube_ows/sql/ows_schema/grants/read_only/001_grant_usage_requires_role.sql new file mode 100644 index 000000000..598fee7a2 --- /dev/null +++ b/datacube_ows/sql/ows_schema/grants/read_only/001_grant_usage_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting usage on schema + +GRANT USAGE ON SCHEMA ows TO {role} diff --git a/datacube_ows/sql/ows_schema/grants/read_write/001_grant_usage_requires_role.sql b/datacube_ows/sql/ows_schema/grants/read_write/001_grant_usage_requires_role.sql new file mode 100644 index 000000000..598fee7a2 --- /dev/null +++ b/datacube_ows/sql/ows_schema/grants/read_write/001_grant_usage_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting usage on schema + +GRANT USAGE ON SCHEMA ows TO {role} diff --git a/datacube_ows/sql/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql b/datacube_ows/sql/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql new file mode 100644 index 000000000..6d7e3f30a --- /dev/null +++ b/datacube_ows/sql/ows_schema/grants/read_write/002_grant_writetables_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting update/insert/delete on all tables in schema + +GRANT update, select, insert, delete ON ALL TABLES IN SCHEMA ows TO {role} diff --git a/datacube_ows/sql/use_space_time.sql b/datacube_ows/sql/use_space_time.sql deleted file mode 100644 index 9687315da..000000000 --- a/datacube_ows/sql/use_space_time.sql +++ /dev/null @@ -1,11 +0,0 @@ ---- Usage of space-time tables to obtain spatial and temporal extents of datasets ---- from indexed materialized views -select dataset_type_ref, ST_Extent(spatial_extent) as bbox, array_agg(temporal_extent) from space_time_view group by dataset_type_ref; -------------------------------------------------------------------------------------------------------------------------------------- ------------------------------------------------------------QUERY PLAN---------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------- ---HashAggregate (cost=316452.75..316453.50 rows=50 width=99) (actual time=9850.712..10340.107 rows=74 loops=1) --- Group Key: dataset_type_ref --- -> Seq Scan on space_time_view (cost=0.00..255246.00 rows=8160900 width=128) (actual time=0.009..2750.915 rows=8160900 loops=1) --- Planning Time: 0.502 ms --- Execution Time: 10364.716 ms diff --git a/datacube_ows/sql/wms_schema/create/001_create_schema.sql b/datacube_ows/sql/wms_schema/create/001_create_schema.sql deleted file mode 100644 index dce631481..000000000 --- a/datacube_ows/sql/wms_schema/create/001_create_schema.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Creating/replacing wms schema - -create schema if not exists wms; diff --git a/datacube_ows/sql/wms_schema/create/003_create_subproduct_rng.sql b/datacube_ows/sql/wms_schema/create/003_create_subproduct_rng.sql deleted file mode 100644 index a11b7e688..000000000 --- a/datacube_ows/sql/wms_schema/create/003_create_subproduct_rng.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Creating/replacing sub-product ranges table - -create table if not exists wms.sub_product_ranges ( - product_id smallint not null references agdc.dataset_type (id), - sub_product_id smallint not null, - - lat_min decimal not null, - lat_max decimal not null, - lon_min decimal not null, - lon_max decimal not null, - - dates jsonb not null, - - bboxes jsonb not null, - constraint pk_sub_product_ranges primary key (product_id, sub_product_id) -); diff --git a/datacube_ows/sql/wms_schema/create/004_create_multiproduct_rng.sql b/datacube_ows/sql/wms_schema/create/004_create_multiproduct_rng.sql deleted file mode 100644 index 0d8f4c577..000000000 --- a/datacube_ows/sql/wms_schema/create/004_create_multiproduct_rng.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Creating/replacing multi-product ranges table - -create table if not exists wms.multiproduct_ranges ( - wms_product_name varchar(128) not null primary key, - - lat_min decimal not null, - lat_max decimal not null, - lon_min decimal not null, - lon_max decimal not null, - - dates jsonb not null, - - bboxes jsonb not null -); diff --git a/datacube_ows/sql/wms_schema/create/005_grant_requires_role.sql b/datacube_ows/sql/wms_schema/create/005_grant_requires_role.sql deleted file mode 100644 index d75837cb9..000000000 --- a/datacube_ows/sql/wms_schema/create/005_grant_requires_role.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Granting usage on schema - -GRANT USAGE ON SCHEMA wms TO {role} diff --git a/datacube_ows/styles/base.py b/datacube_ows/styles/base.py index a372a5143..30cd24b86 100644 --- a/datacube_ows/styles/base.py +++ b/datacube_ows/styles/base.py @@ -589,7 +589,7 @@ def lookup_impl(cls, prod = subs["layer"].get(keyvals["layer"]) if not prod: try: - prod = cfg.product_index[keyvals["layer"]] + prod = cfg.layer_index[keyvals["layer"]] except KeyError: raise OWSEntryNotFound(f"No layer named {keyvals['layer']}") diff --git a/datacube_ows/templates/wcs_capabilities.xml b/datacube_ows/templates/wcs_capabilities.xml index db08e3fb3..25ba8e104 100644 --- a/datacube_ows/templates/wcs_capabilities.xml +++ b/datacube_ows/templates/wcs_capabilities.xml @@ -111,7 +111,7 @@ xsi:schemaLocation="http://www.opengis.net/wcs http://schemas.opengis.net/wcs/1. {% endif %} {% if show_content_metadata %} - {% for product in cfg.product_index.values() %} + {% for product in cfg.layer_index.values() %} {% if product.wcs and product.ready and not product.hide %} {% set product_ranges = product.ranges %} diff --git a/datacube_ows/templates/wms_capabilities.xml b/datacube_ows/templates/wms_capabilities.xml index 764eb6a09..275e24b78 100644 --- a/datacube_ows/templates/wms_capabilities.xml +++ b/datacube_ows/templates/wms_capabilities.xml @@ -279,7 +279,7 @@ http://schemas.opengis.net/wms/1.3.0/capabilities_1_3_0.xsd"> XML - {% for lyr in cfg.product_index.values() %} + {% for lyr in cfg.layer_index.values() %} {% if lyr.ready and not lyr.hide and lyr.user_band_math %} user_band_math diff --git a/datacube_ows/templates/wmts_capabilities.xml b/datacube_ows/templates/wmts_capabilities.xml index 816155871..fa3cc0aa6 100644 --- a/datacube_ows/templates/wmts_capabilities.xml +++ b/datacube_ows/templates/wmts_capabilities.xml @@ -126,7 +126,7 @@ {% if show_contents %} - {% for layer in cfg.product_index.values() %} + {% for layer in cfg.layer_index.values() %} {% if layer.ready and not layer.hide %} {% set product_ranges = layer.ranges %} {% if product_ranges %} diff --git a/datacube_ows/update_ranges_impl.py b/datacube_ows/update_ranges_impl.py index 15f853008..006499014 100755 --- a/datacube_ows/update_ranges_impl.py +++ b/datacube_ows/update_ranges_impl.py @@ -22,40 +22,85 @@ from datacube_ows.product_ranges import add_ranges, get_sqlconn from datacube_ows.startup_utils import initialise_debugging +class AbortRun(Exception): + pass + @click.command() -@click.option("--views", is_flag=True, default=False, help="Refresh the ODC spatio-temporal materialised views.") -@click.option("--schema", is_flag=True, default=False, help="Create or update the OWS database schema, including the spatio-temporal materialised views.") -@click.option("--role", default=None, help="Role to grant database permissions to") -@click.option("--merge-only/--no-merge-only", default=False, help="When used with a multiproduct layer, the ranges for underlying datacube products are not updated.") -@click.option("--version", is_flag=True, default=False, help="Print version string and exit") +@click.option("--views", is_flag=True, default=False, + help="Refresh the ODC spatio-temporal materialised views.") +@click.option("--schema", is_flag=True, default=False, + help="Create or update the OWS database schema, including the spatio-temporal materialised views.") +@click.option("--read-role", multiple=True, + help="(Only valid with --schema) Role(s) to grant read-only database permissions to") +@click.option("--write-role", multiple=True, + help="(Only valid with --schema) Role(s) to grant both read and write/update database permissions to") +@click.option("--cleanup", is_flag=True, default=False, + help="Cleanup up any datacube-ows 1.8.x tables/views") +@click.option("-e", "--env", default=None, + help="(Only valid with --schema or --read-role or --write-role or --cleanup) environment to write to.") +@click.option("--version", is_flag=True, default=False, + help="Print version string and exit") @click.argument("layers", nargs=-1) def main(layers: list[str], - merge_only: bool, - schema: bool, views: bool, role: str | None, version: bool) -> int: - """Manage datacube-ows range tables. + env: str | None, + schema: bool, + read_role: list[str], + write_role: list[str], + cleanup: bool, views: bool, version: bool) -> int: + """Manage datacube-ows range tables. Exposed on setup as datacube-ows-update Valid invocations: - * update_ranges.py --schema --role myrole - Create (re-create) the OWS schema (including materialised views) and grants permission to role myrole + 1. Schema/permissions/migration management + + * datacube-ows-update --schema + Create (re-create) the OWS schema (including materialised views) + + * datacube-ows-update --read-role role1 --read-role role2 --write-role role3 + Grants read or read/write permissions to the OWS tables and views to the indicated role(s). + + The --read-role and --write-role options can also be passed in combination with the --schema option + described above. + + Read permissions are required for the database role that the datacube-ows service uses. + + Write permissions are required for the database role used to run the Data Management actions below. + + (These schema management actions require higher level permissions.) + + * datacube-ows-update --cleanup + Clean up (drop) any datcube-ows 1.8.x database entities. - * update_ranges.py --views + The --cleanup option can also be passed in combination with the --schema option described above. + + All of the above schema management actions can also be used with the --env or -E option: + + * datacube-ows-update --cleanup --env dev + Use the "dev" environment from the ODC configuration for connecting to the database. + (Defaults to env defined in OWS global config, or "default") + + Schema management functions attempt to create or modify database objects and assign permissions over those + objects. They typically need to run with a very high level of database permissions - e.g. depending + on the requested action and the current state of the database schema, they may need to be able to create + schemas, roles and/or extensions. + + 2. Data management (updating OWS indexes) + + * datacube-ows-update --views Refresh the materialised views - * One or more OWS or ODC layer names - Update ranges for the specified LAYERS + * datacube-ows-update layer1 layer2 ... + Update ranges for the specified LAYERS (Note that ODC product names are no longer supported) - * No LAYERS (and neither the --views nor --schema options) - (Update ranges for all configured OWS layers. + * datacube-ows-update + Update ranges for all configured OWS layers. Uses the DATACUBE_OWS_CFG environment variable to find the OWS config file. """ # --version if version: - print("Open Data Cube Open Web Services (datacube-ows) version", - __version__ - ) + print("Open Data Cube Open Web Services (datacube-ows) version", __version__) sys.exit(0) # Handle old-style calls if not layers: @@ -63,72 +108,106 @@ def main(layers: list[str], if schema and layers: print("Sorry, cannot update the schema and ranges in the same invocation.") sys.exit(1) + if schema and views: + print("Sorry, No point in updating materialised views and updating the schema in the same invocation.") + sys.exit(1) + elif cleanup and layers: + print("Sorry, cannot cleanup 1.8.x database entities and update ranges in the same invocation.") + sys.exit(1) + elif views and cleanup: + print("Sorry, cannot update the materialised views and cleanup the database in the same invocation.") + sys.exit(1) elif views and layers: print("Sorry, cannot update the materialised views and ranges in the same invocation.") sys.exit(1) - elif schema and not role: - print("Sorry, cannot update schema without specifying a role, use: '--schema --role myrole'") + elif read_role and (views or layers): + print("Sorry, read-role can't be granted with view or range updates") sys.exit(1) - elif role and not schema: - print("Sorry, role only makes sense for updating the schema") + elif write_role and (views or layers): + print("Sorry, write-role can't be granted with view or range updates") sys.exit(1) initialise_debugging() - dc = Datacube(app="ows_update_ranges") cfg = get_config(called_from_update_ranges=True) - if schema: - assert role is not None # for type checker - print("Checking schema....") - print("Creating or replacing WMS database schema...") - create_schema(dc, role) - print("Creating or replacing materialised views...") - create_views(dc) - print("Done") - return 0 - elif views: - print("Refreshing materialised views...") - refresh_views(dc) - print("Done") + app = cfg.odc_app + "-update" + errors: bool = False + if schema or read_role or write_role or cleanup: + if cfg.default_env and env is None: + dc = Datacube(env=cfg.default_env, app=app) + else: + dc = Datacube(env=env, app=app) + + click.echo(f"Applying database schema updates to the {dc.index.environment.db_database} database:...") + try: + if schema: + click.echo("Creating or replacing OWS database schema:...") + create_schema(dc) + for role in read_role: + click.echo(f"Granting read-only access to role {role}...") + grant_perms(dc, role, read_only=True) + for role in write_role: + click.echo(f"Granting read/write access to role {role}...") + grant_perms(dc, role) + if cleanup: + click.echo("Cleaning up datacube-1.8.x range tables and views...") + cleanup_schema(dc) + except AbortRun: + click.echo("Aborting schema update") + errors = True + click.echo("Done") + if errors: + sys.exit(1) return 0 print("Deriving extents from materialised views") - if not layers: - layers = list(cfg.product_index.keys()) try: - errors = add_ranges(dc, layers, merge_only) - except (psycopg2.errors.UndefinedColumn, - sqlalchemy.exc.ProgrammingError) as e: - print("ERROR: OWS schema or extent materialised views appear to be missing", - "\n", - " Try running with the --schema options first." - ) - sys.exit(1) + errors = add_ranges(cfg, layers) + click.echo("Done.") + except sqlalchemy.exc.ProgrammingError as e: + if isinstance(e.orig, psycopg2.errors.UndefinedColumn): + click.echo("ERROR: OWS schema or extent materialised views appear to be missing") + click.echo("") + click.echo(" Try running with the --schema options first.") + sys.exit(1) + else: + raise e if errors: sys.exit(1) return 0 -def create_views(dc: datacube.Datacube): - from datacube.cfg import ODCConfig - odc_cfg = ODCConfig().get_environment() - dbname = odc_cfg.db_database - run_sql(dc, "extent_views/create", database=dbname) - - def refresh_views(dc: datacube.Datacube): run_sql(dc, "extent_views/refresh") -def create_schema(dc: datacube.Datacube, role: str): - run_sql(dc, "wms_schema/create", role=role) +def create_schema(dc: datacube.Datacube): + click.echo("Creating/updating schema and tables...") + run_sql(dc, "ows_schema/create") + click.echo("Creating/updating materialised views...") + run_sql(dc, "extent_views/create") + click.echo("Setting ownership of materialised views...") + run_sql(dc, "extent_views/grants/refresh_owner") + + +def grant_perms(dc: datacube.Datacube, role: str, read_only: bool = False): + if read_only: + run_sql(dc, "ows_schema/grants/read_only", role=role) + run_sql(dc, "extent_views/grants/read_only", role=role) + else: + run_sql(dc, "ows_schema/grants/read_write", role=role) + run_sql(dc, "extent_views/grants/write_refresh", role=role) + +def cleanup_schema(dc: datacube.Datacube): + run_sql(dc, "ows_schema/cleanup") -def run_sql(dc: datacube.Datacube, path: str, **params: str): + +def run_sql(dc: datacube.Datacube, path: str, **params: str) -> bool: if not importlib.resources.files("datacube_ows").joinpath(f"sql/{path}").is_dir(): print("Cannot find SQL resource directory - check your datacube-ows installation") - return + return False files = sorted( importlib.resources.files("datacube_ows").joinpath(f"sql/{path}").iterdir() # type: ignore[type-var] @@ -137,12 +216,13 @@ def run_sql(dc: datacube.Datacube, path: str, **params: str): filename_req_pattern = re.compile(r"\d+[_a-zA-Z0-9]+_requires_(?P[_a-zA-Z0-9]+)\.sql") filename_pattern = re.compile(r"\d+[_a-zA-Z0-9]+\.sql") conn = get_sqlconn(dc) - + all_ok: bool = True for fi in files: f = fi.name match = filename_pattern.fullmatch(f) if not match: - print(f"Illegal SQL filename: {f} (skipping)") + click.echo(f"Illegal SQL filename: {f} (skipping)") + all_ok = False continue req_match = filename_req_pattern.fullmatch(f) if req_match: @@ -156,19 +236,32 @@ def run_sql(dc: datacube.Datacube, path: str, **params: str): for line in fp: sline = str(line, "utf-8") if first and sline.startswith("--"): - print(sline[2:]) + click.echo(f" - Running {sline[2:]}") else: sql = sql + "\n" + sline - if first: - print(f"Running {f}") first = False if reqs: try: kwargs = {v: params[v] for v in reqs} except KeyError as e: - print(f"Required parameter {e} for file {f} not supplied - skipping") + click.echo(f"Required parameter {e} for file {f} not supplied - skipping") + all_ok = False continue sql = sql.format(**kwargs) - # Special handling of "_raw.sql" scripts no longer required in SQLAlchemy 2? - conn.execute(text(sql)) + try: + conn.execute(text(sql)) + except sqlalchemy.exc.ProgrammingError as e: + if isinstance(e.orig, psycopg2.errors.InsufficientPrivilege): + click.echo( + f"Insufficient Privileges. Schema altering actions should be run by a role with admin privileges" + ) + raise AbortRun() from None + elif isinstance(e.orig, psycopg2.errors.DuplicateObject): + if f.endswith('_ignore_duplicates.sql'): + click.echo(f"Ignoring 'already exists' error") + else: + raise e from None + else: + raise e from e + return all_ok conn.close() diff --git a/datacube_ows/wcs1.py b/datacube_ows/wcs1.py index ade17eaa0..892f7e31b 100644 --- a/datacube_ows/wcs1.py +++ b/datacube_ows/wcs1.py @@ -84,7 +84,7 @@ def desc_coverages(args): if coverages: coverages = coverages.split(",") for c in coverages: - p = cfg.product_index.get(c) + p = cfg.layer_index.get(c) if p and p.wcs: products.append(p) else: @@ -92,7 +92,7 @@ def desc_coverages(args): WCS1Exception.COVERAGE_NOT_DEFINED, locator="Coverage parameter") else: - for p in cfg.product_index.values(): + for p in cfg.layer_index.values(): if p.ready and p.wcs: products.append(p) min_cache_age = min(p.resource_limits.wcs_desc_cache_rule for p in products) diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index 4da96ab2d..2af754680 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -35,14 +35,14 @@ def __init__(self, args): raise WCS1Exception("No coverage specified", WCS1Exception.MISSING_PARAMETER_VALUE, locator="COVERAGE parameter", - valid_keys=list(cfg.product_index)) + valid_keys=list(cfg.layer_index)) self.layer_name = args["coverage"] - self.layer = cfg.product_index.get(self.layer_name) + self.layer = cfg.layer_index.get(self.layer_name) if not self.layer or not self.layer.wcs: raise WCS1Exception("Invalid coverage: %s" % self.layer_name, WCS1Exception.COVERAGE_NOT_DEFINED, locator="COVERAGE parameter", - valid_keys=list(cfg.product_index)) + valid_keys=list(cfg.layer_index)) # Argument: FORMAT (required) -> a supported format if "format" not in args: @@ -149,12 +149,12 @@ def __init__(self, args): continue try: time = parse(t).date() - if time not in self.layer.ranges["time_set"]: + if time not in self.layer.ranges.time_set: raise WCS1Exception( "Time value '%s' not a valid date for coverage %s" % (t, self.layer_name), WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] + valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges.time_set] ) self.times.append(time) except ValueError: @@ -162,7 +162,7 @@ def __init__(self, args): "Time value '%s' not a valid ISO-8601 date" % t, WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] + valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges.time_set] ) self.times.sort() @@ -171,7 +171,7 @@ def __init__(self, args): "No valid ISO-8601 dates", WCS1Exception.INVALID_PARAMETER_VALUE, locator="TIME parameter", - valid_keys = [d.strftime('%Y-%m-%d') for d in self.layer.ranges["time_set"]] + valid_keys=[d.strftime('%Y-%m-%d') for d in self.layer.ranges.time_set] ) elif len(self.times) > 1 and not self.format.multi_time: raise WCS1Exception( diff --git a/datacube_ows/wcs2.py b/datacube_ows/wcs2.py index 5600749ed..3ed782686 100644 --- a/datacube_ows/wcs2.py +++ b/datacube_ows/wcs2.py @@ -114,11 +114,11 @@ def get_capabilities(args): coverage_subtype='RectifiedGridCoverage', title=product.title, wgs84_bbox=WGS84BoundingBox([ - product.ranges['lon']['min'], product.ranges['lat']['min'], - product.ranges['lon']['max'], product.ranges['lat']['max'], + product.ranges.lon.min, product.ranges.lat.min, + product.ranges.lon.max, product.ranges.lat.max, ]) ) - for product in cfg.product_index.values() + for product in cfg.layer_index.values() if product.ready and not product.hide and product.wcs ], formats_supported=[ @@ -156,12 +156,12 @@ def create_coverage_description(cfg, product): label=product.native_CRS_def["horizontal_coord"], index_label='i', lower_bound=min( - product.ranges["bboxes"][product.native_CRS]["left"], - product.ranges["bboxes"][product.native_CRS]["right"], + product.ranges.bboxes[product.native_CRS]["left"], + product.ranges.bboxes[product.native_CRS]["right"], ), upper_bound=max( - product.ranges["bboxes"][product.native_CRS]["left"], - product.ranges["bboxes"][product.native_CRS]["right"], + product.ranges.bboxes[product.native_CRS]["left"], + product.ranges.bboxes[product.native_CRS]["right"], ), resolution=product.resolution_x, uom='deg', @@ -171,12 +171,12 @@ def create_coverage_description(cfg, product): label=product.native_CRS_def["vertical_coord"], index_label='j', lower_bound=min( - product.ranges["bboxes"][product.native_CRS]["top"], - product.ranges["bboxes"][product.native_CRS]["bottom"], + product.ranges.bboxes[product.native_CRS]["top"], + product.ranges.bboxes[product.native_CRS]["bottom"], ), upper_bound=max( - product.ranges["bboxes"][product.native_CRS]["top"], - product.ranges["bboxes"][product.native_CRS]["bottom"], + product.ranges.bboxes[product.native_CRS]["top"], + product.ranges.bboxes[product.native_CRS]["bottom"], ), resolution=product.resolution_y, uom='deg', @@ -210,7 +210,7 @@ def create_coverage_description(cfg, product): index_label='k', positions=[ f'{t.isoformat()}' - for t in product.ranges['times'] + for t in product.ranges.times ], uom='ISO-8601', type=SpatioTemporalType.TEMPORAL, @@ -223,7 +223,7 @@ def create_coverage_description(cfg, product): index_label='k', positions=[ f'{t.isoformat()}T00:00:00.000Z' - for t in product.ranges['times'] + for t in product.ranges.times ], uom='ISO-8601', type=SpatioTemporalType.TEMPORAL, @@ -264,7 +264,7 @@ def desc_coverages(args): products = [] for coverage_id in request_obj.coverage_ids: - product = cfg.product_index.get(coverage_id) + product = cfg.layer_index.get(coverage_id) if product and product.wcs: products.append(product) else: diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index d137805d6..482a10f49 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -51,12 +51,12 @@ def get_coverage_data(request, styles, qprof): cfg = get_config() qprof.start_event("setup") layer_name = request.coverage_id - layer = cfg.product_index.get(layer_name) + layer = cfg.layer_index.get(layer_name) if not layer or not layer.wcs: raise WCS2Exception("Invalid coverage: %s" % layer_name, WCS2Exception.NO_SUCH_COVERAGE, locator="COVERAGE parameter", - valid_keys=list(cfg.product_index)) + valid_keys=list(cfg.layer_index)) # # CRS handling @@ -83,7 +83,7 @@ def get_coverage_data(request, styles, qprof): # scaler = WCSScaler(layer, subsetting_crs) - times = layer.ranges["times"] + times = layer.ranges.times subsets = request.subsets diff --git a/datacube_ows/wms_utils.py b/datacube_ows/wms_utils.py index c5612a0fc..ba2095446 100644 --- a/datacube_ows/wms_utils.py +++ b/datacube_ows/wms_utils.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 import math -from datetime import datetime +from datetime import datetime, date import numpy import regex as re @@ -136,12 +136,12 @@ def get_layer_from_arg(args, argname="layers") -> OWSNamedLayer: layer_chunks = lyr.split("__") lyr = layer_chunks[0] cfg = get_config() - layer = cfg.product_index.get(lyr) + layer = cfg.layer_index.get(lyr) if not layer: raise WMSException("Layer %s is not defined" % lyr, WMSException.LAYER_NOT_DEFINED, locator="Layer parameter", - valid_keys=list(cfg.product_index)) + valid_keys=list(cfg.layer_index)) return layer @@ -165,20 +165,19 @@ def get_arg(args, argname, verbose_name, lower=False, return fmt -def get_times_for_product(product): - ranges = product.ranges - return ranges['times'] +def get_times_for_layer(layer: OWSNamedLayer) -> list[datetime | date]: + return layer.ranges.times -def get_times(args, product: OWSNamedLayer) -> list[datetime]: +def get_times(args, layer: OWSNamedLayer) -> list[datetime | date]: # Time parameter times_raw = args.get('time', '') times = times_raw.split(',') - return list([parse_time_item(item, product) for item in times]) + return list([parse_time_item(item, layer) for item in times]) -def parse_time_item(item: str, product: OWSNamedLayer) -> datetime: +def parse_time_item(item: str, layer: OWSNamedLayer) -> datetime | date: times = item.split('/') # Time range handling follows the implementation described by GeoServer # https://docs.geoserver.org/stable/en/user/services/wms/time.html @@ -186,16 +185,16 @@ def parse_time_item(item: str, product: OWSNamedLayer) -> datetime: # If all times are equal we can proceed if len(times) > 1: # TODO WMS Time range selections (/ notation) are poorly and incompletely implemented. - start, end = parse_wms_time_strings(times, with_tz=product.time_resolution.is_subday()) - if product.time_resolution.is_subday(): - matching_times = [t for t in product.ranges['times'] if start <= t <= end] + start, end = parse_wms_time_strings(times, with_tz=layer.time_resolution.is_subday()) + if layer.time_resolution.is_subday(): + matching_times: list[datetime | date] = [t for t in layer.ranges.times if start <= t <= end] else: start, end = start.date(), end.date() - matching_times = [t for t in product.ranges['times'] if start <= t <= end] + matching_times = [t for t in layer.ranges.times if start <= t <= end] if matching_times: # default to the first matching time return matching_times[0] - elif product.regular_time_axis: + elif layer.regular_time_axis: raise WMSException( "No data available for time dimension range '%s'-'%s' for this layer" % (start, end), WMSException.INVALID_DIMENSION_VALUE, @@ -207,11 +206,11 @@ def parse_time_item(item: str, product: OWSNamedLayer) -> datetime: locator="Time parameter") elif not times[0]: # default to last available time if not supplied. - product_times = get_times_for_product(product) + product_times = get_times_for_layer(layer) return product_times[-1] try: time = parse(times[0]) - if not product.time_resolution.is_subday(): + if not layer.time_resolution.is_subday(): time = time.date() # type: ignore[assignment] except ValueError: raise WMSException( @@ -220,8 +219,8 @@ def parse_time_item(item: str, product: OWSNamedLayer) -> datetime: locator="Time parameter") # Validate time parameter for requested layer. - if product.regular_time_axis: - start, end = product.time_range() + if layer.regular_time_axis: + start, end = layer.time_range() if time < start: raise WMSException( "Time dimension value '%s' not valid for this layer" % times[0], @@ -232,19 +231,19 @@ def parse_time_item(item: str, product: OWSNamedLayer) -> datetime: "Time dimension value '%s' not valid for this layer" % times[0], WMSException.INVALID_DIMENSION_VALUE, locator="Time parameter") - if (time - start).days % product.time_axis_interval != 0: + if (time - start).days % layer.time_axis_interval != 0: raise WMSException( "Time dimension value '%s' not valid for this layer" % times[0], WMSException.INVALID_DIMENSION_VALUE, locator="Time parameter") - elif product.time_resolution.is_subday(): - if not find_matching_date(time, product.ranges["times"]): + elif layer.time_resolution.is_subday(): + if not find_matching_date(time, layer.ranges.times): raise WMSException( "Time dimension value '%s' not valid for this layer" % times[0], WMSException.INVALID_DIMENSION_VALUE, locator="Time parameter") else: - if time not in product.ranges["time_set"]: + if time not in layer.ranges.time_set: raise WMSException( "Time dimension value '%s' not valid for this layer" % times[0], WMSException.INVALID_DIMENSION_VALUE, diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 394e90510..597fe67a2 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -1,2 +1,2 @@ -FROM kartoza/postgis:15 +FROM kartoza/postgis:16 COPY s2_dump.sql /docker-entrypoint-initdb.d/ diff --git a/docs/database.rst b/docs/database.rst index 99cb017f2..030ce43cc 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -7,13 +7,13 @@ Datacube OWS uses three groups of database entities: 1. `ODC native entities <#open-data-cube-native-entities>`_ -2. Public `materialised views over ODC indexes <#materialised-views-over-odc-indexes>`_ +2. `OWS materialised views over ODC indexes <#materialised-views-over-odc-indexes>`_ 3. `OWS range tables <#range-tables-layer-extent-cache>`_. System Architecture Diagram --------------------------- -.. figure:: ./diagrams/ows_diagram.png +.. figure:: diagrams/ows_diagram1.9.png :target: /_images/ows_diagram.png OWS Architecture Diagram, including Database structure. @@ -37,22 +37,83 @@ step to populate the Layer Extent Cache, as described below, and for doing dataset queries for GetMap, GetFeatureInfo and GetCoverage requests. -It is hoped that this layer will eventually be implemented as tables -maintained as part of the core ODC index. Currently it must be -maintained separately using the ``datacube-ows-update`` (``update_ranges)``) +This layer will eventually become folded up into functionality in core, but +in the meantime (and while using the legacy ``postgres`` index driver), +it must be maintained separately using the ``datacube-ows-update`` (``update_ranges)``) command. -=========================== -Creating Materialised Views -=========================== +Range Tables (Layer Extent Cache) +---------------------------------- + +Range tables serve as a cache of full-layer spatio-temporal extents +for generating GetCapabilities documents efficiently. + +Creating/Maintaining the OWS Schema +----------------------------------- + +Creating or updating an OWS schema is performed with following options to ``datacube-ows-update``. + +Note that the options in this schema requires database superuser privileges. + +=================================== +Creating or Updating the OWS Schema +=================================== + +The ``--schema`` option to ``datacube-ows-update`` creates a new OWS schema if it does not exist, or +updates to the form required by the installed version of ``datacube-ows``:: + + datacube-ows-update --schema + +========================================== +Cleaning up an old datacube-ows 1.8 schema +========================================== + +In the 1.8.x series of datacube-ows releases, the OWS specific views and tables were stored somewhat haphazardly +in the misleadingly named ``wms`` schema with some entities in the ``public`` schema. + +After upgrading OWS to a 1.9.x series release, these older database entities can be dropped from the database +with the ``--cleanup`` option:: + + datacube-ows-update --cleanup + +You can combine the ``--schema`` and ``--cleanup`` options in the one invocation:: + + datacube-ows-update --schema --cleanup + +The new schema is always created before dropping the new one, regardless of the order you specify the options. -The materialised views are created (along with the range tables, -as described below) with the ``--schema`` flag: +====================================== +Granting permissions to database roles +====================================== - ``datacube-ows-update --schema --role rolename`` +The ``datacube-ows`` web application requires permissions to read from the various tables and views in the ``ows`` +schema. These can permissions can be granted to a database role with the ``--read-role `` argument:: -where ``rolename`` is the name of the database role that the OWS server -instance will use. + datacube-ows-update --read-role role1 + +The role used by ``datacube-ows`` also needs read-access to the ODC tables. These should managed with the ``datacube users`` +CLI tool - refer to the datacube-core documentation. + +You do not need to use --read-role and --write-role on the same user - granting write permissions automatically +grants read permissions as well. + +The ``datacube-ows-update`` options that update the data in the OWS schema (described below) require read/write +permissions that can be granted with the ``--write-role `` option:: + + datacube-ows-update --write-role role2 + +The ``--read-role`` and ``write-role`` options can be used separately, together, and/or with the ``--schema`` +and ``--cleanup`` options. They can also be used multiple times in the one invocation to grant permissions to +multiple roles:: + + datacube-ows-update --schema --cleanup --read-role role1 --read-role role2 --read-role role3 --write-role admin + + +Updating/Maintaining OWS data +----------------------------- + +Updating/maintaining data in the OWS schema requires the permissions granted with ``--write-role``, +as described above. It is performed with the following ``datacube-ows-update`` options: ============================= Refreshing Materialised Views @@ -60,9 +121,9 @@ Refreshing Materialised Views As datasets are added into or archived out of the ODC database, the materialised views become stale, and need to be periodically -manually refreshed, with the ``--view`` flag. +manually refreshed, with the ``--views`` flag. - ``datacube-ows-update --view`` + ``datacube-ows-update --views`` A lot of the speed of OWS comes from pushing expensive database calculations down into these materialised @@ -80,38 +141,31 @@ ENVIRONMENT. This will leave OWS broken and unable to respond to requests until the refresh is complete. In a production environment you should not be refreshing views -much more than 3 or 4 times a day unless your database is very small. - -Range Tables (Layer Extent Cache) ----------------------------------- - -Range tables serve as a cache of full-layer spatio-temporal extents -for generating GetCapabilities documents efficiently. They are -maintained with the ``datacube-ows-update`` (``update_ranges.py``) -command. - -The range tables are created at the same time as the materialised -views using the ``--schema`` flag, -`as described above <#creating-materialised-views>`_. +much more than 2 or 3 times a day unless your database is small +(e.g. less than a few thousand datasets). ===================== Updating range tables ===================== -The range tables are updated from the materialised views by simply calling: +The range table is updated from the materialised views by simply calling: datacube-ows-update -Note that this operation is very fast and computationally light -compared to refreshing the materialised views. +Note that this operation is very fast and computationally light compared to refreshing the materialised views. -------------------------------------------- +In a production environment, this should be run after refreshing the materialised views, as described above (after +waiting a couple of minutes for the final refresh to complete). + +=========================================== Updating range tables for individual layers -------------------------------------------- +=========================================== Specific layers can be updated using: datacube-ows-update layer1 layer2 layer3 -(You can use OWS layer names or ODC product names here, -but OWS layer names are generally preferred). +(In datacube-ows 1.8.x you could also use ODC product names, but this is no longer supported) + +This will need to be done after adding a new layer to the OWS configuration, or after changing the time +resolution or the ODC product(s) of an existing layer. diff --git a/docs/diagrams/db-relationship-diagram.svg b/docs/diagrams/db-relationship-diagram.svg deleted file mode 100644 index afacf221a..000000000 --- a/docs/diagrams/db-relationship-diagram.svg +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - -largeRelationshipsDiagram - -Generated by SchemaSpy - - -sub_product_ranges - - - - -sub_product_ranges -[table] - - -product_id - - -sub_product_id - -lat_min - -lat_max - -lon_min - -lon_max - -dates - -bboxes - - -< 1 - - -   - - -   - - - - - -agdc.dataset_type - - - - -agdc.dataset_type -[table] - -id - -name - -metadata - -metadata_type_ref - -definition - -added - -added_by - -updated - - -   - - -   - - -2 > - - - - - -sub_product_ranges:w->agdc.dataset_type:e - - - - - - -product_ranges - - - - -product_ranges -[table] - - -id - -lat_min - -lat_max - -lon_min - -lon_max - -dates - -bboxes - - -< 1 - - -   - - -   - - - - - -product_ranges:w->agdc.dataset_type:e - - - - - - - - -orphans - - - -multiproduct_ranges - - - - -multiproduct_ranges -[table] - - -wms_product_name - -varchar[128] - -lat_min - -numeric[0] - -lat_max - -numeric[0] - -lon_min - -numeric[0] - -lon_max - -numeric[0] - -dates - -jsonb[2147483647] - -bboxes - -jsonb[2147483647] - - -< 0 - - -   - - -0 > - - - - - - - diff --git a/docs/environment_variables.rst b/docs/environment_variables.rst index b149f1508..b95a72177 100644 --- a/docs/environment_variables.rst +++ b/docs/environment_variables.rst @@ -23,8 +23,16 @@ environment variable. The format of postgres connection URL is:: postgresql://:@:/ Other valid methods for configuring an OpenDatacube instance (e.g. a ``.datacube.conf`` file) -should also work. However OWS currently expects to use the `default` configuration environment -and only works with legacy/postgres index driver. +should also work. Note that OWS currently only works with legacy/postgres index driver. +Postgis support is hopefully coming soon. + +An ODC environment other than ``default`` can be used by setting the ``env`` option in the global OWS +configuration. + +Note that ``docker-compose`` arrangement used for integration testing on github also redundantly requires +the ``$DB_USERNAME``, ``$DB_PASSWORD``, ``$DB_DATABSE`` and ``$DB_PORT`` environment variables to set up +the generic docker postgres container. If you are connecting to an existing database, these variables +are not required. Configuring AWS Access ---------------------- @@ -93,6 +101,8 @@ Dev Tools PYDEV_DEBUG: If set to anything other than "n", "f", "no" or "false" (case insensitive), activates PyDev remote debugging. + NEVER use in production. + DEFER_CFG_PARSE: If set, the configuration file is not read and parsed at startup. This is mostly useful for creating test fixtures. diff --git a/integration_tests/test_layers.py b/integration_tests/test_layers.py index c3d47f55d..e70862234 100644 --- a/integration_tests/test_layers.py +++ b/integration_tests/test_layers.py @@ -97,7 +97,7 @@ def test_metadata_read(monkeypatch, product_name): assert "Over-ridden" in folder.abstract assert "bunny-rabbit" in folder.abstract - lyr = cfg.product_index[product_name] + lyr = cfg.layer_index[product_name] assert "Over-ridden" in lyr.title assert "chook" in lyr.title diff --git a/integration_tests/test_mv_index.py b/integration_tests/test_mv_index.py index 970e56d19..234f719a1 100644 --- a/integration_tests/test_mv_index.py +++ b/integration_tests/test_mv_index.py @@ -14,14 +14,14 @@ def test_full_layer(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] sel = mv_search(lyr.dc.index, MVSelectOpts.COUNT, products=lyr.products) assert sel > 0 def test_select_all(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] rows = mv_search(lyr.dc.index, MVSelectOpts.ALL, products=lyr.products) for row in rows: assert len(row) > 1 @@ -29,7 +29,7 @@ def test_select_all(): def test_no_products(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] with pytest.raises(Exception) as e: sel = mv_search(lyr.dc.index, MVSelectOpts.COUNT) assert "Must filter by product/layer" in str(e.value) @@ -37,7 +37,7 @@ def test_no_products(): def test_bad_set_opt(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] with pytest.raises(ValueError) as e: sel = MVSelectOpts("INVALID") @@ -51,8 +51,8 @@ def __init__(self, geom): def test_time_search(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] - time = lyr.ranges["times"][-1] + lyr = list(cfg.layer_index.values())[0] + time = lyr.ranges.end_time geom = box( lyr.bboxes["EPSG:4326"]["left"], lyr.bboxes["EPSG:4326"]["bottom"], @@ -70,7 +70,7 @@ def test_time_search(): def test_count(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] count = mv_search(lyr.dc.index, MVSelectOpts.COUNT, products=lyr.products) ids = mv_search(lyr.dc.index, MVSelectOpts.IDS, products=lyr.products) assert len(ids) == count @@ -78,7 +78,7 @@ def test_count(): def test_datasets(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] dss = mv_search(lyr.dc.index, MVSelectOpts.DATASETS, products=lyr.products) ids = mv_search(lyr.dc.index, MVSelectOpts.IDS, products=lyr.products) assert len(ids) == len(dss) @@ -88,7 +88,7 @@ def test_datasets(): def test_extent_and_spatial(): cfg = get_config() - lyr = list(cfg.product_index.values())[0] + lyr = list(cfg.layer_index.values())[0] layer_ext_bbx = ( lyr.bboxes["EPSG:4326"]["left"], lyr.bboxes["EPSG:4326"]["bottom"], diff --git a/integration_tests/test_update_ranges.py b/integration_tests/test_update_ranges.py index 603b3024c..2dbda52ff 100644 --- a/integration_tests/test_update_ranges.py +++ b/integration_tests/test_update_ranges.py @@ -10,8 +10,26 @@ from datacube_ows.update_ranges_impl import main -def test_updates_ranges_schema(runner, role_name): - result = runner.invoke(main, ["--schema", "--role", role_name]) +def test_update_ranges_schema_without_roles(runner): + result = runner.invoke(main, ["--schema"]) + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 + + +def test_update_ranges_schema_with_roles(runner, role_name): + result = runner.invoke(main, ["--schema", "--read-role", role_name, "--write-role", role_name]) + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 + + +def test_update_ranges_roles_only(runner, role_name): + result = runner.invoke(main, ["--read-role", role_name, "--write-role", role_name]) + assert "Cannot find SQL resource" not in result.output + assert result.exit_code == 0 + + +def test_update_ranges_cleanup(runner): + result = runner.invoke(main, ["--cleanup"]) assert "Cannot find SQL resource" not in result.output assert result.exit_code == 0 @@ -37,7 +55,7 @@ def test_update_ranges_product(runner, product_name): def test_update_ranges_bad_product(runner, product_name): result = runner.invoke(main, ["not_a_real_product_name"]) assert "not_a_real_product_name" in result.output - assert "Unrecognised product name" in result.output + assert "does not exist in the OWS configuration - skipping" in result.output assert result.exit_code == 1 @@ -45,21 +63,3 @@ def test_update_ranges(runner): result = runner.invoke(main) assert "ERROR" not in result.output assert result.exit_code == 0 - - -def test_update_ranges_misuse_cases(runner, role_name, product_name): - result = runner.invoke(main, ["--schema"]) - assert "Sorry" in result.output - assert result.exit_code == 1 - - result = runner.invoke(main, ["--role", role_name]) - assert "Sorry" in result.output - assert result.exit_code == 1 - - result = runner.invoke(main, ["--views", product_name]) - assert "Sorry" in result.output - assert result.exit_code == 1 - - result = runner.invoke(main, ["--schema", product_name]) - assert "Sorry" in result.output - assert result.exit_code == 1 diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index 9e3000351..22d35e740 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -206,7 +206,7 @@ def test_wcs1_time_exceptions(ows_server): contents = list(wcs.contents) test_layer_name = contents[0] cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST_TWO ) @@ -269,7 +269,7 @@ def test_wcs1_multi_time_exceptions(ows_server): contents = list(wcs.contents) test_layer_name = contents[0] cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST_TWO ) @@ -299,7 +299,7 @@ def test_wcs1_getcov_no_meas(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -328,7 +328,7 @@ def test_wcs1_getcov_multi_style(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -357,7 +357,7 @@ def test_wcs1_width_height_res_exceptions(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -551,7 +551,7 @@ def test_wcs1_style(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -615,7 +615,7 @@ def test_wcs1_ows_stats(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -647,7 +647,7 @@ def test_wcs1_getcov_bad_meas(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -676,7 +676,7 @@ def test_wcs1_getcov_badexception(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -706,7 +706,7 @@ def test_wcs1_getcov_interp(ows_server): if not test_layer_name.startswith("s2_l"): break cfg = get_config(refresh=True) - layer = cfg.product_index[test_layer_name] + layer = cfg.layer_index[test_layer_name] extents = ODCExtent(layer).wcs1_args( space=ODCExtent.CENTRAL_SUBSET_FOR_TIMES, time=ODCExtent.FIRST ) @@ -868,7 +868,7 @@ def test_extent_utils(): OWSConfig._instance = None cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1062,7 +1062,7 @@ def test_wcs20_getcoverage_geotiff(ows_server): # Ensure that we have at least some layers available contents = list(wcs.contents) - layer = cfg.product_index[contents[0]] + layer = cfg.layer_index[contents[0]] assert layer.ready and not layer.hide extent = ODCExtent(layer) subsets = extent.wcs2_subsets( @@ -1084,7 +1084,7 @@ def test_wcs20_getcoverage_geotiff_bigimage(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="2.0.0", timeout=120) - layer = cfg.product_index.get("s2_l2a_clone") + layer = cfg.layer_index.get("s2_l2a_clone") assert layer.ready and not layer.hide extent = ODCExtent(layer) subsets = extent.wcs2_subsets( @@ -1108,7 +1108,7 @@ def test_wcs20_getcoverage_netcdf(ows_server): # Ensure that we have at least some layers available contents = list(wcs.contents) - layer = cfg.product_index[contents[0]] + layer = cfg.layer_index[contents[0]] extent = ODCExtent(layer) subsets = extent.wcs2_subsets( ODCExtent.CENTRAL_SUBSET_FOR_TIMES, ODCExtent.SECOND, "EPSG:4326" @@ -1135,7 +1135,7 @@ def test_wcs20_getcoverage_crs_alias(ows_server): contents = list(wcs.contents) for lyr_name in contents: if not lyr_name.startswith('s2_l'): - layer = cfg.product_index[lyr_name] + layer = cfg.layer_index[lyr_name] break extent = ODCExtent(layer) subsets = extent.wcs2_subsets( @@ -1163,7 +1163,7 @@ def test_wcs20_getcoverage_multidate_geotiff(ows_server): # Ensure that we have at least some layers available contents = list(wcs.contents) - layer = cfg.product_index[contents[0]] + layer = cfg.layer_index[contents[0]] extent = ODCExtent(layer) subsets = extent.wcs2_subsets( ODCExtent.CENTRAL_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO, crs="EPSG:4326" @@ -1188,7 +1188,7 @@ def test_wcs20_getcoverage_multidate_netcdf(ows_server): # Ensure that we have at least some layers available contents = list(wcs.contents) assert len(contents) == 6 - layer = cfg.product_index[contents[0]] + layer = cfg.layer_index[contents[0]] extent = ODCExtent(layer) subsets = extent.wcs2_subsets( ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO, crs="EPSG:4326" @@ -1211,7 +1211,7 @@ def test_wcs21_server(ows_server): assert r.status_code == 200 cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr assert lyr.name in r.text @@ -1255,7 +1255,7 @@ def test_wcs21_describecoverage(ows_server): def test_wcs21_getcoverage(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1283,7 +1283,7 @@ def test_wcs21_getcoverage(ows_server): def test_wcs21_ows_stats(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1328,7 +1328,7 @@ def test_wcs2_getcov_badcov(ows_server): def test_wcs2_getcov_unpub_subset_crs(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1351,7 +1351,7 @@ def test_wcs2_getcov_unpub_subset_crs(ows_server): def test_wcs2_getcov_unpub_output_crs(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1374,7 +1374,7 @@ def test_wcs2_getcov_unpub_output_crs(ows_server): def test_wcs2_getcov_dup_subset_dims(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1403,7 +1403,7 @@ def test_wcs2_getcov_dup_subset_dims(ows_server): def test_wcs2_getcov_trim_time(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1437,7 +1437,7 @@ def test_wcs2_getcov_trim_time(ows_server): def test_wcs2_getcov_badtrim_time(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1472,7 +1472,7 @@ def test_wcs2_getcov_badtrim_time(ows_server): def test_wcs2_getcov_slice_space(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1502,7 +1502,7 @@ def test_wcs2_getcov_slice_space(ows_server): def test_wcs2_getcov_invalid_space_dim(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1533,7 +1533,7 @@ def test_wcs2_getcov_invalid_space_dim(ows_server): def test_wcs2_getcov_duplicate_scale_dim(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1563,7 +1563,7 @@ def test_wcs2_getcov_duplicate_scale_dim(ows_server): def test_wcs2_getcov_unscalable_dim(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1593,7 +1593,7 @@ def test_wcs2_getcov_unscalable_dim(ows_server): def test_wcs2_getcov_styles(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1669,7 +1669,7 @@ def test_wcs2_getcov_styles(ows_server): def test_wcs2_tiff_multidate(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1700,7 +1700,7 @@ def test_wcs2_tiff_multidate(ows_server): def test_wcs2_getcov_bands(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1730,7 +1730,7 @@ def test_wcs2_getcov_bands(ows_server): def test_wcs2_getcov_band_range(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith("s2_l"): layer = lyr break @@ -1760,7 +1760,7 @@ def test_wcs2_getcov_band_range(ows_server): def test_wcs2_getcov_bad_band(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1791,7 +1791,7 @@ def test_wcs2_getcov_bad_band(ows_server): def test_wcs2_getcov_bad_band_range(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1839,7 +1839,7 @@ def test_wcs2_getcov_bad_band_range(ows_server): def test_wcs2_getcov_native_format(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide and not lyr.name.startswith('s2_l'): layer = lyr break @@ -1867,7 +1867,7 @@ def test_wcs2_getcov_native_format(ows_server): def test_wcs2_getcov_bad_format(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break @@ -1914,7 +1914,7 @@ def test_wcs2_getcov_bad_format(ows_server): def test_wcs2_getcov_bad_multitime_format(ows_server): cfg = get_config(refresh=True) layer = None - for lyr in cfg.product_index.values(): + for lyr in cfg.layer_index.values(): if lyr.ready and not lyr.hide: layer = lyr break diff --git a/integration_tests/test_wms_server.py b/integration_tests/test_wms_server.py index 54bfa3b47..72e0ff0df 100644 --- a/integration_tests/test_wms_server.py +++ b/integration_tests/test_wms_server.py @@ -8,6 +8,7 @@ import pytest import requests +import math from lxml import etree from owslib.wms import WebMapService @@ -107,7 +108,7 @@ def test_getcap_response(ows_server): geo_bbox = layer.findall( "./{http://www.opengis.net/wms}BoundingBox[@CRS='EPSG:4326']" )[0] - assert wLong.text == geo_bbox.attrib["miny"] + assert math.isclose(float(wLong.text), float(geo_bbox.attrib["miny"]), rel_tol=1e-8) def test_wms_server(ows_server): diff --git a/integration_tests/utils.py b/integration_tests/utils.py index a1fd7d7bc..107b1a6c3 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -368,7 +368,7 @@ def subsets( space=SpaceRequestType.CENTRAL_SUBSET_FOR_TIMES, time=TimeRequestTypes.LAST, ): - ext_times = time.slice(self.layer.ranges["times"]) + ext_times = time.slice(self.layer.ranges.times) search_times = [self.layer.search_times(t) for t in ext_times] if space.needs_full_extent() and not self.full_extent: self.full_extent = mv_search( diff --git a/tests/cfg/minimal_cfg.py b/tests/cfg/minimal_cfg.py index 3d4c6bfc7..8a2a66835 100644 --- a/tests/cfg/minimal_cfg.py +++ b/tests/cfg/minimal_cfg.py @@ -9,6 +9,7 @@ "title": "Minimal test config", "allowed_urls": [], "info_url": "http://opendatacube.org", + "env": "nosuchdb", "published_CRSs": { "EPSG:3857": { # Web Mercator "geographic": False, diff --git a/tests/conftest.py b/tests/conftest.py index be7c91e2b..f7ee192c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,11 +114,12 @@ def lookup_measurements(ls): def minimal_global_cfg(minimal_dc): global_cfg = MagicMock() global_cfg.keywords = {"global"} - global_cfg.product_index = {} + global_cfg.layer_index = {} global_cfg.attribution.title = "Global Attribution" global_cfg.contact_org = None global_cfg.contact_position = None global_cfg.abstract = "Global Abstract" + global_cfg.odc_app = "app" global_cfg.dc = minimal_dc global_cfg.authorities = { "auth0": "http://test.url/auth0", @@ -173,8 +174,9 @@ def minimal_global_cfg(minimal_dc): @pytest.fixture -def minimal_parent(): +def minimal_parent(minimal_global_cfg): parent = MagicMock() + parent.dc = minimal_global_cfg.dc parent.abstract = "Parent Abstract" parent.keywords = {"global", "parent"} parent.attribution.title = "Parent Attribution" @@ -248,26 +250,18 @@ def minimal_multiprod_cfg(): @pytest.fixture def mock_range(): + from datacube_ows.product_ranges import LayerExtent, CoordRange times = [datetime.date(2010, 1, 1), datetime.date(2010, 1, 2), datetime.date(2010, 1, 3)] - return { - "lat": { - "min": -0.1, - "max": 0.1, - }, - "lon": { - "min": -0.1, - "max": 0.1, - }, - "times": times, - "start_time": times[0], - "end_time": times[-1], - "time_set": set(times), - "bboxes": { + return LayerExtent( + lat=CoordRange(-0.1, 0.1), + lon=CoordRange(-0.1, 0.1), + times=times, + bboxes={ "EPSG:4326": {"top": 0.1, "bottom": -0.1, "left": -0.1, "right": 0.1, }, "EPSG:3577": {"top": 0.1, "bottom": -0.1, "left": -0.1, "right": 0.1, }, "EPSG:3857": {"top": 0.1, "bottom": -0.1, "left": -0.1, "right": 0.1, }, } - } + ) @pytest.fixture diff --git a/tests/test_cfg_global.py b/tests/test_cfg_global.py index 631fbb34d..05a58b12d 100644 --- a/tests/test_cfg_global.py +++ b/tests/test_cfg_global.py @@ -79,7 +79,7 @@ def fake_dc(*args, **kwargs): "horizontal_coord": "x", "vertical_coord": "y", }, - "EPSG:99899": { # Made up + "EPSG:2154": { # Made up - epsg:2154 is real, but does not have these properties. "geographic": True, "horizontal_coord": "longitude", "vertical_coord": "latitude", @@ -87,7 +87,7 @@ def fake_dc(*args, **kwargs): } cfg = OWSConfig(cfg=minimal_global_raw_cfg) cfg.make_ready(minimal_dc) - assert cfg.default_geographic_CRS == "EPSG:99899" + assert cfg.default_geographic_CRS == "EPSG:2154" def test_contact_details_parse(minimal_global_cfg): diff --git a/tests/test_cfg_layer.py b/tests/test_cfg_layer.py index f79abd4e3..b1f29dc05 100644 --- a/tests/test_cfg_layer.py +++ b/tests/test_cfg_layer.py @@ -13,6 +13,7 @@ from datacube_ows.config_utils import ConfigException from datacube_ows.ows_configuration import OWSFolder, OWSLayer, parse_ows_layer from datacube_ows.resource_limits import ResourceLimited +from datacube_ows.product_ranges import LayerExtent, CoordRange def test_missing_title(minimal_global_cfg): @@ -179,7 +180,7 @@ def test_minimal_named_layer(minimal_layer_cfg, minimal_global_cfg, mock_range): lyr.make_ready() assert lyr.ready assert not lyr.hide - assert lyr.default_time == mock_range["times"][-1] + assert lyr.default_time == mock_range.times[-1] assert "a_layer" in str(lyr) assert len(lyr.low_res_products) == 0 assert lyr.mosaic_date_func is None @@ -433,7 +434,7 @@ def test_resource_limit_checks(minimal_layer_cfg, minimal_global_cfg): assert "too much projected resource requirements" not in str(e.value) assert e.value.wcs_hard minimal_layer_cfg["resource_limits"]["wms"]["min_zoom_level"] = 5 - minimal_global_cfg.product_index = {} + minimal_global_cfg.layer_index = {} lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) with pytest.raises(ResourceLimited) as e: @@ -593,14 +594,17 @@ def test_invalid_native_format(minimal_layer_cfg, minimal_global_cfg): def test_time_range_irreg(minimal_layer_cfg, minimal_global_cfg): lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) - ranges = { - "times": [ + ranges = LayerExtent( + lat=CoordRange(-0.1, 0.1), + lon=CoordRange(-0.1, 0.1), + times= [ datetime.date(2021, 1, 5), datetime.date(2021, 1, 6), datetime.date(2021, 1, 7), datetime.date(2021, 1, 8), - ] - } + ], + bboxes={} + ) start, end = lyr.time_range(ranges) assert start == datetime.date(2021, 1, 5) assert end == datetime.date(2021, 1, 8) @@ -611,14 +615,17 @@ def test_time_range_reg_default(minimal_layer_cfg, minimal_global_cfg): "time_interval": 1 } lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) - ranges = { - "times": [ + ranges = LayerExtent( + lat=CoordRange(-0.1, 0.1), + lon=CoordRange(-0.1, 0.1), + times= [ datetime.date(2021, 1, 5), datetime.date(2021, 1, 6), datetime.date(2021, 1, 7), datetime.date(2021, 1, 8), - ] - } + ], + bboxes={} + ) start, end = lyr.time_range(ranges) assert start == datetime.date(2021, 1, 5) assert end == datetime.date(2021, 1, 8) @@ -715,7 +722,7 @@ def test_earliest_default_time(minimal_layer_cfg, minimal_global_cfg, minimal_dc lyr.make_ready(minimal_dc) assert lyr.ready assert not lyr.hide - assert lyr.default_time == mock_range["times"][0] + assert lyr.default_time == mock_range.times[0] assert "a_layer" in str(lyr) assert len(lyr.low_res_products) == 0 @@ -730,7 +737,7 @@ def test_latest_default_time(minimal_layer_cfg, minimal_global_cfg, minimal_dc, lyr.make_ready(minimal_dc) assert lyr.ready assert not lyr.hide - assert lyr.default_time == mock_range["times"][-1] + assert lyr.default_time == mock_range.times[-1] assert "a_layer" in str(lyr) assert len(lyr.low_res_products) == 0 @@ -751,8 +758,7 @@ def test_valid_default_time(minimal_layer_cfg, minimal_global_cfg, minimal_dc, m def test_missing_default_time(minimal_layer_cfg, minimal_global_cfg, minimal_dc, mock_range): minimal_layer_cfg["default_time"] = "2020-01-22" - lyr = parse_ows_layer(minimal_layer_cfg, - global_cfg=minimal_global_cfg) + lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) assert lyr.name == "a_layer" assert not lyr.ready with patch("datacube_ows.product_ranges.get_ranges") as get_rng: @@ -760,7 +766,7 @@ def test_missing_default_time(minimal_layer_cfg, minimal_global_cfg, minimal_dc, lyr.make_ready(minimal_dc) assert lyr.ready assert not lyr.hide - assert lyr.default_time == mock_range["times"][-1] + assert lyr.default_time == mock_range.times[-1] assert "a_layer" in str(lyr) assert len(lyr.low_res_products) == 0 diff --git a/tests/test_cfg_wcs.py b/tests/test_cfg_wcs.py index ce4eb5043..a65be949b 100644 --- a/tests/test_cfg_wcs.py +++ b/tests/test_cfg_wcs.py @@ -18,7 +18,7 @@ def test_zero_grid(minimal_global_cfg, minimal_layer_cfg, minimal_dc, mock_range minimal_layer_cfg["product_name"] = "foo_nativeres" lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) - mock_range["bboxes"]["EPSG:4326"] = { + mock_range.bboxes["EPSG:4326"] = { "top": 0.1, "bottom": 0.1, "left": -0.1, "right": 0.1, } @@ -30,10 +30,10 @@ def test_zero_grid(minimal_global_cfg, minimal_layer_cfg, minimal_dc, mock_range assert "Grid High y is non-positive" in str(excinfo.value) assert "a_layer" in str(excinfo.value) assert "EPSG:4326" in str(excinfo.value) - minimal_global_cfg.product_index = {} + minimal_global_cfg.layer_index = {} lyr = parse_ows_layer(minimal_layer_cfg, global_cfg=minimal_global_cfg) - mock_range["bboxes"]["EPSG:4326"] = { + mock_range.bboxes["EPSG:4326"] = { "top": 0.1, "bottom": -0.1, "left": -0.1, "right": -0.1, } diff --git a/tests/test_no_db_routes.py b/tests/test_no_db_routes.py index 5190b59c0..1d9220a5a 100644 --- a/tests/test_no_db_routes.py +++ b/tests/test_no_db_routes.py @@ -8,64 +8,58 @@ """ import os import sys +import pytest src_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if src_dir not in sys.path: sys.path.append(src_dir) +def reset_global_config(): + from datacube_ows.ows_configuration import OWSConfig + OWSConfig._instance = None + + +@pytest.fixture def no_db(monkeypatch): monkeypatch.setenv("DATACUBE_OWS_CFG", "tests.cfg.minimal_cfg.ows_cfg") - monkeypatch.setenv("DB_USERNAME", "fakeuser") - monkeypatch.setenv("DB_PASSWORD", "password") - monkeypatch.setenv("DB_HOSTNAME", "localhost") - from datacube_ows.ows_configuration import get_config - cfg = get_config(refresh=True) + reset_global_config() + yield + reset_global_config() -def test_db_connect_fail(monkeypatch, flask_client): +def test_db_connect_fail(no_db, flask_client): """Start with a database connection""" - - no_db(monkeypatch) rv = flask_client.get('/ping') assert rv.status_code == 500 -def test_wcs_fail(monkeypatch, flask_client): +def test_wcs_fail(no_db, flask_client): """WCS endpoint fails""" - - no_db(monkeypatch) rv = flask_client.get('/wcs') assert rv.status_code == 400 -def test_wms_fail(monkeypatch, flask_client): +def test_wms_fail(no_db, flask_client): """WMS endpoint fails""" - - no_db(monkeypatch) rv = flask_client.get('/wms') assert rv.status_code == 400 -def test_wmts_fail(monkeypatch, flask_client): +def test_wmts_fail(no_db, flask_client): """WMTS endpoint fails""" - - no_db(monkeypatch) rv = flask_client.get('/wmts') assert rv.status_code == 400 -def test_legend_fail(monkeypatch, flask_client): +def test_legend_fail(no_db, flask_client): """Fail on legend""" - - no_db(monkeypatch) rv = flask_client.get("/legend/layer/style/legend.png") assert rv.status_code == 404 -def test_index_fail(monkeypatch, flask_client): +def test_index_fail(no_db, flask_client): """Base index endpoint fails""" # Should actually be 200 TODO - no_db(monkeypatch) rv = flask_client.get('/') assert rv.status_code == 500 diff --git a/tests/test_styles.py b/tests/test_styles.py index f78a2244d..3308f11c0 100644 --- a/tests/test_styles.py +++ b/tests/test_styles.py @@ -80,7 +80,7 @@ def __init__(self, pq_name, band): "azure": "red", "fake": "fake", } - product_layer.global_cfg.product_index = { + product_layer.global_cfg.layer_index = { "test_product": product_layer } product_layer.data_manual_merge = False diff --git a/tests/test_update_ranges.py b/tests/test_update_ranges.py new file mode 100644 index 000000000..b605c893a --- /dev/null +++ b/tests/test_update_ranges.py @@ -0,0 +1,73 @@ +# This file is part of datacube-ows, part of the Open Data Cube project. +# See https://opendatacube.org for more information. +# +# Copyright (c) 2017-2024 OWS Contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Test update ranges on DB using Click testing +https://click.palletsprojects.com/en/7.x/testing/ +""" +import pytest +from click.testing import CliRunner +from datacube_ows.update_ranges_impl import main, run_sql + + +@pytest.fixture +def runner(): + return CliRunner() + + +@pytest.fixture +def role_name(): + return "role1" + + +@pytest.fixture +def layer_name(): + return "a_layer" + + +def test_update_ranges_misuse_cases(runner, role_name, layer_name): + result = runner.invoke(main, ["--schema", layer_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--cleanup", layer_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--read-role", role_name, layer_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--write-role", role_name, layer_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--views", "--cleanup"]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--views", layer_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--views", "--schema"]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--views", "--read-role", role_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + result = runner.invoke(main, ["--views", "--write-role", role_name]) + assert "Sorry" in result.output + assert result.exit_code == 1 + + +def test_run_sql(minimal_dc): + assert not run_sql(minimal_dc, "sql/no_such_directory") + + assert not run_sql(minimal_dc, "templates") + + assert not run_sql(minimal_dc, "ows_schema/grants/read_only") diff --git a/tests/test_wms_utils.py b/tests/test_wms_utils.py index 8a8d15450..31e3d3d9c 100644 --- a/tests/test_wms_utils.py +++ b/tests/test_wms_utils.py @@ -13,6 +13,7 @@ import datacube_ows.wms_utils from datacube_ows.ogc_exceptions import WMSException from datacube_ows.ows_configuration import TimeRes +from datacube_ows.product_ranges import LayerExtent, CoordRange def test_parse_time_delta(): @@ -168,17 +169,20 @@ def test_parse_unsorted_colorscalerange(dummy_product): assert "Colorscale range must be two numbers, sorted and separated by a comma." in str(e.value) def test_parse_item_1(dummy_product): - dummy_product.ranges = { - "times": [ + dummy_product.ranges = LayerExtent( + lat=CoordRange(-0.1, 0.1), + lon=CoordRange(-0.1, 0.1), + times = [ datetime.date(2021, 1, 6), datetime.date(2021, 1, 7), datetime.date(2021, 1, 8), datetime.date(2021, 1, 9), datetime.date(2021, 1, 10), - ] - } + ], + bboxes={} + ) dt = datacube_ows.wms_utils.parse_time_item("2010-01-01/2021-01-08", dummy_product) - assert dt == dummy_product.ranges["times"][0] + assert dt == dummy_product.ranges.times[0] dummy_product.regular_time_axis = True with pytest.raises(WMSException) as e: dt = datacube_ows.wms_utils.parse_time_item("2010-01-01/2010-01-08", dummy_product) @@ -189,7 +193,7 @@ def test_parse_item_1(dummy_product): assert "Time dimension range" in str(e.value) assert "not valid for this layer" in str(e.value) dt = datacube_ows.wms_utils.parse_time_item("", dummy_product) - assert dt == dummy_product.ranges["times"][-1] + assert dt == dummy_product.ranges.times[-1] with pytest.raises(WMSException) as e: dt = datacube_ows.wms_utils.parse_time_item("this_is_not_a_date, mate", dummy_product) assert "Time dimension value" in str(e.value) @@ -197,15 +201,17 @@ def test_parse_item_1(dummy_product): def test_parse_item_2(dummy_product): - dummy_product.ranges = { - "times": [ + dummy_product.ranges = LayerExtent( + lat=CoordRange(-0.1, 0.1), + lon=CoordRange(-0.1, 0.1), + times = [ datetime.date(2021, 1, 6), datetime.date(2021, 1, 7), datetime.date(2021, 1, 8), datetime.date(2021, 1, 10), - ] - } - dummy_product.ranges["time_set"] = set(dummy_product.ranges["times"]) + ], + bboxes={} + ) dummy_product.time_range.return_value = ( datetime.date(2021, 1, 6), datetime.date(2021, 1, 10), ) diff --git a/wordlist.txt b/wordlist.txt index 6643f5fa1..63599bb66 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -1,7 +1,9 @@ aa administrivia adocker +ACode ADocker +ADockerfile ae aec aedee @@ -255,7 +257,10 @@ multidate multiproc multiproduct mv +mydb +mypassword mysecretpassword +myuser namespace natively ncols