diff --git a/.env_simple b/.env_simple index 20a1c754f..0ee794fbe 100644 --- a/.env_simple +++ b/.env_simple @@ -5,7 +5,7 @@ DB_PORT=5432 DB_USERNAME=opendatacubeusername DB_PASSWORD=opendatacubepassword DB_DATABASE=opendatacube -OWS_CFG_FILE=./ows_test_cfg.py +OWS_CFG_FILE=./integration_tests/cfg/ows_test_cfg.py AWS_NO_SIGN_REQUEST=yes # If you want to use pydev for interactive debugging PYDEV_DEBUG= diff --git a/datacube_ows/cfg_parser.py b/datacube_ows/cfg_parser.py index e839f4870..f5aaed226 100755 --- a/datacube_ows/cfg_parser.py +++ b/datacube_ows/cfg_parser.py @@ -24,6 +24,7 @@ default=False, help="Only parse the syntax of the config file - do not validate against database", ) + @click.option( "-f", "--folders", @@ -38,6 +39,13 @@ default=False, help="Print the styles for each layer to stdout (format depends on --folders flag).", ) +@click.option( + "-c", + "--cfg-only", + is_flag=True, + default=False, + help="Read metadata from config only - ignore configured metadata message file).", +) @click.option( "-i", "--input-file", @@ -48,9 +56,16 @@ "-o", "--output-file", default=False, - help="Provide an output file name with extension .json", + help="Provide an output inventory file name with extension .json", ) -def main(version, parse_only, folders, styles, input_file, output_file, paths): +@click.option( + "-m", + "--msg-file", + default=False, + help="Write a message file containing the translatable metadata from the configuration.", +) + +def main(version, cfg_only, parse_only, folders, styles, msg_file, input_file, output_file, paths): """Test configuration files Valid invocations: @@ -70,12 +85,12 @@ def main(version, parse_only, folders, styles, input_file, output_file, paths): all_ok = True if not paths: - if parse_path(None, parse_only, folders, styles, input_file, output_file): + if parse_path(None, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file): return 0 else: sys.exit(1) for path in paths: - if not parse_path(path, parse_only, folders, styles, input_file, output_file): + if not parse_path(path, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file): all_ok = False if not all_ok: @@ -83,10 +98,10 @@ def main(version, parse_only, folders, styles, input_file, output_file, paths): return 0 -def parse_path(path, parse_only, folders, styles, input_file, output_file): +def parse_path(path, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file): try: raw_cfg = read_config(path) - cfg = OWSConfig(refresh=True, cfg=raw_cfg) + cfg = OWSConfig(refresh=True, cfg=raw_cfg, ignore_msgfile=cfg_only) if not parse_only: with Datacube() as dc: cfg.make_ready(dc) @@ -110,9 +125,28 @@ def parse_path(path, parse_only, folders, styles, input_file, output_file): print() if input_file or output_file: layers_report(cfg.product_index, input_file, output_file) + if msg_file: + write_msg_file(msg_file, cfg) return True +def write_msg_file(msg_file, cfg): + with open(msg_file, "w") as fp: + for key, value in cfg.export_metadata(): + if value: + print("", file=fp) + lines = list(value.split("\n")) + for line in lines: + print(f"#. {line}", file=fp) + print(f'msgid "{key}"', file=fp) + if len(lines) == 1: + print(f'msgstr "{value}"', file=fp) + else: + print('msgstr ""', file=fp) + for line in lines: + print(f'"{line}\\n"', file=fp) + + def layers_report(config_values, input_file, output_file): report = {"total_layers_count": len(config_values.values()), "layers": []} for lyr in config_values.values(): diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 1af87d721..816270fd3 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -90,6 +90,12 @@ def declare_unready(self, name): raise ConfigException(f"Cannot declare {name} as unready on a ready object") self._unready_attributes.add(name) + def get(self, key, default=None): + try: + return getattr(self, key) + except AttributeError: + return default + def __getattribute__(self, name): if name == "_unready_attributes": pass @@ -110,6 +116,213 @@ def make_ready(self, dc, *args, **kwargs): raise OWSConfigNotReady(f"The following parameters have not been initialised: {self._unready_attributes}") self.ready = True +FLD_TITLE = "title" +FLD_ABSTRACT = "abstract" +FLD_KEYWORDS = "local_keywords" +FLD_FEES = "fees" +FLD_ACCESS_CONSTRAINTS = "access_constraints" +FLD_ATTRIBUTION = "attribution_title" +FLD_CONTACT_ORGANISATION = "contact_org" +FLD_CONTACT_POSITION = "contact_position" + +class OWSMetadataConfig(OWSConfigEntry): + + METADATA_TITLE = True + METADATA_ABSTRACT = True + METADATA_KEYWORDS = False + METADATA_CONTACT_INFO = False + METADATA_FEES = False + METADATA_ACCESS_CONSTRAINTS = False + METADATA_ATTRIBUTION = False + + _metadata_registry = {} + _inheritance_registry = {} + + _msg_src = None + + def get_obj_label(self): + return "global" + + + def can_inherit_from(self): + return None + + default_title = None + default_abstract = None + + _keywords = None + def parse_metadata(self, cfg): + # can_inherit_from() can be over-ridden by subclasses + # pylint: disable=assignment-from-none + inherit_from = self.can_inherit_from() + if self.METADATA_TITLE: + if self.default_title: + self.register_metadata(self.get_obj_label(), FLD_TITLE, cfg.get("title", self.default_title)) + else: + try: + self.register_metadata(self.get_obj_label(), FLD_TITLE, cfg["title"]) + except KeyError: + raise ConfigException(f"Entity {self.get_obj_label()} has no title.") + if self.METADATA_ABSTRACT: + local_abstract = cfg.get("abstract") + if local_abstract is None and inherit_from is not None: + self.register_metadata(self.get_obj_label(), FLD_ABSTRACT, inherit_from.abstract, inherited=True) + elif local_abstract is None and self.default_abstract is not None: + self.register_metadata(self.get_obj_label(), FLD_ABSTRACT, self.default_abstract) + elif local_abstract is None: + raise ConfigException(f"Entity {self.get_obj_label()} has no abstract") + else: + self.register_metadata(self.get_obj_label(), "abstract", local_abstract) + if self.METADATA_KEYWORDS: + local_keyword_set = set(cfg.get("keywords", [])) + self.register_metadata(self.get_obj_label(), FLD_KEYWORDS, ",".join(local_keyword_set)) + if inherit_from: + keyword_set = inherit_from.keywords + else: + keyword_set = set() + self._keywords = keyword_set.union(local_keyword_set) + if self.METADATA_ATTRIBUTION: + inheriting = False + attrib = cfg.get("attribution") + if attrib is None and inherit_from is not None: + attrib = inherit_from.attribution + inheriting = True + if attrib: + attrib_title = attrib.get("title") + else: + attrib_title = None + if attrib_title: + self.register_metadata(self.get_obj_label(), FLD_ATTRIBUTION, attrib_title, inheriting) + if self.METADATA_FEES: + fees = cfg.get("fees") + if not fees: + fees = "none" + self.register_metadata(self.get_obj_label(), FLD_FEES, fees) + if self.METADATA_ACCESS_CONSTRAINTS: + acc = cfg.get("access_contraints") + if not acc: + acc = "none" + self.register_metadata(self.get_obj_label(), FLD_ACCESS_CONSTRAINTS, acc) + if self.METADATA_CONTACT_INFO: + org = cfg.get("contact_info", {}).get("organisation") + position = cfg.get("contact_info", {}).get("position") + if org: + self.register_metadata(self.get_obj_label(), FLD_CONTACT_ORGANISATION, org) + if position: + self.register_metadata(self.get_obj_label(), FLD_CONTACT_POSITION, position) + @property + def keywords(self): + return self._keywords + + @classmethod + def set_msg_src(cls, src): + OWSMetadataConfig._msg_src = src + + def read_metadata(self, lbl, fld): + lookup = ".".join([lbl, fld]) + if self._msg_src is not None: + return self._msg_src.get(lookup, self._metadata_registry.get(lookup)) + return self._metadata_registry.get(lookup) + + def read_inheritance(self, lbl, fld): + lookup = ".".join([lbl, fld]) + return self._inheritance_registry.get(lookup, False) + + def register_metadata(self, lbl, fld, val, inherited=False): + lookup = ".".join([lbl, fld]) + self._metadata_registry[lookup] = val + self._inheritance_registry[lookup] = inherited + + def read_local_metadata(self, fld): + return self.read_metadata(self.get_obj_label(), fld) + + def is_inherited(self, fld): + return self.read_inheritance(self.get_obj_label(), fld) + + def __getattribute__(self, name): + if name in (FLD_TITLE, FLD_ABSTRACT, FLD_FEES, FLD_ACCESS_CONSTRAINTS, FLD_CONTACT_POSITION, FLD_CONTACT_ORGANISATION): + return self.read_local_metadata(name) + elif name == FLD_KEYWORDS: + kw = self.read_local_metadata(FLD_KEYWORDS) + if kw: + return set(kw.split(",")) + else: + return set() + elif name == FLD_ATTRIBUTION: + return self.read_local_metadata(FLD_ATTRIBUTION) + else: + return super().__getattribute__(name) + + +class OWSMessageFile: + # Parser states + START = 0 + GOT_ID = 1 + IN_STR = 2 + + def __init__(self, fp): + self._index = {} + self.state = self.START + self.msgstr = "" + self.msgid = "" + self.line_no = 0 + self.read_file(fp) + + def _new_element(self): + if self.msgid: + self._index[self.msgid] = self.msgstr + self.state = self.START + self.msgstr = "" + self.msgid = "" + + def __getitem__(self, item): + return self._index[item] + + def __contains__(self, item): + return item in self._index + + def get(self, item, default=None): + return self._index.get(item, default) + + def unescape(self, esc_str): + return esc_str.encode("utf-8").decode("unicode-escape") + + def read_file(self, fp): + for line in fp.readlines(): + self.line_no += 1 + if line.startswith("msgid "): + if self.state == self.GOT_ID: + raise ConfigException(f"Unexpected msgid line in message file: at line {self.line_no}") + self._new_element() + self.msgid = line[7:-2] + if self.msgid in self: + raise ConfigException(f"Duplicate msgid: at line {self.line_no}") + self.state = self.GOT_ID + elif line.strip() == "": + if self.state == self.IN_STR: + self._new_element() + elif line.startswith("msgstr "): + if self.state == self.START: + raise ConfigException(f"msgstr without msgid: at line {self.line_no}") + elif self.state == self.IN_STR: + raise ConfigException(f"Badly formatted multi-line msgstr: at line {self.line_no}") + # GOT_ID: + self.msgstr = self.unescape(line[8:-2]) + self.state = self.IN_STR + elif line.startswith('"'): + if self.state == self.START: + raise ConfigException(f"untagged string: at line {self.line_no}") + elif self.state == self.GOT_ID: + raise ConfigException(f"Multiline msgids not supported by OWS: at line {self.line_no}") + # IN_STR + self.msgstr += self.unescape(line[1:-2]) + elif line.startswith("#"): + if self.state == self.IN_STR: + self._new_element() + else: + raise ConfigException(f"Invalid message file: error at line {self.line_no}") + self._new_element() + class OWSEntryNotFound(ConfigException): pass diff --git a/datacube_ows/ows_cfg_example.py b/datacube_ows/ows_cfg_example.py index ee4366731..e1e8ee793 100644 --- a/datacube_ows/ows_cfg_example.py +++ b/datacube_ows/ows_cfg_example.py @@ -1292,6 +1292,29 @@ "fax": "+61 2 1234 6789", "email": "test@example.com", }, + # Attribution. + # + # This provides a way to identify the source of the data used in a WMS layer or layers. + # This entire section is optional. If provided, it is taken as the + # default attribution for any layer that does not override it. + "attribution": { + # Attribution must contain at least one of ("title", "url" and "logo") + # A human readable title for the attribution - e.g. the name of the attributed organisation + "title": "Acme Satellites", + # The associated - e.g. URL for the attributed organisation + "url": "http://www.acme.com/satellites", + # Logo image - e.g. for the attributed organisation + "logo": { + # Image width in pixels (optional) + "width": 370, + # Image height in pixels (optional) + "height": 73, + # URL for the logo image. (required if logo specified) + "url": "https://www.acme.com/satellites/images/acme-370x73.png", + # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.) + "format": "image/png", + } + }, # If fees are charged for the use of the service, these can be described here in free text. # If blank or not included, defaults to "none". "fees": "", @@ -1334,27 +1357,7 @@ # Optional, defaults to 256x256 "max_width": 512, "max_height": 512, - # Attribution. This provides a way to identify the source of the data used in a layer or layers. - # This entire section is optional. If provided, it is taken as the - # default attribution for any layer that does not override it. - "attribution": { - # Attribution must contain at least one of ("title", "url" and "logo") - # A human readable title for the attribution - e.g. the name of the attributed organisation - "title": "Acme Satellites", - # The associated - e.g. URL for the attributed organisation - "url": "http://www.acme.com/satellites", - # Logo image - e.g. for the attributed organisation - "logo": { - # Image width in pixels (optional) - "width": 370, - # Image height in pixels (optional) - "height": 73, - # URL for the logo image. (required if logo specified) - "url": "https://www.acme.com/satellites/images/acme-370x73.png", - # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.) - "format": "image/png", - } - }, + # These define the AuthorityURLs. # They represent the authorities that define the "Identifiers" defined layer by layer below. # The spec allows AuthorityURLs to be defined anywhere on the Layer heirarchy, but datacube_ows treats them diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 0e12e6d96..0d732f164 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -20,6 +20,7 @@ from datacube_ows.config_utils import (FlagProductBands, OWSConfigEntry, OWSEntryNotFound, OWSExtensibleConfigEntry, OWSFlagBand, + OWSMessageFile, OWSMetadataConfig, cfg_expand, import_python_obj, load_json_obj) from datacube_ows.cube_pool import cube, get_cube, release_cube @@ -123,9 +124,9 @@ def band_nodata_vals(self): class AttributionCfg(OWSConfigEntry): - def __init__(self, cfg): + def __init__(self, cfg, owner): super().__init__(cfg) - self.title = cfg.get("title") + self.owner = owner self.url = cfg.get("url") logo = cfg.get("logo") if not self.title and not self.url and not logo: @@ -143,12 +144,16 @@ def __init__(self, cfg): if not self.logo_url or not self.logo_fmt: raise ConfigException("url and format must both be specified in an attribution logo.") + @property + def title(self): + return self.owner.attribution_title + @classmethod - def parse(cls, cfg): + def parse(cls, cfg, owner): if not cfg: return None else: - return cls(cfg) + return cls(cfg, owner) class SuppURL(OWSConfigEntry): @@ -165,40 +170,35 @@ def __init__(self, cfg): -class OWSLayer(OWSConfigEntry): +class OWSLayer(OWSMetadataConfig): + METADATA_KEYWORDS = True + METADATA_ATTRIBUTION = True + named = False - def __init__(self, cfg, parent_layer=None, **kwargs): + def __init__(self, cfg, object_label, parent_layer=None, **kwargs): super().__init__(cfg, **kwargs) + self.object_label = object_label self.global_cfg = kwargs["global_cfg"] self.parent_layer = parent_layer - if "title" not in cfg: - raise ConfigException("Layer without title found under parent layer %s" % str(parent_layer)) - self.title = cfg["title"] - if "abstract" in cfg: - self.abstract = cfg["abstract"] - elif parent_layer: - self.abstract = parent_layer.abstract - else: - raise ConfigException("No abstract supplied for top-level layer %s" % self.title) - # Accumulate keywords - self.keywords = set() - if self.parent_layer: - for word in self.parent_layer.keywords: - self.keywords.add(word) - else: - for word in self.global_cfg.keywords: - self.keywords.add(word) - for word in cfg.get("keywords", []): - self.keywords.add(word) + self.parse_metadata(cfg) # Inherit or override attribution if "attribution" in cfg: - self.attribution = AttributionCfg.parse(cfg.get("attribution")) + self.attribution = AttributionCfg.parse(cfg.get("attribution"), self) elif parent_layer: self.attribution = self.parent_layer.attribution else: self.attribution = self.global_cfg.attribution + def can_inherit_from(self): + if self.parent_layer: + return self.parent_layer + else: + return self.global_cfg + + def get_obj_label(self): + return self.object_label + def layer_count(self): return 0 @@ -210,19 +210,30 @@ def __str__(self): class OWSFolder(OWSLayer): - def __init__(self, cfg, global_cfg, parent_layer=None, **kwargs): - super().__init__(cfg, parent_layer, global_cfg=global_cfg, **kwargs) + def __init__(self, cfg, global_cfg, parent_layer=None, sibling=0, **kwargs): + if "label" in cfg: + obj_lbl = f"folder.{cfg['label']}" + elif parent_layer: + obj_lbl = f"{parent_layer.object_label}.{sibling}" + else: + obj_lbl = f"folder.{sibling}" + if obj_lbl in global_cfg.folder_index: + raise ConfigException(f"Duplicate folder label: {obj_lbl}") + super().__init__(cfg, parent_layer=parent_layer, object_label=obj_lbl, global_cfg=global_cfg, **kwargs) self.slug_name = slugify(self.title, separator="_") self.unready_layers = [] self.child_layers = [] if "layers" not in cfg: raise ConfigException("No layers section in folder layer %s" % self.title) + child = 0 for lyr_cfg in cfg["layers"]: try: - lyr = parse_ows_layer(lyr_cfg, global_cfg, parent_layer=self) + lyr = parse_ows_layer(lyr_cfg, global_cfg=global_cfg, parent_layer=self, sibling=child) self.unready_layers.append(lyr) except ConfigException as e: _LOG.error("Could not parse layer: %s", str(e)) + child += 1 + global_cfg.folder_index[obj_lbl] = self def unready_layer_count(self): return sum([l.layer_count() for l in self.unready_layers]) @@ -308,7 +319,7 @@ class OWSNamedLayer(OWSExtensibleConfigEntry, OWSLayer): def __init__(self, cfg, global_cfg, parent_layer=None, **kwargs): name = cfg["name"] - super().__init__(cfg, global_cfg=global_cfg, parent_layer=parent_layer, + super().__init__(cfg, object_label=f"layer.{name}", global_cfg=global_cfg, parent_layer=parent_layer, keyvals={"layer": name}, **kwargs) self.name = name @@ -384,7 +395,6 @@ def __init__(self, cfg, global_cfg, parent_layer=None, **kwargs): # And finally, add to the global product index. self.global_cfg.product_index[self.name] = self - # pylint: disable=attribute-defined-outside-init def make_ready(self, dc, *args, **kwargs): self.products = [] @@ -850,14 +860,14 @@ def parse_pq_names(self, cfg): } -def parse_ows_layer(cfg, global_cfg, parent_layer=None): +def parse_ows_layer(cfg, global_cfg, parent_layer=None, sibling=0): if cfg.get("name", None): if cfg.get("multi_product", False): return OWSMultiProductLayer(cfg, global_cfg, parent_layer) else: return OWSProductLayer(cfg, global_cfg, parent_layer) else: - return OWSFolder(cfg, global_cfg, parent_layer) + return OWSFolder(cfg, global_cfg, parent_layer=parent_layer, sibling=sibling) class WCSFormat: @@ -914,7 +924,51 @@ def renderer(self, version): return self.renderers[version] -class OWSConfig(OWSConfigEntry): +class ContactInfo(OWSConfigEntry): + def __init__(self, cfg, global_cfg): + super().__init__(cfg) + self.global_cfg = global_cfg + self.person = cfg.get("person") + + class Address(OWSConfigEntry): + def __init__(self, cfg): + super().__init__(cfg) + self.type = cfg.get("type") + self.address = cfg.get("address") + self.city = cfg.get("city") + self.state = cfg.get("state") + self.postcode = cfg.get("postcode") + self.country = cfg.get("country") + + @classmethod + def parse(cls, cfg): + if not cfg: + return None + else: + return cls(cfg) + + self.address = Address.parse(cfg.get("address")) + self.telephone = cfg.get("telephone") + self.fax = cfg.get("fax") + self.email = cfg.get("email") + + @property + def organisation(self): + return self.global_cfg.contact_org + + @property + def position(self): + return self.global_cfg.contact_position + + @classmethod + def parse(cls, cfg, global_cfg): + if cfg: + return cls(cfg, global_cfg) + else: + return None + + +class OWSConfig(OWSMetadataConfig): _instance = None initialised = False @@ -923,14 +977,22 @@ def __new__(cls, *args, **kwargs): cls._instance = super().__new__(cls) return cls._instance - def __init__(self, refresh=False, cfg=None): + METADATA_KEYWORDS = True + METADATA_ATTRIBUTIONS = True + METADATA_FEES = True + METADATA_ACCESS_CONSTRAINTS = True + METADATA_CONTACT_INFO = True + + default_abstract = "" + + def __init__(self, refresh=False, cfg=None, ignore_msgfile=False): if not self.initialised or refresh: + self.msgfile = None if not cfg: cfg = read_config() super().__init__(cfg) - self.initialised = True try: - self.parse_global(cfg["global"]) + self.parse_global(cfg["global"], ignore_msgfile) except KeyError as e: raise ConfigException( "Missing required config entry in 'global' section: %s" % str(e) @@ -967,30 +1029,46 @@ def __init__(self, refresh=False, cfg=None): #pylint: disable=attribute-defined-outside-init def make_ready(self, dc, *args, **kwargs): + if self.msg_file_name: + try: + with open(self.msg_file_name, "r") as fp: + self.set_msg_src(OWSMessageFile(fp)) + except FileNotFoundError: + _LOG.warning("Message file %s does not exist - using metadata from config file", self.msg_file_name) + else: + self.set_msg_src(None) self.native_product_index = {} self.root_layer_folder.make_ready(dc, *args, **kwargs) super().make_ready(dc, *args, **kwargs) - def parse_global(self, cfg): + def export_metadata(self): + for k, v in self._metadata_registry.items(): + if self._inheritance_registry[k]: + continue + if k in [ + "folder.ows_root_hidden.title", + "folder.ows_root_hidden.abstract", + "folder.ows_root_hidden.local_keywords", + ]: + continue + yield k, v + + def parse_global(self, cfg, ignore_msgfile): self._response_headers = cfg.get("response_headers", {}) self.wms = cfg.get("services", {}).get("wms", True) self.wmts = cfg.get("services", {}).get("wmts", True) self.wcs = cfg.get("services", {}).get("wcs", False) if not self.wms and not self.wmts and not self.wcs: raise ConfigException("At least one service must be active.") - self.title = cfg["title"] + if ignore_msgfile: + self.msg_file_name = None + else: + self.msg_file_name = cfg.get("message_file") + self.parse_metadata(cfg) self.allowed_urls = cfg["allowed_urls"] self.info_url = cfg["info_url"] - self.abstract = cfg.get("abstract") - self.contact_info = cfg.get("contact_info", {}) - self.keywords = cfg.get("keywords", []) - self.fees = cfg.get("fees") - self.access_constraints = cfg.get("access_constraints") - # self.use_extent_views = cfg.get("use_extent_views", False) - if not self.fees: - self.fees = "none" - if not self.access_constraints: - self.access_constraints = "none" + self.contact_info = ContactInfo.parse(cfg.get("contact_info"), self) + self.attribution = AttributionCfg.parse(cfg.get("attribution"), self) def make_gml_name(name): if name.startswith("EPSG:"): @@ -1040,9 +1118,10 @@ def parse_wms(self, cfg): self.s3_aws_zone = cfg.get("s3_aws_zone", "") self.wms_max_width = cfg.get("max_width", 256) self.wms_max_height = cfg.get("max_height", 256) - self.attribution = AttributionCfg.parse(cfg.get("attribution")) self.authorities = cfg.get("authorities", {}) self.user_band_math_extension = cfg.get("user_band_math_extension", False) + if "attribution" in cfg: + _LOG.warning("Attribution entry in top level 'wms' section will be ignored. Attribution should be moved to the 'global' section") def parse_wcs(self, cfg): if self.wcs: @@ -1088,13 +1167,14 @@ def parse_wmts(self, cfg): self.tile_matrix_sets[identifier] = TileMatrixSet(identifier, tms, self) def parse_layers(self, cfg): + self.folder_index = {} self.product_index = {} self.declare_unready("native_product_index") self.root_layer_folder = OWSFolder({ "title": "Root Folder (hidden)", - "abstract": ".", + "label": "ows_root_hidden", "layers": cfg - }, self, None) + }, global_cfg=self, parent_layer=None) @property def layers(self): diff --git a/datacube_ows/styles/api.py b/datacube_ows/styles/api.py deleted file mode 100644 index 554960cb6..000000000 --- a/datacube_ows/styles/api.py +++ /dev/null @@ -1,97 +0,0 @@ -from datacube_ows.startup_utils import initialise_ignorable_warnings -from datacube_ows.styles.base import StandaloneProductProxy, StyleDefBase - -initialise_ignorable_warnings() - - -def StandaloneStyle(cfg): - """ - Construct a OWS style object that stands alone, independent of a complete OWS configuration environment. - - :param cfg: A valid OWS Style definition configuration dictionary. - - Refer to the documentation for the valid syntax: - - https://datacube-ows.readthedocs.io/en/latest/cfg_styling.html - - :return: A OWS Style Definition object, prepared to work in standalone mode. - """ - style = StyleDefBase(StandaloneProductProxy(), cfg, stand_alone=True) - style.make_ready(None) - return style - - -def apply_ows_style(style, data, valid_data_mask=None): - """ - Apply an OWS style to an ODC XArray to generate a styled image. - - :param style: An OWS Style object, as created by StandaloneStyle() - :param data: An xarray Dataset, as generated by datacube.load_data() - Note that the Dataset must contain all of the band names referenced by the standalone style - configuration. (The names of the data variables in the dataset must exactly match - the band names in the configuration. None of the band aliasing techniques normally - supported by OWS can work in standalone mode.) - For bands that are used as bitmaps (i.e. either for masking with pq_mask or colour coding - in value_map), the data_variable must have a valid flag_definition attribute. - :param valid_data_mask: (optional) An xarray DataArray mask, with dimensions and coordinates matching data. - :return: An xarray Dataset, with the same dimensions and coordinates as data, and four data_vars of - 8 bit signed integer data named red, green, blue and alpha, representing an 24bit RGBA image. - """ - return style.transform_data( - data, - style.to_mask( - data, - valid_data_mask - ) - ) - - -def apply_ows_style_cfg(cfg, data, valid_data_mask=None): - """ - Apply an OWS style configuration to an ODC XArray to generate a styled image. - - :param cfg: A valid OWS Style definition configuration dictionary. - - Refer to the documentation for the valid syntax: - - https://datacube-ows.readthedocs.io/en/latest/cfg_styling.html - :param data: An xarray Dataset, as generated by datacube.load_data() - Note that the Dataset must contain all of the band names referenced by the standalone style - configuration. (The names of the data variables in the dataset must exactly match - the band names in the configuration. None of the band aliasing techniques normally - supported by OWS can work in standalone mode.) - For bands that are used as bitmaps (i.e. either for masking with pq_mask or colour coding - in value_map), the data_variable must have a valid flag_definition attribute. - :param valid_data_mask: (optional) An xarray DataArray mask, with dimensions and coordinates matching data. - :return: An xarray Dataset, with the same dimensions and coordinates as data, and four data_vars of - 8 bit signed integer data named red, green, blue and alpha, representing an 24bit RGBA image. - """ - return apply_ows_style( - StandaloneStyle(cfg), - data, - valid_data_mask - ) - - -def generate_ows_legend_style(style, ndates=0): - """ - - :param style: An OWS Style object, as created by StandaloneStyle() - :param ndates: (optional) Number of dates (for styles with multi-date handlers) - :return: A PIL Image object. - """ - return style.render_legend(ndates) - - -def generate_ows_legend_style_cfg(cfg, ndates=0): - """ - - :param cfg: A valid OWS Style definition configuration dictionary. - - Refer to the documentation for the valid syntax: - - https://datacube-ows.readthedocs.io/en/latest/cfg_styling.html - :param ndates: (optional) Number of dates (for styles with multi-date handlers) - :return: A PIL Image object. - """ - return generate_ows_legend_style(StandaloneStyle(cfg), ndates) diff --git a/datacube_ows/styles/base.py b/datacube_ows/styles/base.py index edd8f939d..c0882c83a 100644 --- a/datacube_ows/styles/base.py +++ b/datacube_ows/styles/base.py @@ -9,7 +9,8 @@ from datacube_ows.config_utils import (FlagProductBands, OWSConfigEntry, OWSEntryNotFound, OWSExtensibleConfigEntry, - OWSFlagBandStandalone) + OWSFlagBandStandalone, + OWSMetadataConfig) from datacube_ows.legend_utils import get_image_from_url from datacube_ows.ogc_exceptions import WMSException from datacube_ows.ogc_utils import ConfigException, FunctionWrapper @@ -17,7 +18,7 @@ _LOG = logging.getLogger(__name__) -class StyleDefBase(OWSExtensibleConfigEntry): +class StyleDefBase(OWSExtensibleConfigEntry, OWSMetadataConfig): INDEX_KEYS = ["layer", "style"] auto_legend = False include_in_feature_info = False @@ -37,6 +38,9 @@ def __new__(cls, product=None, style_cfg=None, stand_alone=False, defer_multi_da return super().__new__(subclass) return super().__new__(cls) + default_title = "Stand-Alone Style" + default_abstract = "Stand-Alone Style" + def __init__(self, product, style_cfg, stand_alone=False, defer_multi_date=False, user_defined=False): super().__init__(style_cfg, global_cfg=product.global_cfg, @@ -54,17 +58,16 @@ def __init__(self, product, style_cfg, stand_alone=False, defer_multi_date=False }) style_cfg = self._raw_cfg self.stand_alone = stand_alone + if self.stand_alone: + self._metadata_registry = {} self.user_defined = user_defined self.local_band_map = style_cfg.get("band_map", {}) self.product = product if self.stand_alone: self.name = style_cfg.get("name", "stand_alone") - self.title = style_cfg.get("title", "Stand Alone Style") - self.abstract = style_cfg.get("abstract", "Stand Alone Style") else: self.name = style_cfg["name"] - self.title = style_cfg["title"] - self.abstract = style_cfg["abstract"] + self.parse_metadata(style_cfg) self.masks = [StyleMask(mask_cfg, self) for mask_cfg in style_cfg.get("pq_masks", [])] if self.stand_alone: self.flag_products = [] @@ -80,6 +83,9 @@ def __init__(self, product, style_cfg, stand_alone=False, defer_multi_date=False if not defer_multi_date: self.parse_multi_date(style_cfg) + def get_obj_label(self): + return f"style.{self.product.name}.{self.name}" + # pylint: disable=attribute-defined-outside-init def make_ready(self, dc, *args, **kwargs): self.needed_bands = set() diff --git a/datacube_ows/wcs2.py b/datacube_ows/wcs2.py index 15b262d4b..9e67b677f 100644 --- a/datacube_ows/wcs2.py +++ b/datacube_ows/wcs2.py @@ -82,17 +82,17 @@ def get_capabilities(args): access_constraints=[cfg.access_constraints], provider_name='', provider_site='', - individual_name=cfg.contact_info['person'], - organisation_name=cfg.contact_info['organisation'], - position_name=cfg.contact_info['position'], - phone_voice=cfg.contact_info['telephone'], - phone_facsimile=cfg.contact_info['fax'], - delivery_point=cfg.contact_info['address']['address'], - city=cfg.contact_info['address']['city'], - administrative_area=cfg.contact_info['address']['state'], - postal_code=cfg.contact_info['address']['postcode'], - country=cfg.contact_info['address']['country'], - electronic_mail_address=cfg.contact_info['email'], + individual_name=cfg.contact_info.person, + organisation_name=cfg.contact_info.organisation, + position_name=cfg.contact_info.position, + phone_voice=cfg.contact_info.telephone, + phone_facsimile=cfg.contact_info.fax, + delivery_point=cfg.contact_info.address.address, + city=cfg.contact_info.address.city, + administrative_area=cfg.contact_info.address.state, + postal_code=cfg.contact_info.address.postcode, + country=cfg.contact_info.address.country, + electronic_mail_address=cfg.contact_info.email, online_resource=base_url, # hours_of_service=, # contact_instructions=, diff --git a/docs/cfg_global.rst b/docs/cfg_global.rst index 14e793559..cab26a702 100644 --- a/docs/cfg_global.rst +++ b/docs/cfg_global.rst @@ -14,6 +14,16 @@ to all services and all layers/coverages. The Global section is always required and contains the following entries: +Message File (message_file) +=========================== + +The "message_file" entry gives the path to the message file used for +`metadata separation and internationalisation +`_. + +Any metadata fields supplied in the metadata file will over-ride the values +supplied in the configuration. + Service Title (title) ===================== @@ -162,8 +172,70 @@ E.g.: If unsure of an `EPSG` code, search in http://epsg.io/ -Optional Metadata -================= +Default Attribution (attribution) +================================= + +Attributions can be declared at any level of the layer hierarchy, and are +inherited by child layers from the parent layer unless over-ridden. An +over-all default attribution may also be declared in the ``wms`` section, +which will serve as the attribution for top-level layers that do not declare +their own attribution section. + +All attribution sections are optional. + +If provided, the attribution section should be a dictionary containing +the following members: + +title + A user-readable title for the attribution (e.g. the name of the attributed + organisation.) + +url + A url for the attribution (e.g. the website address of the attributed organisation) + +logo + A dictionary (structure described below) describing a logo for the attribution + (e.g. the logo of the attributed organisation.) + +All of the above elements are optional, but at least one must be +provided if the attribution section exists. + +---------------- +Attribution Logo +---------------- + +The structure of the logo section is as follows: + +url + URL of the logo image. (Required if a logo is specified) + +format + The MIME type of the logo image. Should match the file type of + the image pointed to by the url. (Required if a logo is specified) + +width + The width (in pixels) of the logo image (optional) + +height + The height (in pixels) of the logo image (optional) + +E.g. + +:: + + "attribution": { + "title": "Acme Satellites", + "url": "http://www.acme.com/satellites", + "logo": { + "width": 370, + "height": 73, + "url": "https://www.acme.com/satellites/images/acme-370x73.png", + "format": "image/png", + } + }, + +Other Optional Metadata +======================= The remainder of the "global" section contains various metadata entries that are written directly to the various Capabilities documents. All metadata in the "global" section diff --git a/docs/cfg_layers.rst b/docs/cfg_layers.rst index 46f743f37..d9c563c18 100644 --- a/docs/cfg_layers.rst +++ b/docs/cfg_layers.rst @@ -109,6 +109,14 @@ above, folder layers have a "layers" element which is a list of child layers (which may be named layers, folder layers with their own child layers). +A folder layer may also have a `label` element which is used only +for +`metadata separation and internationalisation +`_. +Each folder's layer +must be globally unique. A unique label based on the folder's position +in the folder hierarchy is generated if one is not supplied. + E.g. :: diff --git a/docs/cfg_wms.rst b/docs/cfg_wms.rst index 737e45652..ce41bc1cd 100644 --- a/docs/cfg_wms.rst +++ b/docs/cfg_wms.rst @@ -84,65 +84,3 @@ E.g. "auth": "https://authoritative-authority.com", "idsrus": "https://www.identifiers-r-us.com", }, - -Default Attribution (attribution) -================================= - -Attributions can be declared at any level of the layer hierarchy, and are -inherited by child layers from the parent layer unless over-ridden. An -over-all default attribution may also be declared in the ``wms`` section, -which will serve as the attribution for top-level layers that do not declare -their own attribution section. - -All attribution sections are optional. - -If provided, the attribution section should be a dictionary containing -the following members: - -title - A user-readable title for the attribution (e.g. the name of the attributed - organisation.) - -url - A url for the attribution (e.g. the website address of the attributed organisation) - -logo - A dictionary (structure described below) describing a logo for the attribution - (e.g. the logo of the attributed organisation.) - -All of the above elements are optional, but at least one must be -provided if the attribution section exists. - ----------------- -Attribution Logo ----------------- - -The structure of the logo section is as follows: - -url - URL of the logo image. (Required if a logo is specified) - -format - The MIME type of the logo image. Should match the file type of - the image pointed to by the url. (Required if a logo is specified) - -width - The width (in pixels) of the logo image (optional) - -height - The height (in pixels) of the logo image (optional) - -E.g. - -:: - - "attribution": { - "title": "Acme Satellites", - "url": "http://www.acme.com/satellites", - "logo": { - "width": 370, - "height": 73, - "url": "https://www.acme.com/satellites/images/acme-370x73.png", - "format": "image/png", - } - }, diff --git a/docs/configuration.rst b/docs/configuration.rst index b26fc4ffc..b3dcfb3f9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -263,7 +263,56 @@ This restriction can be avoided using direct inheritance. Care should be taken of the special handling of lists in configuration: 1. If the child entry is an empty list, this will replace the parent entry, resulting in an empty list. -2. If the c +2. If the child entry is a non-empty list, the values in the child list are appended to the parent entry, resulting + in a merged list. + +Metadata Separation and Internationalisation +-------------------------------------------- + +Human-readable metadata can simply be embedded directly in the configuration. However in order to support +use-cases like multi-language internationalisation and integrating metadata with external +content management systems, all human-readable metadata in the OWS configuration can be extracted +into a separate file and managed independently. + +Metadata Separation ++++++++++++++++++++ + +To separate your metadata from config (either as an end in itself, or as preparation for internationalisation/translation): + +1. Add a unique ``label`` to each of your folder layers. + + This step is strongly recommended optional. OWS will autogenerate + a unique but non-obvious label for each folder if you do not supply one. + +2. Run ``python cfg_parser.py -c -m messages.po`` + + This extracts all the translatable/human-readable text from your config file, and writes it to the named file in gettext "po" file + format. + + Add ``"message_file": "/path/to/messages.po"`` to the global section of your OWS config file. + + Subsequently, text in messages.po will over-ride text in the config file. Update as needed, restart wsgi process to take effect. + Any field not included in the message file will be loaded directly from the config, as previously. + +The msgid's in the messages file are symbolic. E.g. + +* ``global.title``: The title for the whole service. +* ``folder.