Merge pull request #587 from opendatacube/separate_metadata_from_config

Separate metadata from config

.env_simple

@@ -5,7 +5,7 @@ DB_PORT=5432
 DB_USERNAME=opendatacubeusername
 DB_PASSWORD=opendatacubepassword
 DB_DATABASE=opendatacube
-OWS_CFG_FILE=./ows_test_cfg.py
+OWS_CFG_FILE=./integration_tests/cfg/ows_test_cfg.py
 AWS_NO_SIGN_REQUEST=yes
 # If you want to use pydev for interactive debugging
 PYDEV_DEBUG=

datacube_ows/cfg_parser.py

@@ -24,6 +24,7 @@
     default=False,
     help="Only parse the syntax of the config file - do not validate against database",
 )
+
 @click.option(
     "-f",
     "--folders",
@@ -38,6 +39,13 @@
     default=False,
     help="Print the styles for each layer to stdout (format depends on --folders flag).",
 )
+@click.option(
+    "-c",
+    "--cfg-only",
+    is_flag=True,
+    default=False,
+    help="Read metadata from config only - ignore configured metadata message file).",
+)
 @click.option(
     "-i",
     "--input-file",
@@ -48,9 +56,16 @@
     "-o",
     "--output-file",
     default=False,
-    help="Provide an output file name with extension .json",
+    help="Provide an output inventory file name with extension .json",
 )
-def main(version, parse_only, folders, styles, input_file, output_file, paths):
+@click.option(
+    "-m",
+    "--msg-file",
+    default=False,
+    help="Write a message file containing the translatable metadata from the configuration.",
+)
+
+def main(version, cfg_only, parse_only, folders, styles, msg_file, input_file, output_file, paths):
     """Test configuration files
 
     Valid invocations:
@@ -70,23 +85,23 @@ def main(version, parse_only, folders, styles, input_file, output_file, paths):
 
     all_ok = True
     if not paths:
-        if parse_path(None, parse_only, folders, styles, input_file, output_file):
+        if parse_path(None, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file):
             return 0
         else:
             sys.exit(1)
     for path in paths:
-        if not parse_path(path, parse_only, folders, styles, input_file, output_file):
+        if not parse_path(path, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file):
             all_ok = False
 
     if not all_ok:
         sys.exit(1)
     return 0
 
 
-def parse_path(path, parse_only, folders, styles, input_file, output_file):
+def parse_path(path, cfg_only, parse_only, folders, styles, input_file, output_file, msg_file):
     try:
         raw_cfg = read_config(path)
-        cfg = OWSConfig(refresh=True, cfg=raw_cfg)
+        cfg = OWSConfig(refresh=True, cfg=raw_cfg, ignore_msgfile=cfg_only)
         if not parse_only:
             with Datacube() as dc:
                 cfg.make_ready(dc)
@@ -110,9 +125,28 @@ def parse_path(path, parse_only, folders, styles, input_file, output_file):
         print()
     if input_file or output_file:
         layers_report(cfg.product_index, input_file, output_file)
+    if msg_file:
+        write_msg_file(msg_file, cfg)
     return True
 
 
+def write_msg_file(msg_file, cfg):
+    with open(msg_file, "w") as fp:
+        for key, value in cfg.export_metadata():
+            if value:
+                print("", file=fp)
+                lines = list(value.split("\n"))
+                for line in lines:
+                    print(f"#. {line}", file=fp)
+                print(f'msgid "{key}"', file=fp)
+                if len(lines) == 1:
+                    print(f'msgstr "{value}"', file=fp)
+                else:
+                    print('msgstr ""', file=fp)
+                    for line in lines:
+                        print(f'"{line}\\n"', file=fp)
+
+
 def layers_report(config_values, input_file, output_file):
     report = {"total_layers_count": len(config_values.values()), "layers": []}
     for lyr in config_values.values():

datacube_ows/config_utils.py

@@ -90,6 +90,12 @@ def declare_unready(self, name):
             raise ConfigException(f"Cannot declare {name} as unready on a ready object")
         self._unready_attributes.add(name)
 
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
     def __getattribute__(self, name):
         if name == "_unready_attributes":
             pass
@@ -110,6 +116,213 @@ def make_ready(self, dc, *args, **kwargs):
             raise OWSConfigNotReady(f"The following parameters have not been initialised: {self._unready_attributes}")
         self.ready = True
 
+FLD_TITLE = "title"
+FLD_ABSTRACT = "abstract"
+FLD_KEYWORDS = "local_keywords"
+FLD_FEES = "fees"
+FLD_ACCESS_CONSTRAINTS = "access_constraints"
+FLD_ATTRIBUTION = "attribution_title"
+FLD_CONTACT_ORGANISATION = "contact_org"
+FLD_CONTACT_POSITION = "contact_position"
+
+class OWSMetadataConfig(OWSConfigEntry):
+
+    METADATA_TITLE = True
+    METADATA_ABSTRACT = True
+    METADATA_KEYWORDS = False
+    METADATA_CONTACT_INFO = False
+    METADATA_FEES = False
+    METADATA_ACCESS_CONSTRAINTS = False
+    METADATA_ATTRIBUTION = False
+
+    _metadata_registry = {}
+    _inheritance_registry = {}
+
+    _msg_src = None
+
+    def get_obj_label(self):
+        return "global"
+
+
+    def can_inherit_from(self):
+        return None
+
+    default_title = None
+    default_abstract = None
+
+    _keywords = None
+    def parse_metadata(self, cfg):
+        # can_inherit_from() can be over-ridden by subclasses
+        # pylint: disable=assignment-from-none
+        inherit_from = self.can_inherit_from()
+        if self.METADATA_TITLE:
+            if self.default_title:
+                self.register_metadata(self.get_obj_label(), FLD_TITLE, cfg.get("title", self.default_title))
+            else:
+                try:
+                    self.register_metadata(self.get_obj_label(), FLD_TITLE, cfg["title"])
+                except KeyError:
+                    raise ConfigException(f"Entity {self.get_obj_label()} has no title.")
+        if self.METADATA_ABSTRACT:
+            local_abstract = cfg.get("abstract")
+            if local_abstract is None and inherit_from is not None:
+                self.register_metadata(self.get_obj_label(), FLD_ABSTRACT, inherit_from.abstract, inherited=True)
+            elif local_abstract is None and self.default_abstract is not None:
+                self.register_metadata(self.get_obj_label(), FLD_ABSTRACT, self.default_abstract)
+            elif local_abstract is None:
+                raise ConfigException(f"Entity {self.get_obj_label()} has no abstract")
+            else:
+                self.register_metadata(self.get_obj_label(), "abstract", local_abstract)
+        if self.METADATA_KEYWORDS:
+            local_keyword_set = set(cfg.get("keywords", []))
+            self.register_metadata(self.get_obj_label(), FLD_KEYWORDS, ",".join(local_keyword_set))
+            if inherit_from:
+                keyword_set = inherit_from.keywords
+            else:
+                keyword_set = set()
+            self._keywords = keyword_set.union(local_keyword_set)
+        if self.METADATA_ATTRIBUTION:
+            inheriting = False
+            attrib = cfg.get("attribution")
+            if attrib is None and inherit_from is not None:
+                attrib = inherit_from.attribution
+                inheriting = True
+            if attrib:
+                attrib_title = attrib.get("title")
+            else:
+                attrib_title = None
+            if attrib_title:
+                self.register_metadata(self.get_obj_label(), FLD_ATTRIBUTION, attrib_title, inheriting)
+        if self.METADATA_FEES:
+            fees = cfg.get("fees")
+            if not fees:
+                fees = "none"
+            self.register_metadata(self.get_obj_label(), FLD_FEES, fees)
+        if self.METADATA_ACCESS_CONSTRAINTS:
+            acc = cfg.get("access_contraints")
+            if not acc:
+                acc = "none"
+            self.register_metadata(self.get_obj_label(), FLD_ACCESS_CONSTRAINTS, acc)
+        if self.METADATA_CONTACT_INFO:
+            org = cfg.get("contact_info", {}).get("organisation")
+            position = cfg.get("contact_info", {}).get("position")
+            if org:
+                self.register_metadata(self.get_obj_label(), FLD_CONTACT_ORGANISATION, org)
+            if position:
+                self.register_metadata(self.get_obj_label(), FLD_CONTACT_POSITION, position)
+    @property
+    def keywords(self):
+        return self._keywords
+
+    @classmethod
+    def set_msg_src(cls, src):
+        OWSMetadataConfig._msg_src = src
+
+    def read_metadata(self, lbl, fld):
+        lookup = ".".join([lbl, fld])
+        if self._msg_src is not None:
+            return self._msg_src.get(lookup, self._metadata_registry.get(lookup))
+        return self._metadata_registry.get(lookup)
+
+    def read_inheritance(self, lbl, fld):
+        lookup = ".".join([lbl, fld])
+        return self._inheritance_registry.get(lookup, False)
+
+    def register_metadata(self, lbl, fld, val, inherited=False):
+        lookup = ".".join([lbl, fld])
+        self._metadata_registry[lookup] = val
+        self._inheritance_registry[lookup] = inherited
+
+    def read_local_metadata(self, fld):
+        return self.read_metadata(self.get_obj_label(), fld)
+
+    def is_inherited(self, fld):
+        return self.read_inheritance(self.get_obj_label(), fld)
+
+    def __getattribute__(self, name):
+        if name in (FLD_TITLE, FLD_ABSTRACT, FLD_FEES, FLD_ACCESS_CONSTRAINTS, FLD_CONTACT_POSITION, FLD_CONTACT_ORGANISATION):
+            return self.read_local_metadata(name)
+        elif name == FLD_KEYWORDS:
+            kw = self.read_local_metadata(FLD_KEYWORDS)
+            if kw:
+                return set(kw.split(","))
+            else:
+                return set()
+        elif name == FLD_ATTRIBUTION:
+            return self.read_local_metadata(FLD_ATTRIBUTION)
+        else:
+            return super().__getattribute__(name)
+
+
+class OWSMessageFile:
+    # Parser states
+    START = 0
+    GOT_ID = 1
+    IN_STR = 2
+
+    def __init__(self, fp):
+        self._index = {}
+        self.state = self.START
+        self.msgstr = ""
+        self.msgid  = ""
+        self.line_no = 0
+        self.read_file(fp)
+
+    def _new_element(self):
+        if self.msgid:
+            self._index[self.msgid] = self.msgstr
+        self.state = self.START
+        self.msgstr = ""
+        self.msgid  = ""
+
+    def __getitem__(self, item):
+        return self._index[item]
+
+    def __contains__(self, item):
+        return item in self._index
+
+    def get(self, item, default=None):
+        return self._index.get(item, default)
+
+    def unescape(self, esc_str):
+        return esc_str.encode("utf-8").decode("unicode-escape")
+
+    def read_file(self, fp):
+        for line in fp.readlines():
+            self.line_no += 1
+            if line.startswith("msgid "):
+                if self.state == self.GOT_ID:
+                    raise ConfigException(f"Unexpected msgid line in message file: at line {self.line_no}")
+                self._new_element()
+                self.msgid = line[7:-2]
+                if self.msgid in self:
+                    raise ConfigException(f"Duplicate msgid: at line {self.line_no}")
+                self.state = self.GOT_ID
+            elif line.strip() == "":
+                if self.state == self.IN_STR:
+                    self._new_element()
+            elif line.startswith("msgstr "):
+                if self.state == self.START:
+                    raise ConfigException(f"msgstr without msgid: at line {self.line_no}")
+                elif self.state == self.IN_STR:
+                    raise ConfigException(f"Badly formatted multi-line msgstr: at line {self.line_no}")
+                # GOT_ID:
+                self.msgstr = self.unescape(line[8:-2])
+                self.state = self.IN_STR
+            elif line.startswith('"'):
+                if self.state == self.START:
+                    raise ConfigException(f"untagged string: at line {self.line_no}")
+                elif self.state == self.GOT_ID:
+                    raise ConfigException(f"Multiline msgids not supported by OWS: at line {self.line_no}")
+                # IN_STR
+                self.msgstr += self.unescape(line[1:-2])
+            elif line.startswith("#"):
+                if self.state == self.IN_STR:
+                    self._new_element()
+            else:
+                raise ConfigException(f"Invalid message file: error at line {self.line_no}")
+        self._new_element()
+
 
 class OWSEntryNotFound(ConfigException):
     pass

datacube_ows/ows_cfg_example.py

@@ -1292,6 +1292,29 @@
             "fax": "+61 2 1234 6789",
             "email": "[email protected]",
         },
+        # Attribution.
+        #
+        # This provides a way to identify the source of the data used in a WMS layer or layers.
+        # This entire section is optional.  If provided, it is taken as the
+        # default attribution for any layer that does not override it.
+        "attribution": {
+            # Attribution must contain at least one of ("title", "url" and "logo")
+            # A human readable title for the attribution - e.g. the name of the attributed organisation
+            "title": "Acme Satellites",
+            # The associated - e.g. URL for the attributed organisation
+            "url": "http://www.acme.com/satellites",
+            # Logo image - e.g. for the attributed organisation
+            "logo": {
+                # Image width in pixels (optional)
+                "width": 370,
+                # Image height in pixels (optional)
+                "height": 73,
+                # URL for the logo image. (required if logo specified)
+                "url": "https://www.acme.com/satellites/images/acme-370x73.png",
+                # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.)
+                "format": "image/png",
+            }
+        },
         # If fees are charged for the use of the service, these can be described here in free text.
         # If blank or not included, defaults to "none".
         "fees": "",
@@ -1334,27 +1357,7 @@
         # Optional, defaults to 256x256
         "max_width": 512,
         "max_height": 512,
-        # Attribution. This provides a way to identify the source of the data used in a layer or layers.
-        # This entire section is optional.  If provided, it is taken as the
-        # default attribution for any layer that does not override it.
-        "attribution": {
-            # Attribution must contain at least one of ("title", "url" and "logo")
-            # A human readable title for the attribution - e.g. the name of the attributed organisation
-            "title": "Acme Satellites",
-            # The associated - e.g. URL for the attributed organisation
-            "url": "http://www.acme.com/satellites",
-            # Logo image - e.g. for the attributed organisation
-            "logo": {
-                # Image width in pixels (optional)
-                "width": 370,
-                # Image height in pixels (optional)
-                "height": 73,
-                # URL for the logo image. (required if logo specified)
-                "url": "https://www.acme.com/satellites/images/acme-370x73.png",
-                # Image MIME type for the logo - should match type referenced in the logo url (required if logo specified.)
-                "format": "image/png",
-            }
-        },
+
         # These define the AuthorityURLs.
         # They represent the authorities that define the "Identifiers" defined layer by layer below.
         # The spec allows AuthorityURLs to be defined anywhere on the Layer heirarchy, but datacube_ows treats them