CMR-7789: update UMM-S to v1.4.1, Related URL Content Type is validat…

…ed against KMS in addition to Type and Subtype (#1382)
nasa · Nov 10, 2021 · 0b07a40 · 0b07a40
1 parent 612ba94
commit 0b07a40
Show file tree

Hide file tree

Showing 15 changed files with 1,983 additions and 24 deletions.
diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj
@@ -43,7 +43,7 @@
 (defconfig service-umm-version
   "Defines the latest service umm version accepted by ingest - it's the latest official version.
    This environment variable needs to be manually set when newer UMM version becomes official"
-  {:default "1.4"})
+  {:default "1.4.1"})
 
 (defconfig tool-umm-version
   "Defines the latest tool umm version accepted by ingest - it's the latest official version.

diff --git a/ingest-app/src/cmr/ingest/services/ingest_service/service.clj b/ingest-app/src/cmr/ingest/services/ingest_service/service.clj
@@ -31,20 +31,20 @@
   (when (seq errors)
     (errors/throw-service-errors error-type errors)))
 
-(defn- match-kms-type-and-subtype
+(defn- match-kms-related-url-content-type-type-and-subtype
   "Create a kms match validator for use by validation-service"
   [context]
   (let [kms-index (kms-fetcher/get-kms-index context)]
     [{:RelatedURLs (validation/match-kms-keywords-validation
                     kms-index
                     :related-urls
-                    msg/related-url-type-subtype-not-matching-kms-keywords)}]))
+                    msg/related-url-content-type-type-subtype-not-matching-kms-keywords)}]))
 
 (defn- validate-all-fields
-  "Check all fields that need to be validated. Currently this is the Type and
+  "Check all fields that need to be validated. Currently this is the Related URL Content Type, Type, and
   Subtype fields. Throws an error if errors are found"
   [context service]
-  (let [errors (seq (umm-spec-validation/validate-service service (match-kms-type-and-subtype context)))]
+  (let [errors (seq (umm-spec-validation/validate-service service (match-kms-related-url-content-type-type-and-subtype context)))]
     (if-errors-throw :bad-request errors)))
 
 (defn-timed save-service

diff --git a/ingest-app/src/cmr/ingest/services/messages.clj b/ingest-app/src/cmr/ingest/services/messages.clj
@@ -82,8 +82,8 @@
    (format "ISO Topic Category [%s] was not a valid keyword."
            iso-topic-category))
 
-(defn related-url-type-subtype-not-matching-kms-keywords
-  "Error msg when Related URL Type and Subtype pair is not in KMS."
+(defn related-url-content-type-type-subtype-not-matching-kms-keywords
+  "Error msg when Related URL Content Type, Type, and Subtype do not match a valid set in KMS."
   [related-url]
   (format "Related URL Content Type, Type, and Subtype [%s>%s>%s] are not a valid set together."
           (:URLContentType related-url)

diff --git a/ingest-app/src/cmr/ingest/validation/validation.clj b/ingest-app/src/cmr/ingest/validation/validation.clj
@@ -95,12 +95,12 @@
   (match-kms-keywords-validation
    kms-index
    :related-urls
-   msg/related-url-type-subtype-not-matching-kms-keywords))
+   msg/related-url-content-type-type-subtype-not-matching-kms-keywords))
 
 (defn- related-url-validator
   "Return a validator that checks a ContentType, Type, and Subtype keywords for
    Related URL field which can be inside a ContactInformation or be a standalone
-   field. ContactInformation can themselfs be found in DataCenters, ContactGroups,
+   field. ContactInformation can themselves be found in DataCenters, ContactGroups,
    and ContactPersons meaning 4 different uses cases for this validator."
   [kms-index]
   {:RelatedUrls (match-related-url-kms-keywords-validations kms-index)})
@@ -112,7 +112,7 @@
 
 (defn- datacenter-contact-url-validator
   "Return a validator for the related urls inside the Contact information fields
-   belolonging to either a :ContactGroups or a ContactPersons."
+   belonging to either a :ContactGroups or a :ContactPersons."
   [kms-index contact-type]
   {contact-type (v/every {:ContactInformation (related-url-validator kms-index)})})
 

diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection_keyword_validation_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection_keyword_validation_test.clj
@@ -188,7 +188,7 @@
         (assert-invalid-keywords
          attribs
          ["RelatedUrls" 0]
-         [(msg/related-url-type-subtype-not-matching-kms-keywords
+         [(msg/related-url-content-type-type-subtype-not-matching-kms-keywords
            (first (:RelatedUrls attribs)))])
 
         "- Missmatched ContentType and Type/Subtype pair"

diff --git a/system-int-test/test/cmr/system_int_test/ingest/service_ingest_test.clj b/system-int-test/test/cmr/system_int_test/ingest/service_ingest_test.clj
@@ -208,11 +208,11 @@
           (is (= 200 status))
           (is (= 3 revision-id)))))))
 
-(deftest related-url-type-and-subtype-check
+(deftest related-url-content-type-type-and-subtype-check
   (testing
-    "Check valid and invalid related type and subtypes. The first URL is good
-    while the second is bad. The error response should represent this showing
-    the second URL type failed to match a valid keyword."
+    "Check valid and invalid related urls url content type, type, and subtype.
+    The first URL is good while the second is bad. The error response should
+    represent this showing the second URL type failed to match a valid keyword."
     (let [related-urls [{"URL" "https://example.gov/good1"
                          "URLContentType" "PublicationURL"
                          "Type" "VIEW RELATED INFORMATION"

diff --git a/system-int-test/test/cmr/system_int_test/search/service/service_search_test.clj b/system-int-test/test/cmr/system_int_test/search/service/service_search_test.clj
@@ -588,8 +588,8 @@
         {:accept (mime-types/with-version mime-types/umm-json umm-version/current-service-version)}
 
         "explicit UMM JSON version through suffix"
-        "1.4"
-        {:url-extension "umm_json_v1_4"})))
+        "1.4.1"
+        {:url-extension "umm_json_v1_4_1"})))
 
   (testing "Searching with non-existent UMM JSON version"
     (are3 [options]

diff --git a/umm-spec-lib/resources/example-data/umm-json/service/v1.4.1/Service_v1.4.1-from-v1.4.json b/umm-spec-lib/resources/example-data/umm-json/service/v1.4.1/Service_v1.4.1-from-v1.4.json
@@ -0,0 +1,241 @@
+{
+  "Name": "AIRX3STD.006_to_130",
+  "LongName": "AIRS/Aqua L3 Daily Standard Physical Retrieval (AIRS+AMSU) 1 degree x 1 degree V006.",
+  "Description": "Access the AIRS/Aqua Level 3 daily standard physical retrieval product (Without HSB) through OPeNDAP",
+  "Type": "NOT PROVIDED",
+  "Version": "1.9",
+  "VersionDescription": "This is a description of version 1.9.",
+  "LastUpdatedDate": "2020-05-10T00:00:00Z",
+  "AccessConstraints": "None",
+  "UseConstraints": {
+    "LicenseURL": "http://www.someurl.com",
+    "LicenseText": "None" },
+  "URL": {
+    "Description": "OPeNDAP Service for AIRX3STD.006",
+    "URLValue": "https://acdisc.gesdisc.eosdis.nasa.gov/opendap/Aqua_AIRS_Level3/AIRX3STD.006/"
+  },
+  "ServiceKeywords": [{
+    "ServiceCategory":"DATA MANAGEMENT/DATA HANDLING",
+    "ServiceTopic":"TRANSFORMATION/REPROJECTION"
+  }],
+  "ServiceOrganizations": [{
+    "Roles": ["SERVICE PROVIDER"],
+    "ShortName": "NASA/GESDISC",
+    "LongName": "GES DISC SERVICE HELP DESK SUPPORT GROUP"
+  }, {
+    "Roles": ["SERVICE PROVIDER"],
+    "ShortName": "NASA/GESDISC-2",
+    "LongName": "GES DISC SERVICE HELP DESK SUPPORT GROUP 2",
+    "OnlineResource": {
+      "Linkage": "https://daacscenter.org",
+      "Name": "HOME PAGE",
+      "Description": "Test the description."
+    }
+  }],
+  "ContactGroups": [{
+    "Roles": ["SCIENCE CONTACT"],
+    "ContactInformation": {
+      "ContactMechanisms": [{
+        "Type": "Email",
+        "Value": "gsfc-help-disc at lists.nasa.gov"
+      }, {
+        "Type": "Telephone",
+        "Value": "301-614-9999"
+      }],
+      "Addresses": [{
+        "StreetAddresses": [
+          "Goddard Earth Sciences Data and Information Systems, Attn: User , NASA Goddard Space Flight Center, Code 610.2"
+        ],
+        "City": "Greenbelt",
+        "StateProvince": "MD",
+        "Country": "USA",
+        "PostalCode": "20771"
+      }]
+    },
+    "GroupName": "Main Level Group Name 1"
+  }, {
+    "Roles": ["TECHNICAL CONTACT"],
+    "ContactInformation": {
+      "OnlineResources": [{
+        "Linkage": "https://someurl1.org",
+        "Protocol": "https",
+        "ApplicationProfile": "Some Application",
+        "Name": "Some Site",
+        "Description": "The description of someurl1.",
+        "Function": "information"
+      }, {
+        "Linkage": "https://someurl2.org",
+        "Protocol": "https",
+        "ApplicationProfile": "Some Application 2",
+        "Name": "Some Site 2",
+        "Description": "The description of someurl2.",
+        "Function": "information"
+      }],
+      "ContactMechanisms": [{
+        "Type": "Email",
+        "Value": "gsfc-help-disc at lists.nasa.gov"
+      }, {
+        "Type": "Telephone",
+        "Value": "301-614-9999"
+      }],
+      "Addresses": [{
+        "StreetAddresses": [
+          "Goddard Earth Sciences Data and Information Systems, Attn: User , NASA Goddard Space Flight Center, Code 610.2"
+        ],
+        "City": "Greenbelt",
+        "StateProvince": "MD",
+        "Country": "USA",
+        "PostalCode": "20771"
+      }]
+    },
+    "GroupName": "Main Level Group Name 2"
+  }],
+  "ContactPersons": [{
+    "Roles": ["SERVICE PROVIDER"],
+    "ContactInformation": {
+      "OnlineResources": [{
+        "Linkage": "https://someurl3.org",
+        "Protocol": "https",
+        "ApplicationProfile": "Some Application 3",
+        "Name": "Some Site",
+        "Description": "The description of someurl3.",
+        "Function": "information"
+      }],
+      "ContactMechanisms": [{
+        "Type": "Email",
+        "Value": "gsfc-help-disc at lists.nasa.gov"
+      }, {
+        "Type": "Telephone",
+        "Value": "301-614-9999"
+      }],
+      "Addresses": [{
+        "StreetAddresses": [
+          "Goddard Earth Sciences Data and Information Systems, Attn: User , NASA Goddard Space Flight Center, Code 610.2"
+        ],
+        "City": "Greenbelt",
+        "StateProvince": "MD",
+        "Country": "USA",
+        "PostalCode": "20771"
+      }]
+    },
+    "FirstName": "FirstName Main Level",
+    "MiddleName": " ",
+    "LastName": "LastName Main Level"
+  }, {
+    "Roles": ["SERVICE PROVIDER"],
+    "ContactInformation": {
+      "ContactMechanisms": [{
+        "Type": "Email",
+        "Value": "gsfc-help-disc at lists.nasa.gov"
+      }, {
+        "Type": "Telephone",
+        "Value": "301-614-9999"
+      }],
+      "Addresses": [{
+        "StreetAddresses": [
+          "Goddard Earth Sciences Data and Information Systems, Attn: User , NASA Goddard Space Flight Center, Code 610.2"
+        ],
+        "City": "Greenbelt",
+        "StateProvince": "MD",
+        "Country": "USA",
+        "PostalCode": "20771"
+      }]
+    },
+    "FirstName": "FirstName Service Org",
+    "MiddleName": "Service Org MiddleName",
+    "LastName": "LastName Service Org"
+  }],
+  "ServiceOptions": {
+    "Subset": {
+      "TemporalSubset": {
+        "AllowMultipleValues": false
+      },
+      "VariableSubset": {
+        "AllowMultipleValues": false
+      }
+    },
+    "SupportedInputProjections": [{
+      "ProjectionName":"Geographic",
+      "ProjectionAuthority":"4326"
+    }],
+    "SupportedOutputProjections": [{
+      "ProjectionName":"Geographic",
+      "ProjectionAuthority":"ep"
+    }],
+    "SupportedReformattings": [{
+      "SupportedInputFormat": "HDF5",
+      "SupportedOutputFormats": ["GEOTIFFFLOAT32","GEOTIFFINT16", "COG"]
+    }, {
+      "SupportedInputFormat": "WKT",
+      "SupportedOutputFormats": ["GEOTIFFFLOAT32","GEOTIFFINT16"]
+    }, {
+      "SupportedInputFormat": "HDF4",
+      "SupportedOutputFormats": ["HDF4","NETCDF-3","NETCDF-4","BINARY","ASCII", "ZARR", "GeoJSON"]
+    }, {
+      "SupportedInputFormat": "Shapefile",
+      "SupportedOutputFormats": ["HDF4","NETCDF-3","NETCDF-4","BINARY","ASCII", "ZARR", "GeoJSON"]
+    }]
+  },
+  "OperationMetadata": [{
+	  "OperationName":"SPATIAL_SUBSETTING",
+	  "DistributedComputingPlatform":["XML","HTTPS"],
+	  "OperationDescription":"Access the AIRS/Aqua Level 3 daily standard physical retrieval product (Without HSB) through OPeNDAP",
+	  "InvocationName":"GES DISC OPeNDAP Service for AIRX3STD.006",
+	  "ConnectPoint": {
+	    "ResourceName":"AIRX3STD.006",
+	   	"ResourceLinkage": "https://acdisc.gesdisc.eosdis.nasa.gov/opendap/Aqua_AIRS_Level3/AIRX3STD.006/",
+	    "ResourceDescription":"AIRS/Aqua L3 Daily Standard Physical Retrieval (AIRS+AMSU) 1 degree x 1 degree V006."
+	  },
+	  "CoupledResource": {
+	    "ScopedName":"AIRX3STD.006",
+	    "DataResourceDOI":"https://doi.org/10.5067/Aqua/AIRS/DATA303",
+	    "CouplingType":"TIGHT",
+	    "DataResource": {
+	      "DataResourceIdentifier":"Aqua_AIRS_Level3/AIRX3STD.006",
+	      "DataResourceSourceType":"Collection",
+	      "DataResourceSpatialType":"BOUNDING_BOX",
+	      "DataResourceSpatialExtent": {
+	        "SpatialBoundingBox": {
+	          "CRSIdentifier":"EPSG:4326",
+	          "WestBoundingCoordinate":-180.00,
+	          "SouthBoundingCoordinate":-90.0,
+	          "EastBoundingCoordinate":180.0,
+	          "NorthBoundingCoordinate":90.0
+	        }
+	      },
+	      "SpatialResolution":1,
+	      "SpatialResolutionUnit":"KM",
+	      "DataResourceTemporalType":"TIME_STAMP",
+	      "DataResourceTemporalExtent": {
+	        "DataResourceTimePoints": [{
+	          "TimeFormat":"%Y%M%D",
+	          "TimeValue":"2009-01-08",
+	          "Description":"Time stamp of the granule within the collection"
+	        }]
+	      },
+	      "TemporalResolution":1,
+	      "TemporalResolutionUnit":"DAY"
+	    }
+    }
+  }, {
+    "OperationName": "GetTile",
+    "CoupledResource": {
+      "DataResource": {
+	      "DataResourceSpatialExtent": {
+	        "SpatialBoundingBox": {
+	          "CRSIdentifier":"Other",
+	          "WestBoundingCoordinate":-180.00,
+	          "SouthBoundingCoordinate":-90.0,
+	          "EastBoundingCoordinate":180.0,
+	          "NorthBoundingCoordinate":90.0
+	        }
+	      }
+      }
+    }
+  }],
+  "MetadataSpecification": {
+    "URL": "https://cdn.earthdata.nasa.gov/umm/service/v1.4.1",
+    "Name": "UMM-S",
+    "Version": "1.4.1"
+  }
+}