CMR-7503: Added error check + doc update (#1352)

nasa · Oct 1, 2021 · c969535 · c969535
1 parent 8d8d306
commit c969535
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 8 deletions.
diff --git a/ingest-app/docs/api.md b/ingest-app/docs/api.md
@@ -1493,9 +1493,9 @@ Supported metadata formats:
 * OPeNDAP url in RelatedUrls for UMM-G format
 * OPeNDAP url in OnlineResources for ECHO10 format
 
-Input for this update type should be a list of granule URs. UMM-G Granules listed will have any `RelatedUrl`s containg the string `"opendap"` updated to include `"Type": "USE SERVICE API"` and `"Subtype": "OPENDAP DATA"`.
+Input for this update type should be a list of granule URs. UMM-G Granules listed will have any `RelatedUrl`s with a *URL* containing the string `"opendap"` updated to include `"Type": "USE SERVICE API"` and `"Subtype": "OPENDAP DATA"`. Echo10 Granules will have any `OnlineResources` with a *type* containing `"opendap"` updated to to include `<Type>USE SERVICE API : OPENDAP DATA</Type>`.
 
-As an alternative to identifying links via the `"opendap"` string method, a subtype string can be supplied with each granule UR as a tuple. If supplied, any links with a subtype matching this input string will be updated instead. As before, the link will be updated to include `"Type": "USE SERVICE API"` and `"Subtype": "OPENDAP DATA"`.
+As an alternative to identifying links via the `"opendap"` string method, a type string can be supplied with each granule UR as a tuple. If supplied, any UMM-G links with a *subtype* matching this input string will be updated instead. As before, the link will be updated to include `"Type": "USE SERVICE API"` and `"Subtype": "OPENDAP DATA"`. Any Echo10 links with a *type* matching this input will have their major typing updated to `USE SERVICE API`.
 
 Examples for each update format are provided below. For the first update, each granule in the list will have any links containing the string `"opendap"` updated to the new Type and Subtype.
 

diff --git a/ingest-app/src/cmr/ingest/services/granule_bulk_update/opendap/echo10.clj b/ingest-app/src/cmr/ingest/services/granule_bulk_update/opendap/echo10.clj
@@ -4,6 +4,7 @@
    [clojure.data.xml :as xml]
    [clojure.string :as string]
    [clojure.zip :as zip]
+   [cmr.common.services.errors :as errors]
    [cmr.common.util :refer [remove-nil-keys]]
    [cmr.common.xml :as cx]
    [cmr.ingest.services.granule-bulk-update.opendap.opendap-util
@@ -74,10 +75,12 @@
   "Returns the updated OPeNDAP type (cloud or on-prem) resource based on the opendap resources,
   the opendap type and the url-map that is parsed from the update url value."
   [opendap-type opendap-resources url-map]
-  (->> opendap-resources
-       (filter #(= opendap-type (opendap-util/url->opendap-type (:url %))))
-       first
-       (update-opendap-resource (first (opendap-type url-map)))))
+  (let [resources (filter #(= opendap-type (opendap-util/url->opendap-type (:url %))) opendap-resources)]
+    (when (< 1 (count resources))
+      (errors/throw-service-errors :invalid-data
+                                   [(str "Cannot update granule - more than one Hyrax-in-the-cloud or"
+                                         " more than one on-prem OPeNDAP link was detected in the granule")]))
+    (update-opendap-resource (first (opendap-type url-map)) (first resources))))
 
 (defn- updated-online-resources
   "Take the parsed online resources in the original metadata, update the existing opendap url

diff --git a/system-int-test/resources/CMR-7503-echo10-bad-data.xml b/system-int-test/resources/CMR-7503-echo10-bad-data.xml
@@ -0,0 +1,62 @@
+<Granule>
+  <GranuleUR>granule_with_duplicate_opendap_types</GranuleUR>
+  <InsertTime>2015-08-19T19:31:03.105Z</InsertTime>
+  <LastUpdate>2019-12-06T16:40:19.803Z</LastUpdate>
+  <Collection>
+    <ShortName>MUR-JPL-L4-GLOB-v4.1</ShortName>
+    <VersionId>4.1</VersionId>
+  </Collection>
+  <DataGranule>
+    <SizeMBDataGranule>331.33482456207275</SizeMBDataGranule>
+    <DayNightFlag>UNSPECIFIED</DayNightFlag>
+    <ProductionDateTime>2021-08-19T10:35:04.000Z</ProductionDateTime>
+  </DataGranule>
+  <Temporal>
+    <RangeDateTime>
+      <BeginningDateTime>2021-08-02T09:00:00.000Z</BeginningDateTime>
+      <EndingDateTime>2021-08-02T09:00:00.000Z</EndingDateTime>
+    </RangeDateTime>
+  </Temporal>
+  <Spatial>
+    <HorizontalSpatialDomain>
+      <Geometry>
+        <BoundingRectangle>
+          <WestBoundingCoordinate>-179.641</WestBoundingCoordinate>
+          <NorthBoundingCoordinate>53.855</NorthBoundingCoordinate>
+          <EastBoundingCoordinate>58.885</EastBoundingCoordinate>
+          <SouthBoundingCoordinate>-87.300</SouthBoundingCoordinate>
+        </BoundingRectangle>
+      </Geometry>
+    </HorizontalSpatialDomain>
+  </Spatial>
+  <OrbitCalculatedSpatialDomains>
+    <OrbitCalculatedSpatialDomain/>
+  </OrbitCalculatedSpatialDomains>
+  <OnlineAccessURLs>
+    <OnlineAccessURL>
+      <URL>https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L4/GLOB/JPL/MUR/v4.1/2002/153/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc</URL>
+      <URLDescription>The HTTP location for the granule.</URLDescription>
+    </OnlineAccessURL>
+  </OnlineAccessURLs>
+  <OnlineResources>
+    <OnlineResource>
+      <URL>https://podaac-opendap.jpl.nasa.gov/opendap/allData/ghrsst/data/GDS2/L4/GLOB/JPL/MUR/v4.1/2002/153/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.html</URL>
+      <Description>The OPENDAP location for the granule.</Description>
+      <Type>OPENDAP</Type>
+      <MimeType>text/html</MimeType>
+    </OnlineResource>
+    <OnlineResource>
+      <URL>https://otherResource.foo</URL>
+      <Description>The OTHER location for the granule.</Description>
+      <Type>OTHER LINK</Type>
+      <MimeType>text/html</MimeType>
+    </OnlineResource>
+    <OnlineResource>
+        <URL>https://link-1</URL>
+        <Description>a generic link with the wrong mime-type</Description>
+        <Type>OPENDAP DATA</Type>
+        <MimeType>text/error+csv</MimeType>
+    </OnlineResource>
+  </OnlineResources>
+  <DataFormat>NETCDF</DataFormat>
+</Granule>
diff --git a/...int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj b/...int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj
@@ -1624,7 +1624,13 @@
                     :concept-type :granule
                     :native-id "test-gran1"
                     :format-key :echo10})
-          {:keys [concept-id revision-id]} granule]
+          {:keys [concept-id revision-id]} granule
+          bad-granule (data-core/ingest-concept-with-metadata-file
+                       "CMR-7503-echo10-bad-data.xml"
+                       {:provider-id "PROV1"
+                        :concept-type :granule
+                        :native-id "test-gran2"
+                        :format-key :echo10})]
 
       (index/wait-until-indexed)
 
@@ -1657,4 +1663,27 @@
 
           (let [latest-metadata (:metadata (mdb/get-concept concept-id))]
             (is (= (slurp (io/resource "CMR-7503-echo10-online-resource-type_updated_by_granuleur_and_type.xml"))
-                   latest-metadata))))))))
+                   latest-metadata)))))
+
+      (testing "error due to duplicate link types in granule"
+        (let [bulk-update {:name "update opendap link online resource type 3"
+                           :operation "UPDATE_TYPE"
+                           :update-field "OPeNDAPLink"
+                           :updates ["granule_with_duplicate_opendap_types"]}
+              {:keys [status task-id errors] :as response} (ingest/bulk-update-granules "PROV1" bulk-update bulk-update-options)]
+
+          (ingest/update-granule-bulk-update-task-statuses)
+          (index/wait-until-indexed)
+
+          (is (= 200 status))
+          (is (some? task-id))
+
+          (let [status-req-options {:query-params {:show_granules "true"}}
+                status-response (ingest/granule-bulk-update-task-status task-id status-req-options)
+                {:keys [task-status status-message granule-statuses]} status-response]
+            (is (= "COMPLETE" task-status))
+            (is (= [{:granule-ur "granule_with_duplicate_opendap_types"
+                     :status "FAILED"
+                     :status-message (str "Cannot update granule - more than one Hyrax-in-the-cloud "
+                                          "or more than one on-prem OPeNDAP link was detected in the granule")}]
+                   granule-statuses))))))))