Merge pull request #1016 from AtlasOfLivingAustralia/feature/issue3292

fixed issue with large GeometryCollection object intersection with spatial layer failing (AtlasOfLivingAustralia/fieldcapture#3292)
AtlasOfLivingAustralia · Sep 18, 2024 · 4fa563b · 4fa563b
2 parents bf1de90 + f08b930
commit 4fa563b
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 31 deletions.
diff --git a/grails-app/conf/ecodata-ehcache.xml b/grails-app/conf/ecodata-ehcache.xml
@@ -38,6 +38,20 @@
             <heap unit="entries">100</heap>
         </resources>
     </cache>
+    <cache alias="spatialPidFidIntersection">
+
+        <key-type>java.io.Serializable</key-type>
+        <value-type>java.io.Serializable</value-type>
+
+        <expiry>
+            <tti unit="days">365</tti>
+        </expiry>
+        <resources>
+            <heap unit="entries">200</heap>
+
+            <disk unit="MB" persistent="true">500</disk>
+        </resources>
+    </cache>
     <cache alias="spatialPidObjectGeometry">
 
         <key-type>java.io.Serializable</key-type>

diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy
@@ -272,6 +272,7 @@ class AdminController {
         dateFormat.setTimeZone(timeZoneUTC)
         def isMERIT = params.getBoolean('isMERIT', true)
         Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate)
+        List siteIds = params.get("siteId")?.split(",")
         def code = 'success'
         def total = 0
         def offset = 0
@@ -281,7 +282,10 @@ class AdminController {
         log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}")
         def totalSites
         List projectIds = []
-        if (isMERIT) {
+        if (siteIds) {
+            totalSites = siteIds.size()
+        }
+        else if (isMERIT) {
             projectIds = projectService.getAllMERITProjectIds()
             totalSites = Site.countByStatusAndProjectsInListAndDateCreatedGreaterThan('active', projectIds, startDate)
         }
@@ -293,7 +297,12 @@ class AdminController {
         while (count == batchSize) {
             batchStartTime = startInterimTime = System.currentTimeMillis()
             def sites
-            if (isMERIT) {
+            if (siteIds) {
+                sites = Site.findAllBySiteIdInList(siteIds, [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
+                    siteService.toMap(it, 'flat')
+                }
+            }
+            else if (isMERIT) {
                 sites = Site.findAllByProjectsInListAndStatusAndDateCreatedGreaterThan(projectIds, 'active', startDate, [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
                     siteService.toMap(it, 'flat')
                 }

diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy
@@ -47,11 +47,25 @@ class SpatialService {
      * supplied geometry
      */
     Map<String,?> intersectGeometry(Map geoJson, List<String> fieldIds = null) {
-        int length = geoJson?.toString().size()
+        // We are using a WKT string instead of geojson as the spatial portal validates geojson - using
+        // WKT allows us to get away with self intersecting polygons that users occasionally draw.
+        String wkt
         int threshold = grailsApplication.config.getProperty('spatial.geoJsonEnvelopeConversionThreshold', Integer)
-        Geometry geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
-        if(length > threshold){
-            geoJson = GeometryUtils.geometryToGeoJsonMap (geo.getEnvelope())
+        Geometry geo
+        // Ignore threshold check for GeometryCollection collection
+        if (geoJson.type != 'GeometryCollection') {
+            int length = geoJson?.toString().size()
+            if (length > threshold) {
+                geoJson = GeometryUtils.geometryToGeoJsonMap(geo.getEnvelope())
+            }
+
+            geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
+            wkt = geo.toText()
+        } else {
+            geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
+            GeometryCollection geometryCollection = (GeometryCollection)geo
+            Geometry convexHullGeometry = geometryCollection.union().convexHull()
+            wkt = convexHullGeometry.toText()
         }
 
         String url = grailsApplication.config.getProperty('spatial.baseUrl')+WKT_INTERSECT_URL_PREFIX
@@ -60,36 +74,16 @@ class SpatialService {
         }
 
         long start = System.currentTimeMillis()
-        // We are using a WKT string instead of geojson as the spatial portal validates geojson - using
-        // WKT allows us to get away with self intersecting polygons that users occasionally draw.
-        String wkt = geo.toText()
         long end = System.currentTimeMillis()
         log.info("Time taken to convert geojson to wkt: ${end-start}ms")
 
 
         Map result = [:]
         fieldIds.each { fid ->
             start = end
-            Map response
-            if (geo.geometryType == 'GeometryCollection') {
-                Map<String, Map> geometryCollectionIntersections = [:]
-                GeometryCollection geometryCollection = (GeometryCollection)geo
-                for (int i=0; i<geometryCollection.numGeometries; i++) {
-                    Geometry geometryN = geometryCollection.getGeometryN(i)
-                    String wktGeometryN = geometryN.toText()
-                    response = webService.doPost(url+fid, wktGeometryN)
-                    if (response.resp && !response.error) {
-                        response.resp?.each {geometryCollectionIntersections[it.pid] = it }
-                    }
-                }
-
-                result[fid] = geometryCollectionIntersections.values()?.toList()
-            }
-            else {
-                response = webService.doPost(url+fid, wkt)
-                if (response.resp && !response.error) {
-                    result[fid] = response.resp
-                }
+            Map response = webService.doPost(url+fid, wkt)
+            if (response.resp && !response.error) {
+                result[fid] = response.resp
             }
             end = System.currentTimeMillis()
             log.info("Time taken to intersect with layer $fid: ${end-start}ms")
@@ -103,7 +97,6 @@ class SpatialService {
             geographicFacets = [:].withDefault{[]}
             GeometryCollection geometryCollection = (GeometryCollection)geo
             for (int i=0; i<geometryCollection.numGeometries; i++) {
-
                 def (filtered, intersectionArea) = filterOutObjectsInBoundary(result, geometryCollection.getGeometryN(i))
                 start = end
                 Map geographicFacetsForGeometry = convertResponsesToGeographicFacets(filtered)
@@ -152,7 +145,7 @@ class SpatialService {
                 result[fid] = lookupTable[pidFid][pid][fid].collect{[name:it]}
             }
             else {
-                Object response = webService.getJson(url+fid+"/"+pid)
+                def response = getPidFidIntersection(url, fid, pid)
                 if (response instanceof List) {
                     result[fid] = response
                 }
@@ -167,6 +160,11 @@ class SpatialService {
         geographicFacets
     }
 
+    @Cacheable(value = "spatialPidFidIntersection")
+    def getPidFidIntersection(String url, String fid, String pid) {
+        webService.getJson(url + fid + "/" + pid)
+    }
+
     private List filterOutObjectsInBoundary(Map response, Map mainObjectGeoJson) {
         Geometry mainGeometry = GeometryUtils.geoJsonMapToGeometry(mainObjectGeoJson)
         filterOutObjectsInBoundary(response, mainGeometry)