AtlasOfLivingAustralia/fieldcapture#3292

- updated boundary intersection logic
- calculates primary and other states and electorates
- intersection logic checks for project's hub to get layers
- bulk site intersection gets geographic layers for intersection from hub of project. Also added start date to query sites
- created hub specific geographic config for MERIT

grails-app/conf/application.groovy

@@ -428,6 +428,7 @@ if(!spatial.geoJsonEnvelopeConversionThreshold){
 }
 
 spatial.intersectionThreshold = 0.05
+spatial.intersectionAreaThresholdInHectare = 10_000
 
 homepageIdx {
     elasticsearch   {
@@ -493,17 +494,11 @@ app {
                 mvg = '/data/nvis_grids/mvg'
                 mvs = '/data/nvis_grids/mvs'
             }
+            checkForBoundaryIntersectionInLayers = [ "cl927", "cl11163" ]
         }
+        displayNames = [elect: "Electorate(s)", state: "State(s)"]
     }
 }
-
-site.check.boundary.layers = [
-        'cl927',
-        'cl10946',
-        'cl10921',
-        'cl2112'
-]
-
 /******************************************************************************\
  *  EXTERNAL SERVERS
  \******************************************************************************/

grails-app/conf/logback.xml

@@ -7,7 +7,7 @@
   <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
     <encoder>
       <charset>UTF-8</charset>
-      <pattern>'%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n%wex'</pattern>
+      <pattern>%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n%wex</pattern>
     </encoder>
   </appender>
 

grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

@@ -229,9 +229,10 @@ class AdminController {
                             log.info("Ignoring site ${site.siteId} due to no associated projects")
                             return
                         }
+                        List fids = metadataService.getSpatialLayerIdsToIntersectForProjects(site.projects)
                         def centroid = site.extent?.geometry?.centre
                         if (!centroid) {
-                            def updatedSite = siteService.populateLocationMetadataForSite(site)
+                            def updatedSite = siteService.populateLocationMetadataForSite(site, fids)
                             siteService.update([extent: updatedSite.extent], site.siteId, false)
                             total++
                             if(total > 0 && (total % 200) == 0) {
@@ -256,18 +257,24 @@ class AdminController {
 
     @AlaSecured(["ROLE_ADMIN"])
     def updateSiteLocationMetadata() {
+        def dateFormat = new SimpleDateFormat("yyyy-MM-dd")
+        def defaultStartDate = "2018-01-01"
+        def timeZoneUTC = TimeZone.getTimeZone("UTC")
+        dateFormat.setTimeZone(timeZoneUTC)
+        def isMERIT = params.getBoolean('isMERIT', true)
+        Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate)
         def code = 'success'
         def total = 0
         def offset = 0
         def batchSize = 100
-        def isMERIT = params.getBoolean('isMERIT', true)
         def startTime = System.currentTimeMillis(), finishTime, startInterimTime, endInterimTime, batchStartTime, batchEndTime
-        List<String> fids = grailsApplication.config.getProperty('site.check.boundary.layers', List<String>)
+        List<String> defaultFids = metadataService.getSpatialLayerIdsToIntersect()
+        log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}")
         def totalSites
         List projectIds = []
         if (isMERIT) {
             projectIds = projectService.getAllMERITProjectIds()
-            totalSites = Site.countByStatusAndProjectsInList('active', projectIds)
+            totalSites = Site.countByStatusAndProjectsInListAndDateCreatedGreaterThan('active', projectIds, startDate)
         }
         else {
             totalSites = Site.countByStatus('active')
@@ -278,7 +285,7 @@ class AdminController {
             batchStartTime = startInterimTime = System.currentTimeMillis()
             def sites
             if (isMERIT) {
-                sites = Site.findAllByProjectsInListAndStatus(projectIds, 'active', [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
+                sites = Site.findAllByProjectsInListAndStatusAndDateCreatedGreaterThan(projectIds, 'active', startDate, [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
                     siteService.toMap(it, 'flat')
                 }
             }
@@ -304,6 +311,9 @@ class AdminController {
                             log.debug("Ignoring site ${site.siteId} due to no associated projects or no extent")
                             return
                         }
+                        def projectsOfSite = site.projects
+                        List hubIds = projectService.findHubIdOfProjects(projectsOfSite)
+                        def fids = hubIds.size() == 1 ? metadataService.getSpatialLayerIdsToIntersect(hubIds[0]) : defaultFids
                         siteService.populateLocationMetadataForSite(site, fids)
                         endInterimTime = System.currentTimeMillis()
                         log.debug("Time taken to update metadata ${site.siteId}: ${endInterimTime - startInterimTime} ms")

grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy

@@ -306,7 +306,7 @@ class MetadataController {
     }
 
     def getGeographicFacetConfig() {
-        render grailsApplication.config.getProperty('app.facets.geographic', Map) as JSON
+        render metadataService.getGeographicConfig() as JSON
     }
 
     /**

grails-app/domain/au/org/ala/ecodata/ExternalId.groovy

@@ -17,6 +17,7 @@ class ExternalId implements Comparable {
 
     static constraints = {
     }
+    static List idTypesByMonitor
 
     IdType idType
     String externalId
@@ -26,4 +27,19 @@ class ExternalId implements Comparable {
         ExternalId other = (ExternalId)otherId
         return (idType.ordinal()+externalId).compareTo(other?.idType?.ordinal()+other?.externalId)
     }
+
+    static List getMonitorIdTypes() {
+        if (!idTypesByMonitor){
+            List monitorIdTypes = []
+            IdType.values().each {
+                if (it.name().startsWith('MONITOR_')) {
+                    monitorIdTypes << it
+                }
+            }
+
+            idTypesByMonitor = monitorIdTypes
+        }
+
+        idTypesByMonitor
+    }
 }

grails-app/domain/au/org/ala/ecodata/Site.groovy

@@ -14,6 +14,9 @@ class Site {
     static String TYPE_PROJECT_AREA = 'projectArea'
     static String TYPE_WORKS_AREA = 'worksArea'
     static String TYPE_SURVEY_AREA = 'surveyArea'
+    static String EMSA_SITE_CODE = 'E'
+    static String REPORTING_SITE_CODE = 'R'
+    static String PLANNING_SITE_CODE = 'P'
 
     static graphql = SiteGraphQLMapper.graphqlMapping()
 

grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy

@@ -441,7 +441,7 @@ class ElasticSearchService {
 
     def buildFacetMapping() {
         def facetList = []
-        Map facetConfig = grailsApplication.config.getProperty('app.facets.geographic', Map)
+        Map facetConfig = metadataService.getGeographicConfig()
         // These groupings of facets determine the way the layers are used with a site, but can be treated the
         // same for the purposes of indexing the results.
         ['contextual', 'grouped', 'special'].each {

grails-app/services/au/org/ala/ecodata/HubService.groovy

@@ -126,9 +126,12 @@ class HubService {
     /**
      * get current hub based on hubId query parameter
      */
-    Hub getCurrentHub () {
-        GrailsWebRequest request = GrailsWebRequest.lookup()
-        String hubId = request?.request?.getParameter('hubId')
+    Hub getCurrentHub (String hubId = null) {
+        if (!hubId) {
+            GrailsWebRequest request = GrailsWebRequest.lookup()
+            hubId = request?.request?.getParameter('hubId')
+        }
+
         if (hubId) {
             return get(hubId)
         }

grails-app/services/au/org/ala/ecodata/MetadataService.groovy

@@ -40,6 +40,7 @@ class MetadataService {
     ActivityFormService activityFormService
     SpeciesReMatchService speciesReMatchService
     HubService hubService
+    ProjectService projectService
 
     /**
      * @deprecated use versioned API to retrieve activity form definitions
@@ -306,7 +307,7 @@ class MetadataService {
     def getNvisClassesForPoint(Double lat, Double lon) {
         def retMap = [:]
 
-        Map nvisLayers = grailsApplication.config.getProperty('app.facets.geographic.special', Map)
+        Map nvisLayers = getGeographicConfig().special
 
         nvisLayers.each { name, path ->
             def classesJsonFile = new File(path + '.json')
@@ -402,8 +403,8 @@ class MetadataService {
     def performLayerIntersect(lat,lng) {
 
 
-        Map contextualLayers = grailsApplication.config.getProperty('app.facets.geographic.contextual', Map)
-        Map groupedFacets = grailsApplication.config.getProperty('app.facets.geographic.grouped', Map)
+        Map contextualLayers = getGeographicConfig().contextual
+        Map groupedFacets = getGeographicConfig().grouped
 
         // Extract all of the layer field ids from the facet configuration so we can make a single web service call to the spatial portal.
         def fieldIds = contextualLayers.collect { k, v -> v }
@@ -471,22 +472,34 @@ class MetadataService {
     }
 
     /** Returns a list of spatial portal layer/field ids that ecodata will intersect every site against to support facetted geographic searches */
-    List<String> getSpatialLayerIdsToIntersect() {
-        def config = getGeographicConfig()
-        def fieldIds = config.contextualLayers.collect { k, v -> v }
-        config.groupedFacets.each { k, v ->
+    List<String> getSpatialLayerIdsToIntersect(String hubId = null) {
+        def config = getGeographicConfig(hubId)
+        def fieldIds = config.contextual.collect { k, v -> v }
+        config.grouped.each { k, v ->
             fieldIds.addAll(v.collect { k1, v1 -> v1 })
         }
         fieldIds
     }
 
+    /**
+     * If all projects belong to a hub, uses geographic configuration of that hub.
+     * Otherwise, check if hubId query parameter is present.
+     * If not, use default configuration present in application.groovy.
+     * @param projectIds
+     * @return
+     */
+    List<String> getSpatialLayerIdsToIntersectForProjects(List projectIds = []) {
+        List hubIds = projectService.findHubIdOfProjects(projectIds)
+        hubIds.size() == 1 ? getSpatialLayerIdsToIntersect(hubIds[0]) : getSpatialLayerIdsToIntersect()
+    }
+
     /**
      * Get layers for intersection from hub if it exists. Otherwise, get from configuration.
      * @return
      */
-    Map getGeographicConfig () {
+    Map getGeographicConfig (String hubId = null) {
         Map config
-        Hub hub = hubService.getCurrentHub()
+        Hub hub = hubService?.getCurrentHub(hubId)
         if (hub?.geographicConfig) {
             config = hub.geographicConfig
         } else {
@@ -502,7 +515,7 @@ class MetadataService {
      * @param fid the field id.
      */
     Map getGeographicFacetConfig(String fid) {
-        Map config = grailsApplication.config.getProperty('app.facets.geographic', Map)
+        Map config = getGeographicConfig()
         Map facetConfig = null
         config.contextual.each { String groupName, String groupFid ->
             if (fid == groupFid) {
@@ -612,8 +625,8 @@ class MetadataService {
             log.error("Missing result for ${lat}, ${lng}")
         }
 
-        Map contextualLayers = grailsApplication.config.getProperty('app.facets.geographic.contextual', Map)
-        Map groupedFacets = grailsApplication.config.getProperty('app.facets.geographic.grouped', Map)
+        Map contextualLayers = getGeographicConfig().contextual
+        Map groupedFacets = getGeographicConfig().grouped
         def facetTerms = [:]
 
         contextualLayers.each { name, fid ->

grails-app/services/au/org/ala/ecodata/ProjectService.groovy

@@ -1150,4 +1150,80 @@ class ProjectService {
         }
     }
 
+    /**
+     * Returns a list of all projects that have been updated since the specified date.
+     * @param date the date to compare against
+     * @return a list of projects
+     */
+    Map orderLayerIntersectionsByAreaOfProjectSites (Map project) {
+        Map<String,Map<String,Double>> sumOfIntersectionsByLayer = [:].withDefault { [:].withDefault { 0 } }
+        Map orderedIntersectionsByArea = [:]
+        // get sites of projects;
+        List projectSites = getRepresentativeSitesOfProject(project)
+        projectSites?.each { Map site ->
+            site.extent?.geometry?.get(SpatialService.INTERSECTION_AREA)?.each { String layerId, Map value ->
+                Set<String> keySet = value.keySet()
+                keySet.each { String layerValue ->
+                    sumOfIntersectionsByLayer[layerId][layerValue] += value[layerValue]
+                }
+            }
+        }
+
+        sumOfIntersectionsByLayer.each { String layerId, Map value ->
+            orderedIntersectionsByArea[layerId] = value.sort { entry ->
+                -entry.value
+            }.keySet().toList()
+        }
+
+        orderedIntersectionsByArea
+    }
+
+    /**
+     * Get representative sites of a project.
+     * 1. Check EMSA/Reporting sites
+     * 2. If there are no EMSA/Reporting sites, return planning/project extent sites
+     * 3. If there are no EMSA/Reporting/Planning/extent sites, return Management Unit boundaries.
+     * 4. Where none exist, return none
+     * @param project
+     * @return
+     */
+    List getRepresentativeSitesOfProject(Map project) {
+        if (project) {
+            List sites = project.sites
+            List projectSites = siteService.filterSitesByPurposeIsReportingOrEMSA(sites) ?: []
+            if (projectSites.isEmpty()) {
+                projectSites =  siteService.filterSitesByPurposeIsPlanning(sites) ?: []
+                if (projectSites.isEmpty() && project.managementUnitId) {
+                    ManagementUnit mu = ManagementUnit.findByManagementUnitId(project.managementUnitId)
+                    String managementUnitSiteId = mu?.managementUnitSiteId
+                    if (managementUnitSiteId) {
+                        Site muSite = Site.findBySiteId(managementUnitSiteId)
+                        if (muSite) {
+                            projectSites = [siteService.toMap(muSite, [SiteService.FLAT])]
+                        }
+                    }
+                }
+            }
+
+            return projectSites
+        }
+
+
+        []
+    }
+
+    /**
+     * Returns a distinct list of hubIds for the supplied projects.
+     * @param projects
+     * @return
+     */
+    List findHubIdOfProjects(List projects) {
+        Project.createCriteria().listDistinct {
+            inList('projectId', projects)
+            projections {
+                property('hubId')
+            }
+        }
+    }
+
 }