update sortby to cover multiple sortables (#1211)

Author: tomkralidis

pycsw/ogc/api/records.py

@@ -34,6 +34,7 @@
 import logging
 from operator import itemgetter
 import os
+from typing import List, Union
 from urllib.parse import urlencode, quote
 
 from owslib.ogcapi.records import Records
@@ -136,7 +137,8 @@ def __init__(self, config: dict):
                 self.config['repository']['database'],
                 self.context,
                 table=self.config['repository']['table'],
-                repo_filter=repo_filter
+                repo_filter=repo_filter,
+                stable_sort=self.config['repository'].get('stable_sort', False)
             )
             LOGGER.debug(f'Repository loaded {self.repository.dbtype}')
         except Exception as err:
@@ -598,9 +600,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
 
             if 'sortby' in json_post_data:
                 LOGGER.debug('Detected sortby')
-                args['sortby'] = json_post_data['sortby'][0]['field']
-                if json_post_data['sortby'][0].get('direction', 'asc') == 'desc':
-                    args['sortby'] = f"-{args['sortby']}"
+                args['sortby'] = json_post_data['sortby']
 
             LOGGER.debug(f'Transformed args: {args}')
 
@@ -757,19 +757,12 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
             sortby = args['sortby']
 
         if sortby is not None:
-            LOGGER.debug('processing sortby')
-            if sortby.startswith('-'):
-                sortby = sortby.lstrip('-')
+            sortbys = sortby_to_order_by(sortby, self.repository.query_mappings)
+            query = query.order_by(*sortbys)
 
-            if sortby not in list(self.repository.query_mappings.keys()):
-                msg = 'Invalid sortby property'
-                LOGGER.exception(msg)
-                return self.get_exception(400, headers_, 'InvalidParameterValue', msg)
-
-            if args['sortby'].startswith('-'):
-                query = query.order_by(self.repository.query_mappings[sortby].desc())
-            else:
-                query = query.order_by(self.repository.query_mappings[sortby])
+        if self.repository.stable_sort:
+            LOGGER.debug('Adding additional stable sort on identifier')
+            query = query.order_by(self.repository.query_mappings['identifier'].asc())
 
         if limit is None and 'limit' in args:
             limit = int(args['limit'])
@@ -1519,3 +1512,38 @@ def build_anytext(name, value):
             predicates.append(f"{name} ILIKE '%{token}%'")
 
     return f"({' OR '.join(predicates)})"
+
+
+def sortby_to_order_by(sortby: Union[str, List[dict]], mappings: dict) -> list:
+
+    sortby_ = []
+    value_list = []
+
+    if isinstance(sortby, str):
+        LOGGER.debug('Normalizing sortby into list of dicts')
+        for s in sortby.split(','):
+            s2 = s.lstrip('-')
+            if s.startswith('-'):
+                s2dir = 'desc'
+            else:
+                s2dir = 'asc'
+
+            sortby_.append({
+                'field': s2,
+                'direction': s2dir
+            })
+    else:
+       sortby_ = sortby
+
+    for sb in sortby_:
+        if sb['field'] not in list(mappings.keys()):
+            msg = 'Invalid sortby property'
+            LOGGER.exception(msg)
+            raise ValueError(msg)
+
+        if sb['direction'] == 'desc':
+            value_list.append(mappings[sb['field']].desc())
+        else:
+            value_list.append(mappings[sb['field']].asc())
+
+    return value_list

pycsw/stac/api.py

@@ -451,9 +451,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
 
         if 'sortby' in json_post_data2:
             LOGGER.debug('Detected sortby parameter')
-            args['sortby'] = json_post_data2['sortby'][0]['field']
-            if json_post_data2['sortby'][0].get('direction', 'asc') == 'desc':
-                args['sortby'] = f"-{args['sortby']}"
+            args['sortby'] = json_post_data2['sortby']
 
         if 'collections' in json_post_data2:
             LOGGER.debug('Detected collections query parameter')

tests/functionaltests/suites/oarec/test_oarec_functional.py

@@ -237,6 +237,24 @@ def test_items(config):
     assert content['numberReturned'] == 10
     assert content['features'][5]['properties']['title'] == 'Lorem ipsum dolor sit amet'  # noqa
 
+    params = {'sortby': '-title,description'}
+    content = json.loads(api.items({}, None, params)[2])
+    assert content['numberMatched'] == 12
+    assert content['numberReturned'] == 10
+    assert content['features'][0]['properties']['title'] == 'Ñunç elementum'
+
+    params = {'sortby': 'title,description'}
+    content = json.loads(api.items({}, None, params)[2])
+    assert content['numberMatched'] == 12
+    assert content['numberReturned'] == 10
+    assert content['features'][5]['properties']['title'] == 'Lorem ipsum'
+
+    params = {'sortby': 'title,-description'}
+    content = json.loads(api.items({}, None, params)[2])
+    assert content['numberMatched'] == 12
+    assert content['numberReturned'] == 10
+    assert content['features'][6]['properties']['title'] == 'Lorem ipsum dolor sit amet'  # noqa
+
     cql_json = {'op': '=', 'args': [{'property': 'title'}, 'Lorem ipsum']}
     content = json.loads(api.items({}, cql_json, {})[2])
     assert content['numberMatched'] == 1

tests/functionaltests/suites/stac_api/test_stac_api_functional.py

@@ -206,6 +206,16 @@ def test_items(config):
     assert content['numberMatched'] == 26
     assert content['features'][6]['properties']['title'] == 'S2B_MSIL1C_20190910T095029_N0208_R079_T33UWQ_20190910T120910.SAFE'  # noqa
 
+    content = json.loads(api.items({}, None, {'sortby': '-description,-title'})[2])  # noqa
+
+    assert content['numberMatched'] == 26
+    assert content['features'][6]['properties']['title'] == 'S2B_MSIL2A_20190910T095029_N0213_R079_T33UXQ_20190910T124513.SAFE'  # noqa
+
+    content = json.loads(api.items({}, None, {'sortby': '-description,title'})[2])  # noqa
+
+    assert content['numberMatched'] == 26
+    assert content['features'][6]['properties']['title'] == 'S2B_MSIL1C_20190910T095029_N0208_R079_T33UWQ_20190910T120910.SAFE'  # noqa
+
     params = {'filter': "title LIKE '%%T33TWN%%'"}
     content = json.loads(api.items({}, None, params)[2])
     assert content['numberMatched'] == 4
@@ -246,6 +256,14 @@ def test_items(config):
     assert content['numberMatched'] == 26
     assert content['features'][5]['properties']['title'] == 'S2B_MSIL2A_20190910T095029_N0500_R079_T33TWN_20230430T083712.SAFE'  # noqa
 
+    content = json.loads(api.items({},
+                                   {'sortby': [{'field': 'title', 'direction': 'asc'},  # noqa
+                                               {'field': 'description', 'direction': 'desc'}  # noqa
+                                              ]},  # noqa
+                                   {})[2])
+    assert content['numberMatched'] == 26
+    assert content['features'][6]['properties']['title'] == 'S2B_MSIL1C_20190910T095029_N0208_R079_T33UWQ_20190910T120910.SAFE'  # noqa
+
     headers, status, content = api.items({}, None, {}, 'foo')
     assert status == 400