Add tile sort order to bulk data manager (#11638)

* Remove unused argument

* Add sortorder to tile_excel

* Make sortorder non-nullable in loadstaging

* Add sortorder to loadstaging model

* Add sortorder to staging_to_tile pg function

* Add hardcoded sortorder of 0 to bulk excel

* Add fixed sortorder of 0 to single csv

* Support sortorder in arches_stage_string_data_for_bulk_edit sql fn

* nit

* Update bulk concept edit to support sortorder

* Add changelog entry

* Check imported tile sort order #11408

---------

Co-authored-by: Johnathan Clementi <[email protected]>

Author: chiatt

arches/app/etl_modules/branch_excel_importer.py

@@ -205,8 +205,9 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                     tile_value_json, passes_validation = self.create_tile_value(
                         cell_values, data_node_lookup, node_lookup, row_details, cursor
                     )
+                    sortorder = 0
                     cursor.execute(
-                        """INSERT INTO load_staging (nodegroupid, legacyid, resourceid, tileid, parenttileid, value, loadid, nodegroup_depth, source_description, passes_validation, operation) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                        """INSERT INTO load_staging (nodegroupid, legacyid, resourceid, tileid, parenttileid, value, loadid, nodegroup_depth, source_description, passes_validation, operation, sortorder) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
                         (
                             row_details["nodegroup_id"],
                             legacyid,
@@ -221,6 +222,7 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                             ),  # source_description
                             passes_validation,
                             operation,
+                            sortorder,
                         ),
                     )
                 except KeyError:

arches/app/etl_modules/bulk_edit_concept.py

@@ -393,8 +393,8 @@ def stage_data(
         try:
             sql = (
                 """
-                INSERT INTO load_staging (value, tileid, nodegroupid, parenttileid, resourceid, loadid, nodegroup_depth, source_description, operation, passes_validation)
-                    (SELECT tiledata, tileid, nodegroupid, parenttileid, resourceinstanceid, %(load_id)s, 0, 'bulk_edit', 'update', true
+                INSERT INTO load_staging (value, tileid, nodegroupid, parenttileid, resourceid, loadid, nodegroup_depth, source_description, operation, passes_validation, sortorder)
+                    (SELECT tiledata, tileid, nodegroupid, parenttileid, resourceinstanceid, %(load_id)s, 0, 'bulk_edit', 'update', true, sortorder
                     FROM tiles
                     WHERE nodegroupid in (SELECT nodegroupid FROM nodes WHERE nodeid = %(node_id)s)
                     AND tiledata -> %(node_id)s ? %(old_id)s

arches/app/etl_modules/import_single_csv.py

@@ -555,8 +555,9 @@ def populate_staging_table(
                                 nodegroup_depth,
                                 source_description,
                                 operation,
-                                passes_validation
-                            ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                                passes_validation,
+                                sortorder
+                            ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
                             (
                                 nodegroup,
                                 legacyid,
@@ -568,6 +569,7 @@ def populate_staging_table(
                                 csv_file_name,
                                 "insert",
                                 passes_validation,
+                                0,
                             ),
                         )
 

arches/app/etl_modules/tile_excel_importer.py

@@ -81,7 +81,6 @@ def run_load_task_async(self, request):
 
     def create_tile_value(
         self,
-        cell_values,
         data_node_lookup,
         node_lookup,
         nodegroup_alias,
@@ -176,6 +175,7 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                 raise ValueError(_("All rows must have a valid resource id"))
 
             node_values = cell_values[3:-3]
+            sortorder = cell_values[-3] if cell_values[-3] else 0
             try:
                 row_count += 1
                 row_details = dict(zip(data_node_lookup[nodegroup_alias], node_values))
@@ -194,7 +194,6 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                 )
                 legacyid, resourceid = self.set_legacy_id(resourceid)
                 tile_value_json, passes_validation = self.create_tile_value(
-                    cell_values,
                     data_node_lookup,
                     node_lookup,
                     nodegroup_alias,
@@ -214,7 +213,7 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                         if TileModel.objects.filter(pk=tileid).exists():
                             operation = "update"
                 cursor.execute(
-                    """INSERT INTO load_staging (nodegroupid, legacyid, resourceid, tileid, parenttileid, value, loadid, nodegroup_depth, source_description, passes_validation, operation) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                    """INSERT INTO load_staging (nodegroupid, legacyid, resourceid, tileid, parenttileid, value, loadid, nodegroup_depth, source_description, passes_validation, operation, sortorder) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
                     (
                         row_details["nodegroup_id"],
                         legacyid,
@@ -229,6 +228,7 @@ def process_worksheet(self, worksheet, cursor, node_lookup, nodegroup_lookup):
                         ),  # source_description
                         passes_validation,
                         operation,
+                        sortorder,
                     ),
                 )
             except KeyError: