From e3815f6ac4263fd36848efdc40a2af7e98bd2448 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 12:08:53 -0500 Subject: [PATCH 01/10] WIP Perserve query column order --- .../backend/stored_queries/batch_edit.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 56f3fedfaed..dae74a47940 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -756,6 +756,8 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): localized_label, ) + logger.debug("====== COLUMN ORDER =======") + logger.debug(self.columns) readonly_fields, readonly_rels = get_readonly_fields(base_table) key_and_fields_and_headers = [ _lookup_in_fields(column.idx, readonly_fields) for column in self.columns @@ -1095,10 +1097,34 @@ def _get_orig_column(string_id: str): # We would have arbitarily sorted the columns, so our columns will not be correct. # Rather than sifting the data, we just add a default visual order. - visual_order = Func.first(headers_enumerated) + caption_to_query_field = {caption: query_field for query_field, caption in query_field_caption_lookup.items()} + visual_order = [None for header in key_and_headers] + # visual_order = Func.first(headers_enumerated) + original_order = enumerate(fields) + columns_at_end = [] + for index, (key, header) in headers_enumerated: + query_field = caption_to_query_field.get(header) + if query_field in fields: + original_place = fields.index(query_field) + visual_order[original_place] = index + else: + columns_at_end.append(key) + + for column, index in enumerate(visual_order): + if column is None: + visual_order[index] = columns_at_end.pop(0) + if len(columns_at_end) == 0: + break headers = Func.second(key_and_headers) + logger.debug("================ COLUMN ORDER ===============") + logger.debug(fields) + # logger.debug(query_fields) # Query fields # Includes hidden fields + # logger.debug(query_field_caption_lookup) # Query field -> header + logger.debug(visual_order) + logger.debug(headers) + json_upload_plan = upload_plan.unparse() validate(json_upload_plan, schema) From 9fc246f15741bfdadea03274632f52a0239ca685 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 12:35:32 -0500 Subject: [PATCH 02/10] Handle new columns --- .../backend/stored_queries/batch_edit.py | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index dae74a47940..3ffdb2b9d72 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1098,19 +1098,20 @@ def _get_orig_column(string_id: str): # We would have arbitarily sorted the columns, so our columns will not be correct. # Rather than sifting the data, we just add a default visual order. caption_to_query_field = {caption: query_field for query_field, caption in query_field_caption_lookup.items()} - visual_order = [None for header in key_and_headers] - # visual_order = Func.first(headers_enumerated) - original_order = enumerate(fields) + visual_order = [None for _ in key_and_headers] columns_at_end = [] - for index, (key, header) in headers_enumerated: + for index, (_, header) in headers_enumerated: + # Find the column's original position if it existed in the origin query query_field = caption_to_query_field.get(header) if query_field in fields: original_place = fields.index(query_field) visual_order[original_place] = index else: - columns_at_end.append(key) + # New field/column. Add it to the end of the dataset. + columns_at_end.append(index) - for column, index in enumerate(visual_order): + # Fill in the gaps with the new columns + for index, column in enumerate(visual_order): if column is None: visual_order[index] = columns_at_end.pop(0) if len(columns_at_end) == 0: @@ -1118,13 +1119,6 @@ def _get_orig_column(string_id: str): headers = Func.second(key_and_headers) - logger.debug("================ COLUMN ORDER ===============") - logger.debug(fields) - # logger.debug(query_fields) # Query fields # Includes hidden fields - # logger.debug(query_field_caption_lookup) # Query field -> header - logger.debug(visual_order) - logger.debug(headers) - json_upload_plan = upload_plan.unparse() validate(json_upload_plan, schema) From 1bd1220c6ec89833d9d1856f9b52db3dff610285 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 14:31:54 -0500 Subject: [PATCH 03/10] WIP handle renamed/duplicate columns --- .../backend/stored_queries/batch_edit.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 3ffdb2b9d72..b870513d2a1 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1094,22 +1094,28 @@ def _get_orig_column(string_id: str): ) headers_enumerated = enumerate(key_and_headers) + logger.info("========== COLUMNS ==========") + logger.info(key_and_headers) + logger.info(extend_row.columns) + logger.info(fields) # We would have arbitarily sorted the columns, so our columns will not be correct. # Rather than sifting the data, we just add a default visual order. - caption_to_query_field = {caption: query_field for query_field, caption in query_field_caption_lookup.items()} visual_order = [None for _ in key_and_headers] columns_at_end = [] - for index, (_, header) in headers_enumerated: + for index, (key, header) in headers_enumerated: # Find the column's original position if it existed in the origin query - query_field = caption_to_query_field.get(header) - if query_field in fields: - original_place = fields.index(query_field) + original_place = key[0] + duplicate_count = key[1] + if original_place < len(fields): visual_order[original_place] = index else: # New field/column. Add it to the end of the dataset. columns_at_end.append(index) + logger.info("----- ORDER 1 ------") + logger.info(visual_order) + # Fill in the gaps with the new columns for index, column in enumerate(visual_order): if column is None: @@ -1117,6 +1123,9 @@ def _get_orig_column(string_id: str): if len(columns_at_end) == 0: break + logger.info("----- ORDER 2 ------") + logger.info(visual_order) + headers = Func.second(key_and_headers) json_upload_plan = upload_plan.unparse() From 4916e66b1b459a0d6ab68f83171295da28d23841 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 14:45:30 -0500 Subject: [PATCH 04/10] cleanup --- .../backend/stored_queries/batch_edit.py | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index b870513d2a1..23c49d331b9 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -756,8 +756,6 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): localized_label, ) - logger.debug("====== COLUMN ORDER =======") - logger.debug(self.columns) readonly_fields, readonly_rels = get_readonly_fields(base_table) key_and_fields_and_headers = [ _lookup_in_fields(column.idx, readonly_fields) for column in self.columns @@ -1094,38 +1092,27 @@ def _get_orig_column(string_id: str): ) headers_enumerated = enumerate(key_and_headers) - logger.info("========== COLUMNS ==========") - logger.info(key_and_headers) - logger.info(extend_row.columns) - logger.info(fields) # We would have arbitarily sorted the columns, so our columns will not be correct. # Rather than sifting the data, we just add a default visual order. - visual_order = [None for _ in key_and_headers] - columns_at_end = [] - for index, (key, header) in headers_enumerated: + visual_order: list[int | None] = [None for _ in key_and_headers] + new_columns: list[int] = [] + for index, (key, _header) in headers_enumerated: # Find the column's original position if it existed in the origin query original_place = key[0] - duplicate_count = key[1] if original_place < len(fields): visual_order[original_place] = index else: # New field/column. Add it to the end of the dataset. - columns_at_end.append(index) - - logger.info("----- ORDER 1 ------") - logger.info(visual_order) + new_columns.append(index) # Fill in the gaps with the new columns for index, column in enumerate(visual_order): if column is None: - visual_order[index] = columns_at_end.pop(0) - if len(columns_at_end) == 0: + visual_order[index] = new_columns.pop(0) + if len(new_columns) == 0: break - logger.info("----- ORDER 2 ------") - logger.info(visual_order) - headers = Func.second(key_and_headers) json_upload_plan = upload_plan.unparse() From f67434d2f2b30058bfa843c9410f8d8014bdd23f Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 15:06:36 -0500 Subject: [PATCH 05/10] Ignore empty new_columns insertion --- specifyweb/backend/stored_queries/batch_edit.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 23c49d331b9..a2bc91f0734 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1107,11 +1107,12 @@ def _get_orig_column(string_id: str): new_columns.append(index) # Fill in the gaps with the new columns - for index, column in enumerate(visual_order): - if column is None: - visual_order[index] = new_columns.pop(0) - if len(new_columns) == 0: - break + if new_columns: + for index, column in enumerate(visual_order): + if column is None: + visual_order[index] = new_columns.pop(0) + if len(new_columns) == 0: + break headers = Func.second(key_and_headers) From 62b67f3bcce71b528344498fbd54b4e18227d8bc Mon Sep 17 00:00:00 2001 From: alesan99 Date: Fri, 24 Apr 2026 15:50:05 -0500 Subject: [PATCH 06/10] Don't overwrite visual_order positions --- specifyweb/backend/stored_queries/batch_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index a2bc91f0734..a09f7b65438 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1100,7 +1100,7 @@ def _get_orig_column(string_id: str): for index, (key, _header) in headers_enumerated: # Find the column's original position if it existed in the origin query original_place = key[0] - if original_place < len(fields): + if original_place < len(fields) and visual_order[original_place] is None: visual_order[original_place] = index else: # New field/column. Add it to the end of the dataset. From 6bb43caafce768b231995b1750aea0ad5a37d6cb Mon Sep 17 00:00:00 2001 From: alesan99 Date: Mon, 27 Apr 2026 09:16:34 -0500 Subject: [PATCH 07/10] Use visible columns to determine old columns --- specifyweb/backend/stored_queries/batch_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index a09f7b65438..e1e88fdba4b 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1100,7 +1100,7 @@ def _get_orig_column(string_id: str): for index, (key, _header) in headers_enumerated: # Find the column's original position if it existed in the origin query original_place = key[0] - if original_place < len(fields) and visual_order[original_place] is None: + if original_place < len(visible_fields) and visual_order[original_place] is None: visual_order[original_place] = index else: # New field/column. Add it to the end of the dataset. From f3edf10abe1a96c398039487872431ae21f736b1 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Mon, 27 Apr 2026 14:31:50 -0500 Subject: [PATCH 08/10] Group duplicates together normally --- .../backend/stored_queries/batch_edit.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index e1e88fdba4b..4e9b6ef154b 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1095,24 +1095,21 @@ def _get_orig_column(string_id: str): # We would have arbitarily sorted the columns, so our columns will not be correct. # Rather than sifting the data, we just add a default visual order. - visual_order: list[int | None] = [None for _ in key_and_headers] - new_columns: list[int] = [] + visual_order_groups: list[list[int]] = [[] for _ in visible_fields] for index, (key, _header) in headers_enumerated: # Find the column's original position if it existed in the origin query original_place = key[0] - if original_place < len(visible_fields) and visual_order[original_place] is None: - visual_order[original_place] = index + duplicate_index = key[1] + if original_place < len(visible_fields): + visual_order_groups[original_place].insert(duplicate_index, index) else: # New field/column. Add it to the end of the dataset. - new_columns.append(index) - - # Fill in the gaps with the new columns - if new_columns: - for index, column in enumerate(visual_order): - if column is None: - visual_order[index] = new_columns.pop(0) - if len(new_columns) == 0: - break + if existing_column is None: + visual_order_groups.append([index]) + + visual_order: list[int] = [] + for bucket in visual_order_groups: + visual_order.extend(bucket) headers = Func.second(key_and_headers) From 23d7288517e4fc78ec9e45ff521cedcfcb87ff6d Mon Sep 17 00:00:00 2001 From: alesan99 Date: Mon, 27 Apr 2026 14:39:58 -0500 Subject: [PATCH 09/10] Remove redundant check --- specifyweb/backend/stored_queries/batch_edit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 4e9b6ef154b..7798f14d197 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1104,8 +1104,7 @@ def _get_orig_column(string_id: str): visual_order_groups[original_place].insert(duplicate_index, index) else: # New field/column. Add it to the end of the dataset. - if existing_column is None: - visual_order_groups.append([index]) + visual_order_groups.append([index]) visual_order: list[int] = [] for bucket in visual_order_groups: From 7bce0165ca678295662f5856cb24f35d09245446 Mon Sep 17 00:00:00 2001 From: alesan99 Date: Wed, 29 Apr 2026 08:40:02 -0500 Subject: [PATCH 10/10] Update batch edit tests to reflect new behavior --- .../stored_queries/tests/test_batch_edit.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/specifyweb/backend/stored_queries/tests/test_batch_edit.py b/specifyweb/backend/stored_queries/tests/test_batch_edit.py index 6372521c6f3..307670a294f 100644 --- a/specifyweb/backend/stored_queries/tests/test_batch_edit.py +++ b/specifyweb/backend/stored_queries/tests/test_batch_edit.py @@ -155,9 +155,9 @@ def test_basic_run(self): "CollectionObject catalogNumber", "CollectionObject integer1", "Agent (formatted)", - "CollectingEvent (formatted)", "Agent firstName", "Agent lastName", + "CollectingEvent (formatted)", "Locality localityName", ], ) @@ -554,6 +554,14 @@ def test_duplicates_flattened(self): "CollectionObject catalogNumber", "CollectionObject integer1", "Agent (formatted)", + "Determination integer1", + "Determination integer1 #2", + "Determination integer1 #3", + "Determination remarks", + "Determination remarks #2", + "Determination remarks #3", + "Preparation countAmt", + "Preparation text1", "Agent firstName", "Agent lastName", "AgentSpecialty specialtyName", @@ -561,29 +569,21 @@ def test_duplicates_flattened(self): "AgentSpecialty specialtyName #3", "AgentSpecialty specialtyName #4", "Collector remarks", - "CollectingEvent stationFieldNumber", "Collector remarks #2", - "CollectingEvent stationFieldNumber #2", "Collector remarks #3", - "CollectingEvent stationFieldNumber #3", "Collector remarks #4", - "CollectingEvent stationFieldNumber #4", "Collector remarks #5", - "CollectingEvent stationFieldNumber #5", "Collector remarks #6", - "CollectingEvent stationFieldNumber #6", "Collector remarks #7", - "CollectingEvent stationFieldNumber #7", "Collector remarks #8", - "CollectingEvent stationFieldNumber #8", - "Determination integer1", - "Determination remarks", - "Determination integer1 #2", - "Determination remarks #2", - "Determination integer1 #3", - "Determination remarks #3", - "Preparation countAmt", - "Preparation text1" + "CollectingEvent stationFieldNumber", + "CollectingEvent stationFieldNumber #2", + "CollectingEvent stationFieldNumber #3", + "CollectingEvent stationFieldNumber #4", + "CollectingEvent stationFieldNumber #5", + "CollectingEvent stationFieldNumber #6", + "CollectingEvent stationFieldNumber #7", + "CollectingEvent stationFieldNumber #8" ], )