diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index c35a33e2890..3c12ab336ed 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -59,38 +59,50 @@ def apply_schema_overrides_for_all_disciplines(_apps): ) apply_schema_defaults_task.apply(args=[discipline.id]) + # PERF: The vast majority of these can be collapsed to a single call to + # update_table_schema_config_with_defaults funcs = [ # usc.update_all_table_schema_config_with_defaults, usc.create_geo_table_schema_config_with_defaults, # specify 0002 usc.create_cotype_splocalecontaineritem, # specify 0003 usc.create_strat_table_schema_config_with_defaults, # specify 0004 - getting skip warnings usc.create_agetype_picklist, # specify 0004 - usc.update_cog_type_fields, # specify 0007 + # BUG: This should really only be run in the context of the migration, + # and not on startup. See the below BUG comment above usc.update_hidden_prop + # usc.update_cog_type_fields, # specify 0007 usc.create_cogtype_picklist, # specify 0007 - usc.update_cogtype_splocalecontaineritem, # specify 0007 - usc.update_systemcogtypes_picklist, # specify 0007 - usc.update_cogtype_type_splocalecontaineritem, # specify 0007 + # BUG: These also shouldn't be run with this suite. These are one way + # data migrations in the contect of migrations meant to resolve + # eariler migrations. + # The functions can be destructive as we can't really discern whether + # or not these functions should be applied + # usc.update_cogtype_splocalecontaineritem, # specify 0007 + # usc.update_systemcogtypes_picklist, # specify 0007 + # usc.update_cogtype_type_splocalecontaineritem, # specify 0007 usc.update_relative_age_fields, # specify 0008 usc.add_cojo_to_schema_config, # specify 0012 usc.update_cog_schema_config, # specify 0013 usc.update_age_schema_config, # specify 0015 - usc.schemaconfig_fixes, # specify 0017 - usc.add_cot_catnum_to_schema, # specify 0018 + # usc.schemaconfig_fixes, # specify 0017 + # usc.add_cot_catnum_to_schema, # specify 0018 usc.add_tectonicunit_to_pc_in_schema_config, # specify 0020 - usc.fix_hidden_geo_prop, # specify 0021 - usc.update_schema_config_field_desc, # specify 0023 - usc.update_hidden_prop, # specify 0023 + # usc.fix_hidden_geo_prop, # specify 0021 + # usc.update_schema_config_field_desc, # specify 0023 + # BUG: We can't reliably run this function at startup, as there is no + # easy way to differentiate Schema Config tables/fields that should or + # should not be updated for already existing Disciplines. + # usc.update_hidden_prop, # specify 0023 usc.update_storage_unique_id_fields, # specify 0024 - usc.update_co_children_fields, # specify 0027 - usc.remove_collectionobject_parentco, # specify 0029 - usc.add_quantities_gift, # specify 0032 - usc.update_paleo_desc, # specify 0033 - usc.update_accession_date_fields, # specify 0034 + # usc.update_co_children_fields, # specify 0027 + # usc.remove_collectionobject_parentco, # specify 0029 + # usc.add_quantities_gift, # specify 0032 + # usc.update_paleo_desc, # specify 0033 + # usc.update_accession_date_fields, # specify 0034 usc.update_loan_and_gift_agent_fields, # specify 0039 - usc.update_loan_and_gift_agents, # specify 0039 - usc.componets_schema_config_migrations, # specify 0040 + usc.remove_componentparent_item, # specify 0040 + usc.create_table_schema_config_with_defaults, # specify 0040 usc.create_discipline_type_picklist, # specify 0042 - usc.update_discipline_type_splocalecontaineritem, # specify 0042 + # usc.update_discipline_type_splocalecontaineritem, # specify 0042 apply_schema_overrides_for_all_disciplines, usc.deduplicate_schema_config_orm, ] diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py index 000b7572053..1259f18b55e 100644 --- a/specifyweb/specify/migration_utils/default_cots.py +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -68,7 +68,7 @@ def create_cogtype_type_picklist(apps, using='default'): Picklistitem = apps.get_model('specify', 'Picklistitem') for collection in Collection.objects.using(using).all(): - cog_type_picklist, _ = Picklist.objects.using(using).get_or_create( + cog_type_picklist, picklist_created = Picklist.objects.using(using).get_or_create( name='SystemCOGTypes', # Default Collection Object Group Types type=0, collection=collection, @@ -77,12 +77,13 @@ def create_cogtype_type_picklist(apps, using='default'): "readonly": False, } ) - for cog_type in DEFAULT_COG_TYPES: - Picklistitem.objects.using(using).get_or_create( - title=cog_type, - value=cog_type, - picklist=cog_type_picklist - ) + if picklist_created: + for cog_type in DEFAULT_COG_TYPES: + Picklistitem.objects.using(using).get_or_create( + title=cog_type, + value=cog_type, + picklist=cog_type_picklist + ) COTYPE_PICKLIST_NAME = 'CollectionObjectType' FIELD_NAME = 'collectionObjectType' diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index 7e4fce0c947..38e5ba48a88 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -411,23 +411,6 @@ 'Gift': ['agent1', 'agent2', 'agent3', 'agent4', 'agent5'], } -MIGRATION_0038_UPDATE_FIELDS = { - 'Loan': [ - ('agent1','Agent 1','Agent 1'), - ('agent2','Agent 2','Agent 2'), - ('agent3','Agent 3','Agent 3'), - ('agent4','Agent 4','Agent 4'), - ('agent5','Agent 5','Agent 5'), - ], - 'Gift': [ - ('agent1','Agent 1','Agent 1'), - ('agent2','Agent 2','Agent 2'), - ('agent3','Agent 3','Agent 3'), - ('agent4','Agent 4','Agent 4'), - ('agent5','Agent 5','Agent 5'), - ] -} - MIGRATION_0040_TABLES = [ ('Component', None), ] diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py index c2afe88c6fa..2769c98754f 100644 --- a/specifyweb/specify/migration_utils/tectonic_ranks.py +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -15,14 +15,28 @@ def create_default_tectonic_ranks(apps): if not tectonic_tree_def: tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) - root, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - title="Root", + root, root_created = TectonicUnitTreeDefItem.objects.get_or_create( rankid=0, parent=None, treedef=tectonic_tree_def, - isenforced=True + defaults={ + "name": "Root", + "title": "Root", + "isenforced": True + } ) + # The root rank already exists in some capacity in the Discipline + # We can assume the user has made modifications to the tree at this + # point, so shouldn't go further with checking/creating lower ranks + if not root_created: + # BUG?: handle setting the tectonicunittreedef on the Discipline + # here? We can probably practically assume it's already set if the + # root node exists. + continue + + # At this point, these get_or_create calls should always be the + # equivalent of create (as we know the root node didn't exist). + # But keeping the get_or_create here just because superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( name="Superstructure", title="Superstructure", @@ -91,23 +105,25 @@ def create_root_tectonic_node(apps): for discipline in Discipline.objects.all(): - tectonic_tree_def = TectonicUnitTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() + tectonic_tree_def = TectonicUnitTreeDef.objects.filter(discipline=discipline).first() if not tectonic_tree_def: tectonic_tree_def, is_created = TectonicUnitTreeDef.objects.get_or_create( name="Tectonic Unit", discipline=discipline ) - tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, name="Root").first() + tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, parent=None).first() if not tectonic_tree_def_item: tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", title="Root", treedef=tectonic_tree_def, + rankid=0, + parent=None, isenforced=True ) - root = TectonicUnit.objects.filter(name="Root", definition=tectonic_tree_def).first() + root = TectonicUnit.objects.filter(definition=tectonic_tree_def, definitionitem=tectonic_tree_def_item, parent=None).first() if not root: root, is_created = TectonicUnit.objects.get_or_create( name="Root", @@ -123,7 +139,7 @@ def create_root_tectonic_node(apps): if is_created: logger.info(f"Created root tectonic unit for discipline {discipline.name}") - TectonicUnitTreeDefItem.objects.filter(rankid=0, isenforced__isnull=True).update(isenforced=True) + TectonicUnitTreeDefItem.objects.filter(parent=None,rankid=0, isenforced__isnull=True).update(isenforced=True) def revert_create_root_tectonic_node(apps, schema_editor=None): TectonicUnit = apps.get_model('specify', 'TectonicUnit') diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index 0693b8ba24d..b91927b90a6 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -1,7 +1,7 @@ import re import json -from typing import NamedTuple, Tuple +from typing import NamedTuple, Tuple, TypedDict, NotRequired import logging from collections import defaultdict from functools import lru_cache @@ -47,7 +47,6 @@ MIGRATION_0034_UPDATE_FIELDS, MIGRATION_0035_FIELDS, MIGRATION_0038_FIELDS, - MIGRATION_0038_UPDATE_FIELDS, MIGRATION_0040_TABLES, MIGRATION_0040_FIELDS, MIGRATION_0040_UPDATE_FIELDS, @@ -235,7 +234,6 @@ def bulk_create_splocaleitemstr_idempotent(Splocaleitemstr, rows: list[dict]) -> key = (r["language"], r[fk_field].pk) desired_by_key[key] = r - rows_to_update = [] ids_to_delete: set[int] = set() to_create = [] for key, desired_row in desired_by_key.items(): @@ -245,32 +243,29 @@ def bulk_create_splocaleitemstr_idempotent(Splocaleitemstr, rows: list[dict]) -> to_create.append(Splocaleitemstr(**desired_row)) continue - keeper = existing_for_key[0] - if keeper.text != desired_row["text"]: - keeper.text = desired_row["text"] - rows_to_update.append(keeper) - for duplicate in existing_for_key[1:]: ids_to_delete.add(duplicate.id) if ids_to_delete: Splocaleitemstr.objects.filter(id__in=ids_to_delete).delete() - if rows_to_update: - Splocaleitemstr.objects.bulk_update(rows_to_update, ["text"]) - if to_create: Splocaleitemstr.objects.bulk_create(to_create) total_created += len(to_create) return total_created +class TableDefaults(TypedDict): + name: NotRequired[str] + desc: NotRequired[str] + items: "NotRequired[dict[str, FieldDefaults]]" + def update_table_schema_config_with_defaults( table_name, discipline_id: int, - description: str = None, + description: str | None = None, apps = global_apps, - defaults: dict = None, + defaults: TableDefaults | None = None, pending_itemstr_rows: list[dict] | None = None, ): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') @@ -293,7 +288,7 @@ def update_table_schema_config_with_defaults( pending_itemstr_rows = [] try: - table_defaults = defaults if defaults is not None else dict() + table_defaults = defaults if defaults is not None else TableDefaults() table_name_str = table_defaults.get('name', camel_to_spaced_title_case(uncapitilize(table.name))) table_desc_str = table_defaults.get('desc', camel_to_spaced_title_case(uncapitilize(table.name))) @@ -311,7 +306,7 @@ def update_table_schema_config_with_defaults( "schematype": table_config.schema_type } - fetched_sp_locale_container = Splocalecontainer.objects.filter(**container_attrs).first() + fetched_sp_locale_container = Splocalecontainer.objects.filter(**container_attrs).order_by("id").first() if fetched_sp_locale_container is None: sp_local_container = Splocalecontainer.objects.create(**{ @@ -345,7 +340,7 @@ def update_table_schema_config_with_defaults( pending_itemstr_rows.extend(item_str_rows) - for field in table.all_fields: + for field in table._all_fields(exclude_id_field=True): field_defaults = None if table_defaults.get('items'): field_defaults = table_defaults['items'].get(field.name.lower()) @@ -379,12 +374,19 @@ def revert_table_schema_config(table_name, apps=global_apps): items.delete() containers.delete() +class FieldDefaults(TypedDict): + name: NotRequired[str] + desc: NotRequired[str] + ishidden: NotRequired[bool] + isrequired: NotRequired[bool] + picklistname: NotRequired[str] + def update_table_field_schema_config_with_defaults( table_name, discipline_id: int, field_name: str, apps = global_apps, - defaults: dict = None, + defaults: FieldDefaults | None = None, pending_itemstr_rows: list[dict] | None = None, ): table = datamodel.get_table(table_name) @@ -457,16 +459,26 @@ def update_table_field_schema_config_with_defaults( language="en" ) - sp_local_container_item, _ = Splocalecontaineritem.objects.get_or_create( - name=field_config.name, - container=sp_local_container, - type=field_config.java_type, - ishidden=field_hidden, - isrequired=field_required, - issystem=table.system, - version=0, - picklistname=picklist_name - ) + container_item_attrs = { + "name": field_config.name, + "container": sp_local_container + } + + fetched_sp_locale_container_item = Splocalecontaineritem.objects.filter(**container_item_attrs).order_by("id").first() + + if fetched_sp_locale_container_item is None: + sp_locale_container_item = Splocalecontaineritem.objects.create(**{ + **container_item_attrs, + "type": field_config.java_type, + "ishidden": field_hidden, + "isrequired": field_required, + "issystem": table.system, + "version": 0, + "picklistname": picklist_name + } + ) + else: + sp_locale_container_item = fetched_sp_locale_container_item itm_str_rows = [] for k, text in { @@ -477,7 +489,7 @@ def update_table_field_schema_config_with_defaults( "text": text, "language": "en", "version": 0, - k: sp_local_container_item, + k: sp_locale_container_item, } itm_str_rows.append(row) @@ -606,14 +618,14 @@ def find_missing_schema_config_fields(discipline_id: int, apps=global_apps): if table_name_lower not in container_names: missing_tables.append(table_name) missing_fields[table_name] = sorted( - field.name for field in table.all_fields if field.name + field.name for field in table._all_fields(exclude_id_field=True) if field.name ) continue existing_fields = existing_fields_by_table.get(table_name_lower, set()) missing_in_table = sorted( # sort for better reproducablity field.name - for field in table.all_fields + for field in table._all_fields(exclude_id_field=True) if field.name and field.name.lower() not in existing_fields ) @@ -829,6 +841,8 @@ def create_geo_table_schema_config_with_defaults(apps): COT_FIELD_NAME = 'collectionObjectType' COT_TEXT = 'Collection Object Type' +# FEAT: Replace this implementation with +# update_table_field_schema_config_with_defaults def create_cotype_splocalecontaineritem(apps): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') @@ -837,23 +851,40 @@ def create_cotype_splocalecontaineritem(apps): # Create a Splocalecontaineritem record for each CollectionObject Splocalecontainer # NOTE: Each discipline has its own CollectionObject Splocalecontainer for container in Splocalecontainer.objects.filter(name='collectionobject', schematype=0): - container_item, _ = Splocalecontaineritem.objects.get_or_create( - name=COT_FIELD_NAME, - picklistname=COT_PICKLIST_NAME, - type='ManyToOne', - container=container, - isrequired=True - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COT_TEXT, - itemname=container_item - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COT_TEXT, - itemdesc=container_item - ) + container_item_attrs = { + "name": COT_FIELD_NAME, + "container": container + } + container_item = Splocalecontaineritem.objects.filter(**container_item_attrs).order_by("id").first() + if container_item is None: + resolved_item = Splocalecontaineritem.objects.create( + **container_item_attrs, + picklistname=COT_PICKLIST_NAME, + type="ManyToOne", + isrequired=True + ) + else: + resolved_item = container_item + + field_label_attrs = { + "language": "en", + "itemname":resolved_item + } + + field_label = Splocaleitemstr.objects.filter(**field_label_attrs).order_by("id").first() + + if field_label is None: + Splocaleitemstr.objects.create(**field_label_attrs, text=COT_TEXT) + + field_desc_attrs = { + "language": "en", + "itemdesc":resolved_item + } + + field_desc = Splocaleitemstr.objects.filter(**field_desc_attrs).order_by("id").first() + + if field_desc is None: + Splocaleitemstr.objects.create(**field_desc_attrs, text=COT_TEXT) # ########################################## # Used in 0004_stratigraphy_age.py @@ -1111,7 +1142,10 @@ def revert_update_cog_schema_config(apps): def update_age_schema_config(apps): # Revert before adding to avoid duplicates - revert_update_age_schema_config(apps) + # BUG: This will delete people's potentially modified Schema Config items + # If we want to avoid duplicates, we should check the creation code and + # prevent duplicates being created there + # revert_update_age_schema_config(apps) Discipline = apps.get_model('specify', 'Discipline') for discipline in Discipline.objects.all(): @@ -1926,90 +1960,26 @@ def revert_version_required(apps): def update_loan_and_gift_agent_fields(apps): Discipline = apps.get_model('specify', 'Discipline') + field_defaults = { + "ishidden": True + } for discipline in Discipline.objects.all(): for table, fields in MIGRATION_0038_FIELDS.items(): for field_name in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field_name, apps) + update_table_field_schema_config_with_defaults(table, discipline.id, field_name, apps, defaults=field_defaults) def revert_loan_and_gift_agent_fields(apps): for table, fields in MIGRATION_0038_FIELDS.items(): for field_name in fields: revert_table_field_schema_config(table, field_name, apps) -def update_loan_and_gift_agents(apps): - """ - Update field descriptions and display names using MIGRATION_0038_UPDATE_FIELDS - (tuple: (fieldName, newLabel, newDesc)). - """ - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - def upsert_single_str(*, itemdesc_id=None, itemname_id=None, text=""): - if (itemdesc_id is None) == (itemname_id is None): - raise ValueError("Exactly one of itemdesc_id or itemname_id must be provided") - - qs = Splocaleitemstr.objects.filter( - itemdesc_id=itemdesc_id, - itemname_id=itemname_id, - ).order_by("id") - - obj = qs.first() - if obj is None: - return Splocaleitemstr.objects.create( - itemdesc_id=itemdesc_id, - itemname_id=itemname_id, - text=text, - ) - - qs.exclude(id=obj.id).delete() - - if obj.text != text: - obj.text = text - obj.save(update_fields=["text"]) - - return obj - - for table, fields in MIGRATION_0038_UPDATE_FIELDS.items(): - containers = Splocalecontainer.objects.filter(name=table.lower()) - - for container in containers: - for (field_name, new_name, new_desc) in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower(), - ) - - for item in items: - # Hide the existing field - if not item.ishidden: - item.ishidden = True - item.save(update_fields=["ishidden"]) - - upsert_single_str(itemdesc_id=item.id, text=new_desc) - upsert_single_str(itemname_id=item.id, text=new_name) - -def revert_loan_and_gift_agents(apps): - """ - Revert the field name/description updates. - """ - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in MIGRATION_0038_UPDATE_FIELDS.items(): - containers = Splocalecontainer.objects.filter(name=table.lower()) - for container in containers: - for (field_name, _, _) in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - # If needed, reset ishidden or revert text - # ########################################## # Used in 0040_components.py # ########################################## +def remove_componentparent_item(apps): + revert_table_field_schema_config("CollectionObject", "componentParent", apps) + def remove_0029_schema_config_fields(apps, schema_editor=None): Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') @@ -2128,14 +2098,6 @@ def reverse_hide_component_fields(apps, schema_editor=None): name=field_name.lower() ) items.update(ishidden=True) - -def componets_schema_config_migrations(apps, schema_editor=None): - remove_0029_schema_config_fields(apps, schema_editor) - create_table_schema_config_with_defaults(apps, schema_editor) - update_schema_config_field_desc(apps, schema_editor) - update_hidden_prop(apps, schema_editor) - create_cotype_splocalecontaineritem(apps) - hide_component_fields(apps, schema_editor) # ########################################## # Used in 0042_discipline_type_picklist.py diff --git a/specifyweb/specify/migrations/0039_agent_fields_for_loan_and_gift.py b/specifyweb/specify/migrations/0039_agent_fields_for_loan_and_gift.py index b6109646b4b..d5f1e0d9435 100644 --- a/specifyweb/specify/migrations/0039_agent_fields_for_loan_and_gift.py +++ b/specifyweb/specify/migrations/0039_agent_fields_for_loan_and_gift.py @@ -8,10 +8,8 @@ def consolidated_0038_forward(apps, schema_editor): usc.update_loan_and_gift_agent_fields(apps) - usc.update_loan_and_gift_agents(apps) def consolidated_0038_backward(apps, schema_editor): - usc.revert_loan_and_gift_agents(apps) usc.revert_loan_and_gift_agent_fields(apps) class Migration(migrations.Migration): diff --git a/specifyweb/specify/models_utils/load_datamodel.py b/specifyweb/specify/models_utils/load_datamodel.py index dbe2c8f8f72..67e1bfbe136 100644 --- a/specifyweb/specify/models_utils/load_datamodel.py +++ b/specifyweb/specify/models_utils/load_datamodel.py @@ -149,19 +149,29 @@ def name(self) -> str: raise ValueError("classname is required to compute the name") return self.classname.split(".")[-1] + def _all_fields(self, exclude_fields=False, exclude_relationships=False, exclude_id_field=False, exclude_virtual_fields=True) -> Iterable[Union["Field", "Relationship"]]: + if not exclude_fields: + yield from self.fields or [] # Handle None by using an empty list + if not exclude_relationships: + yield from self.relationships or [] # Handle None by using an empty list + if not exclude_virtual_fields: + yield from self.virtual_fields or [] + if (not exclude_id_field) and self.idField is not None: + yield self.idField + @property def django_name(self) -> str: return self.name.capitalize() @property def all_fields(self) -> list[Union["Field", "Relationship"]]: - def af() -> Iterable[Union["Field","Relationship"]]: - yield from self.fields or [] # Handle None by using an empty list - yield from self.relationships or [] # Handle None by using an empty list - if self.idField is not None: - yield self.idField - - return list(af()) + """ + A list of all non-virtual fields (including the ID field) and + relationships for the table. + If you need more granularity over which fields to return, use + _all_fields or a filter object + """ + return list(self._all_fields()) def is_virtual_field(self, fieldname: str) -> bool: