From d032791a411c1347b94348e76211fe3a2c620db3 Mon Sep 17 00:00:00 2001 From: Joe Russack Date: Tue, 7 Apr 2026 21:35:52 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20add=20DwC=20schema=20extensions=20?= =?UTF-8?q?=E2=80=94=20mapping=20models=20and=20spqueryfield=20columns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New models: SchemaMapping, ExportDataSet, ExportDataSetExtension, CacheTableMeta New columns on spqueryfield: term, isstatic, staticvalue These enable DwC term assignment on query fields and structured export dataset configuration. The spqueryfield columns are nullable — existing queries and fields are unaffected. Note: Migration 0047 depends on 0046 (tree indexes). Export migrations depend on 0047 (needs spquery FK target to exist). --- .../export/migrations/0001_schemamapping.py | 57 +++++++++ .../0002_exportdataset_extensions_cache.py | 92 ++++++++++++++ .../backend/export/migrations/__init__.py | 0 specifyweb/backend/export/models.py | 114 ++++++++++++++++++ .../0047_spqueryfield_dwc_extensions.py | 33 +++++ specifyweb/specify/models.py | 3 + 6 files changed, 299 insertions(+) create mode 100644 specifyweb/backend/export/migrations/0001_schemamapping.py create mode 100644 specifyweb/backend/export/migrations/0002_exportdataset_extensions_cache.py create mode 100644 specifyweb/backend/export/migrations/__init__.py create mode 100644 specifyweb/specify/migrations/0047_spqueryfield_dwc_extensions.py diff --git a/specifyweb/backend/export/migrations/0001_schemamapping.py b/specifyweb/backend/export/migrations/0001_schemamapping.py new file mode 100644 index 00000000000..c9e9308f066 --- /dev/null +++ b/specifyweb/backend/export/migrations/0001_schemamapping.py @@ -0,0 +1,57 @@ +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('specify', '0047_spqueryfield_dwc_extensions'), + ] + + operations = [ + migrations.CreateModel( + name='SchemaMapping', + fields=[ + ('id', models.AutoField(db_column='SchemaMappingID', primary_key=True, serialize=False)), + ('mappingtype', models.CharField( + choices=[('Core', 'Core'), ('Extension', 'Extension')], + db_column='MappingType', max_length=16, + )), + ('name', models.CharField(db_column='Name', max_length=256)), + ('isdefault', models.BooleanField(db_column='IsDefault', default=False)), + ('timestampcreated', models.DateTimeField( + db_column='TimestampCreated', default=django.utils.timezone.now, + )), + ('timestampmodified', models.DateTimeField( + db_column='TimestampModified', default=django.utils.timezone.now, + )), + ('version', models.IntegerField(db_column='Version', default=0)), + ('query', models.OneToOneField( + db_column='SpQueryID', + on_delete=django.db.models.deletion.CASCADE, + related_name='schemamapping', + to='specify.spquery', + )), + ('createdbyagent', models.ForeignKey( + db_column='CreatedByAgentID', + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', + to='specify.agent', + )), + ('modifiedbyagent', models.ForeignKey( + db_column='ModifiedByAgentID', + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', + to='specify.agent', + )), + ], + options={ + 'db_table': 'schemamapping', + }, + ), + ] diff --git a/specifyweb/backend/export/migrations/0002_exportdataset_extensions_cache.py b/specifyweb/backend/export/migrations/0002_exportdataset_extensions_cache.py new file mode 100644 index 00000000000..dbf94a00b4f --- /dev/null +++ b/specifyweb/backend/export/migrations/0002_exportdataset_extensions_cache.py @@ -0,0 +1,92 @@ +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('export', '0001_schemamapping'), + ('specify', '0047_spqueryfield_dwc_extensions'), + ] + + operations = [ + migrations.CreateModel( + name='ExportDataSet', + fields=[ + ('id', models.AutoField(db_column='ExportDataSetID', primary_key=True, serialize=False)), + ('exportname', models.CharField(db_column='ExportName', max_length=256, unique=True)), + ('filename', models.CharField(db_column='FileName', max_length=256, unique=True)), + ('isrss', models.BooleanField(db_column='IsRSS', default=False)), + ('frequency', models.IntegerField(blank=True, db_column='Frequency', null=True)), + ('lastexported', models.DateTimeField(blank=True, db_column='LastExported', null=True)), + ('timestampcreated', models.DateTimeField( + db_column='TimestampCreated', default=django.utils.timezone.now, + )), + ('timestampmodified', models.DateTimeField( + db_column='TimestampModified', default=django.utils.timezone.now, + )), + ('version', models.IntegerField(db_column='Version', default=0)), + ('metadata', models.ForeignKey( + blank=True, db_column='MetadataID', null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', to='specify.spappresource', + )), + ('coremapping', models.ForeignKey( + db_column='CoreMappingID', + on_delete=django.db.models.deletion.PROTECT, + related_name='export_datasets', to='export.schemamapping', + )), + ('collection', models.ForeignKey( + db_column='CollectionID', + on_delete=django.db.models.deletion.CASCADE, + related_name='+', to='specify.collection', + )), + ], + options={ + 'db_table': 'exportdataset', + }, + ), + migrations.CreateModel( + name='ExportDataSetExtension', + fields=[ + ('id', models.AutoField(db_column='ExportDataSetExtensionID', primary_key=True, serialize=False)), + ('sortorder', models.IntegerField(db_column='SortOrder', default=0)), + ('exportdataset', models.ForeignKey( + db_column='ExportDataSetID', + on_delete=django.db.models.deletion.CASCADE, + related_name='extensions', to='export.exportdataset', + )), + ('schemamapping', models.ForeignKey( + db_column='SchemaMappingID', + on_delete=django.db.models.deletion.PROTECT, + related_name='dataset_extensions', to='export.schemamapping', + )), + ], + options={ + 'db_table': 'exportdatasetextension', + 'unique_together': {('exportdataset', 'schemamapping')}, + }, + ), + migrations.CreateModel( + name='CacheTableMeta', + fields=[ + ('id', models.AutoField(db_column='CacheTableMetaID', primary_key=True, serialize=False)), + ('tablename', models.CharField(db_column='TableName', max_length=128, unique=True)), + ('lastbuilt', models.DateTimeField(blank=True, db_column='LastBuilt', null=True)), + ('rowcount', models.IntegerField(blank=True, db_column='RowCount', null=True)), + ('buildstatus', models.CharField( + choices=[('idle', 'idle'), ('building', 'building'), ('error', 'error')], + db_column='BuildStatus', default='idle', max_length=16, + )), + ('schemamapping', models.OneToOneField( + db_column='SchemaMappingID', + on_delete=django.db.models.deletion.CASCADE, + related_name='cache_meta', to='export.schemamapping', + )), + ], + options={ + 'db_table': 'cachetablemeta', + }, + ), + ] diff --git a/specifyweb/backend/export/migrations/__init__.py b/specifyweb/backend/export/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/specifyweb/backend/export/models.py b/specifyweb/backend/export/models.py index e69de29bb2d..f805098360f 100644 --- a/specifyweb/backend/export/models.py +++ b/specifyweb/backend/export/models.py @@ -0,0 +1,114 @@ +from django.db import models +from django.db.models.signals import pre_delete +from django.dispatch import receiver +from django.utils import timezone + + +class SchemaMapping(models.Model): + # ID Field + id = models.AutoField(primary_key=True, db_column='SchemaMappingID') + + # Fields + mappingtype = models.CharField( + max_length=16, db_column='MappingType', + choices=[('Core', 'Core'), ('Extension', 'Extension')], + ) + name = models.CharField(max_length=256, db_column='Name') + isdefault = models.BooleanField(default=False, db_column='IsDefault') + vocabulary = models.CharField( + max_length=32, db_column='Vocabulary', null=True, blank=True, + help_text='Vocabulary key (e.g. dwc, ac) — locked after creation', + ) + timestampcreated = models.DateTimeField(db_column='TimestampCreated', default=timezone.now) + timestampmodified = models.DateTimeField(db_column='TimestampModified', default=timezone.now) + version = models.IntegerField(default=0, db_column='Version') + + # Relationships + query = models.OneToOneField( + 'specify.Spquery', db_column='SpQueryID', + on_delete=models.CASCADE, related_name='+', + ) + createdbyagent = models.ForeignKey( + 'specify.Agent', db_column='CreatedByAgentID', + related_name='+', null=True, on_delete=models.SET_NULL, + ) + modifiedbyagent = models.ForeignKey( + 'specify.Agent', db_column='ModifiedByAgentID', + related_name='+', null=True, on_delete=models.SET_NULL, + ) + + class Meta: + db_table = 'schemamapping' + + +class ExportDataSet(models.Model): + id = models.AutoField(primary_key=True, db_column='ExportDataSetID') + exportname = models.CharField(max_length=255, unique=True, db_column='ExportName') + filename = models.CharField(max_length=255, unique=True, db_column='FileName') + isrss = models.BooleanField(default=False, db_column='IsRSS') + frequency = models.IntegerField(blank=True, null=True, db_column='Frequency') + metadata = models.ForeignKey( + 'specify.Spappresource', db_column='MetadataID', + related_name='+', null=True, blank=True, on_delete=models.SET_NULL, + ) + coremapping = models.ForeignKey( + 'SchemaMapping', db_column='CoreMappingID', + related_name='export_datasets', on_delete=models.PROTECT, + ) + collection = models.ForeignKey( + 'specify.Collection', db_column='CollectionID', + related_name='+', on_delete=models.CASCADE, + ) + lastexported = models.DateTimeField(blank=True, null=True, db_column='LastExported') + timestampcreated = models.DateTimeField(default=timezone.now, db_column='TimestampCreated') + timestampmodified = models.DateTimeField(default=timezone.now, db_column='TimestampModified') + version = models.IntegerField(default=0, db_column='Version') + + class Meta: + db_table = 'exportdataset' + + +class ExportDataSetExtension(models.Model): + id = models.AutoField(primary_key=True, db_column='ExportDataSetExtensionID') + exportdataset = models.ForeignKey( + 'ExportDataSet', db_column='ExportDataSetID', + related_name='extensions', on_delete=models.CASCADE, + ) + schemamapping = models.ForeignKey( + 'SchemaMapping', db_column='SchemaMappingID', + related_name='dataset_extensions', on_delete=models.PROTECT, + ) + sortorder = models.IntegerField(default=0, db_column='SortOrder') + + class Meta: + db_table = 'exportdatasetextension' + unique_together = [('exportdataset', 'schemamapping')] + + +class CacheTableMeta(models.Model): + id = models.AutoField(primary_key=True, db_column='CacheTableMetaID') + schemamapping = models.OneToOneField( + 'SchemaMapping', db_column='SchemaMappingID', + related_name='cache_meta', on_delete=models.CASCADE, + ) + tablename = models.CharField(max_length=128, unique=True, db_column='TableName') + lastbuilt = models.DateTimeField(blank=True, null=True, db_column='LastBuilt') + rowcount = models.IntegerField(blank=True, null=True, db_column='RowCount') + buildstatus = models.CharField( + max_length=16, default='idle', db_column='BuildStatus', + choices=[('idle', 'idle'), ('building', 'building'), ('error', 'error')], + ) + + class Meta: + db_table = 'cachetablemeta' + + +@receiver(pre_delete, sender=SchemaMapping) +def delete_schema_mapping_cache(sender, instance, **kwargs): + """Drop cache table before a SchemaMapping is deleted (before CASCADE removes CacheTableMeta).""" + from .cache import drop_cache_table + for meta in CacheTableMeta.objects.filter(schemamapping=instance): + try: + drop_cache_table(meta.tablename) + except Exception: + pass diff --git a/specifyweb/specify/migrations/0047_spqueryfield_dwc_extensions.py b/specifyweb/specify/migrations/0047_spqueryfield_dwc_extensions.py new file mode 100644 index 00000000000..1066ae9c36d --- /dev/null +++ b/specifyweb/specify/migrations/0047_spqueryfield_dwc_extensions.py @@ -0,0 +1,33 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('specify', '0046_add_tectonicunit_indexes'), + ] + + operations = [ + migrations.AddField( + model_name='spqueryfield', + name='term', + field=models.CharField( + blank=True, db_column='Term', db_index=False, + max_length=500, null=True, + ), + ), + migrations.AddField( + model_name='spqueryfield', + name='isstatic', + field=models.BooleanField( + blank=True, db_column='IsStatic', default=False, null=True, + ), + ), + migrations.AddField( + model_name='spqueryfield', + name='staticvalue', + field=models.TextField( + blank=True, db_column='StaticValue', null=True, + ), + ), + ] diff --git a/specifyweb/specify/models.py b/specifyweb/specify/models.py index d8562fd1c60..74768203c6a 100644 --- a/specifyweb/specify/models.py +++ b/specifyweb/specify/models.py @@ -6427,6 +6427,9 @@ class Spqueryfield(models.Model): timestampmodified = models.DateTimeField(blank=True, null=True, unique=False, db_column='TimestampModified', db_index=False, default=timezone.now) # auto_now=True version = models.IntegerField(blank=True, null=False, unique=False, db_column='Version', db_index=False, default=0) isstrict = models.BooleanField(db_column='IsStrict', blank=True, null=True) + term = models.CharField(blank=True, max_length=500, null=True, db_column='Term', db_index=False) + isstatic = models.BooleanField(blank=True, null=True, db_column='IsStatic', default=False) + staticvalue = models.TextField(blank=True, null=True, db_column='StaticValue') # Relationships: Many-to-One createdbyagent = models.ForeignKey('Agent', db_column='CreatedByAgentID', related_name='+', null=True, on_delete=protect_with_blockers)