diff --git a/backend/analytics/migrations/0006_rename_prunplanner_materia_62060a_idx_idx_material_ticker_delta_and_more.py b/backend/analytics/migrations/0006_rename_prunplanner_materia_62060a_idx_idx_material_ticker_delta_and_more.py new file mode 100644 index 0000000..94021ca --- /dev/null +++ b/backend/analytics/migrations/0006_rename_prunplanner_materia_62060a_idx_idx_material_ticker_delta_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 6.0.4 on 2026-04-17 06:51 + +from django.db import migrations, models +from decimal import Decimal, ROUND_HALF_UP + +def clean_and_recalculate_precision(apps, schema_editor): + Snapshot = apps.get_model('analytics', 'AnalyticsEmpireMaterialSnapshot') + + batch_size = 1000 + queryset = Snapshot.objects.all() + to_update = [] + + for obj in Snapshot.objects.iterator(chunk_size=batch_size): + # Round to 6 decimal places to kill float noise + p = Decimal(str(obj.production)).quantize(Decimal('0.000001'), rounding=ROUND_HALF_UP) + c = Decimal(str(obj.consumption)).quantize(Decimal('0.000001'), rounding=ROUND_HALF_UP) + + obj.production = p + obj.consumption = c + obj.delta = p - c + + to_update.append(obj) + + # When batch is full, perform one large update + if len(to_update) >= batch_size: + Snapshot.objects.bulk_update(to_update, ['production', 'consumption', 'delta']) + to_update = [] + + if to_update: + Snapshot.objects.bulk_update(to_update, ['production', 'consumption', 'delta']) + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('analytics', '0005_analyticsempirematerialsnapshot'), + ('planning', '0006_planningempire_empire_state'), + ] + + operations = [ + # structure + migrations.AlterField( + model_name='analyticsempirematerialsnapshot', + name='consumption', + field=models.DecimalField(decimal_places=6, default=0.0, max_digits=20), + ), + migrations.AlterField( + model_name='analyticsempirematerialsnapshot', + name='delta', + field=models.DecimalField(decimal_places=6, default=0.0, max_digits=20), + ), + migrations.AlterField( + model_name='analyticsempirematerialsnapshot', + name='production', + field=models.DecimalField(decimal_places=6, default=0.0, max_digits=20), + ), + + # data cleanup + migrations.RunPython(clean_and_recalculate_precision, reverse_code=migrations.RunPython.noop), + + # clean indexes and constraints + migrations.RenameIndex( + model_name='analyticsempirematerialsnapshot', + new_name='idx_material_ticker_delta', + old_name='prunplanner_materia_62060a_idx', + ), + migrations.AlterUniqueTogether( + name='analyticsempirematerialsnapshot', + unique_together=set(), + ), + migrations.AddIndex( + model_name='analyticsempirematerialsnapshot', + index=models.Index(fields=['empire', 'delta'], name='idx_empire_delta'), + ), + migrations.AddConstraint( + model_name='analyticsempirematerialsnapshot', + constraint=models.UniqueConstraint(fields=('empire', 'material_ticker'), name='unique_empire_material_ticker'), + ), + ] diff --git a/backend/analytics/models/empire_material_snapshot.py b/backend/analytics/models/empire_material_snapshot.py index 0527f60..2acea4d 100644 --- a/backend/analytics/models/empire_material_snapshot.py +++ b/backend/analytics/models/empire_material_snapshot.py @@ -9,17 +9,23 @@ class AnalyticsEmpireMaterialSnapshot(models.Model): material_ticker = models.CharField(max_length=3, db_index=True) - production = models.FloatField(default=0.0) - consumption = models.FloatField(default=0.0) - delta = models.FloatField(default=0.0) + production = models.DecimalField(max_digits=20, decimal_places=6, default=0.0) + consumption = models.DecimalField(max_digits=20, decimal_places=6, default=0.0) + delta = models.DecimalField(max_digits=20, decimal_places=6, default=0.0) class Meta: db_table = 'prunplanner_statistics_empire_material_snapshot' - unique_together = ('empire', 'material_ticker') verbose_name = 'Empire Material Snapshot' verbose_name_plural = 'Empire Material Snapshots' - indexes = [models.Index(fields=['material_ticker', 'delta'])] + indexes = [ + models.Index(fields=['material_ticker', 'delta'], name='idx_material_ticker_delta'), + models.Index(fields=['empire', 'delta'], name='idx_empire_delta'), + ] + + constraints = [ + models.UniqueConstraint(fields=['empire', 'material_ticker'], name='unique_empire_material_ticker') + ] def __str__(self) -> str: return f'{self.empire.uuid} | {self.material_ticker}: {self.delta}' diff --git a/backend/planning/services/empire_state_service.py b/backend/planning/services/empire_state_service.py index 4a4e184..cfdd1a8 100644 --- a/backend/planning/services/empire_state_service.py +++ b/backend/planning/services/empire_state_service.py @@ -1,3 +1,5 @@ +from decimal import ROUND_HALF_UP, Decimal + from analytics.models import AnalyticsEmpireMaterialSnapshot from django.db import transaction from planning.models import PlanningEmpire @@ -5,7 +7,6 @@ class EmpireStateService: @staticmethod - @transaction.atomic def sync_empire_state(empire: PlanningEmpire, state_data: dict) -> None: # update the json field @@ -18,7 +19,14 @@ def sync_empire_state(empire: PlanningEmpire, state_data: dict) -> None: snapshot_objs = [] for material_ticker, stats in empire_total.items(): - p, c, d = stats.get('p', 0), stats.get('c', 0), stats.get('d', 0) + p_raw, c_raw = stats.get('p', 0), stats.get('c', 0) + + # decimal conversion + p = Decimal(str(p_raw)).quantize(Decimal('0.000001'), rounding=ROUND_HALF_UP) + c = Decimal(str(c_raw)).quantize(Decimal('0.000001'), rounding=ROUND_HALF_UP) + + d = p - c + if p != 0 or c != 0 or d != 0: active_tickers.append(material_ticker) @@ -28,15 +36,16 @@ def sync_empire_state(empire: PlanningEmpire, state_data: dict) -> None: ) ) - # only remove materials no longer in the empire total delta - AnalyticsEmpireMaterialSnapshot.objects.filter(empire=empire).exclude( - material_ticker__in=active_tickers - ).delete() - - # perform the upsert - AnalyticsEmpireMaterialSnapshot.objects.bulk_create( - snapshot_objs, - update_conflicts=True, - update_fields=['production', 'consumption', 'delta'], - unique_fields=['empire', 'material_ticker'], - ) + with transaction.atomic(): + # only remove materials no longer in the empire total delta + AnalyticsEmpireMaterialSnapshot.objects.filter(empire=empire).exclude( + material_ticker__in=active_tickers + ).delete() + + # perform the upsert + AnalyticsEmpireMaterialSnapshot.objects.bulk_create( + snapshot_objs, + update_conflicts=True, + unique_fields=['empire', 'material_ticker'], + update_fields=['production', 'consumption', 'delta'], + )