diff --git a/backend/analytics/admin.py b/backend/analytics/admin.py index 1e2f00e..1159765 100644 --- a/backend/analytics/admin.py +++ b/backend/analytics/admin.py @@ -5,7 +5,7 @@ from unfold.admin import ModelAdmin from unfold.decorators import action -from analytics.models import AnalyticsPlanAggregate, AppStatistic +from analytics.models import AnalyticsEmpireMaterialSnapshot, AnalyticsPlanAggregate, AppStatistic @admin.register(AppStatistic) @@ -38,7 +38,7 @@ class AnalyticsPlanAggregateAdmin(ModelAdmin): @action(description='Run Aggregator', url_path='analytics-aggregate-all') def action_aggregate_all(self, request): - from analytics.services.PlanInsightAggregatorService import PlanInsightAggregatorService + from analytics.services.planinsight_aggregator_service import PlanInsightAggregatorService try: aggregator = PlanInsightAggregatorService() @@ -52,3 +52,12 @@ def action_aggregate_all(self, request): self.message_user(request, 'Error processing aggregates.', messages.ERROR) return redirect('../') + + +@admin.register(AnalyticsEmpireMaterialSnapshot) +class AnalyticsEmpireMaterialSnapshotAdmin(ModelAdmin): + list_display = ['id', 'empire', 'material_ticker', 'production', 'consumption', 'delta'] + search_fields = ['empire.uuid'] + + def get_queryset(self, request): + return super().get_queryset(request).select_related('empire') diff --git a/backend/analytics/api/urls.py b/backend/analytics/api/urls.py index 39758f4..f84c758 100644 --- a/backend/analytics/api/urls.py +++ b/backend/analytics/api/urls.py @@ -1,4 +1,4 @@ -from analytics.api.viewsets import AnalyticsPlanAggregateViewSet +from analytics.api.viewsets import AnalyticsMarketInsightViewSet, AnalyticsPlanAggregateViewSet from django.urls import path app_name = 'analytics' @@ -8,4 +8,9 @@ AnalyticsPlanAggregateViewSet.as_view({'get': 'retrieve'}), name='planet-insight-detail', ), + path( + 'planning_insights/materials/', + AnalyticsMarketInsightViewSet.as_view({'get': 'get_global_materials'}), + name='planning-insight-materials', + ), ] diff --git a/backend/analytics/api/viewsets.py b/backend/analytics/api/viewsets.py index 9b64891..6a64d6e 100644 --- a/backend/analytics/api/viewsets.py +++ b/backend/analytics/api/viewsets.py @@ -1,11 +1,16 @@ +from datetime import timedelta + from analytics.api.serializer import AnalyticsPlanAggregateSerializer -from analytics.models import AnalyticsPlanAggregate +from analytics.models import AnalyticsEmpireMaterialSnapshot, AnalyticsPlanAggregate +from analytics.services.analytics_cache_manager import AnalyticsCacheManager +from django.db.models import Sum from django.http import Http404 +from django.utils import timezone from drf_spectacular.utils import extend_schema from gamedata.models.game_planet import GamePlanet -from rest_framework import status, viewsets +from rest_framework import viewsets +from rest_framework.decorators import action from rest_framework.exceptions import NotFound -from rest_framework.response import Response class AnalyticsPlanAggregateViewSet(viewsets.ReadOnlyModelViewSet): @@ -13,26 +18,48 @@ class AnalyticsPlanAggregateViewSet(viewsets.ReadOnlyModelViewSet): lookup_field = 'planet_natural_id' serializer_class = AnalyticsPlanAggregateSerializer - @extend_schema(auth=[], summary='Fetch planning insights for planet') + @extend_schema(auth=[], summary='Fetch planet insights by Planet Natural Id') def retrieve(self, request, *args, **kwargs): - planet_id = kwargs.get('planet_natural_id') + planet_id: str = kwargs.get('planet_natural_id', '') if not GamePlanet.objects.filter(planet_natural_id=planet_id).exists(): - return Response({'detail': 'Planet not found.'}, status=status.HTTP_404_NOT_FOUND) - - try: - # try to get aggregate - instance = self.get_object() - serializer = self.get_serializer(instance) - return Response(serializer.data) - except (AnalyticsPlanAggregate.DoesNotExist, Http404, NotFound): - # return a 200 OK, but without any data - return Response( - { + raise NotFound(detail='Planet not found.') + + def fetch_data(planet_natural_id: str): + + try: + # try to get aggregate + instance = self.get_object() + serializer = self.get_serializer(instance) + return serializer.data + except (AnalyticsPlanAggregate.DoesNotExist, Http404): + # return a 200 OK, but without any data + return { 'status': 'below_threshold', - 'planet_natural_id': planet_id, + 'planet_natural_id': planet_natural_id, 'total_plans_analyzed': 0, 'aggregated_data': None, - }, - status=status.HTTP_200_OK, + } + + return AnalyticsCacheManager.get_plan_aggregate_response(planet_id, lambda: fetch_data(planet_id)) + + +class AnalyticsMarketInsightViewSet(viewsets.ViewSet): + @extend_schema(auth=[], summary='Fetch planning insights for materials') + @action(detail=False, methods=['get'], url_path='get-global-tracker') + def get_global_materials(self, request): + + def fetch_data(): + + active_cutoff = timezone.now() - timedelta(days=30) + + stats_queryset = ( + AnalyticsEmpireMaterialSnapshot.objects.filter(empire__modified_at__gte=active_cutoff) + .values('material_ticker') + .annotate(total_p=Sum('production'), total_c=Sum('consumption'), net_d=Sum('delta')) + .order_by('material_ticker') ) + + return list(stats_queryset.values_list('material_ticker', 'total_p', 'total_c', 'net_d')) + + return AnalyticsCacheManager.get_planning_insight_materials(fetch_data) diff --git a/backend/analytics/migrations/0005_analyticsempirematerialsnapshot.py b/backend/analytics/migrations/0005_analyticsempirematerialsnapshot.py new file mode 100644 index 0000000..0002e61 --- /dev/null +++ b/backend/analytics/migrations/0005_analyticsempirematerialsnapshot.py @@ -0,0 +1,61 @@ +# Generated by Django 6.0.4 on 2026-04-16 12:41 + +import django.db.models.deletion +from django.db import migrations, models + +def apply_postgres_tuning(apps, schema_editor): + # Only execute if we are on a PostgreSQL backend + if schema_editor.connection.vendor != 'postgresql': + return + + with schema_editor.connection.cursor() as cursor: + cursor.execute("ALTER TABLE prunplanner_statistics_empire_material_snapshot SET (fillfactor = 80);") + cursor.execute(""" + ALTER TABLE prunplanner_statistics_empire_material_snapshot SET ( + autovacuum_vacuum_scale_factor = 0.05, + autovacuum_vacuum_threshold = 50 + ); + """) + +def reverse_postgres_tuning(apps, schema_editor): + if schema_editor.connection.vendor != 'postgresql': + return + + with schema_editor.connection.cursor() as cursor: + cursor.execute("ALTER TABLE prunplanner_statistics_empire_material_snapshot RESET (fillfactor);") + cursor.execute(""" + ALTER TABLE prunplanner_statistics_empire_material_snapshot RESET ( + autovacuum_vacuum_scale_factor, + autovacuum_vacuum_threshold + ); + """) + + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0004_analyticsplanaggregate'), + ('planning', '0006_planningempire_empire_state'), + ] + + operations = [ + migrations.CreateModel( + name='AnalyticsEmpireMaterialSnapshot', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('material_ticker', models.CharField(db_index=True, max_length=3)), + ('production', models.FloatField(default=0.0)), + ('consumption', models.FloatField(default=0.0)), + ('delta', models.FloatField(default=0.0)), + ('empire', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='material_snapshot', to='planning.planningempire')), + ], + options={ + 'verbose_name': 'Empire Material Snapshot', + 'verbose_name_plural': 'Empire Material Snapshots', + 'db_table': 'prunplanner_statistics_empire_material_snapshot', + 'indexes': [models.Index(fields=['material_ticker', 'delta'], name='prunplanner_materia_62060a_idx')], + 'unique_together': {('empire', 'material_ticker')}, + }, + ), + migrations.RunPython(apply_postgres_tuning, reverse_postgres_tuning), + ] diff --git a/backend/analytics/models/__init__.py b/backend/analytics/models/__init__.py index e5cc281..3effdfb 100644 --- a/backend/analytics/models/__init__.py +++ b/backend/analytics/models/__init__.py @@ -1,2 +1,3 @@ from .app_statistics import AppStatistic from .plan_aggregates import AnalyticsPlanAggregate +from .empire_material_snapshot import AnalyticsEmpireMaterialSnapshot diff --git a/backend/analytics/models/empire_material_snapshot.py b/backend/analytics/models/empire_material_snapshot.py new file mode 100644 index 0000000..0527f60 --- /dev/null +++ b/backend/analytics/models/empire_material_snapshot.py @@ -0,0 +1,25 @@ +from django.db import models +from planning.models import PlanningEmpire + + +class AnalyticsEmpireMaterialSnapshot(models.Model): + id = models.BigAutoField(primary_key=True) + + empire = models.ForeignKey(PlanningEmpire, on_delete=models.CASCADE, related_name='material_snapshot') + + material_ticker = models.CharField(max_length=3, db_index=True) + + production = models.FloatField(default=0.0) + consumption = models.FloatField(default=0.0) + delta = models.FloatField(default=0.0) + + class Meta: + db_table = 'prunplanner_statistics_empire_material_snapshot' + unique_together = ('empire', 'material_ticker') + verbose_name = 'Empire Material Snapshot' + verbose_name_plural = 'Empire Material Snapshots' + + indexes = [models.Index(fields=['material_ticker', 'delta'])] + + def __str__(self) -> str: + return f'{self.empire.uuid} | {self.material_ticker}: {self.delta}' diff --git a/backend/analytics/services/analytics_cache_manager.py b/backend/analytics/services/analytics_cache_manager.py new file mode 100644 index 0000000..fb22388 --- /dev/null +++ b/backend/analytics/services/analytics_cache_manager.py @@ -0,0 +1,31 @@ +from collections.abc import Callable +from typing import Any + +from core.services.cache_manager import CacheManager +from django.http import HttpResponse + + +class AnalyticsCacheManager(CacheManager): + BASE_KEY = 'ANALYTICS' + + CACHE_TIMEOUT_3HOURS = 60 * 60 * 3 + + # Keys + @classmethod + def key_for_plan_aggregate(cls, planet_natural_id: str) -> str: + return cls.make_key('plan_aggregate', planet_natural_id) + + @classmethod + def key_planning_insight_materials(cls) -> str: + return cls.make_key('planning_insight_materials') + + # Operations + @classmethod + def get_plan_aggregate_response(cls, planet_natural_id: str, func: Callable[[], Any]) -> HttpResponse: + key = cls.key_for_plan_aggregate(planet_natural_id) + return cls.get_or_set_response(key, func, timeout=cls.CACHE_TIMEOUT_3HOURS) + + @classmethod + def get_planning_insight_materials(cls, func: Callable[[], Any]) -> HttpResponse: + key = cls.key_planning_insight_materials() + return cls.get_or_set_response(key, func, timeout=cls.CACHE_TIMEOUT_3HOURS) diff --git a/backend/analytics/services/PlanInsightAggregatorService.py b/backend/analytics/services/planinsight_aggregator_service.py similarity index 100% rename from backend/analytics/services/PlanInsightAggregatorService.py rename to backend/analytics/services/planinsight_aggregator_service.py diff --git a/backend/analytics/tasks.py b/backend/analytics/tasks.py index f99b41a..05c9284 100644 --- a/backend/analytics/tasks.py +++ b/backend/analytics/tasks.py @@ -21,7 +21,7 @@ from user.models import User from analytics.models import AppStatistic -from analytics.services.PlanInsightAggregatorService import PlanInsightAggregatorService +from analytics.services.planinsight_aggregator_service import PlanInsightAggregatorService logger = structlog.get_logger(__name__) diff --git a/backend/planning/api/serializers/empire.py b/backend/planning/api/serializers/empire.py index b1ac0de..f9f9aaf 100644 --- a/backend/planning/api/serializers/empire.py +++ b/backend/planning/api/serializers/empire.py @@ -95,10 +95,7 @@ class PlanningEmpireStateUpdateSerializer(serializers.Serializer): empire_total = serializers.DictField(child=PlanningEmpireMaterialIOSerializer()) plan_details = serializers.DictField(child=SinglePlanDetailSerializer()) - @transaction.atomic def update(self, instance, validated_data): - instance.empire_state = validated_data - instance.save(update_fields=['empire_state']) return instance def create(self, validated_data): diff --git a/backend/planning/api/viewsets/empire_viewset.py b/backend/planning/api/viewsets/empire_viewset.py index a26fe8f..db37c71 100644 --- a/backend/planning/api/viewsets/empire_viewset.py +++ b/backend/planning/api/viewsets/empire_viewset.py @@ -13,6 +13,7 @@ from planning.api.serializers.empire import PlanningEmpireStateUpdateSerializer from planning.models import PlanningEmpire, PlanningEmpirePlan, PlanningPlan from planning.planning_cache_manager import PlanningCacheManager +from planning.services.empire_state_service import EmpireStateService from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated @@ -175,9 +176,11 @@ def sync_state(self, request, pk=None): serializer = self.get_serializer(instance, data=request.data) serializer.is_valid(raise_exception=True) - with transaction.atomic(): - self.perform_update(serializer) - PlanningCacheManager.delete_pattern(f'*PLANNING:{request.user.id}:*') + # handle empire + relational snapshot refresh + EmpireStateService.sync_empire_state(instance, serializer.validated_data) + + # clear caches + PlanningCacheManager.delete_pattern(f'*PLANNING:{request.user.id}:*') return Response(PlanningEmpireDetailSerializer(instance).data) diff --git a/backend/planning/services/empire_state_service.py b/backend/planning/services/empire_state_service.py new file mode 100644 index 0000000..25688a8 --- /dev/null +++ b/backend/planning/services/empire_state_service.py @@ -0,0 +1,42 @@ +from analytics.models import AnalyticsEmpireMaterialSnapshot +from django.db import transaction +from planning.models import PlanningEmpire + + +class EmpireStateService: + @staticmethod + @transaction.atomic + def sync_empire_state(empire: PlanningEmpire, state_data: dict) -> None: + + # update the json field + empire.empire_state = state_data + empire.save(update_fields=['empire_state']) + + # upsert / delete stale from EmpireMaterialSnapshot + empire_total = state_data.get('empire_total', {}) + active_tickers = [] + + snapshot_objs = [] + for material_ticker, stats in empire_total.items(): + p, c, d = stats.get('p', 0), stats.get('c', 0), stats.get('d', 0) + if p != 0 or c != 0 or d != 0: + active_tickers.append(material_ticker) + + snapshot_objs.append( + AnalyticsEmpireMaterialSnapshot( + empire=empire, material_ticker=material_ticker, production=p, consumption=c, delta=d + ) + ) + + # only remove materials no longer in the empire total delta + AnalyticsEmpireMaterialSnapshot.objects.filter(empire=empire).exclude( + material_ticker__in=active_tickers + ).delete() + + # perform the upsert + AnalyticsEmpireMaterialSnapshot.objects.bulk_create( + snapshot_objs, + update_conflicts=True, + update_fields=['production', 'consumption', 'delta'], + unique_fields=['empire', 'material_ticker'], + )