feat: CR usage controller - moving CR usage calculation into CRD status, for API usage and Quota usage

api/v1alpha1/committed_resource_types.go

@@ -111,6 +111,18 @@ type CommittedResourceStatus struct {
 	// +kubebuilder:validation:Optional
 	AcceptedAmount *resource.Quantity `json:"acceptedAmount,omitempty"`
 
+	// AcceptedSpec is a snapshot of Spec from the last successful reconcile.
+	// Used by rollbackToAccepted to restore the exact previously-accepted placement (AZ, amount,
+	// project, domain, flavor group) even when the current spec has already been mutated to a new value.
+	// +kubebuilder:validation:Optional
+	AcceptedSpec *CommittedResourceSpec `json:"acceptedSpec,omitempty"`
+
+	// ConsecutiveFailures counts reconcile cycles that ended in a placement failure (applyErr or anyFailed).
+	// Reset to 0 on successful acceptance. Used to compute exponential backoff for the retry interval
+	// and to suppress Reservation watch re-enqueues while backing off.
+	// +kubebuilder:validation:Optional
+	ConsecutiveFailures int32 `json:"consecutiveFailures,omitempty"`
+
 	// AcceptedAt is when the controller last successfully reconciled the spec into Reservation slots.
 	// +kubebuilder:validation:Optional
 	AcceptedAt *metav1.Time `json:"acceptedAt,omitempty"`
@@ -124,20 +136,27 @@ type CommittedResourceStatus struct {
 	// +kubebuilder:validation:Optional
 	LastReconcileAt *metav1.Time `json:"lastReconcileAt,omitempty"`
 
-	// AssignedVMs holds the UUIDs of VMs deterministically assigned to this committed resource.
-	// Populated by the usage reconciler; used to compute UsedAmount and drive the quota controller.
+	// AssignedInstances holds the UUIDs of VM instances deterministically assigned to this committed resource.
+	// Populated by the usage reconciler; used to compute UsedResources and drive the quota controller.
 	// +kubebuilder:validation:Optional
-	AssignedVMs []string `json:"assignedVMs,omitempty"`
+	AssignedInstances []string `json:"assignedInstances,omitempty"`
 
-	// UsedAmount is the sum of assigned VM resources expressed in the same units as Spec.Amount.
-	// Populated by the usage reconciler.
+	// UsedResources is the total resource consumption of assigned VM instances, keyed by resource type
+	// (e.g. "memory" in MiB binary SI, "cpu" as core count). Populated by the usage reconciler.
 	// +kubebuilder:validation:Optional
-	UsedAmount *resource.Quantity `json:"usedAmount,omitempty"`
+	UsedResources map[string]resource.Quantity `json:"usedResources,omitempty"`
 
-	// LastUsageReconcileAt is when the usage reconciler last updated AssignedVMs and UsedAmount.
+	// LastUsageReconcileAt is when the usage reconciler last updated AssignedInstances and UsedResources.
 	// +kubebuilder:validation:Optional
 	LastUsageReconcileAt *metav1.Time `json:"lastUsageReconcileAt,omitempty"`
 
+	// UsageObservedGeneration is the CR generation that the usage reconciler last processed.
+	// Follows the Kubernetes observedGeneration pattern: when this differs from
+	// metadata.generation the cooldown is bypassed so spec changes (e.g. shrink) are reflected
+	// immediately rather than waiting for the next cooldown interval.
+	// +kubebuilder:validation:Optional
+	UsageObservedGeneration *int64 `json:"usageObservedGeneration,omitempty"`
+
 	// Conditions holds the current status conditions.
 	// +kubebuilder:validation:Optional
 	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
@@ -164,7 +183,8 @@ const (
 // +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone"
 // +kubebuilder:printcolumn:name="Amount",type="string",JSONPath=".spec.amount"
 // +kubebuilder:printcolumn:name="AcceptedAmount",type="string",JSONPath=".status.acceptedAmount"
-// +kubebuilder:printcolumn:name="UsedAmount",type="string",JSONPath=".status.usedAmount"
+// +kubebuilder:printcolumn:name="UsedMemory",type="string",JSONPath=".status.usedResources.memory",priority=1
+// +kubebuilder:printcolumn:name="UsedCPU",type="string",JSONPath=".status.usedResources.cpu",priority=1
 // +kubebuilder:printcolumn:name="State",type="string",JSONPath=".spec.state"
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status"
 // +kubebuilder:printcolumn:name="StartTime",type="date",JSONPath=".spec.startTime",priority=1

api/v1alpha1/datasource_types.go

@@ -52,6 +52,7 @@ const (
 	NovaDatasourceTypeFlavors        NovaDatasourceType = "flavors"
 	NovaDatasourceTypeMigrations     NovaDatasourceType = "migrations"
 	NovaDatasourceTypeAggregates     NovaDatasourceType = "aggregates"
+	NovaDatasourceTypeImages         NovaDatasourceType = "images"
 )
 
 type NovaDatasource struct {

cmd/manager/main.go

@@ -548,14 +548,34 @@ func main() {
 			os.Exit(1)
 		}
 
+		crControllerConf := commitmentsConfig.CommittedResourceController
+		crControllerConf.ApplyDefaults()
 		if err := (&commitments.CommittedResourceController{
 			Client: multiclusterClient,
 			Scheme: mgr.GetScheme(),
-			Conf:   commitmentsConfig.CommittedResourceController,
+			Conf:   crControllerConf,
 		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "CommittedResource")
 			os.Exit(1)
 		}
+
+		usageReconcilerMonitor := commitments.NewUsageReconcilerMonitor()
+		metrics.Registry.MustRegister(&usageReconcilerMonitor)
+		if commitmentsUsageDB == nil {
+			setupLog.Error(nil, "UsageReconciler requires a datasource but commitments.datasourceName is not configured — skipping")
+		} else {
+			usageReconcilerConf := commitmentsConfig.UsageReconciler
+			usageReconcilerConf.ApplyDefaults()
+			if err := (&commitments.UsageReconciler{
+				Client:  multiclusterClient,
+				Conf:    usageReconcilerConf,
+				UsageDB: commitmentsUsageDB,
+				Monitor: usageReconcilerMonitor,
+			}).SetupWithManager(mgr, multiclusterClient); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CommittedResourceUsage")
+				os.Exit(1)
+			}
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") {
 		setupLog.Info("enabling controller", "controller", "datasource-controllers")

helm/bundles/cortex-nova/templates/datasources.yaml

@@ -337,6 +337,30 @@ spec:
 ---
 apiVersion: cortex.cloud/v1alpha1
 kind: Datasource
+metadata:
+  name: nova-images
+spec:
+  schedulingDomain: nova
+  databaseSecretRef:
+    name: cortex-nova-postgres
+    namespace: {{ .Release.Namespace }}
+  {{- if .Values.openstack.sso.enabled }}
+  ssoSecretRef:
+    name: cortex-nova-openstack-sso
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  type: openstack
+  openstack:
+    syncInterval: 3600s
+    secretRef:
+      name: cortex-nova-openstack-keystone
+      namespace: {{ .Release.Namespace }}
+    type: nova
+    nova:
+      type: images
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: Datasource
 metadata:
   name: limes-project-commitments
 spec:

helm/bundles/cortex-nova/values.yaml

@@ -158,8 +158,10 @@ cortex-scheduling-controllers:
       # URL of the nova external scheduler API for placement decisions
       schedulerURL: "http://localhost:8080/scheduler/nova/external"
     committedResourceController:
-      # Back-off interval while CommittedResource placement is pending or failed
+      # Back-off interval while CommittedResource placement is pending or failed (base for exponential backoff)
       requeueIntervalRetry: "1m"
+      # Maximum back-off interval cap for the exponential retry delay
+      maxRequeueInterval: "30m"
     committedResourceAPI:
       # Timeout for watching CommittedResource CRDs before rolling back
       watchTimeout: "10s"
@@ -187,6 +189,11 @@ cortex-scheduling-controllers:
             handlesCommitments: false
             hasCapacity: true
             hasQuota: false
+    committedResourceUsageReconciler:
+      # Minimum time between usage reconcile runs for the same CommittedResource.
+      # Also acts as the periodic fallback interval: a successful reconcile schedules
+      # the next run after this duration, so this is also the maximum status staleness.
+      cooldownInterval: "5m"
     # OvercommitMappings is a list of mappings that map hypervisor traits to
     # overcommit ratios. Note that this list is applied in order, so if there
     # are multiple mappings applying to the same hypervisors, the last mapping

helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml

@@ -33,8 +33,13 @@ spec:
     - jsonPath: .status.acceptedAmount
       name: AcceptedAmount
       type: string
-    - jsonPath: .status.usedAmount
-      name: UsedAmount
+    - jsonPath: .status.usedResources.memory
+      name: UsedMemory
+      priority: 1
+      type: string
+    - jsonPath: .status.usedResources.cpu
+      name: UsedCPU
+      priority: 1
       type: string
     - jsonPath: .spec.state
       name: State
@@ -180,10 +185,105 @@ spec:
                   the spec into Reservation slots.
                 format: date-time
                 type: string
-              assignedVMs:
+              acceptedSpec:
                 description: |-
-                  AssignedVMs holds the UUIDs of VMs deterministically assigned to this committed resource.
-                  Populated by the usage reconciler; used to compute UsedAmount and drive the quota controller.
+                  AcceptedSpec is a snapshot of Spec from the last successful reconcile.
+                  Used by rollbackToAccepted to restore the exact previously-accepted placement (AZ, amount,
+                  project, domain, flavor group) even when the current spec has already been mutated to a new value.
+                properties:
+                  allowRejection:
+                    description: |-
+                      AllowRejection controls what the CommittedResource controller does when placement fails
+                      for a guaranteed or confirmed commitment.
+                      true  — controller may reject: on failure, child Reservations are rolled back and the CR
+                              is marked Rejected. Use this when the caller is making a first-time placement
+                              decision and a "no" answer is acceptable (e.g. the change-commitments API).
+                      false — controller must retry: on failure, existing child Reservations are kept and the
+                              CR is set to Reserving so the controller retries later. Use this when the caller
+                              is restoring already-committed state that Cortex must honour (e.g. the syncer).
+                      Only meaningful for state=guaranteed or state=confirmed; ignored for all other states.
+                    type: boolean
+                  amount:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: |-
+                      Amount is the total committed quantity.
+                      memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi").
+                      cores: integer core count (e.g. "40").
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                  availabilityZone:
+                    description: AvailabilityZone specifies the availability zone
+                      for this commitment.
+                    type: string
+                  commitmentUUID:
+                    description: UUID of the commitment this resource corresponds
+                      to.
+                    type: string
+                  confirmedAt:
+                    description: ConfirmedAt is when the commitment was confirmed.
+                    format: date-time
+                    type: string
+                  domainID:
+                    description: DomainID of the OpenStack domain this commitment
+                      belongs to.
+                    type: string
+                  endTime:
+                    description: EndTime is when Reservation slots expire. Nil for
+                      unbounded commitments with no expiry.
+                    format: date-time
+                    type: string
+                  flavorGroupName:
+                    description: FlavorGroupName identifies the flavor group this
+                      commitment targets, e.g. "kvm_v2_hana_s".
+                    type: string
+                  projectID:
+                    description: ProjectID of the OpenStack project this commitment
+                      belongs to.
+                    type: string
+                  resourceType:
+                    description: 'ResourceType identifies the kind of resource committed:
+                      memory drives Reservation slots; cores uses an arithmetic check
+                      only.'
+                    enum:
+                    - memory
+                    - cores
+                    type: string
+                  schedulingDomain:
+                    description: SchedulingDomain specifies the scheduling domain
+                      for this committed resource (e.g., "nova", "ironcore").
+                    type: string
+                  startTime:
+                    description: |-
+                      StartTime is the activation time for Reservation slots.
+                      Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones.
+                    format: date-time
+                    type: string
+                  state:
+                    description: State is the lifecycle state of the commitment.
+                    enum:
+                    - planned
+                    - pending
+                    - guaranteed
+                    - confirmed
+                    - superseded
+                    - expired
+                    type: string
+                required:
+                - amount
+                - availabilityZone
+                - commitmentUUID
+                - domainID
+                - flavorGroupName
+                - projectID
+                - resourceType
+                - state
+                type: object
+              assignedInstances:
+                description: |-
+                  AssignedInstances holds the UUIDs of VM instances deterministically assigned to this committed resource.
+                  Populated by the usage reconciler; used to compute UsedResources and drive the quota controller.
                 items:
                   type: string
                 type: array
@@ -244,6 +344,13 @@ spec:
                   - type
                   type: object
                 type: array
+              consecutiveFailures:
+                description: |-
+                  ConsecutiveFailures counts reconcile cycles that ended in a placement failure (applyErr or anyFailed).
+                  Reset to 0 on successful acceptance. Used to compute exponential backoff for the retry interval
+                  and to suppress Reservation watch re-enqueues while backing off.
+                format: int32
+                type: integer
               lastChanged:
                 description: |-
                   LastChanged is when the spec was last written by the syncer.
@@ -257,18 +364,28 @@ spec:
                 type: string
               lastUsageReconcileAt:
                 description: LastUsageReconcileAt is when the usage reconciler last
-                  updated AssignedVMs and UsedAmount.
+                  updated AssignedInstances and UsedResources.
                 format: date-time
                 type: string
-              usedAmount:
-                anyOf:
-                - type: integer
-                - type: string
+              usageObservedGeneration:
                 description: |-
-                  UsedAmount is the sum of assigned VM resources expressed in the same units as Spec.Amount.
-                  Populated by the usage reconciler.
-                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                x-kubernetes-int-or-string: true
+                  UsageObservedGeneration is the CR generation that the usage reconciler last processed.
+                  Follows the Kubernetes observedGeneration pattern: when this differs from
+                  metadata.generation the cooldown is bypassed so spec changes (e.g. shrink) are reflected
+                  immediately rather than waiting for the next cooldown interval.
+                format: int64
+                type: integer
+              usedResources:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: |-
+                  UsedResources is the total resource consumption of assigned VM instances, keyed by resource type
+                  (e.g. "memory" in MiB binary SI, "cpu" as core count). Populated by the usage reconciler.
+                type: object
             type: object
         required:
         - spec

internal/knowledge/datasources/plugins/openstack/controller_test.go

@@ -104,6 +104,7 @@ func TestNovaDatasourceTypeConstants(t *testing.T) {
 		{v1alpha1.NovaDatasourceTypeFlavors, "flavors"},
 		{v1alpha1.NovaDatasourceTypeMigrations, "migrations"},
 		{v1alpha1.NovaDatasourceTypeAggregates, "aggregates"},
+		{v1alpha1.NovaDatasourceTypeImages, "images"},
 	}
 
 	for _, test := range tests {