diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 561d9fc3c..b7a95599f 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -557,4 +557,44 @@ spec: VM is allocated get a higher weight, encouraging placement on pre-reserved failover capacity. For non-evacuation requests, this weigher has no effect. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: Pipeline +metadata: + name: kvm-report-capacity +spec: + schedulingDomain: nova + description: | + This pipeline is used by the Liquid capacity reporter to determine the + theoretical maximum capacity of each flavor group per availability zone, + as if all hosts were completely empty. It ignores current VM allocations + and all reservation blockings so that only raw hardware capacity is + considered. + type: filter-weigher + createDecisions: false + # Fetch all placement candidates, ignoring nova's preselection. + ignorePreselection: true + filters: + - name: filter_correct_az + description: | + Restricts host candidates to the requested availability zone. + - name: filter_has_enough_capacity + description: | + Filters hosts that cannot fit the flavor based on raw hardware capacity. + VM allocations and all reservation types are ignored to represent an + empty datacenter scenario. + params: + - {key: ignoreAllocations, boolValue: true} + - {key: ignoredReservationTypes, stringListValue: ["CommittedResourceReservation", "FailoverReservation"]} + - name: filter_has_requested_traits + description: | + Ensures hosts have the hardware traits required by the flavor. + - name: filter_capabilities + description: | + Ensures hosts meet the compute capabilities required by the flavor + extra specs (e.g., architecture, maxphysaddr bits). + - name: filter_status_conditions + description: | + Excludes hosts that are not ready or are disabled. + weighers: [] {{- end }} diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index d30d89a4f..397c5133c 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -149,6 +149,9 @@ cortex-scheduling-controllers: "*": "kvm-general-purpose-load-balancing" # Catch-all fallback # Default pipeline for CR reservations when no CommittedResourceFlavorGroupPipelines entry matches committedResourcePipelineDefault: "kvm-general-purpose-load-balancing" + # Pipeline used for capacity reporting: determines eligible hosts per AZ (ignores allocations/reservations). + # Host resource data is then read from Hypervisor CRDs to compute actual multiples. + reportCapacityTotalPipeline: "kvm-report-capacity" # How often to re-verify active reservations # 5m = 300000000000 nanoseconds committedResourceRequeueIntervalActive: 300000000000 diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index e6956609a..2ceb4944f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -26,6 +26,10 @@ type FilterHasEnoughCapacityOpts struct { // When a reservation type is in this list, its capacity is not blocked. // Default: empty (all reservation types are considered) IgnoredReservationTypes []v1alpha1.ReservationType `json:"ignoredReservationTypes,omitempty"` + + // IgnoreAllocations skips subtracting current VM allocations from host capacity. + // When true, only raw hardware capacity is considered (empty datacenter scenario). + IgnoreAllocations bool `json:"ignoreAllocations,omitempty"` } func (FilterHasEnoughCapacityOpts) Validate() error { return nil } @@ -71,18 +75,20 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa freeResourcesByHost[hv.Name] = hv.Status.EffectiveCapacity } - // Subtract allocated resources. - for resourceName, allocated := range hv.Status.Allocation { - free, ok := freeResourcesByHost[hv.Name][resourceName] - if !ok { - traceLog.Error( - "hypervisor with allocation for unknown resource", - "host", hv.Name, "resource", resourceName, - ) - continue + // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). + if !s.Options.IgnoreAllocations { + for resourceName, allocated := range hv.Status.Allocation { + free, ok := freeResourcesByHost[hv.Name][resourceName] + if !ok { + traceLog.Error( + "hypervisor with allocation for unknown resource", + "host", hv.Name, "resource", resourceName, + ) + continue + } + free.Sub(allocated) + freeResourcesByHost[hv.Name][resourceName] = free } - free.Sub(allocated) - freeResourcesByHost[hv.Name][resourceName] = free } } diff --git a/internal/scheduling/reservations/commitments/api_report_capacity.go b/internal/scheduling/reservations/commitments/api_report_capacity.go index 09fc55168..9d084b211 100644 --- a/internal/scheduling/reservations/commitments/api_report_capacity.go +++ b/internal/scheduling/reservations/commitments/api_report_capacity.go @@ -11,7 +11,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/google/uuid" - "github.com/sapcc/go-api-declarations/liquid" ) // handles POST /commitments/v1/report-capacity requests from Limes: @@ -50,16 +49,9 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) logger.V(1).Info("processing report capacity request") - // Parse request body (may be empty or contain ServiceCapacityRequest) - var req liquid.ServiceCapacityRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - // Empty body is acceptable for capacity reports - req = liquid.ServiceCapacityRequest{} - } - // Calculate capacity - calculator := NewCapacityCalculator(api.client) - report, err := calculator.CalculateCapacity(ctx, req) + calculator := NewCapacityCalculator(api.client, api.config) + report, err := calculator.CalculateCapacity(ctx) if err != nil { logger.Error(err, "failed to calculate capacity") statusCode = http.StatusInternalServerError diff --git a/internal/scheduling/reservations/commitments/api_report_capacity_test.go b/internal/scheduling/reservations/commitments/api_report_capacity_test.go index b151382cf..8573ea021 100644 --- a/internal/scheduling/reservations/commitments/api_report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api_report_capacity_test.go @@ -9,24 +9,35 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "slices" "strings" "testing" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/sapcc/go-api-declarations/liquid" + "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) -func TestHandleReportCapacity(t *testing.T) { - // Setup fake client +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatal(err) } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + return scheme +} + +func TestHandleReportCapacity(t *testing.T) { + scheme := testScheme(t) // Create empty flavor groups knowledge so capacity calculation doesn't fail emptyKnowledge := createEmptyFlavorGroupKnowledge() @@ -124,22 +135,15 @@ func TestHandleReportCapacity(t *testing.T) { } func TestCapacityCalculator(t *testing.T) { - // Setup fake client with Knowledge CRD - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } + scheme := testScheme(t) t.Run("CalculateCapacity returns error when no flavor groups knowledge exists", func(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"az-one", "az-two"}, - } - _, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + _, err := calculator.CalculateCapacity(context.Background()) if err == nil { t.Fatal("Expected error when flavor groups knowledge doesn't exist, got nil") } @@ -149,7 +153,6 @@ func TestCapacityCalculator(t *testing.T) { }) t.Run("CalculateCapacity returns empty report when flavor groups knowledge exists but is empty", func(t *testing.T) { - // Create empty flavor groups knowledge emptyKnowledge := createEmptyFlavorGroupKnowledge() fakeClient := fake.NewClientBuilder(). @@ -157,11 +160,8 @@ func TestCapacityCalculator(t *testing.T) { WithObjects(emptyKnowledge). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"az-one", "az-two"}, - } - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -175,18 +175,23 @@ func TestCapacityCalculator(t *testing.T) { } }) - t.Run("CalculateCapacity returns perAZ entries for all AZs from request", func(t *testing.T) { + t.Run("CalculateCapacity returns perAZ entries for all AZs from hypervisors", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") + hvs := createTestHypervisorsWithAZ(map[string]string{ + "host-1": "qa-de-1a", + "host-2": "qa-de-1b", + }) + server := newMockSchedulerServer(t, []string{}) + defer server.Close() + cfg := DefaultConfig() + cfg.SchedulerURL = server.URL fakeClient := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(flavorGroupKnowledge). + WithObjects(flavorGroupKnowledge, hvs[0], hvs[1]). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"qa-de-1a", "qa-de-1b", "qa-de-1d"}, - } - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, cfg) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -195,22 +200,31 @@ func TestCapacityCalculator(t *testing.T) { t.Fatalf("Expected 3 resources (_ram, _cores, _instances), got %d", len(report.Resources)) } - // Verify all resources have exactly the requested AZs - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_ram"], req.AllAZs) - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_cores"], req.AllAZs) - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_instances"], req.AllAZs) + // Verify all resources have entries for the AZs from hypervisors + expectedAZs := []liquid.AvailabilityZone{"qa-de-1a", "qa-de-1b"} + for _, resName := range []string{"hw_version_test-group_ram", "hw_version_test-group_cores", "hw_version_test-group_instances"} { + res := report.Resources[liquid.ResourceName(resName)] + if res == nil { + t.Errorf("resource %s not found", resName) + continue + } + for _, az := range expectedAZs { + if _, ok := res.PerAZ[az]; !ok { + t.Errorf("%s: missing entry for AZ %s", resName, az) + } + } + } }) - t.Run("CalculateCapacity with empty AllAZs returns empty perAZ maps", func(t *testing.T) { + t.Run("CalculateCapacity with no host details returns empty perAZ maps", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorGroupKnowledge). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{}} - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -226,63 +240,332 @@ func TestCapacityCalculator(t *testing.T) { } }) - t.Run("CalculateCapacity responds to different AZ sets correctly", func(t *testing.T) { + t.Run("CalculateCapacity produces perAZ entries matching hypervisor AZs", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") + hvs := createTestHypervisorsWithAZ(map[string]string{ + "host-a": "eu-de-1a", + "host-b": "eu-de-1b", + }) + server := newMockSchedulerServer(t, []string{}) + defer server.Close() + cfg := DefaultConfig() + cfg.SchedulerURL = server.URL + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, hvs[0], hvs[1]). + Build() + + calculator := NewCapacityCalculator(fakeClient, cfg) + report, err := calculator.CalculateCapacity(context.Background()) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + for resName, res := range report.Resources { + if len(res.PerAZ) != 2 { + t.Errorf("%s: expected 2 AZs, got %d", resName, len(res.PerAZ)) + } + for _, az := range []liquid.AvailabilityZone{"eu-de-1a", "eu-de-1b"} { + if _, ok := res.PerAZ[az]; !ok { + t.Errorf("%s: missing entry for AZ %s", resName, az) + } + } + } + }) +} + +func TestCapacityCalculatorWithHypervisors(t *testing.T) { + scheme := testScheme(t) + + const ( + flavorGroup = "test-group" + az = "az-a" + flavorMemMB = uint64(32768) // 32 GiB + flavorVCPUs = uint64(8) + ) + + flavorGroupKnowledge := createTestFlavorGroupKnowledgeWithSmallest(t, flavorGroup, flavorMemMB, flavorVCPUs) + + t.Run("computes capacity as multiples of smallest flavor", func(t *testing.T) { + // Host has 256 GiB effective capacity. Smallest flavor = 32 GiB. + // Total capacity = floor(256 / 32) = 8. + server := newMockSchedulerServer(t, []string{"host-1"}) + defer server.Close() + + hvObj := createTestHypervisorWithAZ("host-1", az, "256Gi", "64Gi") + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, hvObj). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + + hvByName := map[string]hv1.Hypervisor{"host-1": *hvObj} + capacity, err := calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capacity != 8 { + t.Errorf("expected capacity = 8, got %d", capacity) + } + }) + + t.Run("sums multiples across multiple hosts", func(t *testing.T) { + // Host-1: 256 GiB → total=8 + // Host-2: 128 GiB → total=4 + // Combined: total=12 + server := newMockSchedulerServer(t, []string{"host-1", "host-2"}) + defer server.Close() + + host1HV := createTestHypervisorWithAZ("host-1", az, "256Gi", "128Gi") + host2HV := createTestHypervisorWithAZ("host-2", az, "128Gi", "0") + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, host1HV, host2HV). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + + hvByName := map[string]hv1.Hypervisor{"host-1": *host1HV, "host-2": *host2HV} + capacity, err := calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capacity != 12 { + t.Errorf("expected capacity = 12, got %d", capacity) + } + }) + + t.Run("capacity is correct when nothing is allocated", func(t *testing.T) { + server := newMockSchedulerServer(t, []string{"host-1"}) + defer server.Close() + + hvObj := createTestHypervisorWithAZ("host-1", az, "128Gi", "0") + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, hvObj). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + + hvByName := map[string]hv1.Hypervisor{"host-1": *hvObj} + capacity, err := calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capacity != 4 { + t.Errorf("expected capacity = 4, got %d", capacity) + } + }) + + t.Run("host not found in HV CRDs is skipped", func(t *testing.T) { + // Scheduler returns a host with no matching HV CRD — should contribute 0 capacity. + server := newMockSchedulerServer(t, []string{"host-unknown"}) + defer server.Close() + + hostDetails := createTestHypervisorsWithAZ(map[string]string{"host-unknown": az}) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, hostDetails[0]). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + + hvByName := map[string]hv1.Hypervisor{} // empty + capacity, err := calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capacity != 0 { + t.Errorf("expected capacity = 0, got %d", capacity) + } + }) + + t.Run("scheduler failure returns error", func(t *testing.T) { + failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "internal error", http.StatusInternalServerError) + })) + defer failServer.Close() + fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorGroupKnowledge). Build() - calculator := NewCapacityCalculator(fakeClient) + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(failServer.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } - req1 := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"eu-de-1a", "eu-de-1b"}, + hvByName := map[string]hv1.Hypervisor{} + _, err = calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err == nil { + t.Fatal("expected error on scheduler failure, got nil") } - report1, err := calculator.CalculateCapacity(context.Background(), req1) + }) + + t.Run("multiple AZs are reported independently", func(t *testing.T) { + // Scheduler always returns both hosts (mock doesn't filter by AZ). + server := newMockSchedulerServer(t, []string{"host-1", "host-2"}) + defer server.Close() + + host1HV := createTestHypervisorWithAZ("host-1", "az-a", "128Gi", "32Gi") + host2HV := createTestHypervisorWithAZ("host-2", "az-b", "64Gi", "0") + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, host1HV, host2HV). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { - t.Fatalf("Expected no error, got: %v", err) + t.Fatalf("unexpected error: %v", err) } - req2 := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"us-west-1a", "us-west-1b", "us-west-1c", "us-west-1d"}, + res := report.Resources[liquid.ResourceName(ResourceNameRAM(flavorGroup))] + if len(res.PerAZ) != 2 { + t.Errorf("expected 2 AZs, got %d", len(res.PerAZ)) + } + if _, ok := res.PerAZ[liquid.AvailabilityZone("az-a")]; !ok { + t.Error("expected az-a in report") } - report2, err := calculator.CalculateCapacity(context.Background(), req2) + if _, ok := res.PerAZ[liquid.AvailabilityZone("az-b")]; !ok { + t.Error("expected az-b in report") + } + }) + + t.Run("partial memory is floored to full multiples", func(t *testing.T) { + // Host has 100 GiB capacity. Smallest flavor = 32 GiB. + // Total = floor(100 / 32) = 3 (not 3.125). + server := newMockSchedulerServer(t, []string{"host-1"}) + defer server.Close() + + hvObj := createTestHypervisorWithAZ("host-1", az, "100Gi", "0") + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, hvObj). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) if err != nil { - t.Fatalf("Expected no error, got: %v", err) + t.Fatalf("failed to get flavor groups: %v", err) } - // Verify reports have exactly the requested AZs - for _, res := range report1.Resources { - verifyPerAZMatchesRequest(t, res, req1.AllAZs) + hvByName := map[string]hv1.Hypervisor{"host-1": *hvObj} + capacity, err := calculator.calculateInstanceCapacity(context.Background(), groups[flavorGroup], az, hvByName) + if err != nil { + t.Fatalf("unexpected error: %v", err) } - for _, res := range report2.Resources { - verifyPerAZMatchesRequest(t, res, req2.AllAZs) + + if capacity != 3 { + t.Errorf("expected capacity = 3 (floored), got %d", capacity) } }) } -// verifyPerAZMatchesRequest checks that perAZ entries match exactly the requested AZs. -// This follows the same semantics as nova liquid: the response must contain -// entries for all AZs in AllAZs, no more and no less. -func verifyPerAZMatchesRequest(t *testing.T, res *liquid.ResourceCapacityReport, requestedAZs []liquid.AvailabilityZone) { +// newMockSchedulerServer returns a test HTTP server that always returns the given host list. +func newMockSchedulerServer(t *testing.T, hosts []string) *httptest.Server { t.Helper() - if res == nil { - t.Error("resource is nil") - return - } - if len(res.PerAZ) != len(requestedAZs) { - t.Errorf("expected %d AZs, got %d", len(requestedAZs), len(res.PerAZ)) - } - for _, az := range requestedAZs { - if _, ok := res.PerAZ[az]; !ok { - t.Errorf("missing entry for requested AZ %s", az) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: hosts}); err != nil { + t.Errorf("mock scheduler: encode error: %v", err) } + })) +} + +// createTestHypervisor creates an HV CRD with the given effective capacity and allocation. +func createTestHypervisor(name, effectiveCapacity, allocation string) *hv1.Hypervisor { + hv := &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(effectiveCapacity), + }, + }, } - for az := range res.PerAZ { - if !slices.Contains(requestedAZs, az) { - t.Errorf("unexpected AZ %s in response (not in request)", az) + if allocation != "0" && allocation != "" { + hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(allocation), } } + return hv +} + +// createTestHypervisorWithAZ creates an HV CRD with a topology.kubernetes.io/zone label. +func createTestHypervisorWithAZ(name, az, effectiveCapacity, allocation string) *hv1.Hypervisor { + hv := createTestHypervisor(name, effectiveCapacity, allocation) + hv.Labels = map[string]string{"topology.kubernetes.io/zone": az} + return hv } // createEmptyFlavorGroupKnowledge creates an empty flavor groups Knowledge CRD @@ -297,7 +580,6 @@ func createEmptyFlavorGroupKnowledge() *v1alpha1.Knowledge { return &v1alpha1.Knowledge{ ObjectMeta: v1.ObjectMeta{ Name: "flavor-groups", - // No namespace - Knowledge is cluster-scoped }, Spec: v1alpha1.KnowledgeSpec{ SchedulingDomain: v1alpha1.SchedulingDomainNova, @@ -318,7 +600,6 @@ func createEmptyFlavorGroupKnowledge() *v1alpha1.Knowledge { } // createTestFlavorGroupKnowledge creates a test Knowledge CRD with flavor group data -// that accepts commitments (has fixed RAM/core ratio) func createTestFlavorGroupKnowledge(t *testing.T, groupName string) *v1alpha1.Knowledge { t.Helper() @@ -345,12 +626,10 @@ func createTestFlavorGroupKnowledge(t *testing.T, groupName string) *v1alpha1.Kn "memoryMB": 32768, "diskGB": 50, }, - // Fixed RAM/core ratio (4096 MiB per vCPU) - required for group to accept commitments "ramCoreRatio": 4096, }, } - // Use BoxFeatureList to properly format the features raw, err := v1alpha1.BoxFeatureList(features) if err != nil { t.Fatal(err) @@ -359,7 +638,6 @@ func createTestFlavorGroupKnowledge(t *testing.T, groupName string) *v1alpha1.Kn return &v1alpha1.Knowledge{ ObjectMeta: v1.ObjectMeta{ Name: "flavor-groups", - // No namespace - Knowledge is cluster-scoped }, Spec: v1alpha1.KnowledgeSpec{ SchedulingDomain: v1alpha1.SchedulingDomainNova, @@ -378,3 +656,68 @@ func createTestFlavorGroupKnowledge(t *testing.T, groupName string) *v1alpha1.Kn }, } } + +// createTestFlavorGroupKnowledgeWithSmallest creates a Knowledge CRD where smallestFlavor +// is explicitly set so the capacity calculator uses the correct memory unit. +func createTestFlavorGroupKnowledgeWithSmallest(t *testing.T, groupName string, memMB, vcpus uint64) *v1alpha1.Knowledge { + t.Helper() + + features := []map[string]interface{}{ + { + "name": groupName, + "flavors": []map[string]interface{}{ + { + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + }, + "smallestFlavor": map[string]interface{}{ + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + "largestFlavor": map[string]interface{}{ + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + }, + } + + raw, err := v1alpha1.BoxFeatureList(features) + if err != nil { + t.Fatal(err) + } + + return &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: "flavor-groups"}, + Spec: v1alpha1.KnowledgeSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + Extractor: v1alpha1.KnowledgeExtractorSpec{Name: "flavor_groups"}, + }, + Status: v1alpha1.KnowledgeStatus{ + Conditions: []v1.Condition{{Type: v1alpha1.KnowledgeConditionReady, Status: "True"}}, + Raw: raw, + }, + } +} + +// createTestHypervisorsWithAZ creates HV CRDs with topology.kubernetes.io/zone labels +// from a host→AZ map. Hypervisors have no capacity data (used only for AZ discovery). +func createTestHypervisorsWithAZ(hostToAZ map[string]string) []*hv1.Hypervisor { + hvs := make([]*hv1.Hypervisor, 0, len(hostToAZ)) + for host, az := range hostToAZ { + hv := &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{ + Name: host, + Labels: map[string]string{"topology.kubernetes.io/zone": az}, + }, + } + hvs = append(hvs, hv) + } + return hvs +} diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 8cd3a7159..e65edac48 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -6,28 +6,38 @@ package commitments import ( "context" "fmt" + "sort" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/google/uuid" . "github.com/majewsky/gg/option" "github.com/sapcc/go-api-declarations/liquid" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) // CapacityCalculator computes capacity reports for Limes LIQUID API. type CapacityCalculator struct { - client client.Client + client client.Client + schedulerClient *reservations.SchedulerClient + totalPipeline string } -func NewCapacityCalculator(client client.Client) *CapacityCalculator { - return &CapacityCalculator{client: client} +func NewCapacityCalculator(client client.Client, config Config) *CapacityCalculator { + return &CapacityCalculator{ + client: client, + schedulerClient: reservations.NewSchedulerClient(config.SchedulerURL), + totalPipeline: config.ReportCapacityTotalPipeline, + } } // CalculateCapacity computes per-AZ capacity for all flavor groups. // For each flavor group, three resources are reported: _ram, _cores, _instances. // All flavor groups are included, not just those with fixed RAM/core ratio. -// The request provides the list of all AZs from Limes that must be included in the report. -func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { +// AZs are derived from Hypervisor CRDs via the topology.kubernetes.io/zone label. +func (c *CapacityCalculator) CalculateCapacity(ctx context.Context) (liquid.ServiceCapacityReport, error) { // Get all flavor groups from Knowledge CRDs knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) @@ -41,6 +51,19 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S infoVersion = knowledgeCRD.Status.LastContentChange.Unix() } + // Pre-fetch all Hypervisor CRDs once (shared across all flavor groups and AZs) + var hvList hv1.HypervisorList + if err := c.client.List(ctx, &hvList); err != nil { + return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to list hypervisors: %w", err) + } + hvByName := make(map[string]hv1.Hypervisor, len(hvList.Items)) + for _, hv := range hvList.Items { + hvByName[hv.Name] = hv + } + + // Derive AZs from Hypervisor CRDs via topology.kubernetes.io/zone label + azs := getAvailabilityZones(hvList.Items) + // Build capacity report for all flavor groups report := liquid.ServiceCapacityReport{ InfoVersion: infoVersion, @@ -48,10 +71,11 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S } for groupName, groupData := range flavorGroups { - // All flavor groups are included in capacity reporting (not just those with fixed ratio). - - // Calculate per-AZ capacity (placeholder: capacity=0 for all resources) - azCapacity := c.calculateAZCapacity(groupName, groupData, req.AllAZs) + // Calculate per-AZ capacity using scheduler + HV CRDs + azCapacity, err := c.calculateAZCapacity(ctx, groupName, groupData, azs, hvByName) + if err != nil { + return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to calculate capacity for flavor group %s: %w", groupName, err) + } // === 1. RAM Resource === ramResourceName := liquid.ResourceName(ResourceNameRAM(groupName)) @@ -60,17 +84,14 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S } // === 2. Cores Resource === - // NOTE: Copying RAM capacity is only valid while capacity=0 (placeholder). - // When real capacity is implemented, derive cores capacity with unit conversion - // (e.g., cores = RAM / ramCoreRatio). See calculateAZCapacity for details. + // All three resources express capacity in units of "multiples of the smallest flavor", + // so the same number applies to ram, cores, and instances. coresResourceName := liquid.ResourceName(ResourceNameCores(groupName)) report.Resources[coresResourceName] = &liquid.ResourceCapacityReport{ PerAZ: c.copyAZCapacity(azCapacity), } // === 3. Instances Resource === - // NOTE: Same as cores - copying is only valid while capacity=0 (placeholder). - // When real capacity is implemented, derive instances capacity appropriately. instancesResourceName := liquid.ResourceName(ResourceNameInstances(groupName)) report.Resources[instancesResourceName] = &liquid.ResourceCapacityReport{ PerAZ: c.copyAZCapacity(azCapacity), @@ -96,30 +117,129 @@ func (c *CapacityCalculator) copyAZCapacity( return result } +// calculateAZCapacity computes capacity per AZ for a flavor group. +// Uses one scheduler call per AZ to get eligible hosts, then reads HV CRDs for resource data. +// On scheduler failure for an AZ, that AZ still gets an entry with capacity=0. +// If all AZs fail, returns an error instead of a zero-capacity report. func (c *CapacityCalculator) calculateAZCapacity( - _ string, // groupName - reserved for future use - _ compute.FlavorGroupFeature, // groupData - reserved for future use - allAZs []liquid.AvailabilityZone, // list of all AZs from Limes request -) map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport { + ctx context.Context, + groupName string, + groupData compute.FlavorGroupFeature, + azs []string, + hvByName map[string]hv1.Hypervisor, +) (map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport, error) { - // Create report entry for each AZ with placeholder capacity=0. - // - // NOTE: When implementing real capacity calculation here, you MUST also update - // the copying logic in CalculateCapacity() for _cores and _instances resources. - // Those resources use different units (vCPUs and VM count) than _ram (memory multiples), - // so the capacity values cannot be simply copied - they require unit conversion: - // - _cores capacity = RAM capacity / ramCoreRatio - // - _instances capacity = needs its own derivation logic - // - // TODO: Calculate actual capacity from Reservation CRDs or host resources - // TODO: Calculate actual usage from VM allocations result := make(map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport) - for _, az := range allAZs { - result[az] = &liquid.AZResourceCapacityReport{ - Capacity: 0, // Placeholder: capacity=0 until actual calculation is implemented + failures := 0 + for _, az := range azs { + capacity, err := c.calculateInstanceCapacity(ctx, groupData, az, hvByName) + if err != nil { + failures++ + LoggerFromContext(ctx).Error(err, "failed to calculate capacity for AZ, reporting 0", + "flavorGroup", groupName, "az", az) + // On failure, report az with capacity=0 rather than aborting entirely. + result[liquid.AvailabilityZone(az)] = &liquid.AZResourceCapacityReport{ + Capacity: 0, + Usage: Some[uint64](0), // Placeholder: usage=0 until actual calculation is implemented + } + continue + } + result[liquid.AvailabilityZone(az)] = &liquid.AZResourceCapacityReport{ + Capacity: capacity, Usage: Some[uint64](0), // Placeholder: usage=0 until actual calculation is implemented } } - return result + // If all AZs failed, return an error instead of a zero-capacity report + if failures == len(azs) && len(azs) > 0 { + return nil, fmt.Errorf("failed to calculate capacity for all AZs in flavor group %s", groupName) + } + + return result, nil } + +// calculateInstanceCapacity returns the total capacity for a flavor group in an AZ. +// Capacity is expressed in multiples of the smallest flavor's memory. +// Usage tracking (VM allocations + reservations) is not yet implemented — see PR 2. +// +// 1. One scheduler call (kvm-report-capacity pipeline, ignoring allocations) → list of eligible hosts +// 2. For each eligible host, read EffectiveCapacity from HV CRDs +// 3. Total capacity = sum(floor(EffectiveCapacity.Memory / smallestFlavorMemory)) +func (c *CapacityCalculator) calculateInstanceCapacity( + ctx context.Context, + groupData compute.FlavorGroupFeature, + az string, + hvByName map[string]hv1.Hypervisor, +) (capacity uint64, err error) { + + smallestFlavor := groupData.SmallestFlavor + smallestFlavorBytes := int64(smallestFlavor.MemoryMB) * 1024 * 1024 //nolint:gosec // flavor memory from Nova, realistically bounded + if smallestFlavorBytes <= 0 { + return 0, fmt.Errorf("smallest flavor %q has invalid memory %d MB", smallestFlavor.Name, smallestFlavor.MemoryMB) + } + + // Scheduler call: get eligible hosts (ignoring allocations and reservations). + resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ + InstanceUUID: uuid.New().String(), + ProjectID: "cortex-capacity-check", + FlavorName: smallestFlavor.Name, + MemoryMB: smallestFlavor.MemoryMB, + VCPUs: smallestFlavor.VCPUs, + FlavorExtraSpecs: smallestFlavor.ExtraSpecs, + AvailabilityZone: az, + Pipeline: c.totalPipeline, + }) + if err != nil { + return 0, fmt.Errorf("scheduler call failed: %w", err) + } + + // For each eligible host, look up HV CRD and compute multiples. + var totalCapacity uint64 + for _, hostName := range resp.Hosts { + hv, ok := hvByName[hostName] + if !ok { + LoggerFromContext(ctx).Info("scheduler host not found in hypervisor CRDs, skipping", + "host", hostName, "az", az) + continue + } + + // Use EffectiveCapacity if available, fall back to Capacity. + effectiveCap := hv.Status.EffectiveCapacity + if effectiveCap == nil { + effectiveCap = hv.Status.Capacity + } + if effectiveCap == nil { + continue + } + + memCapacity, ok := effectiveCap[hv1.ResourceMemory] + if !ok { + continue + } + + // Total: floor(effectiveCapacity / smallestFlavorMemory) + capBytes := memCapacity.Value() + if capBytes > 0 { + totalCapacity += uint64(capBytes / smallestFlavorBytes) //nolint:gosec // both values are positive, result fits uint64 + } + } + + return totalCapacity, nil +} + +// getAvailabilityZones returns a sorted, deduplicated list of AZs from Hypervisor CRDs. +// AZ is read from the topology.kubernetes.io/zone label on each Hypervisor. +func getAvailabilityZones(hvs []hv1.Hypervisor) []string { + azSet := make(map[string]struct{}) + for _, hv := range hvs { + if az, ok := hv.Labels["topology.kubernetes.io/zone"]; ok && az != "" { + azSet[az] = struct{}{} + } + } + azs := make([]string, 0, len(azSet)) + for az := range azSet { + azs = append(azs, az) + } + sort.Strings(azs) + return azs +} \ No newline at end of file diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index 937449e34..e41153cb6 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -31,6 +31,11 @@ type Config struct { // Secret ref to the database credentials for querying VM state. DatabaseSecretRef *corev1.SecretReference `json:"databaseSecretRef,omitempty"` + // ReportCapacityTotalPipeline is the pipeline used to determine eligible hosts for capacity calculation. + // This pipeline ignores VM allocations and reservations (empty datacenter scenario). + // Host resource data is then read from Hypervisor CRDs to compute actual multiples. + ReportCapacityTotalPipeline string `json:"reportCapacityTotalPipeline"` + // FlavorGroupPipelines maps flavor group names to pipeline names. // Example: {"2152": "kvm-hana-bin-packing", "2101": "kvm-general-purpose-load-balancing", "*": "kvm-general-purpose-load-balancing"} // Used to select different scheduling pipelines based on flavor group characteristics. @@ -81,6 +86,9 @@ func (c *Config) ApplyDefaults() { if c.SchedulerURL == "" { c.SchedulerURL = defaults.SchedulerURL } + if c.ReportCapacityTotalPipeline == "" { + c.ReportCapacityTotalPipeline = defaults.ReportCapacityTotalPipeline + } if c.ChangeAPIWatchReservationsTimeout == 0 { c.ChangeAPIWatchReservationsTimeout = defaults.ChangeAPIWatchReservationsTimeout } @@ -99,6 +107,7 @@ func DefaultConfig() Config { AllocationGracePeriod: 15 * time.Minute, PipelineDefault: "kvm-general-purpose-load-balancing", SchedulerURL: "http://localhost:8080/scheduler/nova/external", + ReportCapacityTotalPipeline: "kvm-report-capacity", ChangeAPIWatchReservationsTimeout: 10 * time.Second, ChangeAPIWatchReservationsPollInterval: 500 * time.Millisecond, EnableChangeCommitmentsAPI: true,