diff --git a/.claude/agents/postgres-bumper.md b/.claude/agents/postgres-bumper.md new file mode 100644 index 000000000..7c9ba6ab5 --- /dev/null +++ b/.claude/agents/postgres-bumper.md @@ -0,0 +1,194 @@ +--- +allowed-tools: Read, Write, Edit, Bash(*), WebSearch, WebFetch, Agent +description: Checks upstream docker-library/postgres for newer PG versions and debian digests, updates the Dockerfile and helm chart, and opens a PR. +--- + +# Postgres Bumper + +You are a postgres-bumper subagent. Your job is to check if the cortex-postgres Dockerfile and helm chart are up-to-date with upstream, apply any needed updates, and open a pull request. You handle both patch updates (same PG major, new minor/digest) and major upgrades (new PG major version). + +--- + +## Setup + +Before doing any work, read the `AGENTS.md` file in the repository root. Follow all conventions described there. + +--- + +## Phase 1: Determine latest upstream versions + +### 1a. Identify current values + +Read `postgres/Dockerfile` and extract: +- The current `FROM debian:-slim@sha256:` line (codename and digest) +- The current `ENV PG_MAJOR` value +- The current `ENV PG_VERSION` value + +Read `helm/library/cortex-postgres/values.yaml` and extract the current `major` value. + +### 1b. Check what major versions are available upstream + +Fetch the upstream repository structure to determine the latest available PG major: + +``` +curl -sL https://api.github.com/repos/docker-library/postgres/contents/ | jq -r '.[].name' | grep -E '^[0-9]+$' | sort -n | tail -1 +``` + +This gives the highest available major version (e.g. `18`). + +### 1c. Determine the target major + +- If a new major version exists upstream that is higher than the current PG_MAJOR, target the new major (major upgrade path). +- Otherwise, stay on the current major (patch update path). + +### 1d. Fetch the upstream Dockerfile for the target major + +Determine the debian codename used by upstream for the target major. The upstream directory contains multiple variants (e.g. bookworm, trixie, plus alpine). Select the codename deterministically by preferring the newest non-alpine Debian suite. Use this approach: + +```bash +# List available variants for the target major +VARIANTS=$(curl -sL https://api.github.com/repos/docker-library/postgres/contents/ | jq -r '.[].name' | grep -v alpine) + +# Prefer the newest Debian codename (sorted alphabetically, last is newest for current naming) +# Known Debian suites in order: bookworm (12), trixie (13), forky (14) +CODENAME=$(echo "$VARIANTS" | grep -m1 'trixie' || echo "$VARIANTS" | grep -m1 'forky' || echo "$VARIANTS" | grep -m1 'bookworm' || echo "$VARIANTS" | tail -1) +``` + +If the current Dockerfile already uses a codename that is available for the target major, prefer that codename to minimize churn. Only switch codenames when the current one is no longer available upstream. + +Then fetch the upstream Dockerfile: + +```bash +curl -sL https://raw.githubusercontent.com/docker-library/postgres/master///Dockerfile +``` + +Extract from it: +- The debian codename (from the path and FROM line) +- `ENV PG_MAJOR` value +- `ENV PG_VERSION` value + +### 1e. Get the latest debian digest + +``` +docker pull debian:-slim +docker inspect --format='{{index .RepoDigests 0}}' debian:-slim +``` + +Extract the `sha256:...` digest. + +--- + +## Phase 2: Compare and classify + +Compare current values with upstream: + +- If PG_MAJOR, PG_VERSION, and the debian digest are all unchanged → **no update needed**. Report this and stop. +- If PG_MAJOR is unchanged but PG_VERSION or digest changed → **patch update**. +- If PG_MAJOR changed → **major upgrade**. + +--- + +## Phase 3: Apply updates + +### 3a. Check for existing PR + +Before making changes, check if there's already an open PR for this: + +``` +gh pr list --head chore/bump-postgres --state open --json number,url +``` + +If one exists, report it and stop (don't create duplicates). + +### 3b. Update the Dockerfile + +For **both** patch and major updates: +1. Update the `FROM` line with the new codename (if changed) and digest. +2. Update `ENV PG_MAJOR` (if changed). +3. Update `ENV PG_VERSION` with the new version string. + +For **major upgrades** additionally: +4. Diff the upstream Dockerfile structure against ours to identify new or removed apt packages. The key differences to preserve in our Dockerfile: + - We install `gosu` via apt (`apt-get install ... gosu`) instead of downloading from GitHub releases with GPG verification. + - We do NOT set `ENV GOSU_VERSION` or download gosu binaries. +5. If the debian codename changed, update the `aptRepo` line in the postgres installation RUN command (e.g. `trixie-pgdg` → `forky-pgdg`). +6. If new system packages are needed (visible in upstream's Dockerfile), add them to the appropriate `apt-get install` block. +7. If packages were removed upstream, remove them from ours too. + +### 3c. Update the helm chart (major upgrades only) + +If PG_MAJOR changed: +1. Update `major` in `helm/library/cortex-postgres/values.yaml` to the new major (e.g. `"18"`). +2. Check each bundle chart's values.yaml (cortex-nova, cortex-manila, cortex-cinder) — if they override `cortex-postgres.major`, update those too. +3. Update the `postgres.host` documentation defaults in each bundle (e.g. `cortex-nova-postgresql-v18`). + +--- + +## Phase 4: Verify the build + +Run a docker build to confirm the image builds successfully: + +``` +docker build -t cortex-postgres-test postgres/ +``` + +If the build fails, investigate and fix. Common issues: +- Package version not yet available for the new codename +- Missing dependencies + +--- + +## Phase 5: Open a Pull Request + +1. Create branch and commit: +``` +git checkout -b chore/bump-postgres +git add postgres/Dockerfile helm/ +git commit -m "Bump postgres to PG ." +git push -u origin chore/bump-postgres +``` + +2. Use the **pull-request-creator** agent to open a PR. Provide the motivation including: + - What was updated (debian digest, PG_VERSION, PG_MAJOR) + - Old → new values + - Whether this is a patch or major upgrade + - For major upgrades, include the following IMPORTANT note prominently in the motivation so it appears in the PR description: + + IMPORTANT: This is a major PostgreSQL upgrade. The helm chart's versioned naming will create a NEW StatefulSet and Service (e.g. cortex-nova-postgresql-v18) alongside the old one (cortex-nova-postgresql-v17). The old deployment will NOT be removed automatically. After deploying this change and confirming the new instance is healthy and re-populated by the knowledge module, operators must manually delete the old StatefulSet and its PVC (e.g. `kubectl delete statefulset cortex-nova-postgresql-v17 && kubectl delete pvc data-cortex-nova-postgresql-v17-0`). + +--- + +## Phase 6: Report + +Return a structured report: + +``` +## Postgres Bumper Results + +### Update Type +[Patch / Major / No update needed] + +### Changes +- Debian codename: (or "unchanged") +- Debian digest: (or "unchanged") +- PG_MAJOR: (or "unchanged") +- PG_VERSION: (or "unchanged") +- Helm major: (or "unchanged") + +### PR +- PR #NNN: (or "skipped — already up-to-date" / "skipped — existing PR found") + +### Notes + +``` + +If no update is needed: + +``` +## Postgres Bumper Results + +No update needed. Current versions match upstream. +- PG_MAJOR: +- PG_VERSION: +- Debian: -slim@sha256: +``` diff --git a/.claude/agents/pull-request-creator.md b/.claude/agents/pull-request-creator.md new file mode 100644 index 000000000..1586719e2 --- /dev/null +++ b/.claude/agents/pull-request-creator.md @@ -0,0 +1,53 @@ +--- +name: pull-request-creator +description: Use this agent to create clean pull requests. It reviews the diff, takes an optional motivation or summary, and opens a PR with a concise description suitable for a commit message. No markdown, no file change summaries, no artificial linebreaks. +tools: Bash, Read +model: inherit +--- + +You are a pull request creator. Your job is to review the current branch's diff against the base branch, accept an optional motivation or summary from the caller, and open a clean pull request. + +## Workflow + +1. Determine the base branch (usually `main`). +2. Run `git log main..HEAD` and `git diff main...HEAD --stat` to understand what changed. +3. Read the diff carefully to understand the substance of the changes. +4. Write a PR title (imperative, under 70 characters). +5. Write a PR description following the rules below. +6. Push the branch if needed and create the PR using `gh pr create`. + +## PR Description Rules + +The description will be used directly as a commit message body. Follow these rules strictly: + +- No markdown formatting (no headers, no bold, no bullet points, no code blocks). +- No artificial linebreaks within paragraphs. Let text flow naturally. +- No file change summaries or lists of modified files. +- Concise: explain what changed and why in a few sentences. Focus on motivation and effect, not mechanics. +- End the description with a blank line followed by an Assisted-by trailer. + +## Assisted-by Trailer + +Add the following trailer at the end of the PR description, separated by a blank line. This follows the linux kernel convention for AI-assisted contributions: + +``` +Assisted-by: AGENT_NAME:MODEL_VERSION [TOOL1] [TOOL2] ... +``` + +Use your own agent name and model version, and list the tools you actually used. + +## Example Description + +``` +Refactor traits API from two-ConfigMap model to a single shim-owned ConfigMap with a Syncer interface. The Helm-managed static ConfigMap is removed; the shim now creates and owns the ConfigMap on startup and syncs from upstream placement periodically. This simplifies the deployment model and removes the merge logic that combined two sources at query time. + +Assisted-by: Claude Code:claude-opus-4-20250514 [Bash] [Read] +``` + +## Important + +- If the caller provides a motivation or summary, incorporate it into the description naturally. +- If no motivation is given, derive it from the diff. +- Never invent changes that aren't in the diff. +- Always push the branch before creating the PR. +- Use `gh pr create` with `--body` for the description. diff --git a/.claude/commands/release.md b/.claude/commands/release.md new file mode 100644 index 000000000..d770d7e24 --- /dev/null +++ b/.claude/commands/release.md @@ -0,0 +1,184 @@ +--- +allowed-tools: Read, Write, Edit, Bash(*), WebSearch, WebFetch, Agent +description: Release orchestrator — builds a digest of what changed in a release PR, opens a changelog PR, and references the bump PR. Usage: /release PR_NUMBER +--- + +# Release Orchestrator + +Your job is to orchestrate the release process for a given PR. This involves analyzing the PR's commits and changed files to build a structured digest of what changed, determining if there are any breaking changes, preparing a changelog, opening a PR to bump chart versions if needed, and updating the original PR description with the changelog and references to the new PRs. + +--- + +## Phase 1: Collect — Build the release digest + +1. Fetch PR metadata: + ``` + gh pr view $ARGUMENTS --json number,title,body,commits,files + ``` + +2. For each commit SHA in the PR, inspect the changed files: + ``` + git show --name-only --format="%H %s" + ``` + +3. Classify each commit to a component: + - Cortex shim: code touching the shim layer (internal/shim and cmd/shim) + - Cortex postgres: code touching the postgres docker image, or its helm chart + - Cortex core: core code touching anything else: the manager or external scheduler logic of cortex + - General: CI, tooling, docs, or other non-code changes + +4. Finally, read through the cortex helm charts in the helm/ folder, and check which ones have updated appVersions, indicating a new Docker image is available and that the chart should be included in the release notes. + +Produce a structured digest in this exact format — the subagents depend on it: + +``` +## Release Digest — PR #NNN "{title}" + +### Changed Charts +- cortex v1.2.3 (sha-xxxxxxxx) +- cortex-postgres v1.2.3 (sha-xxxxxxxx) +- cortex-nova v1.2.3 — includes cortex v1.2.3, cortex-postgres v1.2.3 + +### Commits by Component + +#### cortex core +- + +#### cortex postgres +- + +#### cortex shim +- + +#### General +- +``` + +**Important**: Do NOT skip or shallow this phase. Read actual file diffs. The subagents depend entirely on the quality of this digest. + +--- + +## Phase 2: Determine Breaking Changes and Prepare a Changelog + +Reason for each change by looking at the commit's diff, if it is a breaking change that requires special attention. + +**Important**: Do NOT skip or shallow this phase. Read actual file diffs. The PR reviewers depend entirely on the quality of this analysis to know what to focus on in their review. + +### When is a change "breaking"? + +A change should be classified as "breaking" if it meets any of the following criteria: + +- It changes or removes the public API of any component (e.g., CRD schemas, CLI flags, or REST API endpoints). Note: additions to the public API are not breaking. +- It requires a config format change (e.g., renaming or removing a values.yaml key, changing the expected format of a value, etc) + +Once the digest is complete, read each agent file, then dispatch all three **in parallel** using the Agent tool in a single message. Each subagent operates independently — do not wait for one before starting the others. + +### Prepare the changelog + +Generate a changelog following this template: + +```markdown +# Changelog + +## YYYY-MM-DD — [#NNN]() + +### v () + +Breaking changes: +- + +Non-breaking changes: +- + +... repeat for each changed chart ... + +### General + +Breaking changes: +- + +Non-breaking changes: +- +``` + +One `###` section per changed chart only. For bundle sections, list which library versions they include, then any bundle-specific changes (values.yaml keys, template/CRD changes). Omit `### General` if empty. No commit SHAs, one line per bullet. + +Example: +```markdown +# Changelog + +## 2026-04-24 — [#123](https://github.com/cobaltcore-dev/cortex/pull/123) + +### cortex v0.0.43 (sha-xxxxxxxx) + +Breaking changes: +- Check hypervisor resources against reservations + +Non-breaking changes: +- Commitments usage API uses postgres database instead of calling nova + +### cortex-postgres v0.5.14 (sha-xxxxxxxx) + +Non-breaking changes: +- Add commitments table migration + +### cortex-nova v0.0.56 (sha-xxxxxxxx) + +Includes updated charts cortex v0.0.43 and cortex-postgres v0.5.14. + +Non-breaking changes: +- values.yaml: added `reservations.enabled` (default: false) + +### General + +Non-breaking changes: +- Update golangci-lint to v2.1.0 +``` + +## Phase 3: Bump Chart Versions + +Prepare chart version bumps so GitHub pushes bumped charts to the registry immediately after the release PR is merged. + +For each changed library chart, patch-bump its `version` in `helm/library//Chart.yaml` (e.g. `0.0.43` → `0.1.0`), if there was no breaking change, otherwise minor-bump it. Do not touch `appVersion`. Then update the matching `dependencies[].version` entry in every `helm/bundles/*/Chart.yaml` that references it. + +### Check for existing bump PR + +Before creating a new PR, check if one already exists for this release: + +``` +gh pr list --head release/bump-charts- --state open --json number,url +``` + +- **If a PR already exists**: check out the existing `release/bump-charts-` branch, reset it to `main` (`git reset --hard origin/main`), apply the version bumps on top, force-push the branch. Then update the existing PR title and body with `gh pr edit` to reflect the latest changes. +- **If no PR exists**: create branch `release/bump-charts-` from `main`, apply the bumps, and open a new PR noting in the body that it should be merged before the release PR. Use the pull-request-creator agent for this subtask, and include the chart changes in the motivation so they are included in the PR description. + +## Phase 4: Update the PR Description + +Use `gh pr edit` with `--body` to update the PR description with the changelog. It is fine for release pull request descriptions to utilize markdown formatting. Reference the opened bump PR in the description as well as a dependency. + +## Phase 5: Create a Changelog PR + +If the CHANGELOG.md does not exist, create it with a `# Changelog` header. Then prepend the new changelog entry below the header. + +### Check for existing changelog PR + +Before creating a new PR, check if one already exists for this release: + +``` +gh pr list --head release/changelog- --state open --json number,url +``` + +- **If a PR already exists**: check out the existing `release/changelog-` branch, reset it to `main` (`git reset --hard origin/main`), apply the changelog update on top, force-push the branch. Then update the existing PR title and body with `gh pr edit` to reflect the latest changes. +- **If no PR exists**: create branch `release/changelog-` from `main`, apply the changelog, and open a new PR to `main` with title `Update changelog for release PR #` and a body noting it should be merged after the release PR. Use the pull-request-creator agent for this subtask. + +## Phase 6: Summarize — Report what happened + +After all subagents return, produce a short summary: + +``` +## Release #NNN Post-Open Summary + +- PR description updated with changelog and bump PR reference +- Bump PR #XXX opened/updated to update chart versions +- Changelog PR #YYY opened/updated to update CHANGELOG.md +``` diff --git a/.claude/commands/review-pr.md b/.claude/commands/review.md similarity index 100% rename from .claude/commands/review-pr.md rename to .claude/commands/review.md diff --git a/.claude/commands/weekly.md b/.claude/commands/weekly.md index 16e6a18dd..23256bad6 100644 --- a/.claude/commands/weekly.md +++ b/.claude/commands/weekly.md @@ -58,7 +58,7 @@ Before dispatching subagents, gather all currently open pull requests so finding ## Phase 4: Dispatch — Hand off to subagents in parallel -Dispatch both subagents **in parallel** using the Agent tool. Each subagent investigates and reports findings — they do NOT open pull requests. +Dispatch all subagents **in parallel** using the Agent tool. The bug detective and docs expert investigate and report findings — they do NOT open pull requests. The postgres bumper is self-contained and opens its own PR if an update is needed. ### Subagent 1: Bug Detective @@ -76,6 +76,15 @@ Read the instructions from `.claude/agents/docs-expert.md`. Send the agent a pro 1. The full digest from Phase 2 2. The full instructions from the docs-expert agent file +### Subagent 3: Postgres Bumper + +Use `subagent_type: "general-purpose"`. + +Read the instructions from `.claude/agents/postgres-bumper.md`. Send the agent a prompt that includes: +1. The full instructions from the postgres-bumper agent file + +This agent does NOT need the weekly digest — it checks upstream independently and opens its own PR if an update is available. + --- ## Phase 5: Deduplicate and filter findings @@ -136,6 +145,9 @@ After all work is done, produce a short summary: - Skipped (already covered by open PRs): N - PRs opened: list PR numbers/titles, or "none" +### Postgres Bumper +- Result: <"no update needed" / "patch update PR #NNN" / "major upgrade PR #NNN" / "skipped — existing PR found"> + ### Backlog (for future runs) - — <one-line description> (items that were deprioritized this run) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index d158def60..e9c45c1bc 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,2 +1,2 @@ reviews: - high_level_summary: false + high_level_summary: false \ No newline at end of file diff --git a/.github/actions/setup-claude-code-action/action.yml b/.github/actions/setup-claude-code-action/action.yml index 840fbdbd9..dbc832aba 100644 --- a/.github/actions/setup-claude-code-action/action.yml +++ b/.github/actions/setup-claude-code-action/action.yml @@ -39,7 +39,7 @@ runs: echo "$HOME/.bun/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.14" diff --git a/.github/scripts/bump-chart.sh b/.github/scripts/bump-chart.sh new file mode 100644 index 000000000..b57f7965b --- /dev/null +++ b/.github/scripts/bump-chart.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Bumps appVersion in a Helm Chart.yaml, skipping if a newer code commit already +# covers this component. +# +# Usage: bump-chart.sh <chart-yaml> <short-sha> <trigger-sha> [path-filter ...] +# +# path-filter args scope the freshness check to specific paths (e.g. postgres/). +# Omit them for an unconditional bump (the main cortex chart). +set -euo pipefail + +CHART=$1; SHORT_SHA=$2; TRIGGER_SHA=$3; shift 3 +PATHS=("$@") + +git config user.name "github-actions[bot]" +git config user.email "github-actions[bot]@users.noreply.github.com" +git fetch origin main +git reset --hard origin/main + +# Exclude bump commits ([skip ci]) so earlier steps in this same run don't +# falsely count as "newer code". Only real code commits trigger a skip. +if [ ${#PATHS[@]} -gt 0 ]; then + NEWER=$(git log --oneline --invert-grep --grep='\[skip ci\]' "$TRIGGER_SHA..HEAD" -- "${PATHS[@]}") +else + NEWER=$(git log --oneline --invert-grep --grep='\[skip ci\]' "$TRIGGER_SHA..HEAD") +fi + +if [ -n "$NEWER" ]; then + echo "Skipping $CHART: newer code commits exist on main for this component" + exit 0 +fi + +CHART_NAME=$(basename "$(dirname "$CHART")") +sed -i 's/^\([ ]*appVersion:[ ]*\).*/\1"'"$SHORT_SHA"'"/' "$CHART" +git add "$CHART" +git diff --cached --quiet && { echo "No changes to commit for $CHART_NAME"; exit 0; } +git commit -m "Bump $CHART_NAME chart appVersions to $SHORT_SHA [skip ci]" +git push origin HEAD:main diff --git a/.github/workflows/claude-release.yaml b/.github/workflows/claude-release.yaml new file mode 100644 index 000000000..216716809 --- /dev/null +++ b/.github/workflows/claude-release.yaml @@ -0,0 +1,52 @@ +name: Claude Code Release Orchestrator + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - release + +jobs: + release: + if: false # Temporarily disabled + runs-on: ubuntu-latest + concurrency: + group: changelog-release + cancel-in-progress: false + permissions: + contents: write + pull-requests: write + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - uses: ./.github/actions/setup-claude-code-action + + - uses: ./.github/actions/start-litellm-proxy + env: + AICORE_RESOURCE_GROUP: ${{ secrets.AICORE_RESOURCE_GROUP }} + AICORE_BASE_URL: ${{ secrets.AICORE_BASE_URL }} + AICORE_AUTH_URL: ${{ secrets.AICORE_AUTH_URL }} + AICORE_CLIENT_ID: ${{ secrets.AICORE_CLIENT_ID }} + AICORE_CLIENT_SECRET: ${{ secrets.AICORE_CLIENT_SECRET }} + + - uses: ./.claude-code-action + with: + claude_args: | + --max-turns 1000 + --permission-mode auto + --allowedTools "Read,Write,Edit,Bash(*),WebSearch,WebFetch,Agent" + use_litellm: "true" + litellm_model: "sap/anthropic--claude-4.6-opus" + github_token: ${{ secrets.GITHUB_TOKEN }} + show_full_output: "true" + prompt: "/release ${{ github.event.pull_request.number }}" + + - uses: ./.github/actions/stop-litellm-proxy + if: always() diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 37022816b..85021eda2 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -12,6 +12,7 @@ env: jobs: helm-lint: + if: github.event.pull_request.draft == false runs-on: ubuntu-latest steps: - name: Checkout PR diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b0bd45e41..41f83e1c3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -21,6 +21,20 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: 'go.mod' + - name: Run make crds deepcopy lint-fix + run: make crds deepcopy lint-fix + - name: Check for diff + run: | + set -e + if ! git diff --exit-code; then + echo "::error::Generated files are out of date. Run 'make crds deepcopy lint-fix' and commit the changes." + exit 1 + fi + if git ls-files --others --exclude-standard | grep -q .; then + echo "::error::Untracked files after generation:" + git ls-files --others --exclude-standard + exit 1 + fi - name: Run golangci-lint run: | set -eux diff --git a/.github/workflows/update-appversion.yml b/.github/workflows/update-appversion.yml index 20087fa80..89aba9f88 100644 --- a/.github/workflows/update-appversion.yml +++ b/.github/workflows/update-appversion.yml @@ -11,6 +11,13 @@ jobs: if: >- ${{ github.event.workflow_run.conclusion == 'success' && !contains(github.event.workflow_run.head_commit.message, '[skip ci]') }} runs-on: ubuntu-latest + # Serialize runs so concurrent merges don't race on git push. + # Example: PR A changes shim, PR B changes cortex — both trigger this + # workflow. Without serialization, one push fails and that chart bump is + # lost permanently (no future run will retry it). + concurrency: + group: update-appversion + cancel-in-progress: false steps: - name: Checkout repository uses: actions/checkout@v6 @@ -30,21 +37,6 @@ jobs: files: | postgres/** - # Only bumped if there are changes in the postgres directory. - - name: Update appVersion in cortex-postgres Chart.yaml - if: steps.changed_postgres_files.outputs.all_changed_files != '' - run: | - sed -i 's/^\([ ]*appVersion:[ ]*\).*/\1"${{ steps.vars.outputs.sha }}"/' helm/library/cortex-postgres/Chart.yaml - - name: Commit and push changes for cortex-postgres - if: steps.changed_postgres_files.outputs.all_changed_files != '' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add helm/library/cortex-postgres/Chart.yaml - git commit -m "Bump cortex-postgres chart appVersions to ${{ steps.vars.outputs.sha }} [skip ci]" || echo "No changes to commit" - git push origin HEAD:main - - # Only bumped if there are changes in shim-related directories - name: Get all changed shim files id: changed_shim_files uses: tj-actions/changed-files@v47 @@ -52,26 +44,14 @@ jobs: files: | internal/shim/** cmd/shim/** - - name: Update appVersion in cortex-shim Chart.yaml - if: steps.changed_shim_files.outputs.all_changed_files != '' - run: | - sed -i 's/^\([ ]*appVersion:[ ]*\).*/\1"${{ steps.vars.outputs.sha }}"/' helm/library/cortex-shim/Chart.yaml - - name: Commit and push changes for cortex-shim + + - name: Bump and push cortex-postgres appVersion + if: steps.changed_postgres_files.outputs.all_changed_files != '' + run: bash .github/scripts/bump-chart.sh helm/library/cortex-postgres/Chart.yaml "${{ steps.vars.outputs.sha }}" "${{ github.event.workflow_run.head_sha }}" postgres/ + + - name: Bump and push cortex-shim appVersion if: steps.changed_shim_files.outputs.all_changed_files != '' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add helm/library/cortex-shim/Chart.yaml - git commit -m "Bump cortex-shim chart appVersions to ${{ steps.vars.outputs.sha }} [skip ci]" || echo "No changes to commit" - git push origin HEAD:main + run: bash .github/scripts/bump-chart.sh helm/library/cortex-shim/Chart.yaml "${{ steps.vars.outputs.sha }}" "${{ github.event.workflow_run.head_sha }}" internal/shim/ cmd/shim/ - - name: Update appVersion in helm/library/cortex/Chart.yaml - run: | - sed -i 's/^\([ ]*appVersion:[ ]*\).*/\1"${{ steps.vars.outputs.sha }}"/' helm/library/cortex/Chart.yaml - - name: Commit and push changes for cortex - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add helm/library/cortex/Chart.yaml - git commit -m "Bump cortex chart appVersions to ${{ steps.vars.outputs.sha }} [skip ci]" || echo "No changes to commit" - git push origin HEAD:main + - name: Bump and push cortex appVersion + run: bash .github/scripts/bump-chart.sh helm/library/cortex/Chart.yaml "${{ steps.vars.outputs.sha }}" "${{ github.event.workflow_run.head_sha }}" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..e9386d04a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,74 @@ +# Changelog + +## 2026-05-04 — [#779](https://github.com/cobaltcore-dev/cortex/pull/779) + +### cortex v0.0.45 (sha-1fb35660) + +Non-breaking changes: +- Add CommittedResource CRD definition and controller that watches CommittedResource objects and manages child Reservation CRUD +- Add `AllowRejection` field to CommittedResourceSpec for controlling placement failure behavior +- Add vmware project utilization KPI tracking instances per project/flavor and capacity per host +- Move vmware resource commitments KPI to new infrastructure plugins package with shared utilities +- Move vmware host capacity KPI to infrastructure plugins package +- Add basic support for flavor groups for failover reservation with consolidation weigher +- Add `useFlavorGroupResources` values.yaml key for cortex-nova (default: false) +- Update external dependencies (controller-runtime v0.24.0, go-sqlite3 v1.14.44, zap v1.28.0) +- Alert only on new vm faults (avoid re-alerting on historical faults) + +### cortex-shim v0.1.0 (sha-d8bb12ef) + +Breaking changes: +- Remove `traits.static` values.yaml key and Helm-managed static traits ConfigMap template — traits are now fully managed by the shim at runtime via a single ConfigMap + +Non-breaking changes: +- Add per-request feature mode override via `X-Cortex-Feature-Mode` header +- Refactor /traits API to single-ConfigMap model with reusable Syncer interface pattern +- Implement feature-gated /resource_classes API with ConfigMap storage (passthrough, hybrid, crd modes) +- Add ResourceClassSyncer for periodic upstream sync into local ConfigMap +- Add `resourceClasses.configMapName` values.yaml key for configuring the resource classes ConfigMap name +- Support traits and aggregates endpoints per resource provider with three feature modes (passthrough, hybrid, crd) +- Exercise all three feature modes in placement shim e2e tests +- Fix nil pointer panic in feature mode override guard + +### cortex-postgres v0.6.0 (sha-88f03a41) + +Breaking changes: +- Upgrade PostgreSQL from 17.9 to 18.3 — resource names now include a `-v{major}` suffix for zero-downtime upgrades (e.g., `cortex-nova-postgresql-v18`). After deploy, operators must remove old StatefulSets and PVCs manually. + +Non-breaking changes: +- Add versioned resource naming with `cortex-postgres.versionedFullname` helper for zero-downtime PG major upgrades +- Add `major` values.yaml key (default: "18") to control version suffix +- Set PGDATA to subdirectory to avoid lost+found conflict + +### cortex-nova v0.0.58 (sha-1fb35660) + +Includes updated charts cortex v0.0.45 and cortex-postgres v0.6.0. + +Non-breaking changes: +- Reorganize KPI CRD templates for infrastructure dashboard metrics +- Add `useFlavorGroupResources` values.yaml key for failover reservations (default: false) +- Restructure committedResource config keys into nested objects (`committedResourceReservationController`, `committedResourceController`, `committedResourceAPI`) +- Add `committedResourceSyncInterval` config key for syncer reconciliation interval + +### cortex-placement-shim v0.1.0 (sha-d8bb12ef) + +Includes updated chart cortex-shim v0.1.0. + +Breaking changes: +- Remove `traits.static` values.yaml key (inherited from cortex-shim breaking change) + +Non-breaking changes: +- Add `resourceClasses.configMapName` values.yaml key + +### General + +Non-breaking changes: +- Fix bump-artifact workflow to handle concurrent changes on main with concurrency groups and freshness checks +- Add reusable `bump-chart.sh` script for CI chart version bumps +- Add pull-request-creator Claude agent +- Add changelog update command and workflow for release PRs +- Add linting workflow for scaffold completeness checks +- Make /release claude command idempotent +- Don't run helm-lint workflow when release PR is in draft +- Update actions/setup-python action to v6 +- Fix stale documentation: traits model, pipeline name, and API path diff --git a/Tiltfile b/Tiltfile index 87d8d026d..ef1ee3b02 100644 --- a/Tiltfile +++ b/Tiltfile @@ -83,7 +83,7 @@ local('kubectl wait --namespace cert-manager --for=condition=available deploymen ########### Dependency CRDs # Make sure the local cluster is running if you are running into startup issues here. -url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/refs/heads/main/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml' +url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/d35f2bc2c5d4fd634b17e7a8dd77ff3025758fbb/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml' local('curl -L ' + url + ' | kubectl apply -f -') ########### Cortex Manager & CRDs diff --git a/api/external/ironcore/v1alpha1/zz_generated.deepcopy.go b/api/external/ironcore/v1alpha1/zz_generated.deepcopy.go index b6e6b7bce..e098f8bfb 100644 --- a/api/external/ironcore/v1alpha1/zz_generated.deepcopy.go +++ b/api/external/ironcore/v1alpha1/zz_generated.deepcopy.go @@ -9,7 +9,7 @@ package v1alpha1 import ( corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index a401df269..e82568941 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -151,6 +151,11 @@ const ( ReserveForFailoverIntent v1alpha1.SchedulingIntent = "reserve_for_failover" // ReserveForCommittedResourceIntent indicates that the request is for CR reservation scheduling. ReserveForCommittedResourceIntent v1alpha1.SchedulingIntent = "reserve_for_committed_resource" + + // HintKeyResourceGroup is the scheduler hint key used to pass the resource group + // (e.g., flavor group name) for failover reservation scheduling. + // The weigher uses this to compare against existing reservations' ResourceGroup. + HintKeyResourceGroup = "_cortex_resource_group" ) // GetIntent analyzes the request spec and determines the intent of the scheduling request. diff --git a/api/v1alpha1/committed_resource_types.go b/api/v1alpha1/committed_resource_types.go index 5ed61a11a..31365887f 100644 --- a/api/v1alpha1/committed_resource_types.go +++ b/api/v1alpha1/committed_resource_types.go @@ -90,6 +90,18 @@ type CommittedResourceSpec struct { // +kubebuilder:validation:Enum=planned;pending;guaranteed;confirmed;superseded;expired // +kubebuilder:validation:Required State CommitmentStatus `json:"state"` + + // AllowRejection controls what the CommittedResource controller does when placement fails + // for a guaranteed or confirmed commitment. + // true — controller may reject: on failure, child Reservations are rolled back and the CR + // is marked Rejected. Use this when the caller is making a first-time placement + // decision and a "no" answer is acceptable (e.g. the change-commitments API). + // false — controller must retry: on failure, existing child Reservations are kept and the + // CR is set to Reserving so the controller retries later. Use this when the caller + // is restoring already-committed state that Cortex must honour (e.g. the syncer). + // Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + // +kubebuilder:validation:Optional + AllowRejection bool `json:"allowRejection,omitempty"` } // CommittedResourceStatus defines the observed state of CommittedResource. @@ -131,6 +143,18 @@ type CommittedResourceStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +const ( + // CommittedResourceConditionReady indicates whether the CommittedResource has been + // successfully reconciled into active Reservation CRDs. + CommittedResourceConditionReady = "Ready" + + // Condition reasons set by the CommittedResource controller. + CommittedResourceReasonAccepted = "Accepted" + CommittedResourceReasonPlanned = "Planned" + CommittedResourceReasonReserving = "Reserving" + CommittedResourceReasonRejected = "Rejected" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster diff --git a/api/v1alpha1/reservation_types.go b/api/v1alpha1/reservation_types.go index aee4a165d..988d4b97d 100644 --- a/api/v1alpha1/reservation_types.go +++ b/api/v1alpha1/reservation_types.go @@ -5,6 +5,7 @@ package v1alpha1 import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -80,6 +81,15 @@ type CommittedResourceReservationSpec struct { // +kubebuilder:validation:Optional Creator string `json:"creator,omitempty"` + // ParentGeneration is the Generation of the CommittedResource CRD at the time this + // reservation was last written by the CommittedResource controller. The Reservation + // controller echoes it to Status.CommittedResourceReservation.ObservedParentGeneration + // once it has processed the reservation, allowing the CR controller to wait until + // all child reservations are up-to-date before accepting. + // Zero means the field is not set (syncer-created reservations, no parent CR). + // +kubebuilder:validation:Optional + ParentGeneration int64 `json:"parentGeneration,omitempty"` + // Allocations maps workload identifiers to their allocation details. // Key: Workload UUID (VM UUID for Nova, Pod UID for Pods, Machine UID for IronCore, etc.) // Value: allocation state and metadata @@ -147,6 +157,12 @@ const ( // CommittedResourceReservationStatus defines the status fields specific to committed resource reservations. type CommittedResourceReservationStatus struct { + // ObservedParentGeneration is the Spec.CommittedResourceReservation.ParentGeneration value + // that this Reservation controller last processed. When it matches ParentGeneration in spec, + // the CR controller knows this reservation is up-to-date for the current CR spec version. + // +kubebuilder:validation:Optional + ObservedParentGeneration int64 `json:"observedParentGeneration,omitempty"` + // Allocations maps VM/instance UUIDs to the host they are currently running on. // Key: VM/instance UUID, Value: Host name where the VM is currently running. // +kubebuilder:validation:Optional @@ -212,6 +228,7 @@ type ReservationStatus struct { // +kubebuilder:printcolumn:name="Host",type="string",JSONPath=".status.host" // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" // +kubebuilder:printcolumn:name="ResourceGroup",type="string",JSONPath=".spec.committedResourceReservation.resourceGroup" +// +kubebuilder:printcolumn:name="HA ResourceGroup",type="string",JSONPath=".spec.failoverReservation.resourceGroup",priority=1 // +kubebuilder:printcolumn:name="Project",type="string",JSONPath=".spec.committedResourceReservation.projectID" // +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" // +kubebuilder:printcolumn:name="StartTime",type="string",JSONPath=".spec.startTime",priority=1 @@ -248,6 +265,11 @@ type ReservationList struct { Items []Reservation `json:"items"` } +// IsReady returns true if the reservation has the Ready condition set to True. +func (r *Reservation) IsReady() bool { + return meta.IsStatusConditionTrue(r.Status.Conditions, ReservationConditionReady) +} + func init() { SchemeBuilder.Register(&Reservation{}, &ReservationList{}) } diff --git a/cmd/manager/main.go b/cmd/manager/main.go index b74b21d1b..e031366f8 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -367,7 +367,7 @@ func main() { if commitmentsConfig.DatasourceName != "" { commitmentsUsageDB = commitments.NewDBUsageClient(multiclusterClient, commitmentsConfig.DatasourceName) } - commitmentsAPI := commitmentsapi.NewAPIWithConfig(multiclusterClient, commitmentsConfig, commitmentsUsageDB) + commitmentsAPI := commitmentsapi.NewAPIWithConfig(multiclusterClient, commitmentsConfig.API, commitmentsUsageDB) commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") { @@ -538,16 +538,24 @@ func main() { monitor := reservations.NewMonitor(multiclusterClient) metrics.Registry.MustRegister(&monitor) commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() - commitmentsConfig.ApplyDefaults() if err := (&commitments.CommitmentReservationController{ Client: multiclusterClient, Scheme: mgr.GetScheme(), - Conf: commitmentsConfig, + Conf: commitmentsConfig.ReservationController, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation") os.Exit(1) } + + if err := (&commitments.CommittedResourceController{ + Client: multiclusterClient, + Scheme: mgr.GetScheme(), + Conf: commitmentsConfig.CommittedResourceController, + }).SetupWithManager(mgr, multiclusterClient); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CommittedResource") + os.Exit(1) + } } if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") { setupLog.Info("enabling controller", "controller", "datasource-controllers") @@ -707,13 +715,12 @@ func main() { os.Exit(1) } - syncerMonitor := commitments.NewSyncerMonitor() - must.Succeed(metrics.Registry.Register(syncerMonitor)) if slices.Contains(mainConfig.EnabledTasks, "commitments-sync-task") { setupLog.Info("starting commitments syncer") + syncerMonitor := commitments.NewSyncerMonitor() + must.Succeed(metrics.Registry.Register(syncerMonitor)) syncer := commitments.NewSyncer(multiclusterClient, syncerMonitor) syncerConfig := conf.GetConfigOrDie[commitments.SyncerConfig]() - syncerConfig.ApplyDefaults() if err := (&task.Runner{ Client: multiclusterClient, Interval: syncerConfig.SyncInterval, diff --git a/docs/architecture.md b/docs/architecture.md index 0e91385c2..6c6ba83aa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,20 +120,19 @@ Both `hybrid` and `crd` modes require a `versioning` config block with `id`, `mi ### Traits -When `features.traits` is set to `hybrid` or `crd`, the shim serves OpenStack Placement traits from a pair of Kubernetes ConfigMaps instead of forwarding to upstream: +When `features.traits` is set to `hybrid` or `crd`, the shim serves OpenStack Placement traits from a single Kubernetes ConfigMap instead of forwarding to upstream. The ConfigMap name is set by `traits.configMapName` in the shim config and is owned by the shim. -- **Static ConfigMap** (Helm-managed): Contains the standard OpenStack traits deployed via Helm. Its name is set by `traits.configMapName` in the shim config. -- **Custom ConfigMap** (shim-managed): Stores `CUSTOM_*` traits created at runtime through PUT requests. Named `{configMapName}-custom`. +On startup, a `TraitSyncer` initializes the ConfigMap (creating it if it does not exist). In the background, the syncer periodically fetches traits from upstream placement (every 60 seconds with jitter) and writes them into the ConfigMap, keeping the local view in sync. The trait endpoints support the full OpenStack Placement traits API: -- `GET /traits` returns a sorted, merged list from both ConfigMaps, with optional filtering via the `name` query parameter (`in:TRAIT_A,TRAIT_B` or `startswith:CUSTOM_`). -- `GET /traits/{name}` checks both ConfigMaps for existence. +- `GET /traits` returns a sorted list from the ConfigMap, with optional filtering via the `name` query parameter (`in:TRAIT_A,TRAIT_B` or `startswith:CUSTOM_`). +- `GET /traits/{name}` checks the ConfigMap for existence. - `PUT /traits/{name}` creates custom traits (only `CUSTOM_*` prefixed names are allowed). - `DELETE /traits/{name}` removes custom traits. -Writes to the custom ConfigMap are serialized across replicas using a Kubernetes Lease-backed distributed lock (see `pkg/resourcelock`). This prevents concurrent writes from corrupting the ConfigMap data. +Writes to the ConfigMap are serialized across replicas using a Kubernetes Lease-backed distributed lock (see `pkg/resourcelock`). This prevents concurrent writes from corrupting the ConfigMap data. -In **hybrid** mode, `GET`, `PUT`, and `DELETE` trait requests are forwarded to upstream Placement (so upstream always has the latest data), and a **periodic sync loop** runs in the background (every 60 seconds with jitter) to fetch traits from upstream and write them into the static ConfigMap. This keeps the local view in sync with upstream and prepares for cutover to `crd` mode. In **crd** mode, traits are served exclusively from the local ConfigMaps with no upstream dependency. +In **hybrid** mode, PUT and DELETE requests are forwarded to upstream placement via the `forwardWithHook` pattern; on success, the trait is eagerly added to or removed from the local ConfigMap so the local view is immediately consistent. GET requests in hybrid mode are also forwarded to upstream. In **crd** mode, traits are served exclusively from the local ConfigMap with no upstream dependency. ### Authentication diff --git a/docs/reservations/committed-resource-reservations.md b/docs/reservations/committed-resource-reservations.md index 52890bf75..cccd55cbf 100644 --- a/docs/reservations/committed-resource-reservations.md +++ b/docs/reservations/committed-resource-reservations.md @@ -7,17 +7,20 @@ Cortex reserves hypervisor capacity for customers who pre-commit resources (comm - [Configuration and Observability](#configuration-and-observability) - [Lifecycle Management](#lifecycle-management) - [State (CRDs)](#state-crds) - - [CR Reservation Lifecycle](#cr-reservation-lifecycle) - - [VM Lifecycle](#vm-lifecycle) - - [Capacity Blocking](#capacity-blocking) + - [CR Commitment Lifecycle](#cr-commitment-lifecycle) + - [CommittedResource Controller](#committedresource-controller) + - [Reservation Lifecycle](#reservation-lifecycle) + - [VM Lifecycle](#vm-lifecycle) + - [Capacity Blocking](#capacity-blocking) + - [Reservation Controller](#reservation-controller) - [Change-Commitments API](#change-commitments-api) - [Syncer Task](#syncer-task) - - [Controller (Reconciliation)](#controller-reconciliation) - [Usage API](#usage-api) The CR reservation implementation is located in `internal/scheduling/reservations/commitments/`. Key components include: -- Controller logic (`controller.go`) -- API handlers in the `api/` subpackage (`change_commitments.go`, `report_capacity.go`, `report_usage.go`) +- `CommittedResource` controller (`committed_resource_controller.go`) — acceptance, rejection, child Reservation CRUD +- `Reservation` controller (`reservation_controller.go`) — placement, VM allocation verification +- API endpoints (`api/`) - Capacity and usage calculation logic (`capacity.go`, `usage.go`) - Syncer for periodic state sync (`syncer.go`) @@ -35,47 +38,122 @@ The CR reservation implementation is located in `internal/scheduling/reservation ## Lifecycle Management -### State (CRDs) -Defined in `api/v1alpha1/reservation_types.go`, which contains definitions for CR reservations and failover reservations (see [./failover-reservations.md](./failover-reservations.md)). - -A reservation CRD represents a single reservation slot on a hypervisor, which holds multiple VMs. -A single CR entry typically refers to multiple reservation CRDs (slots). - - -### CR Reservation Lifecycle +The system is organized around two CRD types and two controllers. `CommittedResource` CRDs represent customer commitments; `Reservation` CRDs represent individual hypervisor capacity slots. Each has its own controller with a well-defined responsibility boundary. ```mermaid flowchart LR subgraph State + CR[(CommittedResource CRDs)] Res[(Reservation CRDs)] end - + Syncer[Syncer Task] ChangeAPI[Change API] CapacityAPI[Capacity API] - Controller[Controller] + CRCtrl[CommittedResource Controller] + ResCtrl[Reservation Controller] UsageAPI[Usage API] Scheduler[Scheduler API] - - ChangeAPI -->|CRUD| Res - Syncer -->|CRUD| Res + + ChangeAPI -->|upsert + poll status| CR + Syncer -->|upsert| CR + UsageAPI -->|read| CR UsageAPI -->|read| Res CapacityAPI -->|read| Res CapacityAPI -->|capacity request| Scheduler - Res -->|watch| Controller - Controller -->|update spec/status| Res - Controller -->|reservation placement request| Scheduler + CR -->|watch| CRCtrl + CRCtrl -->|CRUD child Reservation slots| Res + CRCtrl -->|update status| CR + Res -->|watch| CRCtrl + Res -->|watch| ResCtrl + ResCtrl -->|placement request| Scheduler + ResCtrl -->|update status| Res ``` -Reservations are managed through the Change API, Syncer Task, and Controller reconciliation. +### State (CRDs) + +**`CommittedResource` CRD** (`committed_resource_types.go`) — primary source of truth for a commitment accepted by Cortex. One CRD per commitment UUID. Spec holds the commitment identity (project, flavor group, ...). Status holds the acceptance outcome (`Ready` condition with reason `Planned`/`Reserving`/`Rejected`) and the accepted amount. + +**`Reservation` CRD** (`reservation_types.go`) — a single reservation slot on a hypervisor, owned by a `CommittedResource`. One `CommittedResource` typically drives multiple `Reservation` CRDs (one per flavor-sized slot). See [./failover-reservations.md](./failover-reservations.md) for the failover reservation type. + +### CR Commitment Lifecycle + +The CR commitment lifecycle covers everything from a commitment being accepted by Limes through to Cortex confirming or rejecting it. The `CommittedResource` CRD is the entry point; the `CommittedResource` controller owns the acceptance decision. + +**Limes state → Cortex action:** + +| Limes State | Meaning | Cortex action | +|---|---|---| +| `planned` | Future start, no guarantee yet | No Reservations — capacity not blocked | +| `pending` | Limes asking for a yes/no decision now | One-shot attempt — accept or reject; no retry | +| `guaranteed` / `confirmed` | Capacity must be honoured | Place Reservations and keep them in sync; see failure handling below | +| `superseded` / `expired` | Commitment no longer active | Remove all child Reservations | + +**CommittedResource status conditions (Cortex-side):** + +```mermaid +stateDiagram-v2 + direction LR + state "Planned (Ready=False)" as Planned + state "Reserving (Ready=False)" as Reserving + state "Active (Ready=True)" as Active + state "Rejected (Ready=False)" as Rejected + + [*] --> Planned : state=planned + [*] --> Reserving : state=pending / guaranteed / confirmed + Planned --> Reserving : state changes to pending/guaranteed/confirmed + Reserving --> Active : placement succeeded + Reserving --> Rejected : placement failed — pending, or AllowRejection=true + Reserving --> Reserving : placement failed — retrying (AllowRejection=false) + Active --> Reserving : spec changed (e.g. resize) + Active --> [*] : state=superseded / expired + Rejected --> [*] : deleted + Planned --> [*] : deleted +``` + +#### CommittedResource Controller + +The controller's job is to keep child `Reservation` CRDs in sync with the desired state expressed in `Spec.Amount`. The key rules: + +- **`pending`**: Cortex is being asked for a yes/no decision. If placement fails for any reason, child Reservations are removed and the CR is marked Rejected. The caller (e.g. the change-commitments API) reads the outcome and reports back to Limes. No retry. + +- **`guaranteed` / `confirmed`**: Cortex is expected to honour the commitment. The default is to keep retrying until placement succeeds (`Ready=False, Reason=Reserving`). Callers that can accept "no" as an answer set `Spec.AllowRejection=true` (the change-commitments API sets this for confirming requests — new commitments, resizes); the controller then rejects on failure instead of retrying. + +- **On rejection**: rolls back child Reservations to the last successfully placed quantity (`Status.AcceptedAmount`). For a CR that was never accepted, this means removing all child Reservations. + +The controller communicates with the Reservation controller only through CRDs — no direct calls. + +**Reconcile trigger flow:** + +```mermaid +sequenceDiagram + participant API as Change-Commitments API + participant CRCtrl as CR Controller + participant CRCRD as CommittedResource CRD + participant ResCRD as Reservation CRD + participant ResCtrl as Reservation Controller + + API->>CRCRD: write (create/update) + CRCRD-->>CRCtrl: watch fires + CRCtrl->>ResCRD: create/update child slots + ResCRD-->>ResCtrl: watch fires + ResCtrl->>ResCRD: update (ObservedParentGeneration, Ready=True/False) + ResCRD-->>CRCtrl: watch fires (Reservation→parent CR lookup) + CRCtrl->>CRCRD: update status (Accepted / Reserving / Rejected) +``` + +### Reservation Lifecycle | Component | Event | Timing | Action | |-----------|-------|--------|--------| -| **Change API / Syncer** | CR Create, Resize, Delete | Immediate/Hourly | Create/update/delete Reservation CRDs | -| **Controller** | Placement | On creation | Find host via scheduler API, set `TargetHost` | -| **Controller** | Optimize unused slots | >> minutes | Assign PAYG VMs or re-place reservations | +| **Reservation Controller** | `Reservation` created | Immediate (watch) | Find host via scheduler API, set `TargetHost` | +| **Scheduling Pipeline** | VM Create, Migrate, Resize | Immediate | Add VM to `Spec.Allocations` | +| **Reservation Controller** | Reservation CRD updated | `committedResourceRequeueIntervalGracePeriod` (default: 1 min) | Defer verification for new VMs still spawning; update `Status.Allocations` | +| **Reservation Controller** | Hypervisor CRD updated (VM appeared/disappeared) | Immediate (event-driven) | Verify allocations via Hypervisor CRD; remove gone VMs from `Spec.Allocations` | +| **Reservation Controller** | Periodic safety-net | `committedResourceRequeueIntervalActive` (default: 5 min) | Same as above; catches any missed events | +| **Reservation Controller** | Optimize unused slots | >> minutes | Assign PAYG VMs or re-place reservations | -### VM Lifecycle +#### VM Lifecycle VM allocations are tracked within reservations: @@ -87,19 +165,12 @@ flowchart LR end A[Nova Scheduler] -->|VM Create/Migrate/Resize| B[Scheduling Pipeline] B -->|update Spec.Allocations| Res - Res -->|watch| C[Controller] + Res -->|watch| C[Reservation Controller] HV -->|watch - instance changes| C Res -->|periodic safety-net requeue| C C -->|update Spec/Status.Allocations| Res ``` -| Component | Event | Timing | Action | -|-----------|-------|--------|--------| -| **Scheduling Pipeline** | VM Create, Migrate, Resize | Immediate | Add VM to `Spec.Allocations` | -| **Controller** | Reservation CRD updated | `committedResourceRequeueIntervalGracePeriod` (default: 1 min) | Defer verification for new VMs still spawning; update `Status.Allocations` | -| **Controller** | Hypervisor CRD updated (VM appeared/disappeared) | Immediate (event-driven) | Verify allocations via Hypervisor CRD; remove gone VMs from `Spec.Allocations` | -| **Controller** | Periodic safety-net | `committedResourceRequeueIntervalActive` (default: 5 min) | Same as above; catches any missed events | - **Allocation fields**: - `Spec.Allocations` — Expected VMs (written by the scheduling pipeline on placement) - `Status.Allocations` — Confirmed VMs (written by the controller after verifying the VM is on the expected host) @@ -124,7 +195,7 @@ stateDiagram-v2 **Note**: VM allocations may not consume all resources of a reservation slot. A reservation with 128 GB may have VMs totaling only 96 GB if that fits the project's needs. Allocations may exceed reservation capacity (e.g., after VM resize). -### Capacity Blocking +#### Capacity Blocking **Blocking rules by allocation state:** @@ -161,33 +232,35 @@ When a reservation is being migrated to a new host, block the full `max(Spec.Res - **VM live migration within a reservation** (VM moves away from the reservation's host): handled implicitly by `hv.Status.Allocation`. Libvirt reports resource consumption on both source and target during live migration, so both hosts' `hv.Status.Allocation` already reflects the in-flight state. No special filter logic needed. The reservation controller will eventually remove the VM from the reservation once it's confirmed on the wrong host past the grace period. -### Change-Commitments API +#### Reservation Controller -The change-commitments API receives batched commitment changes from Limes and manages reservations accordingly. +The `Reservation` controller (`CommitmentReservationController`) watches `Reservation` CRDs and `Hypervisor` CRDs. `MaxConcurrentReconciles=1` prevents overbooking during concurrent placements. -**Request Semantics**: A request can contain multiple commitment changes across different projects and flavor groups. The semantic is **all-or-nothing** — if any commitment in the batch cannot be fulfilled (e.g., insufficient capacity), the entire request is rejected and rolled back. +**Placement** — finds hosts for new reservations (calls scheduler API) + +**Allocation Verification** — tracks VM lifecycle on reservations. The controller uses the Hypervisor CRD as the sole source of truth, with two triggers: +- New VMs (within `committedResourceAllocationGracePeriod`, default: 15 min): verification deferred — VM may still be spawning; requeued every `committedResourceRequeueIntervalGracePeriod` (default: 1 min) +- Established VMs: verified reactively when the Hypervisor CRD changes (VM appeared or disappeared in `Status.Instances`), with `committedResourceRequeueIntervalActive` (default: 5 min) as a safety-net fallback +- Missing VMs: removed from `Spec.Allocations` when not found on the Hypervisor CRD after the grace period -**Operations**: Cortex performs CRUD operations on local Reservation CRDs to match the new desired state: -- Creates new reservations for increased commitment amounts -- Deletes existing reservations for decreased commitments -- Preserves existing reservations that already have VMs allocated when possible +**Reservation migration is not supported yet.** -### Syncer Task +### Change-Commitments API -The syncer task runs periodically and syncs local Reservation CRD state to match Limes' view of commitments, correcting drift from missed API calls or restarts. +The change-commitments API receives batched commitment changes from Limes and applies them using a **write-intent, watch-for-outcome** pattern: the handler creates or updates `CommittedResource` CRDs and polls their `Status.Conditions` until each reaches a terminal state — it does not interact with `Reservation` CRDs directly. -### Controller (Reconciliation) +**Request Semantics**: A request can contain multiple commitment changes across different projects and flavor groups. The semantic is **all-or-nothing** — if any commitment in the batch cannot be fulfilled (e.g., insufficient capacity), the entire request is rejected and rolled back. -The controller watches Reservation CRDs and performs two types of reconciliation: +**Operations**: +1. For each commitment in the batch, create or update a `CommittedResource` CRD. `Spec.AllowRejection` mirrors the request's `RequiresConfirmation` flag: `true` for changes where Limes needs a yes/no answer (new commitments, resizes), `false` for non-confirming changes (deletions, status-only transitions) where Limes doesn't act on the rejection reason +2. Poll `CommittedResource.Status.Conditions[Ready]` until each reaches a terminal state: `Reason=Accepted` (success), `Reason=Planned` (deferred; accepted), or `Reason=Rejected` (failure) — only for confirming changes; non-confirming changes return immediately without polling +3. On any failure or timeout, restore all modified `CommittedResource` CRDs to their pre-request specs (or delete newly-created ones) -**Placement** - Finds hosts for new reservations (calls scheduler API) +The `CommittedResource` controller handles all downstream `Reservation` CRUD. `AllowRejection=true` tells it to reject and roll back child Reservations on placement failure rather than retrying indefinitely. -**Allocation Verification** - Tracks VM lifecycle on reservations. The controller uses the Hypervisor CRD as the sole source of truth, with two triggers: -- New VMs (within `committedResourceAllocationGracePeriod`, default: 15 min): verification deferred — VM may still be spawning; requeued every `committedResourceRequeueIntervalGracePeriod` (default: 1 min) -- Established VMs: verified reactively when the Hypervisor CRD changes (VM appeared or disappeared in `Status.Instances`), with `committedResourceRequeueIntervalActive` (default: 5 min) as a safety-net fallback -- Missing VMs: removed from `Spec.Allocations` when not found on the Hypervisor CRD after the grace period +### Syncer Task -**Reservation migration is not supported yet.** +The syncer task runs periodically and syncs local `CommittedResource` CRD state to match Limes' view of commitments, correcting drift from missed API calls or restarts. It writes `CommittedResource` CRDs only — Reservation CRUD is the controller's responsibility. ### Usage API @@ -196,4 +269,4 @@ For each flavor group `X` that accepts commitments, Cortex exposes three resourc - `hw_version_X_cores` — CPU cores derived from RAM via fixed ratio (`HandlesCommitments=false`) - `hw_version_X_instances` — instance count (`HandlesCommitments=false`) -For each VM, the API reports whether it accounts to a specific commitment or PAYG. This assignment is deterministic and may differ from the actual Cortex internal assignment used for scheduling. \ No newline at end of file +For each VM, the API reports whether it accounts to a specific commitment or PAYG. This assignment is deterministic and may differ from the actual Cortex internal assignment used for scheduling. diff --git a/docs/reservations/failover-reservations.md b/docs/reservations/failover-reservations.md index 1fa36d79d..14f785ae1 100644 --- a/docs/reservations/failover-reservations.md +++ b/docs/reservations/failover-reservations.md @@ -144,7 +144,7 @@ We use three different scheduler pipelines for failover reservations, each servi **Why:** When reusing a reservation, capacity is already reserved on the target host. We only need to verify that the VM is compatible with the host (traits, capabilities, AZ, etc.) without checking if there's enough free capacity. -### `kvm-valid-host-new-failover-reservation` +### `kvm-new-failover-reservation` **Used when:** Creating a new failover reservation. **Why:** When creating a new reservation, we need to find a host that: diff --git a/go.mod b/go.mod index a23aa4ff8..52da7fef1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cobaltcore-dev/cortex go 1.26.0 require ( - github.com/cobaltcore-dev/openstack-hypervisor-operator v1.0.2-0.20260423190401-f34871697a61 + github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.0 github.com/go-gorp/gorp v2.2.0+incompatible github.com/gophercloud/gophercloud/v2 v2.12.0 github.com/ironcore-dev/ironcore v0.3.0 @@ -14,7 +14,7 @@ require ( k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 - sigs.k8s.io/controller-runtime v0.23.3 + sigs.k8s.io/controller-runtime v0.24.0 ) require ( @@ -73,7 +73,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lib/pq v1.12.3 - github.com/mattn/go-sqlite3 v1.14.42 + github.com/mattn/go-sqlite3 v1.14.44 github.com/moby/sys/user v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect @@ -104,7 +104,7 @@ require ( go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 + go.uber.org/zap v1.28.0 go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect @@ -125,16 +125,16 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible // indirect - k8s.io/apiextensions-apiserver v0.36.0-beta.0 // indirect - k8s.io/apiserver v0.36.0-beta.0 // indirect - k8s.io/component-base v0.36.0-beta.0 // indirect + k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiserver v0.36.0 // indirect + k8s.io/component-base v0.36.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index f26c7a92b..638b047bd 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cobaltcore-dev/openstack-hypervisor-operator v1.0.2-0.20260423190401-f34871697a61 h1:I0qmFydo/Bibw0JLRypLmLnlZOx5fl4NNPaOiLKUfmU= -github.com/cobaltcore-dev/openstack-hypervisor-operator v1.0.2-0.20260423190401-f34871697a61/go.mod h1:fTJ5LAHj8NJ0AuQtsEX16Z1LXtCKqJfg+UhGfEnwImA= +github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.0 h1:XYVIKTC19dj4jck2uinYzTNXcoED5HNTvv+BJ75M2E0= +github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.0/go.mod h1:iuhqhW6ozxfYWbGlEeh9rW9xyTb/EgelkDJqzJXBclk= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -155,8 +155,8 @@ github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/majewsky/gg v1.6.0 h1:QyUP+a1YHlCRmcvAlyVhOnqdpeDQogmAygQaeGU0VPc= github.com/majewsky/gg v1.6.0/go.mod h1:KC7qUlln1VBY90OE0jXMNjXW2b9B4jJ1heYQ08OzeAg= -github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= -github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= +github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -174,8 +174,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= -github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps= +github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -259,8 +259,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -315,16 +315,16 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= -k8s.io/apiextensions-apiserver v0.36.0-beta.0 h1:KstD2m0/HuCoyKpUo3rswTzMYB6rlliCS7bxeXleXP8= -k8s.io/apiextensions-apiserver v0.36.0-beta.0/go.mod h1:puKb28c0ZYFZv/s+YYgzk7JIaOAhtaLAPnkuvVsVvkw= +k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= +k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= -k8s.io/apiserver v0.36.0-beta.0 h1:p1ow5wWRkn8e2vLhInHv1vLEgsEK0rEoCiH+d/dcuc8= -k8s.io/apiserver v0.36.0-beta.0/go.mod h1:ogUX8mFvFY1xoTFrd16A1P5IP9RM6cVPIee4L4ILOw0= +k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= +k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= -k8s.io/component-base v0.36.0-beta.0 h1:jQIJsypS9vwTg1y2YCSovj5wr3ua4yFq/FT7rhV6/7M= -k8s.io/component-base v0.36.0-beta.0/go.mod h1:WHn3mIHRrU6cfgf4vt/XsBBS0H/0eXwrDqosH4O43Ow= +k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= +k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= @@ -335,13 +335,13 @@ k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbe k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.23.1-0.20260418192536-e4a998cc6b09 h1:wGxxs0wawYxrBIc8BrwZCugW+4lx7SkPBi+70sJI7KE= -sigs.k8s.io/controller-runtime v0.23.1-0.20260418192536-e4a998cc6b09/go.mod h1:bTZXYvH6eWv12M5PgRYYTSMw/LDN1xBf0oNTycNo1YY= +sigs.k8s.io/controller-runtime v0.24.0 h1:Ck6N2LdS8Lovy1o25BB4r1xjvLEKUl1s2o9kU+KWDE4= +sigs.k8s.io/controller-runtime v0.24.0/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index eed40eef0..c1a93f75e 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.5.14 + version: 0.6.0 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-cinder/templates/secrets.yaml b/helm/bundles/cortex-cinder/templates/secrets.yaml index d7384c6a3..0d3ba89a6 100644 --- a/helm/bundles/cortex-cinder/templates/secrets.yaml +++ b/helm/bundles/cortex-cinder/templates/secrets.yaml @@ -4,7 +4,7 @@ kind: Secret metadata: name: cortex-cinder-postgres data: - host: {{ .Values.postgres.host | b64enc | quote }} + host: {{ printf "%s-v%s" (index .Values "cortex-postgres" "fullnameOverride") (index .Values "cortex-postgres" "major") | b64enc | quote }} user: {{ .Values.postgres.user | b64enc | quote }} password: {{ .Values.postgres.password | b64enc | quote }} database: {{ .Values.postgres.database | b64enc | quote }} diff --git a/helm/bundles/cortex-cinder/values.yaml b/helm/bundles/cortex-cinder/values.yaml index b3853af04..300322880 100644 --- a/helm/bundles/cortex-cinder/values.yaml +++ b/helm/bundles/cortex-cinder/values.yaml @@ -38,7 +38,7 @@ sharedSSOCert: &sharedSSOCert selfSigned: "false" postgres: - host: cortex-cinder-postgresql + host: cortex-cinder-postgresql-v18 user: postgres password: secret database: postgres @@ -138,3 +138,4 @@ cortex-knowledge-controllers: # Custom configuration for the cortex postgres chart. cortex-postgres: fullnameOverride: cortex-cinder-postgresql + major: "18" diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index ba8715c21..4972527e3 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 06413a2d6..2f97392d9 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index aac18d818..484789b26 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.5.14 + version: 0.6.0 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-manila/templates/secrets.yaml b/helm/bundles/cortex-manila/templates/secrets.yaml index 3f3e93d0c..59c21b425 100644 --- a/helm/bundles/cortex-manila/templates/secrets.yaml +++ b/helm/bundles/cortex-manila/templates/secrets.yaml @@ -4,7 +4,7 @@ kind: Secret metadata: name: cortex-manila-postgres data: - host: {{ .Values.postgres.host | b64enc | quote }} + host: {{ printf "%s-v%s" (index .Values "cortex-postgres" "fullnameOverride") (index .Values "cortex-postgres" "major") | b64enc | quote }} user: {{ .Values.postgres.user | b64enc | quote }} password: {{ .Values.postgres.password | b64enc | quote }} database: {{ .Values.postgres.database | b64enc | quote }} diff --git a/helm/bundles/cortex-manila/values.yaml b/helm/bundles/cortex-manila/values.yaml index e6be31d4b..66c1c5f2d 100644 --- a/helm/bundles/cortex-manila/values.yaml +++ b/helm/bundles/cortex-manila/values.yaml @@ -38,7 +38,7 @@ sharedSSOCert: &sharedSSOCert selfSigned: "false" postgres: - host: cortex-manila-postgresql + host: cortex-manila-postgresql-v18 user: postgres password: secret database: postgres @@ -138,3 +138,4 @@ cortex-knowledge-controllers: # Custom configuration for the cortex postgres chart. cortex-postgres: fullnameOverride: cortex-manila-postgresql + major: "18" diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index dca30d2e6..e0b941ee1 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.5.14 + version: 0.6.0 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/alerts/nova.alerts.yaml b/helm/bundles/cortex-nova/alerts/nova.alerts.yaml index 75e6deb65..f48f0cb28 100644 --- a/helm/bundles/cortex-nova/alerts/nova.alerts.yaml +++ b/helm/bundles/cortex-nova/alerts/nova.alerts.yaml @@ -505,52 +505,10 @@ groups: CRD retrieval. Limes scrapes may time out, affecting capacity reporting. # Committed Resource Syncer Alerts - - alert: CortexNovaCommittedResourceSyncerErrorsHigh - expr: increase(cortex_committed_resource_syncer_errors_total{service="cortex-nova-metrics"}[1h]) > 3 - for: 5m - labels: - context: committed-resource-syncer - dashboard: cortex-status-dashboard/cortex-status-dashboard - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Committed Resource syncer experiencing errors" - description: > - The committed resource syncer has encountered multiple errors in the last hour. - This may indicate connectivity issues with Limes, malformed API responses, - or failures writing reservation CRDs. Check the syncer logs for error details. - - - alert: CortexNovaCommittedResourceSyncerUnitMismatchRateHigh - expr: | - ( - sum(rate(cortex_committed_resource_syncer_commitments_skipped_total{service="cortex-nova-metrics", reason="unit_mismatch"}[1h])) - / sum(rate(cortex_committed_resource_syncer_commitments_total{service="cortex-nova-metrics"}[1h])) - ) > 0.05 - and on() sum(rate(cortex_committed_resource_syncer_commitments_total{service="cortex-nova-metrics"}[1h])) > 0 - for: 15m - labels: - context: committed-resource-syncer - dashboard: cortex-status-dashboard/cortex-status-dashboard - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Committed Resource syncer unit mismatch rate >5%" - description: > - More than 5% of commitments are being skipped due to unit mismatches between - Limes and Cortex flavor groups. This happens when Limes has not yet been - updated to use the new unit format after a flavor group change. The affected - commitments will keep their existing reservations until Limes notices the update. - Check the logs if this error persists for longer time. - - - alert: CortexNovaCommittedResourceSyncerUnknownFlavorGroupRateHigh - expr: | - ( - sum(rate(cortex_committed_resource_syncer_commitments_skipped_total{service="cortex-nova-metrics", reason="unknown_flavor_group"}[1h])) - / sum(rate(cortex_committed_resource_syncer_commitments_total{service="cortex-nova-metrics"}[1h])) - ) > 0 - and on() sum(rate(cortex_committed_resource_syncer_commitments_total{service="cortex-nova-metrics"}[1h])) > 0 + # These alerts only fire when the syncer is enabled (metrics are only registered when enabled). + # Absent metrics = syncer disabled = alerts inactive by design. + - alert: CortexNovaCommittedResourceSyncerNotRunning + expr: increase(cortex_committed_resource_syncer_duration_seconds_count{service="cortex-nova-metrics"}[3h]) < 1 for: 15m labels: context: committed-resource-syncer @@ -559,46 +517,15 @@ groups: severity: warning support_group: workload-management annotations: - summary: "Committed Resource syncer unknown flavor group rate >0%" + summary: "Committed Resource syncer has not run in 3 hours" description: > - Some commitments reference flavor groups that don't exist in - Cortex Knowledge (anymore). This may indicate that flavor group configuration is - out of sync between Limes and Cortex, or that Knowledge extraction is failing. - Check the flavor group Knowledge CRD and history to see what was changed. + No commitment sync has completed in the last 3 hours. The syncer runs hourly, + so at least 2 runs should appear in this window. Check that the syncer task + is healthy and Limes is reachable. - - alert: CortexNovaCommittedResourceSyncerLocalChangeRateHigh - expr: | - ( - ( - rate(cortex_committed_resource_syncer_reservations_created_total{service="cortex-nova-metrics"}[1h]) + - rate(cortex_committed_resource_syncer_reservations_deleted_total{service="cortex-nova-metrics"}[1h]) + - rate(cortex_committed_resource_syncer_reservations_repaired_total{service="cortex-nova-metrics"}[1h]) - ) / rate(cortex_committed_resource_syncer_commitments_processed_total{service="cortex-nova-metrics"}[1h]) - ) > 0.01 - and on() rate(cortex_committed_resource_syncer_commitments_processed_total{service="cortex-nova-metrics"}[1h]) > 0 - for: 15m - labels: - context: committed-resource-syncer - dashboard: cortex-status-dashboard/cortex-status-dashboard - service: cortex - severity: warning - support_group: workload-management - annotations: - summary: "Committed Resource syncer local change rate >1%" - description: > - More than 1% of synced commitments are requiring reservation changes - (creates, deletes, or repairs). This is higher than expected for steady-state - operation and may indicate data inconsistencies, external modifications to - reservations, or issues with the CRDs. Check Cortex logs for details. - - - alert: CortexNovaCommittedResourceSyncerRepairRateHigh - expr: | - ( - rate(cortex_committed_resource_syncer_reservations_repaired_total{service="cortex-nova-metrics"}[1h]) - / rate(cortex_committed_resource_syncer_commitments_processed_total{service="cortex-nova-metrics"}[1h]) - ) > 0 - and on() rate(cortex_committed_resource_syncer_commitments_processed_total{service="cortex-nova-metrics"}[1h]) > 0 - for: 15m + - alert: CortexNovaCommittedResourceSyncerErrors + expr: increase(cortex_committed_resource_syncer_errors_total{service="cortex-nova-metrics"}[1h]) > 3 + for: 5m labels: context: committed-resource-syncer dashboard: cortex-status-dashboard/cortex-status-dashboard @@ -606,16 +533,14 @@ groups: severity: warning support_group: workload-management annotations: - summary: "Committed Resource syncer repair rate >0%" + summary: "Committed Resource syncer is repeatedly failing" description: > - Some commitments have reservations that needed repair - (wrong metadata like project ID or flavor group). This may indicate data - corruption, bugs in reservation creation, or external modifications. - Reservations are automatically repaired, but the root cause should be - investigated if this alert persists. + The committed resource syncer has encountered more than 3 errors in the last + hour. Check syncer logs for details; common causes are connectivity issues + with Limes or failures writing CommittedResource CRDs. - alert: CortexNovaDoesntFindValidKVMHosts - expr: sum by (az, hvtype) (cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*"}) > 0 + expr: sum by (az, hvtype) (increase(cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*"}[5m])) > 0 for: 5m labels: context: scheduling @@ -627,7 +552,7 @@ groups: annotations: summary: "Nova scheduling cannot find valid KVM hosts" description: > - Cortex is seeing faulty vms in `{{$labels.az}}` where Nova scheduling + Cortex is seeing new faulty vms in `{{$labels.az}}` where Nova scheduling failed to find a valid `{{$labels.hvtype}}` host. This may indicate capacity issues, misconfigured filters, or resource constraints in the datacenter. Investigate the affected VMs and hypervisor availability. diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index a84989b7b..6979b0e29 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -29,20 +29,6 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI -metadata: - name: vmware-host-capacity -spec: - schedulingDomain: nova - impl: vmware_host_capacity_kpi - dependencies: - knowledges: - - name: host-details - - name: host-utilization - description: | - This KPI tracks the total, utilized, reserved and failover capacity of VMware hosts. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: KPI metadata: name: host-running-vms spec: @@ -188,14 +174,45 @@ spec: apiVersion: cortex.cloud/v1alpha1 kind: KPI metadata: - name: vmware-commitments + name: vmware-project-utilization spec: schedulingDomain: nova - impl: vmware_commitments_kpi + impl: vmware_project_utilization_kpi dependencies: datasources: - - name: limes-project-commitments + - name: nova-servers - name: nova-flavors + - name: identity-projects + knowledges: + - name: host-details + description: | + This KPI tracks the resource utilization of projects running VMs on VMware hosts. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: vmware-resource-commitments +spec: + schedulingDomain: nova + impl: vmware_resource_commitments_kpi + dependencies: + datasources: - name: nova-servers + - name: nova-flavors + - name: limes-project-commitments + description: | + This KPI tracks the resource commitments of projects running VMs on VMware hosts. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: vmware-host-capacity +spec: + schedulingDomain: nova + impl: vmware_host_capacity_kpi + dependencies: + knowledges: + - name: host-details + - name: host-utilization description: | - This KPI tracks unused VMware commitments based on project commitments and usage. \ No newline at end of file + This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. \ No newline at end of file diff --git a/helm/bundles/cortex-nova/templates/secrets.yaml b/helm/bundles/cortex-nova/templates/secrets.yaml index 382fe3e0a..50ae310ff 100644 --- a/helm/bundles/cortex-nova/templates/secrets.yaml +++ b/helm/bundles/cortex-nova/templates/secrets.yaml @@ -4,7 +4,7 @@ kind: Secret metadata: name: cortex-nova-postgres data: - host: {{ .Values.postgres.host | b64enc | quote }} + host: {{ printf "%s-v%s" (index .Values "cortex-postgres" "fullnameOverride") (index .Values "cortex-postgres" "major") | b64enc | quote }} user: {{ .Values.postgres.user | b64enc | quote }} password: {{ .Values.postgres.password | b64enc | quote }} database: {{ .Values.postgres.database | b64enc | quote }} diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index c40849739..f4e9d5725 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -38,7 +38,7 @@ sharedSSOCert: &sharedSSOCert selfSigned: "false" postgres: - host: cortex-nova-postgresql + host: cortex-nova-postgresql-v18 user: postgres password: secret database: postgres @@ -141,35 +141,39 @@ cortex-scheduling-controllers: # Number of top hosts to shuffle for evacuation requests. # Set to 0 or negative to disable shuffling. evacuationShuffleK: 3 - # CommittedResourceFlavorGroupPipelines maps flavor group IDs to pipeline names for CR reservations - # This allows different scheduling strategies per flavor group (e.g., HANA vs GP) - committedResourceFlavorGroupPipelines: - "2152": "kvm-hana-bin-packing" # HANA flavor group - "2101": "kvm-general-purpose-load-balancing" # General Purpose flavor group - "*": "kvm-general-purpose-load-balancing" # Catch-all fallback - # Default pipeline for CR reservations when no CommittedResourceFlavorGroupPipelines entry matches - committedResourcePipelineDefault: "kvm-general-purpose-load-balancing" - # How often to re-verify active reservations - # 5m = 300000000000 nanoseconds - committedResourceRequeueIntervalActive: 300000000000 - # How often to retry when knowledge is not ready - # 1m = 60000000000 nanoseconds - committedResourceRequeueIntervalRetry: 60000000000 - # Timeout for watching reservations to become ready before rolling back - # 10s = 10000000000 nanoseconds - committedResourceChangeAPIWatchReservationsTimeout: 10000000000 - # How often to poll reservation status during watch - # 500ms = 500000000 nanoseconds - committedResourceChangeAPIWatchReservationsPollInterval: 500000000 - # Whether the change-commitments API endpoint is active - # When false, the endpoint returns HTTP 503. The info endpoint remains available. - committedResourceEnableChangeCommitmentsAPI: true - # Whether the report-usage API endpoint is active - # When false, the endpoint returns HTTP 503. - committedResourceEnableReportUsageAPI: true - # Whether the report-capacity API endpoint is active - # When false, the endpoint returns HTTP 503. - committedResourceEnableReportCapacityAPI: true + committedResourceReservationController: + # Maps flavor group IDs to pipeline names; "*" acts as catch-all fallback + flavorGroupPipelines: + "2152": "kvm-hana-bin-packing" # HANA flavor group + "2101": "kvm-general-purpose-load-balancing" # General Purpose flavor group + "*": "kvm-general-purpose-load-balancing" # Catch-all fallback + # Fallback pipeline when no flavorGroupPipelines entry matches + pipelineDefault: "kvm-general-purpose-load-balancing" + # How often to re-verify active Reservation CRDs (healthy state) + requeueIntervalActive: "5m" + # Back-off interval when knowledge is unavailable + requeueIntervalRetry: "1m" + # Back-off interval while a VM allocation is still within allocationGracePeriod + requeueIntervalGracePeriod: "1m" + # How long after a VM is allocated to a reservation before it is expected to appear + # on the target host; allocations not confirmed within this window are removed + allocationGracePeriod: "15m" + # URL of the nova external scheduler API for placement decisions + schedulerURL: "http://localhost:8080/scheduler/nova/external" + committedResourceController: + # Back-off interval while CommittedResource placement is pending or failed + requeueIntervalRetry: "1m" + committedResourceAPI: + # Timeout for watching CommittedResource CRDs before rolling back + watchTimeout: "10s" + # How often to poll CommittedResource CRD conditions during watch + watchPollInterval: "500ms" + # When false, the endpoint returns HTTP 503; the info endpoint remains available. + enableChangeCommitments: true + # When false, the endpoint returns HTTP 503. + enableReportUsage: true + # When false, the endpoint returns HTTP 503. + enableReportCapacity: true # OvercommitMappings is a list of mappings that map hypervisor traits to # overcommit ratios. Note that this list is applied in order, so if there # are multiple mappings applying to the same hypervisors, the last mapping @@ -178,7 +182,7 @@ cortex-scheduling-controllers: # Failover reservations controller configuration # Name of the Datasource CRD that provides database connection info for Nova VM data datasourceName: nova-servers - # URL of the nova external scheduler API for placement decisions + # URL of the nova external scheduler API for placement decisions (used by failover controller) schedulerURL: "http://localhost:8080/scheduler/nova/external" # Maps flavor name patterns (glob) to required failover count # Example: {"hana_*": 2, "m1.xlarge": 1} @@ -203,6 +207,10 @@ cortex-scheduling-controllers: revalidationInterval: 30m # Prevents creating multiple new reservations on the same hypervisor per cycle limitOneNewReservationPerHypervisor: false + # Size failover reservations based on LargestFlavor in the flavor group + useFlavorGroupResources: false + # How often the commitments syncer reconciles Limes commitments to Reservation CRDs + committedResourceSyncInterval: "1h" cortex-knowledge-controllers: <<: *cortex @@ -226,3 +234,4 @@ cortex-knowledge-controllers: # Custom configuration for the cortex postgres chart. cortex-postgres: fullnameOverride: cortex-nova-postgresql + major: "18" diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 7cf70c100..d15e63025 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.0.3 +version: 0.1.0 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.3 + version: 0.1.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-placement-shim/templates/configmap-traits.yaml b/helm/bundles/cortex-placement-shim/templates/configmap-traits.yaml deleted file mode 100644 index b6969aaa7..000000000 --- a/helm/bundles/cortex-placement-shim/templates/configmap-traits.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if ne ((index .Values "cortex-shim").conf.features.traits | default "passthrough") "passthrough" }} -{{- $cmName := (index .Values "cortex-shim").conf.traits.configMapName }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ $cmName }} - labels: - {{- include "chart.labels" . | nindent 4 }} -data: - traits: {{ (index .Values "cortex-shim").conf.traits.static | toJson | quote }} -{{- end }} diff --git a/helm/bundles/cortex-placement-shim/values.yaml b/helm/bundles/cortex-placement-shim/values.yaml index 7e1818e9e..eaf725d07 100644 --- a/helm/bundles/cortex-placement-shim/values.yaml +++ b/helm/bundles/cortex-placement-shim/values.yaml @@ -62,9 +62,8 @@ cortex-shim: status: "CURRENT" traits: configMapName: "cortex-placement-shim-traits" - # Static traits included in every Helm install/upgrade. The shim - # merges them with dynamic CUSTOM_* traits at request time. - static: [] + resourceClasses: + configMapName: "cortex-placement-shim-resource-classes" auth: tokenCacheTTL: "5m" policies: diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 2368851c4..e5f17d322 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.57 +version: 0.0.58 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.0.44 + version: 0.0.45 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 7fdaa8932..fb2ea1e8d 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 84.0.0 + version: 84.4.0 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 7e76ef0fb..2e0ab6abb 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.5.14 -appVersion: "sha-bdda1892" +version: 0.6.0 +appVersion: "sha-88f03a41" diff --git a/helm/library/cortex-postgres/templates/_helpers.tpl b/helm/library/cortex-postgres/templates/_helpers.tpl index d3dea331c..48135f745 100644 --- a/helm/library/cortex-postgres/templates/_helpers.tpl +++ b/helm/library/cortex-postgres/templates/_helpers.tpl @@ -63,3 +63,13 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Versioned fully qualified app name (appends -v<major> to the fullname). +Truncates the base name to leave room for the suffix within the 63-char DNS limit. +*/}} +{{- define "cortex-postgres.versionedFullname" -}} +{{- $suffix := printf "-v%s" .Values.major -}} +{{- $base := include "cortex-postgres.fullname" . -}} +{{- printf "%s%s" ($base | trunc (int (sub 63 (len $suffix)))) $suffix | trimSuffix "-" }} +{{- end }} diff --git a/helm/library/cortex-postgres/templates/configmap.yaml b/helm/library/cortex-postgres/templates/configmap.yaml index 4aaa7a434..e785a30b2 100644 --- a/helm/library/cortex-postgres/templates/configmap.yaml +++ b/helm/library/cortex-postgres/templates/configmap.yaml @@ -1,6 +1,8 @@ kind: ConfigMap apiVersion: v1 metadata: - name: {{ include "cortex-postgres.fullname" . }}-max-conns + name: {{ include "cortex-postgres.versionedFullname" . }}-max-conns + labels: + {{- include "cortex-postgres.labels" . | nindent 4 }} data: max_conns.sql: "ALTER SYSTEM SET max_connections = 256;" diff --git a/helm/library/cortex-postgres/templates/secret.yaml b/helm/library/cortex-postgres/templates/secret.yaml index 58544f2ab..cf9b92388 100644 --- a/helm/library/cortex-postgres/templates/secret.yaml +++ b/helm/library/cortex-postgres/templates/secret.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ include "cortex-postgres.fullname" . }}-secret + name: {{ include "cortex-postgres.versionedFullname" . }}-secret labels: {{- include "cortex-postgres.labels" . | nindent 4 }} type: Opaque diff --git a/helm/library/cortex-postgres/templates/service.yaml b/helm/library/cortex-postgres/templates/service.yaml index 6fdc2c3e3..67937a2ca 100644 --- a/helm/library/cortex-postgres/templates/service.yaml +++ b/helm/library/cortex-postgres/templates/service.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "cortex-postgres.fullname" . }} + name: {{ include "cortex-postgres.versionedFullname" . }} labels: {{- include "cortex-postgres.labels" . | nindent 4 }} spec: @@ -16,3 +16,4 @@ spec: name: postgresql selector: {{- include "cortex-postgres.selectorLabels" . | nindent 4 }} + app.kubernetes.io/pg-major: {{ .Values.major | quote }} diff --git a/helm/library/cortex-postgres/templates/statefulset.yaml b/helm/library/cortex-postgres/templates/statefulset.yaml index de96f38e5..2a770ddf7 100644 --- a/helm/library/cortex-postgres/templates/statefulset.yaml +++ b/helm/library/cortex-postgres/templates/statefulset.yaml @@ -1,27 +1,29 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "cortex-postgres.fullname" . }} + name: {{ include "cortex-postgres.versionedFullname" . }} annotations: checksum/image: "{{ .Values.image.tag }}" - kubectl.kubernetes.io/default-container: {{ include "cortex-postgres.fullname" . }} + kubectl.kubernetes.io/default-container: {{ include "cortex-postgres.versionedFullname" . }} labels: - app: {{ include "cortex-postgres.fullname" . }} + app: {{ include "cortex-postgres.versionedFullname" . }} {{- include "cortex-postgres.labels" . | nindent 4 }} spec: selector: matchLabels: {{- include "cortex-postgres.selectorLabels" . | nindent 6 }} - serviceName: 'postgresql-svc' + app.kubernetes.io/pg-major: {{ .Values.major | quote }} + serviceName: {{ include "cortex-postgres.versionedFullname" . }} template: metadata: labels: - app: {{ include "cortex-postgres.fullname" . }} + app: {{ include "cortex-postgres.versionedFullname" . }} {{- include "cortex-postgres.labels" . | nindent 8 }} + app.kubernetes.io/pg-major: {{ .Values.major | quote }} spec: terminationGracePeriodSeconds: 10 containers: - - name: {{ include "cortex-postgres.fullname" . }} + - name: {{ include "cortex-postgres.versionedFullname" . }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} resources: @@ -35,21 +37,21 @@ spec: - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: {{ include "cortex-postgres.fullname" . }}-secret + name: {{ include "cortex-postgres.versionedFullname" . }}-secret key: postgres-password ports: - containerPort: 5432 name: postgresql volumeMounts: - name: data - mountPath: /var/lib/postgresql/data + mountPath: /var/lib/postgresql - name: max-conns mountPath: /docker-entrypoint-initdb.d/max_conns.sql subPath: max_conns.sql volumes: - name: max-conns configMap: - name: {{ include "cortex-postgres.fullname" . }}-max-conns + name: {{ include "cortex-postgres.versionedFullname" . }}-max-conns volumeClaimTemplates: - metadata: name: data diff --git a/helm/library/cortex-postgres/templates/vpa.yaml b/helm/library/cortex-postgres/templates/vpa.yaml index 011cf1b54..488549a8d 100644 --- a/helm/library/cortex-postgres/templates/vpa.yaml +++ b/helm/library/cortex-postgres/templates/vpa.yaml @@ -5,22 +5,22 @@ apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: - name: {{ include "cortex-postgres.fullname" . }}-vpa + name: {{ include "cortex-postgres.versionedFullname" . }}-vpa labels: - app: {{ include "cortex-postgres.fullname" . }} + app: {{ include "cortex-postgres.versionedFullname" . }} {{- include "cortex-postgres.labels" . | nindent 4 }} spec: targetRef: apiVersion: "apps/v1" - kind: "Deployment" - name: {{ include "cortex-postgres.fullname" . }} + kind: "StatefulSet" + name: {{ include "cortex-postgres.versionedFullname" . }} updatePolicy: updateMode: "Initial" minReplicas: 1 maxReplicas: 1 resourcePolicy: containerPolicies: - - containerName: {{ include "cortex-postgres.fullname" . }} + - containerName: {{ include "cortex-postgres.versionedFullname" . }} mode: "Auto" minAllowed: cpu: {{ .Values.resources.requests.cpu }} diff --git a/helm/library/cortex-postgres/values.yaml b/helm/library/cortex-postgres/values.yaml index d318ee97e..2d4cb4b40 100644 --- a/helm/library/cortex-postgres/values.yaml +++ b/helm/library/cortex-postgres/values.yaml @@ -3,6 +3,8 @@ fullnameOverride: cortex-postgresql +major: "18" + image: repository: ghcr.io/cobaltcore-dev/cortex-postgres pullPolicy: IfNotPresent diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index b95c5c282..df4c688e6 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.0.3 -appVersion: "sha-166b515f" +version: 0.1.0 +appVersion: "sha-d8bb12ef" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 84bf46232..463af7d34 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.0.44 -appVersion: "sha-28311dec" +version: 0.0.45 +appVersion: "sha-07d25e89" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml new file mode 100644 index 000000000..092827edd --- /dev/null +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml @@ -0,0 +1,279 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: committedresources.cortex.cloud +spec: + group: cortex.cloud + names: + kind: CommittedResource + listKind: CommittedResourceList + plural: committedresources + singular: committedresource + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.projectID + name: Project + type: string + - jsonPath: .spec.flavorGroupName + name: FlavorGroup + type: string + - jsonPath: .spec.resourceType + name: ResourceType + type: string + - jsonPath: .spec.availabilityZone + name: AZ + type: string + - jsonPath: .spec.amount + name: Amount + type: string + - jsonPath: .status.acceptedAmount + name: AcceptedAmount + type: string + - jsonPath: .status.usedAmount + name: UsedAmount + type: string + - jsonPath: .spec.state + name: State + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .spec.startTime + name: StartTime + priority: 1 + type: date + - jsonPath: .spec.endTime + name: EndTime + priority: 1 + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CommittedResource is the Schema for the committedresources API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CommittedResourceSpec defines the desired state of CommittedResource, + properties: + allowRejection: + description: |- + AllowRejection controls what the CommittedResource controller does when placement fails + for a guaranteed or confirmed commitment. + true — controller may reject: on failure, child Reservations are rolled back and the CR + is marked Rejected. Use this when the caller is making a first-time placement + decision and a "no" answer is acceptable (e.g. the change-commitments API). + false — controller must retry: on failure, existing child Reservations are kept and the + CR is set to Reserving so the controller retries later. Use this when the caller + is restoring already-committed state that Cortex must honour (e.g. the syncer). + Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + type: boolean + amount: + anyOf: + - type: integer + - type: string + description: |- + Amount is the total committed quantity. + memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi"). + cores: integer core count (e.g. "40"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + availabilityZone: + description: AvailabilityZone specifies the availability zone for + this commitment. + type: string + commitmentUUID: + description: UUID of the commitment this resource corresponds to. + type: string + confirmedAt: + description: ConfirmedAt is when the commitment was confirmed. + format: date-time + type: string + domainID: + description: DomainID of the OpenStack domain this commitment belongs + to. + type: string + endTime: + description: EndTime is when Reservation slots expire. Nil for unbounded + commitments with no expiry. + format: date-time + type: string + flavorGroupName: + description: FlavorGroupName identifies the flavor group this commitment + targets, e.g. "kvm_v2_hana_s". + type: string + projectID: + description: ProjectID of the OpenStack project this commitment belongs + to. + type: string + resourceType: + description: 'ResourceType identifies the kind of resource committed: + memory drives Reservation slots; cores uses an arithmetic check + only.' + enum: + - memory + - cores + type: string + schedulingDomain: + description: SchedulingDomain specifies the scheduling domain for + this committed resource (e.g., "nova", "ironcore"). + type: string + startTime: + description: |- + StartTime is the activation time for Reservation slots. + Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones. + format: date-time + type: string + state: + description: State is the lifecycle state of the commitment. + enum: + - planned + - pending + - guaranteed + - confirmed + - superseded + - expired + type: string + required: + - amount + - availabilityZone + - commitmentUUID + - domainID + - flavorGroupName + - projectID + - resourceType + - state + type: object + status: + description: CommittedResourceStatus defines the observed state of CommittedResource. + properties: + acceptedAmount: + anyOf: + - type: integer + - type: string + description: |- + AcceptedAmount is the quantity the controller last successfully provisioned as Reservation slots. + Nil if the spec has never been successfully reconciled. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + acceptedAt: + description: AcceptedAt is when the controller last successfully reconciled + the spec into Reservation slots. + format: date-time + type: string + assignedVMs: + description: |- + AssignedVMs holds the UUIDs of VMs deterministically assigned to this committed resource. + Populated by the usage reconciler; used to compute UsedAmount and drive the quota controller. + items: + type: string + type: array + conditions: + description: Conditions holds the current status conditions. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + lastChanged: + description: |- + LastChanged is when the spec was last written by the syncer. + When AcceptedAt is older than LastChanged, the controller has pending work. + format: date-time + type: string + lastReconcileAt: + description: LastReconcileAt is when the controller last ran its reconcile + loop for this resource. + format: date-time + type: string + lastUsageReconcileAt: + description: LastUsageReconcileAt is when the usage reconciler last + updated AssignedVMs and UsedAmount. + format: date-time + type: string + usedAmount: + anyOf: + - type: integer + - type: string + description: |- + UsedAmount is the sum of assigned VM resources expressed in the same units as Spec.Amount. + Populated by the usage reconciler. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml index 8ab0fade9..a30b7d221 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml @@ -27,6 +27,10 @@ spec: - jsonPath: .spec.committedResourceReservation.resourceGroup name: ResourceGroup type: string + - jsonPath: .spec.failoverReservation.resourceGroup + name: HA ResourceGroup + priority: 1 + type: string - jsonPath: .spec.committedResourceReservation.projectID name: Project type: string @@ -134,6 +138,16 @@ spec: type: string domainID: type: string + parentGeneration: + description: |- + ParentGeneration is the Generation of the CommittedResource CRD at the time this + reservation was last written by the CommittedResource controller. The Reservation + controller echoes it to Status.CommittedResourceReservation.ObservedParentGeneration + once it has processed the reservation, allowing the CR controller to wait until + all child reservations are up-to-date before accepting. + Zero means the field is not set (syncer-created reservations, no parent CR). + format: int64 + type: integer projectID: type: string resourceGroup: @@ -208,6 +222,13 @@ spec: Allocations maps VM/instance UUIDs to the host they are currently running on. Key: VM/instance UUID, Value: Host name where the VM is currently running. type: object + observedParentGeneration: + description: |- + ObservedParentGeneration is the Spec.CommittedResourceReservation.ParentGeneration value + that this Reservation controller last processed. When it matches ParentGeneration in spec, + the CR controller knows this reservation is up-to-date for the current CR spec version. + format: int64 + type: integer type: object conditions: description: |- diff --git a/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm.go b/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm.go index 7d3bb33ef..4a4040fd5 100644 --- a/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm.go +++ b/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm.go @@ -9,9 +9,7 @@ import ( "strconv" "strings" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -177,8 +175,7 @@ func aggregateReservationsByHost(reservations []v1alpha1.Reservation) ( continue } - readyCondition := meta.FindStatusCondition(reservation.Status.Conditions, v1alpha1.ReservationConditionReady) - if readyCondition == nil || readyCondition.Status != metav1.ConditionTrue { + if !reservation.IsReady() { continue } diff --git a/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm_test.go b/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm_test.go index 6e9d38c7b..c233cfd4c 100644 --- a/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm_test.go +++ b/internal/knowledge/kpis/plugins/compute/resource_capacity_kvm_test.go @@ -4,6 +4,7 @@ package compute import ( + "regexp" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -38,6 +39,16 @@ type kvmMetricLabels struct { Maintenance string } +var fqNameRe = regexp.MustCompile(`fqName: "([^"]+)"`) + +func getMetricName(desc string) string { + match := fqNameRe.FindStringSubmatch(desc) + if len(match) > 1 { + return match[1] + } + return "" +} + type kvmExpectedMetric struct { Name string // metric family name (e.g. "cortex_kvm_host_capacity_total") Labels kvmMetricLabels diff --git a/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware.go b/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware.go deleted file mode 100644 index 8bd2d4177..000000000 --- a/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package compute - -import ( - "context" - "log/slog" - "strconv" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" -) - -type VMwareResourceCapacityKPI struct { - // Common base for all KPIs that provides standard functionality. - plugins.BaseKPI[struct{}] // No options passed through yaml config - - availableCapacityPerHost *prometheus.Desc - totalCapacityPerHost *prometheus.Desc -} - -func (VMwareResourceCapacityKPI) GetName() string { - return "vmware_host_capacity_kpi" -} - -func (k *VMwareResourceCapacityKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { - if err := k.BaseKPI.Init(db, client, opts); err != nil { - return err - } - k.availableCapacityPerHost = prometheus.NewDesc( - "cortex_vmware_host_capacity_available", - "Available capacity per resource on the hosts currently (individually by host).", - []string{ - "compute_host", - "resource", - "availability_zone", - "cpu_architecture", - "workload_type", - "enabled", - "decommissioned", - "external_customer", - "pinned_projects", - "disabled_reason", - "pinned_project_ids", - }, - nil, - ) - k.totalCapacityPerHost = prometheus.NewDesc( - "cortex_vmware_host_capacity_total", - "Total resources available on the hosts currently (individually by host).", - []string{ - "compute_host", - "resource", - "availability_zone", - "cpu_architecture", - "workload_type", - "enabled", - "decommissioned", - "external_customer", - "pinned_projects", - "pinned_project_ids", - }, - nil, - ) - return nil -} - -func (k *VMwareResourceCapacityKPI) Describe(ch chan<- *prometheus.Desc) { - ch <- k.availableCapacityPerHost - ch <- k.totalCapacityPerHost -} - -func (k *VMwareResourceCapacityKPI) Collect(ch chan<- prometheus.Metric) { - hostDetailsKnowledge := &v1alpha1.Knowledge{} - if err := k.Client.Get( - context.Background(), - client.ObjectKey{Name: "host-details"}, - hostDetailsKnowledge, - ); err != nil { - slog.Error("failed to get knowledge host-details", "err", err) - return - } - hostDetails, err := v1alpha1. - UnboxFeatureList[compute.HostDetails](hostDetailsKnowledge.Status.Raw) - if err != nil { - slog.Error("failed to unbox storage pool cpu usage", "err", err) - return - } - detailsByComputeHost := make(map[string]compute.HostDetails) - for _, detail := range hostDetails { - detailsByComputeHost[detail.ComputeHost] = detail - } - - hostUtilizationKnowledge := &v1alpha1.Knowledge{} - if err := k.Client.Get( - context.Background(), - client.ObjectKey{Name: "host-utilization"}, - hostUtilizationKnowledge, - ); err != nil { - slog.Error("failed to get knowledge host-utilization", "err", err) - return - } - hostUtilizations, err := v1alpha1. - UnboxFeatureList[compute.HostUtilization](hostUtilizationKnowledge.Status.Raw) - if err != nil { - slog.Error("failed to unbox host utilization", "err", err) - return - } - - for _, utilization := range hostUtilizations { - detail, exists := detailsByComputeHost[utilization.ComputeHost] - if !exists { - slog.Warn("host_available_capacity: missing host details for compute host", "compute_host", utilization.ComputeHost) - continue - } - if detail.HypervisorType == "ironic" { - continue // Ironic hosts do not run VMs/instances - } - - if detail.HypervisorFamily != "vmware" { - continue - } - - if utilization.TotalRAMAllocatableMB == 0 || utilization.TotalVCPUsAllocatable == 0 || utilization.TotalDiskAllocatableGB == 0 { - slog.Info( - "Skipping host since placement is reporting zero allocatable resources", - "metric", "cortex_available_capacity_per_host", - "host", utilization.ComputeHost, - "cpu", utilization.TotalVCPUsAllocatable, - "ram", utilization.TotalRAMAllocatableMB, - "disk", utilization.TotalDiskAllocatableGB, - ) - continue - } - - availableCPUs := float64(utilization.TotalVCPUsAllocatable - utilization.VCPUsUsed) - availableRAMMB := float64(utilization.TotalRAMAllocatableMB - utilization.RAMUsedMB) - availableDiskGB := float64(utilization.TotalDiskAllocatableGB - utilization.DiskUsedGB) - - k.exportCapacityMetricVMware(ch, "cpu", availableCPUs, utilization.TotalVCPUsAllocatable, detail) - k.exportCapacityMetricVMware(ch, "ram", availableRAMMB, utilization.TotalRAMAllocatableMB, detail) - k.exportCapacityMetricVMware(ch, "disk", availableDiskGB, utilization.TotalDiskAllocatableGB, detail) - } -} - -func (k *VMwareResourceCapacityKPI) exportCapacityMetricVMware(ch chan<- prometheus.Metric, resource string, available, total float64, host compute.HostDetails) { - enabled := strconv.FormatBool(host.Enabled) - decommissioned := strconv.FormatBool(host.Decommissioned) - externalCustomer := strconv.FormatBool(host.ExternalCustomer) - pinnedProjectIds := "" - pinnedProjects := "false" - if host.PinnedProjects != nil { - pinnedProjectIds = *host.PinnedProjects - pinnedProjects = "true" - } - - disabledReason := "-" - if host.DisabledReason != nil { - disabledReason = *host.DisabledReason - } - - ch <- prometheus.MustNewConstMetric( - k.availableCapacityPerHost, - prometheus.GaugeValue, - available, - host.ComputeHost, - resource, - host.AvailabilityZone, - host.CPUArchitecture, - host.WorkloadType, - enabled, - decommissioned, - externalCustomer, - pinnedProjects, - disabledReason, - pinnedProjectIds, - ) - - ch <- prometheus.MustNewConstMetric( - k.totalCapacityPerHost, - prometheus.GaugeValue, - total, - host.ComputeHost, - resource, - host.AvailabilityZone, - host.CPUArchitecture, - host.WorkloadType, - enabled, - decommissioned, - externalCustomer, - pinnedProjects, - pinnedProjectIds, - ) -} diff --git a/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware_test.go b/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware_test.go deleted file mode 100644 index 875be6357..000000000 --- a/internal/knowledge/kpis/plugins/compute/resource_capacity_vmware_test.go +++ /dev/null @@ -1,503 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package compute - -import ( - "reflect" - "regexp" - "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/pkg/conf" - testlib "github.com/cobaltcore-dev/cortex/pkg/testing" - "github.com/prometheus/client_golang/prometheus" - prometheusgo "github.com/prometheus/client_model/go" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestVMwareResourceCapacityKPI_Init(t *testing.T) { - kpi := &VMwareResourceCapacityKPI{} - if err := kpi.Init(nil, nil, conf.NewRawOpts("{}")); err != nil { - t.Fatalf("expected no error, got %v", err) - } -} - -var fqNameRe = regexp.MustCompile(`fqName: "([^"]+)"`) - -func getMetricName(desc string) string { - match := fqNameRe.FindStringSubmatch(desc) - if len(match) > 1 { - return match[1] - } - return "" -} - -func TestVMwareResourceCapacityKPI_Collect_AbsoluteMetric(t *testing.T) { - scheme, err := v1alpha1.SchemeBuilder.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hostDetails, err := v1alpha1.BoxFeatureList([]any{ - &compute.HostDetails{ - ComputeHost: "vmware-host", - AvailabilityZone: "az1", - CPUArchitecture: "cascade-lake", - HypervisorType: "vcenter", - HypervisorFamily: "vmware", - WorkloadType: "general-purpose", - Enabled: true, - Decommissioned: true, - ExternalCustomer: true, - DisabledReason: nil, - PinnedProjects: nil, - }, - // Skip this because it's not a VMware host - &compute.HostDetails{ - ComputeHost: "kvm-host", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "qemu", - HypervisorFamily: "kvm", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1,project2"), - }, - // Skip this because placement doesn't report any capacity for this host - &compute.HostDetails{ - ComputeHost: "vmware-host-2", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "qemu", - HypervisorFamily: "vmware", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1,project2"), - }, - // Skip this because it's a ironic host - &compute.HostDetails{ - ComputeHost: "ironic-host", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "ironic", - HypervisorFamily: "vmware", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1"), - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hostUtilizations, err := v1alpha1.BoxFeatureList([]any{ - &compute.HostUtilization{ - ComputeHost: "vmware-host", - TotalVCPUsAllocatable: 100, - TotalRAMAllocatableMB: 200, - TotalDiskAllocatableGB: 300, - VCPUsUsed: 40, - RAMUsedMB: 40, - DiskUsedGB: 40, - }, - &compute.HostUtilization{ - ComputeHost: "kvm-host", - TotalVCPUsAllocatable: 100, - TotalRAMAllocatableMB: 100, - TotalDiskAllocatableGB: 100, - VCPUsUsed: 75, - RAMUsedMB: 80, - DiskUsedGB: 85, - }, - &compute.HostUtilization{ - ComputeHost: "ironic-host", - TotalVCPUsAllocatable: 0, - TotalRAMAllocatableMB: 0, - TotalDiskAllocatableGB: 0, - VCPUsUsed: 0, - RAMUsedMB: 0, - DiskUsedGB: 0, - }, - // No Capacity reported for host kvm-host-2 - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - kpi := &VMwareResourceCapacityKPI{} - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(&v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: "host-details"}, - Status: v1alpha1.KnowledgeStatus{Raw: hostDetails}, - }, &v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: "host-utilization"}, - Status: v1alpha1.KnowledgeStatus{Raw: hostUtilizations}, - }). - Build() - if err := kpi.Init(nil, client, conf.NewRawOpts("{}")); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan prometheus.Metric, 100) - kpi.Collect(ch) - close(ch) - - type HostResourceMetric struct { - ComputeHost string - Resource string - AvailabilityZone string - Enabled string - Decommissioned string - ExternalCustomer string - CPUArchitecture string - WorkloadType string - DisabledReason string - PinnedProjects string - PinnedProjectIds string - Value float64 - } - - actualMetrics := make(map[string]HostResourceMetric, 0) - - for metric := range ch { - desc := metric.Desc().String() - metricName := getMetricName(desc) - - // Only consider cortex_vmware_host_capacity_available metric in this test - if metricName != "cortex_vmware_host_capacity_available" { - continue - } - - var m prometheusgo.Metric - if err := metric.Write(&m); err != nil { - t.Fatalf("failed to write metric: %v", err) - } - - labels := make(map[string]string) - for _, label := range m.Label { - labels[label.GetName()] = label.GetValue() - } - - key := labels["compute_host"] + "-" + labels["resource"] - - actualMetrics[key] = HostResourceMetric{ - ComputeHost: labels["compute_host"], - Resource: labels["resource"], - AvailabilityZone: labels["availability_zone"], - Enabled: labels["enabled"], - Decommissioned: labels["decommissioned"], - ExternalCustomer: labels["external_customer"], - CPUArchitecture: labels["cpu_architecture"], - WorkloadType: labels["workload_type"], - DisabledReason: labels["disabled_reason"], - PinnedProjects: labels["pinned_projects"], - PinnedProjectIds: labels["pinned_project_ids"], - Value: m.GetGauge().GetValue(), - } - } - - expectedMetrics := map[string]HostResourceMetric{ - "vmware-host-cpu": { - ComputeHost: "vmware-host", - Resource: "cpu", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - DisabledReason: "-", - PinnedProjects: "false", - PinnedProjectIds: "", - Value: 60, // 100 - 40 - }, - "vmware-host-ram": { - ComputeHost: "vmware-host", - Resource: "ram", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - DisabledReason: "-", - PinnedProjects: "false", - PinnedProjectIds: "", - Value: 160, // 200 - 40 - }, - "vmware-host-disk": { - ComputeHost: "vmware-host", - Resource: "disk", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - DisabledReason: "-", - PinnedProjects: "false", - PinnedProjectIds: "", - Value: 260, // 300 - 40 - }, - } - - if len(expectedMetrics) != len(actualMetrics) { - t.Errorf("expected %d metrics, got %d", len(expectedMetrics), len(actualMetrics)) - } - - for key, expected := range expectedMetrics { - actual, ok := actualMetrics[key] - if !ok { - t.Errorf("expected metric %q not found", key) - continue - } - - if !reflect.DeepEqual(expected, actual) { - t.Errorf("metric %q: expected %+v, got %+v", key, expected, actual) - } - } -} - -func TestVMwareResourceCapacityKPI_Collect_TotalMetric(t *testing.T) { - scheme, err := v1alpha1.SchemeBuilder.Build() - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hostDetails, err := v1alpha1.BoxFeatureList([]any{ - &compute.HostDetails{ - ComputeHost: "vmware-host", - AvailabilityZone: "az1", - CPUArchitecture: "cascade-lake", - HypervisorType: "vcenter", - HypervisorFamily: "vmware", - WorkloadType: "general-purpose", - Enabled: true, - Decommissioned: true, - ExternalCustomer: true, - DisabledReason: nil, - PinnedProjects: testlib.Ptr("project1,project2"), - }, - // Skip this because it's not a VMware host - &compute.HostDetails{ - ComputeHost: "kvm-host", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "qemu", - HypervisorFamily: "kvm", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1,project2"), - }, - // Skip this because placement doesn't report any capacity for this host - &compute.HostDetails{ - ComputeHost: "vmware-host-2", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "qemu", - HypervisorFamily: "vmware", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1,project2"), - }, - // Skip this because it's a ironic host - &compute.HostDetails{ - ComputeHost: "ironic-host", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "ironic", - HypervisorFamily: "vmware", - WorkloadType: "hana", - Enabled: false, - Decommissioned: false, - ExternalCustomer: false, - DisabledReason: testlib.Ptr("test"), - PinnedProjects: testlib.Ptr("project1"), - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - hostUtilizations, err := v1alpha1.BoxFeatureList([]any{ - &compute.HostUtilization{ - ComputeHost: "vmware-host", - TotalVCPUsAllocatable: 100, - TotalRAMAllocatableMB: 200, - TotalDiskAllocatableGB: 300, - VCPUsUsed: 40, - RAMUsedMB: 40, - DiskUsedGB: 40, - }, - &compute.HostUtilization{ - ComputeHost: "kvm-host", - TotalVCPUsAllocatable: 100, - TotalRAMAllocatableMB: 100, - TotalDiskAllocatableGB: 100, - VCPUsUsed: 75, - RAMUsedMB: 80, - DiskUsedGB: 85, - }, - &compute.HostUtilization{ - ComputeHost: "ironic-host", - TotalVCPUsAllocatable: 0, - TotalRAMAllocatableMB: 0, - TotalDiskAllocatableGB: 0, - VCPUsUsed: 0, - RAMUsedMB: 0, - DiskUsedGB: 0, - }, - // No Capacity reported for host kvm-host-2 - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - kpi := &VMwareResourceCapacityKPI{} - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects(&v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: "host-details"}, - Status: v1alpha1.KnowledgeStatus{Raw: hostDetails}, - }, &v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: "host-utilization"}, - Status: v1alpha1.KnowledgeStatus{Raw: hostUtilizations}, - }). - Build() - if err := kpi.Init(nil, client, conf.NewRawOpts("{}")); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan prometheus.Metric, 100) - kpi.Collect(ch) - close(ch) - - type HostResourceMetric struct { - ComputeHost string - Resource string - AvailabilityZone string - Enabled string - Decommissioned string - ExternalCustomer string - CPUArchitecture string - WorkloadType string - PinnedProjects string - PinnedProjectIds string - Value float64 - } - - actualMetrics := make(map[string]HostResourceMetric, 0) - - for metric := range ch { - desc := metric.Desc().String() - metricName := getMetricName(desc) - - // Only consider cortex_vmware_host_capacity_total metric in this test - if metricName != "cortex_vmware_host_capacity_total" { - continue - } - - var m prometheusgo.Metric - if err := metric.Write(&m); err != nil { - t.Fatalf("failed to write metric: %v", err) - } - - labels := make(map[string]string) - for _, label := range m.Label { - labels[label.GetName()] = label.GetValue() - } - - key := labels["compute_host"] + "-" + labels["resource"] - - actualMetrics[key] = HostResourceMetric{ - ComputeHost: labels["compute_host"], - Resource: labels["resource"], - AvailabilityZone: labels["availability_zone"], - Enabled: labels["enabled"], - Decommissioned: labels["decommissioned"], - ExternalCustomer: labels["external_customer"], - CPUArchitecture: labels["cpu_architecture"], - WorkloadType: labels["workload_type"], - PinnedProjects: labels["pinned_projects"], - PinnedProjectIds: labels["pinned_project_ids"], - Value: m.GetGauge().GetValue(), - } - } - - expectedMetrics := map[string]HostResourceMetric{ - "vmware-host-cpu": { - ComputeHost: "vmware-host", - Resource: "cpu", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - PinnedProjects: "true", - PinnedProjectIds: "project1,project2", - Value: 100, - }, - "vmware-host-ram": { - ComputeHost: "vmware-host", - Resource: "ram", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - PinnedProjects: "true", - PinnedProjectIds: "project1,project2", - Value: 200, - }, - "vmware-host-disk": { - ComputeHost: "vmware-host", - Resource: "disk", - AvailabilityZone: "az1", - Enabled: "true", - Decommissioned: "true", - ExternalCustomer: "true", - CPUArchitecture: "cascade-lake", - WorkloadType: "general-purpose", - PinnedProjects: "true", - PinnedProjectIds: "project1,project2", - Value: 300, - }, - } - - if len(expectedMetrics) != len(actualMetrics) { - t.Errorf("expected %d metrics, got %d", len(expectedMetrics), len(actualMetrics)) - } - - for key, expected := range expectedMetrics { - actual, ok := actualMetrics[key] - if !ok { - t.Errorf("expected metric %q not found", key) - continue - } - - if !reflect.DeepEqual(expected, actual) { - t.Errorf("metric %q: expected %+v, got %+v", key, expected, actual) - } - } -} diff --git a/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware.go b/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware.go deleted file mode 100644 index 74cde06a0..000000000 --- a/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package compute - -import ( - "log/slog" - "strings" - - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type VMwareResourceCommitmentsKPI struct { - // Common base for all KPIs that provides standard functionality. - plugins.BaseKPI[struct{}] // No options passed through yaml config - - unusedInstanceCommitments *prometheus.Desc -} - -func (VMwareResourceCommitmentsKPI) GetName() string { - return "vmware_commitments_kpi" -} - -func (k *VMwareResourceCommitmentsKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { - if err := k.BaseKPI.Init(db, client, opts); err != nil { - return err - } - k.unusedInstanceCommitments = prometheus.NewDesc( - "cortex_vmware_hana_unused_instance_commitments", - "Unused instance commitment capacity summed across all projects (vcpus / ram_mb / disk_gb).", - []string{ - "resource", // "cpu", "ram", "disk" - "availability_zone", - "cpu_architecture", // "sapphire-rapids" (_v2 suffix) or "cascade-lake" - }, - nil, - ) - return nil -} - -func (k *VMwareResourceCommitmentsKPI) Describe(ch chan<- *prometheus.Desc) { - ch <- k.unusedInstanceCommitments -} - -func (k *VMwareResourceCommitmentsKPI) Collect(ch chan<- prometheus.Metric) { - k.collectUnusedCommitments(ch) -} - -// getRunningHANAServers loads all running HANA servers from the database. We consider a server "running" if its status is not DELETED or ERROR. -func (k *VMwareResourceCommitmentsKPI) getRunningHANAServers() ([]nova.Server, error) { - // Load running HANA servers (non-deleted, non-error). - var servers []nova.Server - if _, err := k.DB.Select(&servers, ` - SELECT * FROM `+nova.Server{}.TableName()+` - WHERE flavor_name LIKE 'hana_%' - AND status NOT IN ('DELETED', 'ERROR') - `); err != nil { - return nil, err - } - return servers, nil -} - -// getFlavorsByName loads all flavors from the database and returns a map of flavor name to flavor struct for easy lookup. -func (k *VMwareResourceCommitmentsKPI) getFlavorsByName() (map[string]nova.Flavor, error) { - var flavors []nova.Flavor - if _, err := k.DB.Select(&flavors, "SELECT * FROM "+nova.Flavor{}.TableName()); err != nil { - return nil, err - } - flavorsByName := make(map[string]nova.Flavor, len(flavors)) - for _, flavor := range flavors { - flavorsByName[flavor.Name] = flavor - } - return flavorsByName, nil -} - -// getInstanceCommitments loads all confirmed or guaranteed instance commitments from the database. -func (k *VMwareResourceCommitmentsKPI) getInstanceCommitments() ([]limes.Commitment, error) { - var commitments []limes.Commitment - if _, err := k.DB.Select(&commitments, ` - SELECT * FROM `+limes.Commitment{}.TableName()+` - WHERE service_type = 'compute' - AND resource_name LIKE 'instances_%' - AND status IN ('confirmed', 'guaranteed') - `); err != nil { - return nil, err - } - return commitments, nil -} - -// cpuArchitectureForFlavor returns the CPU architecture label for a HANA flavor name. -// Flavors with a "_v2" suffix run on sapphire-rapids; all others are cascade-lake. -func cpuArchitectureForFlavor(flavorName string) string { - if strings.HasSuffix(flavorName, "_v2") { - return "sapphire-rapids" - } - return "cascade-lake" -} - -// resourceKey identifies an aggregated capacity bucket by (resource, az, architecture). -type resourceKey struct{ resource, az, architecture string } - -// calculateUnusedInstanceCapacity computes per-(resource, az, architecture) capacity sums for unused -// HANA VMware commitments. It filters out non-HANA and KVM (hana_k_) commitments, then for each -// (project, flavor, az, architecture) bucket subtracts running servers from committed amount; over-used -// buckets are clamped to zero and omitted from the result. -func calculateUnusedInstanceCapacity( - commitments []limes.Commitment, - servers []nova.Server, - flavorsByName map[string]nova.Flavor, -) map[resourceKey]float64 { - // running: (project_id, flavor_name, az) -> count of non-deleted/non-error servers. - type serverCountKey struct{ projectID, flavorName, az string } - running := make(map[serverCountKey]uint64, len(servers)) - for _, s := range servers { - running[serverCountKey{s.TenantID, s.FlavorName, s.OSEXTAvailabilityZone}]++ - } - - // committed: (project_id, flavor_name, az, cpuArchitecture) -> total committed amount. - type commitmentKey struct{ projectID, flavorName, az, cpuArchitecture string } - committed := make(map[commitmentKey]uint64) - for _, c := range commitments { - flavorName := strings.TrimPrefix(c.ResourceName, "instances_") - if !strings.HasPrefix(flavorName, "hana_") { - continue - } - if strings.HasPrefix(flavorName, "hana_k_") { - slog.Debug("unused_commitments: skipping hana kvm commitment", "flavor", flavorName, "project_id", c.ProjectID) - continue - } - key := commitmentKey{c.ProjectID, flavorName, c.AvailabilityZone, cpuArchitectureForFlavor(flavorName)} - committed[key] += c.Amount - } - - sum := make(map[resourceKey]float64) - for ck, total := range committed { - run := running[serverCountKey{ck.projectID, ck.flavorName, ck.az}] - if run >= total { - continue - } - unused := total - run - flavor, ok := flavorsByName[ck.flavorName] - if !ok { - slog.Warn("unused_commitments: flavor not found in flavor table", "flavor", ck.flavorName) - continue - } - sum[resourceKey{"cpu", ck.az, ck.cpuArchitecture}] += float64(unused) * float64(flavor.VCPUs) - sum[resourceKey{"ram", ck.az, ck.cpuArchitecture}] += float64(unused) * float64(flavor.RAM) - sum[resourceKey{"disk", ck.az, ck.cpuArchitecture}] += float64(unused) * float64(flavor.Disk) - } - return sum -} - -func (k *VMwareResourceCommitmentsKPI) collectUnusedCommitments(ch chan<- prometheus.Metric) { - if k.DB == nil { - return - } - - // Load confirmed/guaranteed instance commitments. - commitments, err := k.getInstanceCommitments() - if err != nil { - slog.Error("unused_commitments: failed to load commitments", "err", err) - return - } - - // Load flavors for capacity lookup. - flavorsByName, err := k.getFlavorsByName() - if err != nil { - slog.Error("unused_commitments: failed to load flavors", "err", err) - return - } - - // Load running HANA servers. - servers, err := k.getRunningHANAServers() - if err != nil { - slog.Error("unused_commitments: failed to get running HANA servers", "err", err) - return - } - - sumByResource := calculateUnusedInstanceCapacity(commitments, servers, flavorsByName) - - for rk, value := range sumByResource { - ch <- prometheus.MustNewConstMetric( - k.unusedInstanceCommitments, - prometheus.GaugeValue, - value, - rk.resource, - rk.az, - rk.architecture, - ) - } -} diff --git a/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware_test.go b/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware_test.go deleted file mode 100644 index 90a1abd3b..000000000 --- a/internal/knowledge/kpis/plugins/compute/resource_commitments_vmware_test.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package compute - -import ( - "reflect" - "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - testlibDB "github.com/cobaltcore-dev/cortex/internal/knowledge/db/testing" - "github.com/cobaltcore-dev/cortex/pkg/conf" - "github.com/prometheus/client_golang/prometheus" - prometheusgo "github.com/prometheus/client_model/go" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestCPUArchitectureForFlavor(t *testing.T) { - tests := []struct { - flavorName string - want string - }{ - {"hana_small", "cascade-lake"}, - {"hana_large", "cascade-lake"}, - {"hana_small_v2", "sapphire-rapids"}, - {"hana_large_v2", "sapphire-rapids"}, - {"hana_v2_extra", "cascade-lake"}, // _v2 must be a suffix - {"hana_x_v2", "sapphire-rapids"}, - } - for _, tt := range tests { - t.Run(tt.flavorName, func(t *testing.T) { - got := cpuArchitectureForFlavor(tt.flavorName) - if got != tt.want { - t.Errorf("cpuArchitectureForFlavor(%q) = %q, want %q", tt.flavorName, got, tt.want) - } - }) - } -} - -func TestCalculateUnusedInstanceCapacity(t *testing.T) { - flavors := map[string]nova.Flavor{ - "hana_small": {VCPUs: 4, RAM: 16384, Disk: 100}, - "hana_large_v2": {VCPUs: 16, RAM: 65536, Disk: 400}, - } - - t.Run("basic unused", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 3}, - } - servers := []nova.Server{ - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, // 1 running -> 2 unused - } - got := calculateUnusedInstanceCapacity(commitments, servers, flavors) - - if got[resourceKey{"cpu", "az1", "cascade-lake"}] != 8 { // 2 * 4 - t.Errorf("expected cpu=8, got %v", got[resourceKey{"cpu", "az1", "cascade-lake"}]) - } - if got[resourceKey{"ram", "az1", "cascade-lake"}] != 32768 { // 2 * 16384 - t.Errorf("expected ram=32768, got %v", got[resourceKey{"ram", "az1", "cascade-lake"}]) - } - if got[resourceKey{"disk", "az1", "cascade-lake"}] != 200 { // 2 * 100 - t.Errorf("expected disk=200, got %v", got[resourceKey{"disk", "az1", "cascade-lake"}]) - } - }) - - t.Run("non-hana and kvm commitments are skipped", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_k_foo", AvailabilityZone: "az1", Amount: 5}, - {ProjectID: "p2", ResourceName: "instances_general_medium", AvailabilityZone: "az1", Amount: 3}, - } - got := calculateUnusedInstanceCapacity(commitments, nil, flavors) - if len(got) != 0 { - t.Errorf("expected no metrics for kvm/non-hana commitments, got %v", got) - } - }) - - t.Run("amounts for the same key are summed", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 3}, - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2}, - } - got := calculateUnusedInstanceCapacity(commitments, nil, flavors) // nil servers -> all unused - if got[resourceKey{"cpu", "az1", "cascade-lake"}] != 20 { // 5 * 4 - t.Errorf("expected cpu=20 for summed commitments, got %v", got[resourceKey{"cpu", "az1", "cascade-lake"}]) - } - }) - - t.Run("over-used bucket emits no metric", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2}, - } - servers := []nova.Server{ // 5 running > 2 committed - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - } - got := calculateUnusedInstanceCapacity(commitments, servers, flavors) - if len(got) != 0 { - t.Errorf("expected no metrics for over-used bucket, got %v", got) - } - }) - - t.Run("exactly-used bucket emits no metric", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 3}, - } - servers := []nova.Server{ // 3 running == 3 committed - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - {TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1"}, - } - got := calculateUnusedInstanceCapacity(commitments, servers, flavors) - if len(got) != 0 { - t.Errorf("expected no metrics for fully-used bucket, got %v", got) - } - }) - - t.Run("unknown flavor is skipped", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_unknown", AvailabilityZone: "az1", Amount: 3}, - } - got := calculateUnusedInstanceCapacity(commitments, nil, flavors) - if len(got) != 0 { - t.Errorf("expected no metrics for unknown flavor, got %v", got) - } - }) - - t.Run("multiple keys aggregated correctly", func(t *testing.T) { - commitments := []limes.Commitment{ - {ProjectID: "p1", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2}, - {ProjectID: "p2", ResourceName: "instances_hana_large_v2", AvailabilityZone: "az1", Amount: 1}, - } - got := calculateUnusedInstanceCapacity(commitments, nil, flavors) // nil running -> all unused - - if got[resourceKey{"cpu", "az1", "cascade-lake"}] != 8 { // 2 * 4 - t.Errorf("expected cpu cascade-lake=8, got %v", got[resourceKey{"cpu", "az1", "cascade-lake"}]) - } - if got[resourceKey{"cpu", "az1", "sapphire-rapids"}] != 16 { // 1 * 16 - t.Errorf("expected cpu sapphire-rapids=16, got %v", got[resourceKey{"cpu", "az1", "sapphire-rapids"}]) - } - }) -} - -func TestVMwareResourceCommitmentsKPI_CollectHanaUnusedCommitments(t *testing.T) { - scheme, err := v1alpha1.SchemeBuilder.Build() - if err != nil { - t.Fatalf("expected no error building scheme, got %v", err) - } - - dbEnv := testlibDB.SetupDBEnv(t) - testDB := db.DB{DbMap: dbEnv.DbMap} - defer dbEnv.Close() - - if err := testDB.CreateTable( - testDB.AddTable(limes.Commitment{}), - testDB.AddTable(nova.Flavor{}), - testDB.AddTable(nova.Server{}), - ); err != nil { - t.Fatalf("expected no error creating tables, got %v", err) - } - - // Flavors: hana_small (4 vcpu, 16384 MB ram, 100 GB disk) - // hana_large_v2 (16 vcpu, 65536 MB ram, 400 GB disk) - if err := testDB.Insert( - &nova.Flavor{ID: "f1", Name: "hana_small", VCPUs: 4, RAM: 16384, Disk: 100}, - &nova.Flavor{ID: "f2", Name: "hana_large_v2", VCPUs: 16, RAM: 65536, Disk: 400}, - &nova.Flavor{ID: "f3", Name: "general_medium", VCPUs: 8, RAM: 32768, Disk: 200}, - ); err != nil { - t.Fatalf("expected no error inserting flavors, got %v", err) - } - - // Commitments across two AZs to verify per-AZ aggregation: - // project-A: 3 x hana_small in az1 (cascade-lake) - // project-B: 2 x hana_large_v2 in az1 (sapphire-rapids) - // project-A: 4 x hana_small in az2 (cascade-lake) — separate AZ bucket - // project-C: 1 x hana_k_foo in az1 — hana_k_ prefix, should be skipped - // project-D: 1 x general_medium — not hana_, should be skipped - // project-A: 10 x hana_small pending — should be excluded (wrong status) - // project-E: 2 x hana_small in az1 — running will exceed this (over-used, no metric) - // project-F: 3 x hana_large_v2 in az2 — running exactly equals this (fully used, no metric) - if err := testDB.Insert( - &limes.Commitment{ID: 1, ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 3, Status: "confirmed", ProjectID: "project-A"}, - &limes.Commitment{ID: 2, ServiceType: "compute", ResourceName: "instances_hana_large_v2", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "project-B"}, - &limes.Commitment{ID: 3, ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az2", Amount: 4, Status: "guaranteed", ProjectID: "project-A"}, - &limes.Commitment{ID: 4, ServiceType: "compute", ResourceName: "instances_hana_k_foo", AvailabilityZone: "az1", Amount: 5, Status: "confirmed", ProjectID: "project-C"}, - &limes.Commitment{ID: 5, ServiceType: "compute", ResourceName: "instances_general_medium", AvailabilityZone: "az1", Amount: 1, Status: "confirmed", ProjectID: "project-D"}, - &limes.Commitment{ID: 6, ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 10, Status: "pending", ProjectID: "project-A"}, - &limes.Commitment{ID: 7, ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "project-E"}, - &limes.Commitment{ID: 8, ServiceType: "compute", ResourceName: "instances_hana_large_v2", AvailabilityZone: "az2", Amount: 3, Status: "confirmed", ProjectID: "project-F"}, - ); err != nil { - t.Fatalf("expected no error inserting commitments, got %v", err) - } - - // Running servers: - // project-A/az1: 1 hana_small ACTIVE, 1 DELETED (ignored) -> 2 unused in az1 - // project-B/az1: 0 hana_large_v2 -> 2 unused in az1 - // project-A/az2: 1 hana_small ACTIVE -> 3 unused in az2 - // project-E/az1: 5 hana_small ACTIVE -> 5 > 2 committed -> 0 unused (over-used, clamped) - // project-F/az2: 3 hana_large_v2 ACTIVE -> 3 == 3 committed -> 0 unused (fully used, clamped) - if err := testDB.Insert( - &nova.Server{ID: "s1", TenantID: "project-A", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s2", TenantID: "project-A", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "DELETED"}, - &nova.Server{ID: "s3", TenantID: "project-A", FlavorName: "hana_small", OSEXTAvailabilityZone: "az2", Status: "ACTIVE"}, - &nova.Server{ID: "s4", TenantID: "project-E", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s5", TenantID: "project-E", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s6", TenantID: "project-E", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s7", TenantID: "project-E", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s8", TenantID: "project-E", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, - &nova.Server{ID: "s9", TenantID: "project-F", FlavorName: "hana_large_v2", OSEXTAvailabilityZone: "az2", Status: "ACTIVE"}, - &nova.Server{ID: "s10", TenantID: "project-F", FlavorName: "hana_large_v2", OSEXTAvailabilityZone: "az2", Status: "ACTIVE"}, - &nova.Server{ID: "s11", TenantID: "project-F", FlavorName: "hana_large_v2", OSEXTAvailabilityZone: "az2", Status: "ACTIVE"}, - ); err != nil { - t.Fatalf("expected no error inserting servers, got %v", err) - } - - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithRuntimeObjects( - &v1alpha1.Knowledge{ObjectMeta: v1.ObjectMeta{Name: "host-details"}}, - ). - Build() - - kpi := &VMwareResourceCommitmentsKPI{} - if err := kpi.Init(&testDB, k8sClient, conf.NewRawOpts("{}")); err != nil { - t.Fatalf("expected no error, got %v", err) - } - - ch := make(chan prometheus.Metric, 100) - kpi.Collect(ch) - close(ch) - - type UnusedMetric struct { - Resource string - AZ string - Arch string - Value float64 - } - - actual := make(map[string]UnusedMetric) - for metric := range ch { - if getMetricName(metric.Desc().String()) != "cortex_vmware_hana_unused_instance_commitments" { - continue - } - var m prometheusgo.Metric - if err := metric.Write(&m); err != nil { - t.Fatalf("failed to write metric: %v", err) - } - labels := make(map[string]string) - for _, lbl := range m.Label { - labels[lbl.GetName()] = lbl.GetValue() - } - key := labels["resource"] + "/" + labels["availability_zone"] + "/" + labels["cpu_architecture"] - if _, exists := actual[key]; exists { - t.Fatalf("duplicate metric key %q (resource=%q, availability_zone=%q, cpu_architecture=%q)", - key, labels["resource"], labels["availability_zone"], labels["cpu_architecture"]) - } - actual[key] = UnusedMetric{ - Resource: labels["resource"], - AZ: labels["availability_zone"], - Arch: labels["cpu_architecture"], - Value: m.GetGauge().GetValue(), - } - } - - // project-A/az1: 2 unused hana_small (cascade-lake) -> cpu=2*4=8, ram=2*16384=32768, disk=2*100=200 - // project-B/az1: 2 unused hana_large_v2 (sapphire-rapids) -> cpu=2*16=32, ram=2*65536=131072, disk=2*400=800 - // project-A/az2: 3 unused hana_small (cascade-lake) -> cpu=3*4=12, ram=3*16384=49152, disk=3*100=300 - // project-E/az1: 5 running > 2 committed hana_small -> clamped to 0, no metric emitted - // project-F/az2: 3 running == 3 committed hana_large_v2 -> clamped to 0, no metric emitted - expected := map[string]UnusedMetric{ - "cpu/az1/cascade-lake": {Resource: "cpu", AZ: "az1", Arch: "cascade-lake", Value: 8}, - "ram/az1/cascade-lake": {Resource: "ram", AZ: "az1", Arch: "cascade-lake", Value: 32768}, - "disk/az1/cascade-lake": {Resource: "disk", AZ: "az1", Arch: "cascade-lake", Value: 200}, - "cpu/az1/sapphire-rapids": {Resource: "cpu", AZ: "az1", Arch: "sapphire-rapids", Value: 32}, - "ram/az1/sapphire-rapids": {Resource: "ram", AZ: "az1", Arch: "sapphire-rapids", Value: 131072}, - "disk/az1/sapphire-rapids": {Resource: "disk", AZ: "az1", Arch: "sapphire-rapids", Value: 800}, - "cpu/az2/cascade-lake": {Resource: "cpu", AZ: "az2", Arch: "cascade-lake", Value: 12}, - "ram/az2/cascade-lake": {Resource: "ram", AZ: "az2", Arch: "cascade-lake", Value: 49152}, - "disk/az2/cascade-lake": {Resource: "disk", AZ: "az2", Arch: "cascade-lake", Value: 300}, - } - - if len(actual) != len(expected) { - t.Errorf("expected %d metrics, got %d: %v", len(expected), len(actual), actual) - } - for key, exp := range expected { - got, ok := actual[key] - if !ok { - t.Errorf("missing metric %q", key) - continue - } - if !reflect.DeepEqual(exp, got) { - t.Errorf("metric %q: expected %+v, got %+v", key, exp, got) - } - } -} diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared.go b/internal/knowledge/kpis/plugins/infrastructure/shared.go new file mode 100644 index 000000000..62eb44e9c --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/shared.go @@ -0,0 +1,134 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "fmt" + "regexp" + "strconv" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" +) + +const ( + hostDetailsKnowledgeName = "host-details" + hostUtilizationKnowledgeName = "host-utilization" + vmwareIronicHypervisorType = "ironic" + hypervisorFamilyVMware = "vmware" + vmwareComputeHostPattern = "nova-compute-%" + vmwareIronicComputeHostPattern = "nova-compute-ironic-%" +) + +// vmwareHost wraps HostDetails with Prometheus metric helpers. +type vmwareHost struct { + compute.HostDetails +} + +func (h vmwareHost) getHostLabels() []string { + pinnedProjectIds := "" + pinnedProjects := false + if h.PinnedProjects != nil { + pinnedProjectIds = *h.PinnedProjects + pinnedProjects = true + } + disabledReason := "-" + if h.DisabledReason != nil { + disabledReason = *h.DisabledReason + } + return []string{ + h.AvailabilityZone, + h.ComputeHost, + h.CPUArchitecture, + h.WorkloadType, + strconv.FormatBool(h.Enabled), + strconv.FormatBool(h.Decommissioned), + strconv.FormatBool(h.ExternalCustomer), + disabledReason, + strconv.FormatBool(pinnedProjects), + pinnedProjectIds, + } +} + +var vmwareHostLabels = []string{ + "availability_zone", + "compute_host", + "cpu_architecture", + "workload_type", + "enabled", + "decommissioned", + "external_customer", + "disabled_reason", + "pinned_projects", + "pinned_project_ids", +} + +var fqNameRe = regexp.MustCompile(`fqName: "([^"]+)"`) + +func getMetricName(desc string) string { + match := fqNameRe.FindStringSubmatch(desc) + if len(match) > 1 { + return match[1] + } + return "" +} + +type collectedVMwareMetric struct { + Name string + Labels map[string]string + Value float64 +} + +// kvmFlavorPattern matches KVM flavors where the second underscore-delimited +// segment is "k" (e.g. "m1_k_small", "hana_k_large"). +var kvmFlavorPattern = regexp.MustCompile(`^[^_]+_k_`) + +// isKVMFlavor reports whether flavorName belongs to a KVM hypervisor. +func isKVMFlavor(name string) bool { + return kvmFlavorPattern.MatchString(name) +} + +// cpuArchitectureRule maps a flavor name regex to a CPU architecture label. +type cpuArchitectureRule struct { + pattern *regexp.Regexp + arch string +} + +// flavorCPUArchitectureRules maps flavor name patterns to CPU architecture labels in priority order. +// The first matching rule wins; defaultCPUArch is used when none match. +var flavorCPUArchitectureRules = []cpuArchitectureRule{ + {regexp.MustCompile(`_v2$`), "sapphire-rapids"}, +} + +const defaultCPUArchitecture = "cascade-lake" + +// flavorCPUArchitecture derives the CPU architecture label from a flavor name. +func flavorCPUArchitecture(flavorName string) string { + for _, rule := range flavorCPUArchitectureRules { + if rule.pattern.MatchString(flavorName) { + return rule.arch + } + } + return defaultCPUArchitecture +} + +// bytesPerUnit maps memory unit strings to their byte multipliers. +var bytesPerUnit = map[string]float64{ + "": 1, + "B": 1, + "KiB": 1024, + "MB": 1024 * 1024, + "MiB": 1024 * 1024, + "GB": 1024 * 1024 * 1024, + "GiB": 1024 * 1024 * 1024, + "TiB": 1024 * 1024 * 1024 * 1024, +} + +// bytesFromUnit converts an amount in the given unit to bytes. +func bytesFromUnit(amount float64, unit string) (float64, error) { + multiplier, ok := bytesPerUnit[unit] + if !ok { + return 0, fmt.Errorf("unknown memory unit: %s", unit) + } + return amount * multiplier, nil +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared_test.go b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go new file mode 100644 index 000000000..351fedc50 --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go @@ -0,0 +1,165 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" +) + +func mockVMwareHostLabels(computeHost, az string) map[string]string { + return map[string]string{ + "availability_zone": az, + "compute_host": computeHost, + "cpu_architecture": "", + "workload_type": "", + "enabled": "false", + "decommissioned": "false", + "external_customer": "false", + "disabled_reason": "-", + "pinned_projects": "false", + "pinned_project_ids": "", + } +} + +func TestVMwareHostGetHostLabels(t *testing.T) { + str := func(s string) *string { return &s } + + tests := []struct { + name string + host vmwareHost + want []string + }{ + { + name: "all optional fields nil", + host: vmwareHost{compute.HostDetails{ + AvailabilityZone: "az1", + ComputeHost: "nova-compute-1", + CPUArchitecture: "cascade-lake", + WorkloadType: "general-purpose", + Enabled: true, + Decommissioned: false, + ExternalCustomer: false, + DisabledReason: nil, + PinnedProjects: nil, + }}, + want: []string{"az1", "nova-compute-1", "cascade-lake", "general-purpose", "true", "false", "false", "-", "false", ""}, + }, + { + name: "disabled reason set", + host: vmwareHost{compute.HostDetails{ + AvailabilityZone: "az2", + ComputeHost: "nova-compute-2", + DisabledReason: str("scheduled-maintenance"), + }}, + want: []string{"az2", "nova-compute-2", "", "", "false", "false", "false", "scheduled-maintenance", "false", ""}, + }, + { + name: "pinned projects set", + host: vmwareHost{compute.HostDetails{ + AvailabilityZone: "az1", + ComputeHost: "nova-compute-3", + PinnedProjects: str("proj-a,proj-b"), + }}, + want: []string{"az1", "nova-compute-3", "", "", "false", "false", "false", "-", "true", "proj-a,proj-b"}, + }, + { + name: "decommissioned and external customer", + host: vmwareHost{compute.HostDetails{ + AvailabilityZone: "az3", + ComputeHost: "nova-compute-4", + Decommissioned: true, + ExternalCustomer: true, + }}, + want: []string{"az3", "nova-compute-4", "", "", "false", "true", "true", "-", "false", ""}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.host.getHostLabels() + if len(got) != len(vmwareHostLabels) { + t.Fatalf("getHostLabels() returned %d values, want %d (matching vmwareHostLabels)", len(got), len(vmwareHostLabels)) + } + for i, want := range tt.want { + if got[i] != want { + t.Errorf("label[%d] (%s) = %q, want %q", i, vmwareHostLabels[i], got[i], want) + } + } + }) + } +} + +func TestIsKVMFlavor(t *testing.T) { + tests := []struct { + flavor string + want bool + }{ + {"m1_k_small", true}, + {"hana_k_large", true}, + {"hana_small", false}, + {"hana_c128_m1600", false}, + {"hana_c128_m1600_v2", false}, + {"small", false}, + {"m1_large", false}, + } + for _, tt := range tests { + if got := isKVMFlavor(tt.flavor); got != tt.want { + t.Errorf("isKVMFlavor(%q) = %v, want %v", tt.flavor, got, tt.want) + } + } +} + +func TestFlavorCPUArchitecture(t *testing.T) { + tests := []struct { + flavor string + want string + }{ + {"hana_c128_m1600_v2", "sapphire-rapids"}, + {"hana_c256_m3200_v2", "sapphire-rapids"}, + {"hana_c128_m1600", "cascade-lake"}, + {"hana_small", "cascade-lake"}, + } + for _, tt := range tests { + if got := flavorCPUArchitecture(tt.flavor); got != tt.want { + t.Errorf("flavorCPUArchitecture(%q) = %q, want %q", tt.flavor, got, tt.want) + } + } +} + +func TestVmwareBytesFromUnit(t *testing.T) { + tests := []struct { + amount float64 + unit string + want float64 + errMsg string + }{ + {1024, "MiB", 1024 * 1024 * 1024, ""}, + {1024, "MB", 1024 * 1024 * 1024, ""}, + {2, "GiB", 2 * 1024 * 1024 * 1024, ""}, + {2, "GB", 2 * 1024 * 1024 * 1024, ""}, + {1, "TiB", 1024 * 1024 * 1024 * 1024, ""}, + {512, "KiB", 512 * 1024, ""}, + {100, "B", 100, ""}, + {100, "", 100, ""}, + {1, "TB", 0, "unknown memory unit: TB"}, + } + for _, tt := range tests { + got, err := bytesFromUnit(tt.amount, tt.unit) + if tt.errMsg != "" { + if err == nil || err.Error() != tt.errMsg { + t.Errorf("vmwareBytesFromUnit(%v, %q): expected error %q, got %v", tt.amount, tt.unit, tt.errMsg, err) + } + continue + } + if err != nil { + t.Errorf("vmwareBytesFromUnit(%v, %q): unexpected error: %v", tt.amount, tt.unit, err) + continue + } + if got != tt.want { + t.Errorf("vmwareBytesFromUnit(%v, %q) = %f, want %f", tt.amount, tt.unit, got, tt.want) + } + } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go new file mode 100644 index 000000000..c7976db3a --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go @@ -0,0 +1,119 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "context" + "log/slog" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type VMwareHostCapacityKPI struct { + plugins.BaseKPI[struct{}] + + capacityUsagePerHost *prometheus.Desc + capacityTotalPerHost *prometheus.Desc +} + +func (k *VMwareHostCapacityKPI) GetName() string { + return "vmware_host_capacity_kpi" +} + +func (k *VMwareHostCapacityKPI) Init(dbConn *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(dbConn, c, opts); err != nil { + return err + } + k.capacityUsagePerHost = prometheus.NewDesc( + "cortex_vmware_host_capacity_usage", + "Capacity usage per VMware host. CPU in vCPUs, memory and disk in bytes.", + append(vmwareHostLabels, "resource"), nil, + ) + k.capacityTotalPerHost = prometheus.NewDesc( + "cortex_vmware_host_capacity_total", + "Total allocatable capacity per VMware host. CPU in vCPUs, memory and disk in bytes.", + append(vmwareHostLabels, "resource"), nil, + ) + return nil +} + +func (k *VMwareHostCapacityKPI) Describe(ch chan<- *prometheus.Desc) { + ch <- k.capacityUsagePerHost + ch <- k.capacityTotalPerHost +} + +func (k *VMwareHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { + hosts, err := k.getVMwareHosts() + if err != nil { + slog.Error("vmware_host_capacity: failed to get vmware hosts", "error", err) + return + } + utilizations, err := k.getHostUtilizations() + if err != nil { + slog.Error("vmware_host_capacity: failed to get host utilizations", "error", err) + return + } + for _, host := range hosts { + util, ok := utilizations[host.ComputeHost] + if !ok { + slog.Warn("vmware_host_capacity: missing utilization for host", "compute_host", host.ComputeHost) + continue + } + + labels := host.getHostLabels() + + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerHost, prometheus.GaugeValue, util.VCPUsUsed, append(labels, "cpu")...) + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerHost, prometheus.GaugeValue, util.RAMUsedMB*1024*1024, append(labels, "ram")...) + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerHost, prometheus.GaugeValue, util.DiskUsedGB*1024*1024*1024, append(labels, "disk")...) + + ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalVCPUsAllocatable, append(labels, "cpu")...) + ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalRAMAllocatableMB*1024*1024, append(labels, "ram")...) + ch <- prometheus.MustNewConstMetric(k.capacityTotalPerHost, prometheus.GaugeValue, util.TotalDiskAllocatableGB*1024*1024*1024, append(labels, "disk")...) + } +} + +func (k *VMwareHostCapacityKPI) getVMwareHosts() ([]vmwareHost, error) { + knowledge := &v1alpha1.Knowledge{} + if err := k.Client.Get(context.Background(), client.ObjectKey{Name: hostDetailsKnowledgeName}, knowledge); err != nil { + return nil, err + } + details, err := v1alpha1.UnboxFeatureList[compute.HostDetails](knowledge.Status.Raw) + if err != nil { + return nil, err + } + hosts := make([]vmwareHost, 0, len(details)) + for _, d := range details { + if d.HypervisorType == vmwareIronicHypervisorType || d.HypervisorFamily != hypervisorFamilyVMware { + continue + } + hosts = append(hosts, vmwareHost{HostDetails: d}) + } + return hosts, nil +} + +func (k *VMwareHostCapacityKPI) getHostUtilizations() (map[string]compute.HostUtilization, error) { + knowledge := &v1alpha1.Knowledge{} + if err := k.Client.Get(context.Background(), client.ObjectKey{Name: hostUtilizationKnowledgeName}, knowledge); err != nil { + return nil, err + } + utils, err := v1alpha1.UnboxFeatureList[compute.HostUtilization](knowledge.Status.Raw) + if err != nil { + return nil, err + } + m := make(map[string]compute.HostUtilization, len(utils)) + for _, u := range utils { + if u.TotalVCPUsAllocatable == 0 || u.TotalRAMAllocatableMB == 0 || u.TotalDiskAllocatableGB == 0 { + slog.Warn("vmware_host_capacity: skipping host with zero allocatable resources", "compute_host", u.ComputeHost) + continue + } + m[u.ComputeHost] = u + } + return m, nil +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go new file mode 100644 index 000000000..f0a025db4 --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go @@ -0,0 +1,335 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "reflect" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + testlibDB "github.com/cobaltcore-dev/cortex/internal/knowledge/db/testing" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + prometheusgo "github.com/prometheus/client_model/go" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func buildHostCapacityClient(t *testing.T, hostDetails []compute.HostDetails, utilizations []compute.HostUtilization) *fake.ClientBuilder { + t.Helper() + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("failed to build scheme: %v", err) + } + rawDetails, err := v1alpha1.BoxFeatureList(hostDetails) + if err != nil { + t.Fatalf("failed to box host details: %v", err) + } + rawUtils, err := v1alpha1.BoxFeatureList(utilizations) + if err != nil { + t.Fatalf("failed to box host utilizations: %v", err) + } + return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects( + &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: hostDetailsKnowledgeName}, + Status: v1alpha1.KnowledgeStatus{Raw: rawDetails}, + }, + &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: hostUtilizationKnowledgeName}, + Status: v1alpha1.KnowledgeStatus{Raw: rawUtils}, + }, + ) +} + +func TestVMwareHostCapacityKPI_Init(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + kpi := &VMwareHostCapacityKPI{} + if err := kpi.Init(&testDB, nil, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestVMwareHostCapacityKPI_getVMwareHosts(t *testing.T) { + hostDetails := []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware}, + {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware}, + {ComputeHost: "nova-compute-ironic-1", HypervisorType: vmwareIronicHypervisorType, HypervisorFamily: hypervisorFamilyVMware}, + {ComputeHost: "nova-compute-3", HypervisorFamily: "other"}, + } + + client := buildHostCapacityClient(t, hostDetails, nil) + kpi := &VMwareHostCapacityKPI{} + kpi.Client = client.Build() + + hosts, err := kpi.getVMwareHosts() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(hosts) != 2 { + t.Fatalf("expected 2 hosts, got %d", len(hosts)) + } + seen := make(map[string]bool) + for _, h := range hosts { + seen[h.ComputeHost] = true + } + for _, name := range []string{"nova-compute-1", "nova-compute-2"} { + if !seen[name] { + t.Errorf("expected host %q in result", name) + } + } +} + +func TestVMwareHostCapacityKPI_getHostUtilizations(t *testing.T) { + tests := []struct { + name string + utilizations []compute.HostUtilization + expectedHosts []string + }{ + { + name: "normal utilizations are returned", + utilizations: []compute.HostUtilization{ + {ComputeHost: "h1", TotalVCPUsAllocatable: 10, TotalRAMAllocatableMB: 1024, TotalDiskAllocatableGB: 100}, + {ComputeHost: "h2", TotalVCPUsAllocatable: 20, TotalRAMAllocatableMB: 2048, TotalDiskAllocatableGB: 200}, + }, + expectedHosts: []string{"h1", "h2"}, + }, + { + name: "zero TotalVCPUsAllocatable is skipped", + utilizations: []compute.HostUtilization{ + {ComputeHost: "h1", TotalVCPUsAllocatable: 0, TotalRAMAllocatableMB: 1024, TotalDiskAllocatableGB: 100}, + }, + expectedHosts: []string{}, + }, + { + name: "zero TotalRAMAllocatableMB is skipped", + utilizations: []compute.HostUtilization{ + {ComputeHost: "h1", TotalVCPUsAllocatable: 10, TotalRAMAllocatableMB: 0, TotalDiskAllocatableGB: 100}, + }, + expectedHosts: []string{}, + }, + { + name: "zero TotalDiskAllocatableGB is skipped", + utilizations: []compute.HostUtilization{ + {ComputeHost: "h1", TotalVCPUsAllocatable: 10, TotalRAMAllocatableMB: 1024, TotalDiskAllocatableGB: 0}, + }, + expectedHosts: []string{}, + }, + { + name: "mix of valid and zero-allocatable entries", + utilizations: []compute.HostUtilization{ + {ComputeHost: "h1", TotalVCPUsAllocatable: 10, TotalRAMAllocatableMB: 1024, TotalDiskAllocatableGB: 100}, + {ComputeHost: "h2", TotalVCPUsAllocatable: 0, TotalRAMAllocatableMB: 1024, TotalDiskAllocatableGB: 100}, + }, + expectedHosts: []string{"h1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := buildHostCapacityClient(t, nil, tt.utilizations) + kpi := &VMwareHostCapacityKPI{} + kpi.Client = client.Build() + + m, err := kpi.getHostUtilizations() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(m) != len(tt.expectedHosts) { + t.Fatalf("expected %d entries, got %d: %v", len(tt.expectedHosts), len(m), m) + } + for _, host := range tt.expectedHosts { + if _, ok := m[host]; !ok { + t.Errorf("expected host %q in result", host) + } + } + }) + } +} + +func TestVMwareHostCapacityKPI_Collect(t *testing.T) { + tests := []struct { + name string + hostDetails []compute.HostDetails + utilizations []compute.HostUtilization + expectedMetrics []collectedVMwareMetric + }{ + { + name: "single host emits usage and total metrics", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + utilizations: []compute.HostUtilization{ + { + ComputeHost: "nova-compute-1", + VCPUsUsed: 4, + TotalVCPUsAllocatable: 16, + RAMUsedMB: 2048, + TotalRAMAllocatableMB: 8192, + DiskUsedGB: 50, + TotalDiskAllocatableGB: 500, + }, + }, + expectedMetrics: []collectedVMwareMetric{ + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 4}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 50 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 16}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 8192 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 500 * 1024 * 1024 * 1024}, + }, + }, + { + name: "multiple hosts each emit their own metrics", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + }, + utilizations: []compute.HostUtilization{ + {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, + {ComputeHost: "nova-compute-2", VCPUsUsed: 6, TotalVCPUsAllocatable: 12, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, + }, + expectedMetrics: []collectedVMwareMetric{ + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "cpu"), Value: 6}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "ram"), Value: 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 20 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "cpu"), Value: 12}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "ram"), Value: 4096 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 200 * 1024 * 1024 * 1024}, + }, + }, + { + name: "ironic hosts are excluded", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-ironic-1", HypervisorType: vmwareIronicHypervisorType, HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + utilizations: []compute.HostUtilization{ + {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, + {ComputeHost: "nova-compute-ironic-1", VCPUsUsed: 4, TotalVCPUsAllocatable: 16, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, + }, + expectedMetrics: []collectedVMwareMetric{ + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, + }, + }, + { + name: "non-vmware hosts are excluded", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", HypervisorFamily: "kvm", AvailabilityZone: "az1"}, + }, + utilizations: []compute.HostUtilization{ + {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, + {ComputeHost: "nova-compute-2", VCPUsUsed: 4, TotalVCPUsAllocatable: 16, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, + }, + expectedMetrics: []collectedVMwareMetric{ + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, + {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, + }, + }, + { + name: "host without matching utilization produces no metrics", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + utilizations: []compute.HostUtilization{}, + expectedMetrics: []collectedVMwareMetric{}, + }, + { + name: "utilization with zero allocatable resources is skipped", + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + utilizations: []compute.HostUtilization{ + {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 0, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, + }, + expectedMetrics: []collectedVMwareMetric{}, + }, + { + name: "no hosts produces no metrics", + hostDetails: []compute.HostDetails{}, + utilizations: []compute.HostUtilization{}, + expectedMetrics: []collectedVMwareMetric{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + + client := buildHostCapacityClient(t, tt.hostDetails, tt.utilizations) + kpi := &VMwareHostCapacityKPI{} + if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error on Init, got %v", err) + } + + ch := make(chan prometheus.Metric, 200) + kpi.Collect(ch) + close(ch) + + actual := make(map[string]collectedVMwareMetric) + for m := range ch { + var pm prometheusgo.Metric + if err := m.Write(&pm); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + labels := make(map[string]string) + for _, lbl := range pm.Label { + labels[lbl.GetName()] = lbl.GetValue() + } + name := getMetricName(m.Desc().String()) + key := name + "|" + labels["compute_host"] + "|" + labels["resource"] + if _, exists := actual[key]; exists { + t.Fatalf("duplicate metric key %q", key) + } + actual[key] = collectedVMwareMetric{Name: name, Labels: labels, Value: pm.GetGauge().GetValue()} + } + + if len(actual) != len(tt.expectedMetrics) { + t.Errorf("expected %d metrics, got %d: actual=%v", len(tt.expectedMetrics), len(actual), actual) + } + for _, exp := range tt.expectedMetrics { + key := exp.Name + "|" + exp.Labels["compute_host"] + "|" + exp.Labels["resource"] + got, ok := actual[key] + if !ok { + t.Errorf("missing metric %q", key) + continue + } + if got.Value != exp.Value { + t.Errorf("metric %q value: expected %v, got %v", key, exp.Value, got.Value) + } + if !reflect.DeepEqual(exp.Labels, got.Labels) { + t.Errorf("metric %q labels: expected %v, got %v", key, exp.Labels, got.Labels) + } + } + }) + } +} + +func hostCapacityLabels(computeHost, az, resource string) map[string]string { + labels := mockVMwareHostLabels(computeHost, az) + labels["resource"] = resource + return labels +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go new file mode 100644 index 000000000..16fcac857 --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go @@ -0,0 +1,217 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "context" + "log/slog" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/identity" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type vmwareProjectInstanceCount struct { + ProjectID string `db:"project_id"` + ProjectName string `db:"project_name"` + ComputeHost string `db:"compute_host"` + FlavorName string `db:"flavor_name"` + AvailabilityZone string `db:"availability_zone"` + InstanceCount float64 `db:"instance_count"` +} + +type vmwareProjectCapacityUsage struct { + ProjectID string `db:"project_id"` + ProjectName string `db:"project_name"` + ComputeHost string `db:"compute_host"` + AvailabilityZone string `db:"availability_zone"` + TotalVCPUs float64 `db:"total_vcpus"` + TotalRAMMB float64 `db:"total_ram_mb"` + TotalDiskGB float64 `db:"total_disk_gb"` +} + +type VMwareProjectUtilizationKPI struct { + // BaseKPI provides common fields and methods for all KPIs, such as database connection and Kubernetes client. + plugins.BaseKPI[struct{}] + + // instanceCountPerProjectAndHostAndFlavor is a Prometheus descriptor for the number of running instances per project, hypervisor, and flavor on VMware. + instanceCountPerProjectAndHostAndFlavor *prometheus.Desc + + // capacityUsagePerProjectAndHost is a Prometheus descriptor for the resource capacity used by a project per VMware hypervisor and flavor. CPU in vCPUs, memory and disk in bytes. + capacityUsagePerProjectAndHost *prometheus.Desc +} + +func (k *VMwareProjectUtilizationKPI) GetName() string { + return "vmware_project_utilization_kpi" +} + +func (k *VMwareProjectUtilizationKPI) Init(dbConn *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(dbConn, c, opts); err != nil { + return err + } + + k.instanceCountPerProjectAndHostAndFlavor = prometheus.NewDesc( + "cortex_vmware_project_instances", + "Number of running instances per project, hypervisor, and flavor on VMware.", + append(vmwareHostLabels, "project_id", "project_name", "flavor_name"), nil, + ) + k.capacityUsagePerProjectAndHost = prometheus.NewDesc( + "cortex_vmware_project_capacity_usage", + "Resource capacity used by a project per VMware hypervisor and flavor. CPU in vCPUs, memory and disk in bytes.", + append(vmwareHostLabels, "project_id", "project_name", "resource"), nil, + ) + return nil +} + +func (k *VMwareProjectUtilizationKPI) Describe(ch chan<- *prometheus.Desc) { + ch <- k.instanceCountPerProjectAndHostAndFlavor + ch <- k.capacityUsagePerProjectAndHost +} + +func (k *VMwareProjectUtilizationKPI) Collect(ch chan<- prometheus.Metric) { + hosts, err := k.getVMwareHosts() + if err != nil { + // Log the error and return early to avoid panicking. The KPI will be retried on the next scrape. + slog.Error("vmware_project_utilization: Failed to get VMware hosts for project utilization KPI", "error", err) + return + } + + // Export project x flavor x compute_host instance count metric + projectInstanceCounts, err := k.queryProjectInstanceCount() + if err != nil { + slog.Error("vmware_project_utilization: Failed to query project instance count for project utilization KPI", "error", err) + return + } + for _, projectInstanceCount := range projectInstanceCounts { + host, ok := hosts[projectInstanceCount.ComputeHost] + if !ok { + slog.Warn("vmware_project_utilization: Compute host not found for project instance count", "compute_host", projectInstanceCount.ComputeHost) + continue + } + hostLabels := host.getHostLabels() + hostLabels = append(hostLabels, projectInstanceCount.ProjectID, projectInstanceCount.ProjectName, projectInstanceCount.FlavorName) + ch <- prometheus.MustNewConstMetric(k.instanceCountPerProjectAndHostAndFlavor, prometheus.GaugeValue, projectInstanceCount.InstanceCount, hostLabels...) + } + + // Export project x compute_host x resource capacity usage metric + projectCapacityUsages, err := k.queryProjectCapacityUsage() + if err != nil { + slog.Error("vmware_project_utilization: Failed to query project capacity usage for project utilization KPI", "error", err) + return + } + for _, projectCapacityUsage := range projectCapacityUsages { + host, ok := hosts[projectCapacityUsage.ComputeHost] + if !ok { + slog.Warn("vmware_project_utilization: Compute host not found for project capacity usage", "compute_host", projectCapacityUsage.ComputeHost) + continue + } + hostLabels := host.getHostLabels() + hostLabels = append(hostLabels, projectCapacityUsage.ProjectID, projectCapacityUsage.ProjectName) + + memoryUsageBytes, err := bytesFromUnit(projectCapacityUsage.TotalRAMMB, "MB") + if err != nil { + slog.Error("vmware_project_utilization: failed to convert memory to bytes", "err", err) + continue + } + diskUsageBytes, err := bytesFromUnit(projectCapacityUsage.TotalDiskGB, "GB") + if err != nil { + slog.Error("vmware_project_utilization: failed to convert disk to bytes", "err", err) + continue + } + + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerProjectAndHost, prometheus.GaugeValue, projectCapacityUsage.TotalVCPUs, append(hostLabels, "vcpu")...) + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerProjectAndHost, prometheus.GaugeValue, memoryUsageBytes, append(hostLabels, "memory")...) + ch <- prometheus.MustNewConstMetric(k.capacityUsagePerProjectAndHost, prometheus.GaugeValue, diskUsageBytes, append(hostLabels, "disk")...) + } +} + +// getVMwareHosts retrieves the mapping of VMware hypervisors to their corresponding host information +func (k *VMwareProjectUtilizationKPI) getVMwareHosts() (map[string]vmwareHost, error) { + knowledge := &v1alpha1.Knowledge{} + if err := k.Client.Get(context.Background(), client.ObjectKey{Name: hostDetailsKnowledgeName}, knowledge); err != nil { + return nil, err + } + + hostDetails, err := v1alpha1.UnboxFeatureList[compute.HostDetails](knowledge.Status.Raw) + if err != nil { + return nil, err + } + + hostMapping := make(map[string]vmwareHost) + for _, host := range hostDetails { + if host.HypervisorType == vmwareIronicHypervisorType || host.HypervisorFamily != hypervisorFamilyVMware { + continue + } + hostMapping[host.ComputeHost] = vmwareHost{HostDetails: host} + } + + return hostMapping, nil +} + +// queryProjectInstanceCount retrieves the number of running instances per project, hypervisor, and flavor on VMware from the database. +func (k *VMwareProjectUtilizationKPI) queryProjectCapacityUsage() ([]vmwareProjectCapacityUsage, error) { + // This query will fetch all active instances. It will perform a join with the openstack projects to get the project name. + // It will also join with the flavors table to get the flavor information, which is needed for the capacity usage metrics. + // The results will be grouped by project, compute host, and availability zone to get the total capacity usage per project and hypervisor. + // We will filter the results to only include instances that are running on VMware hypervisors by checking the compute host name pattern. + // This assumes that all VMware hypervisors have a compute host name that starts with "nova-compute-", + // which is a naming convention in SAP Cloud Infrastructure and may need to be adjusted based on the actual environment. + query := ` + SELECT + s.tenant_id AS project_id, + COALESCE(p.name, '') AS project_name, + s.os_ext_srv_attr_host AS compute_host, + s.os_ext_az_availability_zone AS availability_zone, + COALESCE(SUM(f.vcpus), 0) AS total_vcpus, + COALESCE(SUM(f.ram), 0) AS total_ram_mb, + COALESCE(SUM(f.disk), 0) AS total_disk_gb + FROM ` + nova.Server{}.TableName() + ` s + LEFT JOIN ` + nova.Flavor{}.TableName() + ` f ON s.flavor_name = f.name + LEFT JOIN ` + identity.Project{}.TableName() + ` p ON p.id = s.tenant_id + WHERE s.status NOT IN ('DELETED', 'ERROR') + AND s.os_ext_srv_attr_host LIKE '` + vmwareComputeHostPattern + `' + AND s.os_ext_srv_attr_host NOT LIKE '` + vmwareIronicComputeHostPattern + `' + GROUP BY s.tenant_id, p.name, s.os_ext_srv_attr_host, s.os_ext_az_availability_zone + ` + var usages []vmwareProjectCapacityUsage + if _, err := k.DB.Select(&usages, query); err != nil { + return nil, err + } + return usages, nil +} + +// queryProjectInstanceCount retrieves the number of running instances per project, hypervisor, and flavor on VMware. +func (k *VMwareProjectUtilizationKPI) queryProjectInstanceCount() ([]vmwareProjectInstanceCount, error) { + // This query will fetch all active instances. It will perform a join with the openstack projects to get the project name. + // The results will be grouped by project, hypervisor, flavor, and availability zone to get the instance count. + // We will filter the results to only include instances that are running on VMware hypervisors by checking the compute host name pattern. + // This assumes that all VMware hypervisors have a compute host name that starts with "nova-compute-", + // which is a naming convention in SAP Cloud Infrastructure and may need to be adjusted based on the actual environment. + query := ` + SELECT + s.tenant_id AS project_id, + COALESCE(p.name, '') AS project_name, + s.os_ext_srv_attr_host AS compute_host, + s.os_ext_az_availability_zone AS availability_zone, + s.flavor_name, + COUNT(*) AS instance_count + FROM ` + nova.Server{}.TableName() + ` s + LEFT JOIN ` + identity.Project{}.TableName() + ` p ON p.id = s.tenant_id + WHERE s.status NOT IN ('DELETED', 'ERROR') + AND s.os_ext_srv_attr_host LIKE '` + vmwareComputeHostPattern + `' + AND s.os_ext_srv_attr_host NOT LIKE '` + vmwareIronicComputeHostPattern + `' + GROUP BY s.tenant_id, p.name, s.os_ext_srv_attr_host, s.flavor_name, s.os_ext_az_availability_zone + ` + var usages []vmwareProjectInstanceCount + if _, err := k.DB.Select(&usages, query); err != nil { + return nil, err + } + return usages, nil +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go new file mode 100644 index 000000000..4c43c893b --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go @@ -0,0 +1,697 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "reflect" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/identity" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + testlibDB "github.com/cobaltcore-dev/cortex/internal/knowledge/db/testing" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + prometheusgo "github.com/prometheus/client_model/go" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func buildMetricKey(name string, labels map[string]string) string { + switch name { + case "cortex_vmware_project_instances": + return name + "|" + labels["compute_host"] + "|" + labels["project_id"] + + "|" + labels["flavor_name"] + "|" + labels["availability_zone"] + case "cortex_vmware_project_capacity_usage": + return name + "|" + labels["compute_host"] + "|" + labels["project_id"] + + "|" + labels["availability_zone"] + "|" + labels["resource"] + default: + return name + } +} + +func instanceMetric(computeHost, az, projectID, projectName, flavorName string, value float64) collectedVMwareMetric { + labels := mockVMwareHostLabels(computeHost, az) + labels["project_id"] = projectID + labels["project_name"] = projectName + labels["flavor_name"] = flavorName + return collectedVMwareMetric{Name: "cortex_vmware_project_instances", Labels: labels, Value: value} +} + +func capacityMetric(computeHost, az, projectID, projectName, resource string, value float64) collectedVMwareMetric { + labels := mockVMwareHostLabels(computeHost, az) + labels["project_id"] = projectID + labels["project_name"] = projectName + labels["resource"] = resource + return collectedVMwareMetric{Name: "cortex_vmware_project_capacity_usage", Labels: labels, Value: value} +} + +func buildVMwareHostDetailsClient(t *testing.T, hostDetails []compute.HostDetails) *fake.ClientBuilder { + t.Helper() + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("failed to build scheme: %v", err) + } + raw, err := v1alpha1.BoxFeatureList(hostDetails) + if err != nil { + t.Fatalf("failed to box host details: %v", err) + } + return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects( + &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: "host-details"}, + Status: v1alpha1.KnowledgeStatus{Raw: raw}, + }, + ) +} + +func TestVMwareProjectUtilizationKPI_Init(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + kpi := &VMwareProjectUtilizationKPI{} + if err := kpi.Init(&testDB, nil, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestVMwareProjectUtilizationKPI_getVMwareHosts(t *testing.T) { + hostDetails := []compute.HostDetails{ + { + ComputeHost: "nova-compute-1", + HypervisorFamily: hypervisorFamilyVMware, + }, + { + ComputeHost: "nova-compute-2", + HypervisorFamily: hypervisorFamilyVMware, + }, + { + ComputeHost: "nova-compute-ironic-1", + HypervisorType: vmwareIronicHypervisorType, + HypervisorFamily: hypervisorFamilyVMware, + }, + { + ComputeHost: "nova-compute-3", + HypervisorFamily: "other", + }, + } + + clientBuilder := buildVMwareHostDetailsClient(t, hostDetails) + kpi := &VMwareProjectUtilizationKPI{} + kpi.Client = clientBuilder.Build() + + hostMapping, err := kpi.getVMwareHosts() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expectedHosts := map[string]vmwareHost{ + "nova-compute-1": {HostDetails: hostDetails[0]}, + "nova-compute-2": {HostDetails: hostDetails[1]}, + } + + if len(hostMapping) != len(expectedHosts) { + t.Fatalf("expected %d hosts, got %d", len(expectedHosts), len(hostMapping)) + } + + for computeHost, expectedHost := range expectedHosts { + host, ok := hostMapping[computeHost] + if !ok { + t.Fatalf("expected host %s not found in mapping", computeHost) + } + if host.ComputeHost != expectedHost.ComputeHost || host.HypervisorFamily != expectedHost.HypervisorFamily { + t.Errorf("host details mismatch for %s: expected %+v, got %+v", computeHost, expectedHost, host) + } + } +} + +func TestVMwareProjectUtilizationKPI_queryProjectInstanceCount(t *testing.T) { + tests := []struct { + name string + servers []nova.Server + projects []identity.Project + expectedCounts map[string]vmwareProjectInstanceCount + }{ + { + name: "single instance in one project", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-1", AvailabilityZone: "az1", InstanceCount: 1}, + }, + }, + { + name: "multiple instances across projects and hosts", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-3", TenantID: "project-2", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + {ID: "server-4", TenantID: "project-2", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + }, + projects: []identity.Project{ + {ID: "project-1", Name: "Project One"}, + {ID: "project-2", Name: "Project Two"}, + }, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-1", AvailabilityZone: "az1", InstanceCount: 1}, + "project-1|nova-compute-1|flavor-2|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-2", AvailabilityZone: "az1", InstanceCount: 1}, + "project-2|nova-compute-2|flavor-1|az2": {ProjectID: "project-2", ProjectName: "Project Two", ComputeHost: "nova-compute-2", FlavorName: "flavor-1", AvailabilityZone: "az2", InstanceCount: 1}, + "project-2|nova-compute-2|flavor-2|az2": {ProjectID: "project-2", ProjectName: "Project Two", ComputeHost: "nova-compute-2", FlavorName: "flavor-2", AvailabilityZone: "az2", InstanceCount: 1}, + }, + }, + { + name: "instances on non-VMware hosts are excluded", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "node-3", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-ironic-1", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-1", AvailabilityZone: "az1", InstanceCount: 1}, + }, + }, + { + name: "instances with non-ACTIVE status are excluded", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "DELETED", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-2", Status: "ERROR", OSEXTAvailabilityZone: "az1"}, + {ID: "server-3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-3", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-3|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-3", AvailabilityZone: "az1", InstanceCount: 1}, + }, + }, + { + name: "multiple instances with same key are counted correctly", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + {ID: "server-4", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", FlavorName: "flavor-1", AvailabilityZone: "az1", InstanceCount: 2}, + "project-1|nova-compute-2|flavor-1|az2": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-2", FlavorName: "flavor-1", AvailabilityZone: "az2", InstanceCount: 2}, + }, + }, + { + name: "missing project entry results in empty project_name", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{}, + expectedCounts: map[string]vmwareProjectInstanceCount{ + "project-1|nova-compute-1|flavor-1|az1": {ProjectID: "project-1", ProjectName: "", ComputeHost: "nova-compute-1", FlavorName: "flavor-1", AvailabilityZone: "az1", InstanceCount: 1}, + }, + }, + { + name: "no instances returns empty result", + servers: []nova.Server{}, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + expectedCounts: map[string]vmwareProjectInstanceCount{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + + if err := testDB.CreateTable( + testDB.AddTable(nova.Server{}), + testDB.AddTable(identity.Project{}), + ); err != nil { + t.Fatalf("failed to create tables: %v", err) + } + + var mockData []any + for i := range tt.servers { + mockData = append(mockData, &tt.servers[i]) + } + for i := range tt.projects { + mockData = append(mockData, &tt.projects[i]) + } + if len(mockData) > 0 { + if err := testDB.Insert(mockData...); err != nil { + t.Fatalf("expected no error, got %v", err) + } + } + + client := buildVMwareHostDetailsClient(t, []compute.HostDetails{}) + kpi := &VMwareProjectUtilizationKPI{} + if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error on Init, got %v", err) + } + counts, err := kpi.queryProjectInstanceCount() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(counts) != len(tt.expectedCounts) { + t.Fatalf("expected %d counts, got %d", len(tt.expectedCounts), len(counts)) + } + for _, got := range counts { + key := got.ProjectID + "|" + got.ComputeHost + "|" + got.FlavorName + "|" + got.AvailabilityZone + exp, ok := tt.expectedCounts[key] + if !ok { + t.Errorf("unexpected count for key %q: %+v", key, got) + continue + } + if got != exp { + t.Errorf("count mismatch for key %q: expected %+v, got %+v", key, exp, got) + } + } + }) + } +} + +func TestVMwareProjectUtilizationKPI_queryProjectCapacityUsage(t *testing.T) { + tests := []struct { + name string + servers []nova.Server + projects []identity.Project + flavors []nova.Flavor + expectedUsages map[string]vmwareProjectCapacityUsage + }{ + { + name: "single instance with flavor details", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{ + "project-1|nova-compute-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", AvailabilityZone: "az1", TotalVCPUs: 2, TotalRAMMB: 4096, TotalDiskGB: 1}, + }, + }, + { + name: "multiple instances with different flavors and projects", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-3", TenantID: "project-2", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + }, + projects: []identity.Project{ + {ID: "project-1", Name: "Project One"}, + {ID: "project-2", Name: "Project Two"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, + {ID: "f2", Name: "flavor-2", VCPUs: 4, RAM: 8192, Disk: 2}, + }, + expectedUsages: map[string]vmwareProjectCapacityUsage{ + "project-1|nova-compute-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", AvailabilityZone: "az1", TotalVCPUs: 6, TotalRAMMB: 12288, TotalDiskGB: 3}, + "project-2|nova-compute-2|az2": {ProjectID: "project-2", ProjectName: "Project Two", ComputeHost: "nova-compute-2", AvailabilityZone: "az2", TotalVCPUs: 2, TotalRAMMB: 4096, TotalDiskGB: 1}, + }, + }, + { + name: "missing flavor entry results in zero capacity", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-missing", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{ + "project-1|nova-compute-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", AvailabilityZone: "az1", TotalVCPUs: 0, TotalRAMMB: 0, TotalDiskGB: 0}, + }, + }, + { + name: "instances on non-VMware hosts are excluded", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "node-3", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{}, + }, + { + name: "instances with non-ACTIVE status are excluded", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "DELETED", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{}, + }, + { + name: "no instances returns empty capacity usage", + servers: []nova.Server{}, + projects: []identity.Project{ + {ID: "project-1", Name: "Project One"}, + }, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{}, + }, + { + name: "multiple instances with same flavor aggregate capacity correctly", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "server-2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{ + "project-1|nova-compute-1|az1": {ProjectID: "project-1", ProjectName: "Project One", ComputeHost: "nova-compute-1", AvailabilityZone: "az1", TotalVCPUs: 4, TotalRAMMB: 8192, TotalDiskGB: 2}, + }, + }, + { + name: "ironic host instances are excluded", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-ironic-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{}, + }, + { + name: "missing project entry results in empty project_name", + servers: []nova.Server{ + {ID: "server-1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + expectedUsages: map[string]vmwareProjectCapacityUsage{ + "project-1|nova-compute-1|az1": {ProjectID: "project-1", ProjectName: "", ComputeHost: "nova-compute-1", AvailabilityZone: "az1", TotalVCPUs: 2, TotalRAMMB: 4096, TotalDiskGB: 1}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + + if err := testDB.CreateTable( + testDB.AddTable(nova.Server{}), + testDB.AddTable(identity.Project{}), + testDB.AddTable(nova.Flavor{}), + ); err != nil { + t.Fatalf("failed to create tables: %v", err) + } + + var mockData []any + for i := range tt.servers { + mockData = append(mockData, &tt.servers[i]) + } + for i := range tt.projects { + mockData = append(mockData, &tt.projects[i]) + } + for i := range tt.flavors { + mockData = append(mockData, &tt.flavors[i]) + } + if len(mockData) > 0 { + if err := testDB.Insert(mockData...); err != nil { + t.Fatalf("expected no error, got %v", err) + } + } + + client := buildVMwareHostDetailsClient(t, []compute.HostDetails{}) + kpi := &VMwareProjectUtilizationKPI{} + if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error on Init, got %v", err) + } + usages, err := kpi.queryProjectCapacityUsage() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(usages) != len(tt.expectedUsages) { + t.Fatalf("expected %d usages, got %d", len(tt.expectedUsages), len(usages)) + } + for _, got := range usages { + key := got.ProjectID + "|" + got.ComputeHost + "|" + got.AvailabilityZone + exp, ok := tt.expectedUsages[key] + if !ok { + t.Errorf("unexpected usage for key %q: %+v", key, got) + continue + } + if got != exp { + t.Errorf("usage mismatch for key %q: expected %+v, got %+v", key, exp, got) + } + } + }) + } +} + +func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { + tests := []struct { + name string + servers []nova.Server + projects []identity.Project + flavors []nova.Flavor + hostDetails []compute.HostDetails + expectedMetrics []collectedVMwareMetric + }{ + { + name: "single instance in one project", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-1", 1), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 2), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 4096*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 1*1024*1024*1024), + }, + }, + { + name: "multiple instances across hosts, projects, and flavors", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-2", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s3", TenantID: "project-2", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + }, + projects: []identity.Project{ + {ID: "project-1", Name: "Project One"}, + {ID: "project-2", Name: "Project Two"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, + {ID: "f2", Name: "flavor-2", VCPUs: 4, RAM: 8192, Disk: 2}, + }, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-1", 1), + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-2", 1), + instanceMetric("nova-compute-2", "az2", "project-2", "Project Two", "flavor-1", 1), + // nova-compute-1/project-1: 1*flavor-1 + 1*flavor-2 + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 6), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 12288*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 3*1024*1024*1024), + // nova-compute-2/project-2: 1*flavor-1 + capacityMetric("nova-compute-2", "az2", "project-2", "Project Two", "vcpu", 2), + capacityMetric("nova-compute-2", "az2", "project-2", "Project Two", "memory", 4096*1024*1024), + capacityMetric("nova-compute-2", "az2", "project-2", "Project Two", "disk", 1*1024*1024*1024), + }, + }, + { + name: "non-VMware and ironic hosts are excluded", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s2", TenantID: "project-1", OSEXTSRVATTRHost: "node-3", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-ironic-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-1", 1), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 2), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 4096*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 1*1024*1024*1024), + }, + }, + { + name: "DELETED and ERROR instances are excluded", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "DELETED", OSEXTAvailabilityZone: "az1"}, + {ID: "s2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-2", Status: "ERROR", OSEXTAvailabilityZone: "az1"}, + {ID: "s3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-3", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{ + {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, + {ID: "f2", Name: "flavor-2", VCPUs: 4, RAM: 8192, Disk: 2}, + {ID: "f3", Name: "flavor-3", VCPUs: 8, RAM: 16384, Disk: 4}, + }, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-3", 1), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 8), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 16384*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 4*1024*1024*1024), + }, + }, + { + name: "multiple instances with same flavor are aggregated correctly", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s2", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + {ID: "s3", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + {ID: "s4", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-2", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az2"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-1", 2), + instanceMetric("nova-compute-2", "az2", "project-1", "Project One", "flavor-1", 2), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 4), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 2*4096*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 2*1024*1024*1024), + capacityMetric("nova-compute-2", "az2", "project-1", "Project One", "vcpu", 4), + capacityMetric("nova-compute-2", "az2", "project-1", "Project One", "memory", 2*4096*1024*1024), + capacityMetric("nova-compute-2", "az2", "project-1", "Project One", "disk", 2*1024*1024*1024), + }, + }, + { + name: "missing project entry results in empty project_name label", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-1", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{}, + flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "", "flavor-1", 1), + capacityMetric("nova-compute-1", "az1", "project-1", "", "vcpu", 2), + capacityMetric("nova-compute-1", "az1", "project-1", "", "memory", 4096*1024*1024), + capacityMetric("nova-compute-1", "az1", "project-1", "", "disk", 1*1024*1024*1024), + }, + }, + { + name: "missing flavor entry results in zero capacity", + servers: []nova.Server{ + {ID: "s1", TenantID: "project-1", OSEXTSRVATTRHost: "nova-compute-1", FlavorName: "flavor-missing", Status: "ACTIVE", OSEXTAvailabilityZone: "az1"}, + }, + projects: []identity.Project{{ID: "project-1", Name: "Project One"}}, + flavors: []nova.Flavor{}, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{ + instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "flavor-missing", 1), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "vcpu", 0), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "memory", 0), + capacityMetric("nova-compute-1", "az1", "project-1", "Project One", "disk", 0), + }, + }, + { + name: "no instances produces no metrics", + servers: []nova.Server{}, + projects: []identity.Project{ + {ID: "project-1", Name: "Project One"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, + }, + hostDetails: []compute.HostDetails{ + {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + }, + expectedMetrics: []collectedVMwareMetric{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + + if err := testDB.CreateTable( + testDB.AddTable(nova.Server{}), + testDB.AddTable(identity.Project{}), + testDB.AddTable(nova.Flavor{}), + ); err != nil { + t.Fatalf("failed to create tables: %v", err) + } + + var mockData []any + for i := range tt.servers { + mockData = append(mockData, &tt.servers[i]) + } + for i := range tt.projects { + mockData = append(mockData, &tt.projects[i]) + } + for i := range tt.flavors { + mockData = append(mockData, &tt.flavors[i]) + } + if len(mockData) > 0 { + if err := testDB.Insert(mockData...); err != nil { + t.Fatalf("expected no error inserting data, got %v", err) + } + } + + client := buildVMwareHostDetailsClient(t, tt.hostDetails) + kpi := &VMwareProjectUtilizationKPI{} + if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error on Init, got %v", err) + } + + ch := make(chan prometheus.Metric, 100) + kpi.Collect(ch) + close(ch) + + actual := make(map[string]collectedVMwareMetric) + for m := range ch { + var pm prometheusgo.Metric + if err := m.Write(&pm); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + labels := make(map[string]string) + for _, lbl := range pm.Label { + labels[lbl.GetName()] = lbl.GetValue() + } + name := getMetricName(m.Desc().String()) + key := buildMetricKey(name, labels) + if _, exists := actual[key]; exists { + t.Fatalf("duplicate metric key %q", key) + } + actual[key] = collectedVMwareMetric{Name: name, Labels: labels, Value: pm.GetGauge().GetValue()} + } + + if len(actual) != len(tt.expectedMetrics) { + t.Errorf("expected %d metrics, got %d: actual=%v", len(tt.expectedMetrics), len(actual), actual) + } + for _, exp := range tt.expectedMetrics { + key := buildMetricKey(exp.Name, exp.Labels) + got, ok := actual[key] + if !ok { + t.Errorf("missing metric %q", key) + continue + } + if got.Value != exp.Value { + t.Errorf("metric %q value: expected %v, got %v", key, exp.Value, got.Value) + } + if !reflect.DeepEqual(exp.Labels, got.Labels) { + t.Errorf("metric %q labels: expected %v, got %v", key, exp.Labels, got.Labels) + } + } + }) + } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments.go new file mode 100644 index 000000000..0d3d5d3ed --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments.go @@ -0,0 +1,271 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "log/slog" + "strings" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// This KPI tracks committed resources in the VMware environment, based on commitments provided by Limes. +// For KVM we can map a commitment to a reservation on a specific host. In VMware this is not possible. +// For general purpose workload customer can specific amounts of resources. +// For HANA workloads customers commit a certain number of HANA instances (based on flavor). +// Like this it is possible to determine the workload type of a commitment. +// For general purpose workloads its not possible to differentiate the cpu architecture. To avoid weird behavior in a dashboard we don't export this label for the metric. +// For HANA flavors the cpu architecture is part of the flavor name (_v2 suffix for sapphire rapids, without suffix for cascade lake). +// For both types of workload however we can not determine on which host the commitment is fulfilled. +type VMwareResourceCommitmentsKPI struct { + // BaseKPI provides common fields and methods for all KPIs, such as database connection and Kubernetes client. + plugins.BaseKPI[struct{}] + + unusedGeneralPurposeCommitmentsPerProject *prometheus.Desc + unusedHanaCommittedResourcesPerProject *prometheus.Desc +} + +func (k *VMwareResourceCommitmentsKPI) GetName() string { + return "vmware_resource_commitments_kpi" +} + +func (k *VMwareResourceCommitmentsKPI) Init(dbConn *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(dbConn, c, opts); err != nil { + return err + } + + k.unusedGeneralPurposeCommitmentsPerProject = prometheus.NewDesc( + "cortex_vmware_commitments_general_purpose", + "Committed general purpose resources that are currently unused. CPU (resource=cpu) in vCPUs, memory (resource=ram) in bytes.", + []string{"availability_zone", "resource", "project_id"}, nil, + ) + k.unusedHanaCommittedResourcesPerProject = prometheus.NewDesc( + "cortex_vmware_commitments_hana_resources", + "Total committed HANA instances capacity that is currently unused, translated to resources. CPU in vCPUs, memory and disk in bytes.", + []string{"availability_zone", "cpu_architecture", "resource", "project_id"}, nil, + ) + return nil +} + +func (k *VMwareResourceCommitmentsKPI) Describe(ch chan<- *prometheus.Desc) { + ch <- k.unusedGeneralPurposeCommitmentsPerProject + ch <- k.unusedHanaCommittedResourcesPerProject +} + +func (k *VMwareResourceCommitmentsKPI) Collect(ch chan<- prometheus.Metric) { + if k.DB == nil { + return + } + + flavorsByName, err := k.getFlavorsByName() + if err != nil { + slog.Error("vmware_resource_commitments: failed to load flavors", "err", err) + return + } + + k.collectGeneralPurpose(ch, flavorsByName) + k.collectHana(ch, flavorsByName) +} + +// getFlavorsByName loads all flavors and returns them keyed by name. +func (k *VMwareResourceCommitmentsKPI) getFlavorsByName() (map[string]nova.Flavor, error) { + var flavors []nova.Flavor + if _, err := k.DB.Select(&flavors, "SELECT * FROM "+nova.Flavor{}.TableName()); err != nil { + return nil, err + } + byName := make(map[string]nova.Flavor, len(flavors)) + for _, f := range flavors { + byName[f.Name] = f + } + return byName, nil +} + +// getGeneralPurposeCommitments loads confirmed/guaranteed cores and ram commitments. +func (k *VMwareResourceCommitmentsKPI) getGeneralPurposeCommitments() ([]limes.Commitment, error) { + var commitments []limes.Commitment + if _, err := k.DB.Select(&commitments, ` + SELECT * FROM `+limes.Commitment{}.TableName()+` + WHERE service_type = 'compute' + AND resource_name IN ('cores', 'ram') + AND status IN ('confirmed', 'guaranteed') + `); err != nil { + return nil, err + } + return commitments, nil +} + +// getGeneralPurposeServers loads running non-HANA servers for general purpose usage accounting. +// KVM-specific flavors are filtered out in Go since SQL LIKE cannot express the segment-exact pattern. +func (k *VMwareResourceCommitmentsKPI) getGeneralPurposeServers() ([]nova.Server, error) { + var servers []nova.Server + if _, err := k.DB.Select(&servers, ` + SELECT * FROM `+nova.Server{}.TableName()+` + WHERE status NOT IN ('DELETED', 'ERROR') + AND flavor_name NOT LIKE 'hana_%' + `); err != nil { + return nil, err + } + result := make([]nova.Server, 0, len(servers)) + for _, s := range servers { + if !isKVMFlavor(s.FlavorName) { + result = append(result, s) + } + } + return result, nil +} + +// getHanaInstanceCommitments loads confirmed/guaranteed HANA instance commitments. +func (k *VMwareResourceCommitmentsKPI) getHanaInstanceCommitments() ([]limes.Commitment, error) { + var commitments []limes.Commitment + if _, err := k.DB.Select(&commitments, ` + SELECT * FROM `+limes.Commitment{}.TableName()+` + WHERE service_type = 'compute' + AND resource_name LIKE 'instances_hana_%' + AND status IN ('confirmed', 'guaranteed') + `); err != nil { + return nil, err + } + return commitments, nil +} + +// getRunningHanaServers loads all running HANA VMware servers (KVM HANA flavors excluded in Go). +func (k *VMwareResourceCommitmentsKPI) getRunningHanaServers() ([]nova.Server, error) { + var servers []nova.Server + if _, err := k.DB.Select(&servers, ` + SELECT * FROM `+nova.Server{}.TableName()+` + WHERE status NOT IN ('DELETED', 'ERROR') + AND flavor_name LIKE 'hana_%' + `); err != nil { + return nil, err + } + result := make([]nova.Server, 0, len(servers)) + for _, s := range servers { + if !isKVMFlavor(s.FlavorName) { + result = append(result, s) + } + } + return result, nil +} + +// collectGeneralPurpose computes and emits unused general purpose committed resources per project. +// Unused = committed - in-use (clamped to zero; zero values are not emitted). +func (k *VMwareResourceCommitmentsKPI) collectGeneralPurpose(ch chan<- prometheus.Metric, flavorsByName map[string]nova.Flavor) { + commitments, err := k.getGeneralPurposeCommitments() + if err != nil { + slog.Error("vmware_resource_commitments: failed to load gp commitments", "err", err) + return + } + servers, err := k.getGeneralPurposeServers() + if err != nil { + slog.Error("vmware_resource_commitments: failed to load gp servers", "err", err) + return + } + + type gpKey struct{ projectID, az, resource string } + + committed := make(map[gpKey]float64) + for _, c := range commitments { + switch c.ResourceName { + case "cores": + committed[gpKey{c.ProjectID, c.AvailabilityZone, "cpu"}] += float64(c.Amount) + case "ram": + bytes, err := bytesFromUnit(float64(c.Amount), c.Unit) + if err != nil { + slog.Warn("vmware_resource_commitments: unknown ram unit", "unit", c.Unit, "err", err) + continue + } + committed[gpKey{c.ProjectID, c.AvailabilityZone, "ram"}] += bytes + } + } + + used := make(map[gpKey]float64) + for _, s := range servers { + flavor, ok := flavorsByName[s.FlavorName] + if !ok { + slog.Warn("vmware_resource_commitments: gp flavor not found", "flavor", s.FlavorName) + continue + } + used[gpKey{s.TenantID, s.OSEXTAvailabilityZone, "cpu"}] += float64(flavor.VCPUs) + used[gpKey{s.TenantID, s.OSEXTAvailabilityZone, "ram"}] += float64(flavor.RAM) * 1024 * 1024 + } + + for key, committedAmt := range committed { + unused := committedAmt - used[key] + if unused <= 0 { + continue + } + ch <- prometheus.MustNewConstMetric( + k.unusedGeneralPurposeCommitmentsPerProject, + prometheus.GaugeValue, + unused, + key.az, key.resource, key.projectID, + ) + } +} + +// collectHana computes and emits unused committed HANA instance resources per project. +// Each HANA instance commitment is compared against running servers; the remainder is +// translated to cpu/ram/disk capacity using the flavor spec. +func (k *VMwareResourceCommitmentsKPI) collectHana(ch chan<- prometheus.Metric, flavorsByName map[string]nova.Flavor) { + commitments, err := k.getHanaInstanceCommitments() + if err != nil { + slog.Error("vmware_resource_commitments: failed to load hana commitments", "err", err) + return + } + servers, err := k.getRunningHanaServers() + if err != nil { + slog.Error("vmware_resource_commitments: failed to load hana servers", "err", err) + return + } + + type serverKey struct{ projectID, flavorName, az string } + running := make(map[serverKey]uint64, len(servers)) + for _, s := range servers { + running[serverKey{s.TenantID, s.FlavorName, s.OSEXTAvailabilityZone}]++ + } + + type commitKey struct{ projectID, flavorName, az, cpuArch string } + committedInstances := make(map[commitKey]uint64) + for _, c := range commitments { + flavorName := strings.TrimPrefix(c.ResourceName, "instances_") + if isKVMFlavor(flavorName) { + continue + } + key := commitKey{c.ProjectID, flavorName, c.AvailabilityZone, flavorCPUArchitecture(flavorName)} + committedInstances[key] += c.Amount + } + + type resourceKey struct{ projectID, az, cpuArch, resource string } + totals := make(map[resourceKey]float64) + for ck, total := range committedInstances { + run := running[serverKey{ck.projectID, ck.flavorName, ck.az}] + if run >= total { + continue + } + unused := total - run + flavor, ok := flavorsByName[ck.flavorName] + if !ok { + slog.Warn("vmware_resource_commitments: hana flavor not found", "flavor", ck.flavorName) + continue + } + totals[resourceKey{ck.projectID, ck.az, ck.cpuArch, "cpu"}] += float64(unused) * float64(flavor.VCPUs) + totals[resourceKey{ck.projectID, ck.az, ck.cpuArch, "ram"}] += float64(unused) * float64(flavor.RAM) * 1024 * 1024 + totals[resourceKey{ck.projectID, ck.az, ck.cpuArch, "disk"}] += float64(unused) * float64(flavor.Disk) * 1024 * 1024 * 1024 + } + + for key, value := range totals { + ch <- prometheus.MustNewConstMetric( + k.unusedHanaCommittedResourcesPerProject, + prometheus.GaugeValue, + value, + key.az, key.cpuArch, key.resource, key.projectID, + ) + } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments_test.go new file mode 100644 index 000000000..6616dc558 --- /dev/null +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_resource_commitments_test.go @@ -0,0 +1,523 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package infrastructure + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + testlibDB "github.com/cobaltcore-dev/cortex/internal/knowledge/db/testing" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + prometheusgo "github.com/prometheus/client_model/go" +) + +func setupResourceCommitmentsDB(t *testing.T) (testDB *db.DB, cleanup func()) { + t.Helper() + dbEnv := testlibDB.SetupDBEnv(t) + testDB = &db.DB{DbMap: dbEnv.DbMap} + if err := testDB.CreateTable( + testDB.AddTable(limes.Commitment{}), + testDB.AddTable(nova.Server{}), + testDB.AddTable(nova.Flavor{}), + ); err != nil { + t.Fatalf("failed to create tables: %v", err) + } + return testDB, dbEnv.Close +} + +// collectResourceCommitmentsMetrics runs the KPI and returns all emitted metrics keyed by +// "metricName|az|cpu_architecture|resource|project_id". GP metrics have an empty cpu_architecture +// segment since the descriptor does not include that label. +func collectResourceCommitmentsMetrics(t *testing.T, testDB *db.DB) map[string]float64 { + t.Helper() + kpi := &VMwareResourceCommitmentsKPI{} + if err := kpi.Init(testDB, nil, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("failed to init KPI: %v", err) + } + ch := make(chan prometheus.Metric, 200) + kpi.Collect(ch) + close(ch) + + result := make(map[string]float64) + for m := range ch { + var pm prometheusgo.Metric + if err := m.Write(&pm); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + lbls := make(map[string]string) + for _, lp := range pm.Label { + lbls[lp.GetName()] = lp.GetValue() + } + name := getMetricName(m.Desc().String()) + key := name + "|" + lbls["availability_zone"] + "|" + lbls["cpu_architecture"] + "|" + lbls["resource"] + "|" + lbls["project_id"] + result[key] = pm.GetGauge().GetValue() + } + return result +} + +// gpKey builds the expected map key for a general-purpose metric. +// cpu_architecture is always empty because the GP metric descriptor omits that label. +func gpKey(az, resource, projectID string) string { + return "cortex_vmware_commitments_general_purpose|" + az + "||" + resource + "|" + projectID +} + +// hKey builds the expected map key for a HANA metric. +func hKey(az, cpuArch, resource, projectID string) string { + return "cortex_vmware_commitments_hana_resources|" + az + "|" + cpuArch + "|" + resource + "|" + projectID +} + +func TestVMwareResourceCommitmentsKPI_Init(t *testing.T) { + dbEnv := testlibDB.SetupDBEnv(t) + testDB := db.DB{DbMap: dbEnv.DbMap} + defer dbEnv.Close() + kpi := &VMwareResourceCommitmentsKPI{} + if err := kpi.Init(&testDB, nil, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} +func TestVMwareResourceCommitmentsKPI_Collect_GeneralPurpose(t *testing.T) { + tests := []struct { + name string + commitments []limes.Commitment + servers []nova.Server + flavors []nova.Flavor + want map[string]float64 + }{ + { + name: "no commitments produces no metrics", + want: map[string]float64{}, + }, + { + name: "fully unused cores commitment", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 10, + }, + }, + { + name: "fully unused ram commitment with MiB unit", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "ram", AvailabilityZone: "az1", Amount: 1024, Unit: "MiB", Status: "confirmed", ProjectID: "p1"}, + }, + want: map[string]float64{ + gpKey("az1", "ram", "p1"): 1024 * 1024 * 1024, + }, + }, + { + name: "fully unused ram commitment with GiB unit", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "ram", AvailabilityZone: "az1", Amount: 2, Unit: "GiB", Status: "confirmed", ProjectID: "p1"}, + }, + want: map[string]float64{ + gpKey("az1", "ram", "p1"): 2 * 1024 * 1024 * 1024, + }, + }, + { + name: "partial cpu usage reduces unused", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + {ID: "s2", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "small", VCPUs: 3, RAM: 0, Disk: 0}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 4, // 10 - 2×3 = 4 + }, + }, + { + name: "fully covered cpu produces no metric", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 4, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "small", VCPUs: 4, RAM: 0, Disk: 0}, + }, + want: map[string]float64{}, + }, + { + name: "over-used cpu produces no metric", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "large", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "large", VCPUs: 8, RAM: 0, Disk: 0}, + }, + want: map[string]float64{}, + }, + { + name: "hana servers not counted against gp commitments", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 8, RAM: 0, Disk: 0}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 10, + }, + }, + { + name: "kvm servers not counted against gp commitments", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "m1_k_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "m1_k_small", VCPUs: 4, RAM: 0, Disk: 0}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 10, + }, + }, + { + name: "DELETED and ERROR servers excluded from usage", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "DELETED"}, + {ID: "s2", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "ERROR"}, + {ID: "s3", TenantID: "p1", FlavorName: "small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "small", VCPUs: 2, RAM: 0, Disk: 0}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 8, // only 1 ACTIVE × 2 subtracted + }, + }, + { + name: "guaranteed commitments counted", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 5, Status: "guaranteed", ProjectID: "p1"}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 5, + }, + }, + { + name: "pending commitments excluded", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 100, Status: "pending", ProjectID: "p1"}, + }, + want: map[string]float64{}, + }, + { + name: "non-compute service type excluded", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "network", ResourceName: "cores", AvailabilityZone: "az1", Amount: 100, Status: "confirmed", ProjectID: "p1"}, + }, + want: map[string]float64{}, + }, + { + name: "multiple commitments per project and AZ summed", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 10, Status: "confirmed", ProjectID: "p1"}, + {ID: 2, UUID: "c2", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 5, Status: "confirmed", ProjectID: "p1"}, + {ID: 3, UUID: "c3", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az2", Amount: 20, Status: "confirmed", ProjectID: "p1"}, + {ID: 4, UUID: "c4", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 8, Status: "confirmed", ProjectID: "p2"}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 15, + gpKey("az2", "cpu", "p1"): 20, + gpKey("az1", "cpu", "p2"): 8, + }, + }, + { + name: "cpu and ram unused reported separately", + commitments: []limes.Commitment{ + {ID: 1, UUID: "c1", ServiceType: "compute", ResourceName: "cores", AvailabilityZone: "az1", Amount: 8, Status: "confirmed", ProjectID: "p1"}, + {ID: 2, UUID: "c2", ServiceType: "compute", ResourceName: "ram", AvailabilityZone: "az1", Amount: 512, Unit: "MiB", Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "medium", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "medium", VCPUs: 2, RAM: 256, Disk: 0}, + }, + want: map[string]float64{ + gpKey("az1", "cpu", "p1"): 6, // 8 - 1×2 + gpKey("az1", "ram", "p1"): (512 - 256) * 1024 * 1024, // 512MiB - 256MB (flavor.RAM is in MB) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testDB, cleanup := setupResourceCommitmentsDB(t) + defer cleanup() + + var rows []any + for i := range tt.commitments { + rows = append(rows, &tt.commitments[i]) + } + for i := range tt.servers { + rows = append(rows, &tt.servers[i]) + } + for i := range tt.flavors { + rows = append(rows, &tt.flavors[i]) + } + if len(rows) > 0 { + if err := testDB.Insert(rows...); err != nil { + t.Fatalf("failed to insert test data: %v", err) + } + } + + got := collectResourceCommitmentsMetrics(t, testDB) + + if len(got) != len(tt.want) { + t.Errorf("expected %d metrics, got %d: %v", len(tt.want), len(got), got) + } + for k, wantVal := range tt.want { + gotVal, ok := got[k] + if !ok { + t.Errorf("missing metric %q", k) + continue + } + if gotVal != wantVal { + t.Errorf("metric %q: expected %f, got %f", k, wantVal, gotVal) + } + } + }) + } +} + +func TestVMwareResourceCommitmentsKPI_Collect_HANA(t *testing.T) { + tests := []struct { + name string + commitments []limes.Commitment + servers []nova.Server + flavors []nova.Flavor + want map[string]float64 + }{ + { + name: "no commitments produces no metrics", + want: map[string]float64{}, + }, + { + name: "fully unused hana instance commitment", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_c128_m1600", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_c128_m1600", VCPUs: 128, RAM: 1638400, Disk: 100}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 2 * 128, + hKey("az1", "cascade-lake", "ram", "p1"): 2 * 1638400 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 2 * 100 * 1024 * 1024 * 1024, + }, + }, + { + name: "partial hana usage reduces unused instances", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_c128_m1600", AvailabilityZone: "az1", Amount: 3, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "hana_c128_m1600", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_c128_m1600", VCPUs: 128, RAM: 1638400, Disk: 100}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 2 * 128, + hKey("az1", "cascade-lake", "ram", "p1"): 2 * 1638400 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 2 * 100 * 1024 * 1024 * 1024, + }, + }, + { + name: "fully used hana produces no metric", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + {ID: "s2", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{}, + }, + { + name: "over-used hana produces no metric", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 1, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + {ID: "s2", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{}, + }, + { + name: "sapphire-rapids arch from _v2 suffix", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_c256_m3200_v2", AvailabilityZone: "az1", Amount: 1, Status: "confirmed", ProjectID: "p1"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_c256_m3200_v2", VCPUs: 256, RAM: 3276800, Disk: 200}, + }, + want: map[string]float64{ + hKey("az1", "sapphire-rapids", "cpu", "p1"): 256, + hKey("az1", "sapphire-rapids", "ram", "p1"): 3276800 * 1024 * 1024, + hKey("az1", "sapphire-rapids", "disk", "p1"): 200 * 1024 * 1024 * 1024, + }, + }, + { + name: "cascade-lake and sapphire-rapids aggregated separately", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_c128_m1600", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + {ID: 2, UUID: "h2", ServiceType: "compute", ResourceName: "instances_hana_c128_m1600_v2", AvailabilityZone: "az1", Amount: 1, Status: "confirmed", ProjectID: "p1"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_c128_m1600", VCPUs: 128, RAM: 1638400, Disk: 100}, + {ID: "f2", Name: "hana_c128_m1600_v2", VCPUs: 128, RAM: 1638400, Disk: 100}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 2 * 128, + hKey("az1", "cascade-lake", "ram", "p1"): 2 * 1638400 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 2 * 100 * 1024 * 1024 * 1024, + hKey("az1", "sapphire-rapids", "cpu", "p1"): 1 * 128, + hKey("az1", "sapphire-rapids", "ram", "p1"): 1 * 1638400 * 1024 * 1024, + hKey("az1", "sapphire-rapids", "disk", "p1"): 1 * 100 * 1024 * 1024 * 1024, + }, + }, + { + name: "kvm hana commitments excluded", + commitments: []limes.Commitment{ + // hana_k_large is a KVM HANA flavor — must be filtered out + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_k_large", AvailabilityZone: "az1", Amount: 5, Status: "confirmed", ProjectID: "p1"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_k_large", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{}, + }, + { + name: "DELETED and ERROR hana servers excluded from running count", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 3, Status: "confirmed", ProjectID: "p1"}, + }, + servers: []nova.Server{ + {ID: "s1", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "DELETED"}, + {ID: "s2", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ERROR"}, + {ID: "s3", TenantID: "p1", FlavorName: "hana_small", OSEXTAvailabilityZone: "az1", Status: "ACTIVE"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 2 * 64, // 3 committed - 1 ACTIVE = 2 unused + hKey("az1", "cascade-lake", "ram", "p1"): 2 * 819200 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 2 * 50 * 1024 * 1024 * 1024, + }, + }, + { + name: "guaranteed hana commitments counted", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 1, Status: "guaranteed", ProjectID: "p1"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 64, + hKey("az1", "cascade-lake", "ram", "p1"): 819200 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 50 * 1024 * 1024 * 1024, + }, + }, + { + name: "unknown flavor is skipped without panic", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_nonexistent", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + }, + want: map[string]float64{}, + }, + { + name: "multiple projects and AZs aggregated per bucket", + commitments: []limes.Commitment{ + {ID: 1, UUID: "h1", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 2, Status: "confirmed", ProjectID: "p1"}, + {ID: 2, UUID: "h2", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az2", Amount: 3, Status: "confirmed", ProjectID: "p1"}, + {ID: 3, UUID: "h3", ServiceType: "compute", ResourceName: "instances_hana_small", AvailabilityZone: "az1", Amount: 1, Status: "confirmed", ProjectID: "p2"}, + }, + flavors: []nova.Flavor{ + {ID: "f1", Name: "hana_small", VCPUs: 64, RAM: 819200, Disk: 50}, + }, + want: map[string]float64{ + hKey("az1", "cascade-lake", "cpu", "p1"): 2 * 64, + hKey("az1", "cascade-lake", "ram", "p1"): 2 * 819200 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p1"): 2 * 50 * 1024 * 1024 * 1024, + hKey("az2", "cascade-lake", "cpu", "p1"): 3 * 64, + hKey("az2", "cascade-lake", "ram", "p1"): 3 * 819200 * 1024 * 1024, + hKey("az2", "cascade-lake", "disk", "p1"): 3 * 50 * 1024 * 1024 * 1024, + hKey("az1", "cascade-lake", "cpu", "p2"): 1 * 64, + hKey("az1", "cascade-lake", "ram", "p2"): 1 * 819200 * 1024 * 1024, + hKey("az1", "cascade-lake", "disk", "p2"): 1 * 50 * 1024 * 1024 * 1024, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testDB, cleanup := setupResourceCommitmentsDB(t) + defer cleanup() + + var rows []any + for i := range tt.commitments { + rows = append(rows, &tt.commitments[i]) + } + for i := range tt.servers { + rows = append(rows, &tt.servers[i]) + } + for i := range tt.flavors { + rows = append(rows, &tt.flavors[i]) + } + if len(rows) > 0 { + if err := testDB.Insert(rows...); err != nil { + t.Fatalf("failed to insert test data: %v", err) + } + } + + got := collectResourceCommitmentsMetrics(t, testDB) + + if len(got) != len(tt.want) { + t.Errorf("expected %d metrics, got %d: %v", len(tt.want), len(got), got) + } + for k, wantVal := range tt.want { + gotVal, ok := got[k] + if !ok { + t.Errorf("missing metric %q", k) + continue + } + if gotVal != wantVal { + t.Errorf("metric %q: expected %f, got %f", k, wantVal, gotVal) + } + } + }) + } +} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go index a812943e1..63a35866b 100644 --- a/internal/knowledge/kpis/supported_kpis.go +++ b/internal/knowledge/kpis/supported_kpis.go @@ -7,6 +7,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/deployment" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/infrastructure" "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/storage" ) @@ -15,14 +16,16 @@ var supportedKPIs = map[string]plugins.KPI{ "kvm_host_capacity_kpi": &compute.KVMResourceCapacityKPI{}, "vmware_host_contention_kpi": &compute.VMwareHostContentionKPI{}, "vmware_project_noisiness_kpi": &compute.VMwareProjectNoisinessKPI{}, - "vmware_host_capacity_kpi": &compute.VMwareResourceCapacityKPI{}, "host_running_vms_kpi": &compute.HostRunningVMsKPI{}, "flavor_running_vms_kpi": &compute.FlavorRunningVMsKPI{}, "vm_migration_statistics_kpi": &compute.VMMigrationStatisticsKPI{}, "vm_life_span_kpi": &compute.VMLifeSpanKPI{}, "vm_commitments_kpi": &compute.VMCommitmentsKPI{}, "vm_faults_kpi": &compute.VMFaultsKPI{}, - "vmware_commitments_kpi": &compute.VMwareResourceCommitmentsKPI{}, + + "vmware_project_utilization_kpi": &infrastructure.VMwareProjectUtilizationKPI{}, + "vmware_resource_commitments_kpi": &infrastructure.VMwareResourceCommitmentsKPI{}, + "vmware_host_capacity_kpi": &infrastructure.VMwareHostCapacityKPI{}, "netapp_storage_pool_cpu_usage_kpi": &storage.NetAppStoragePoolCPUUsageKPI{}, diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index e6956609a..5b471f789 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -13,7 +13,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" ) @@ -92,7 +91,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa return nil, err } for _, reservation := range reservations.Items { - if !meta.IsStatusConditionTrue(reservation.Status.Conditions, v1alpha1.ReservationConditionReady) { + if !reservation.IsReady() { continue // Only consider active reservations (Ready=True). } diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go index 1f404f6b5..dcbcbf8bd 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_evacuation.go @@ -10,7 +10,6 @@ import ( api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" - "k8s.io/apimachinery/pkg/api/meta" ) // Options for the KVM failover evacuation weigher. @@ -72,8 +71,7 @@ func (s *KVMFailoverEvacuationStep) Run(traceLog *slog.Logger, request api.Exter failoverHosts := make(map[string]bool) for _, reservation := range reservations.Items { // Only consider active failover reservations (Ready condition is True) - readyCondition := meta.FindStatusCondition(reservation.Status.Conditions, v1alpha1.ReservationConditionReady) - if readyCondition == nil || readyCondition.Status != "True" { + if !reservation.IsReady() { continue } if reservation.Spec.Type != v1alpha1.ReservationTypeFailover { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go new file mode 100644 index 000000000..727afce33 --- /dev/null +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation.go @@ -0,0 +1,164 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package weighers + +import ( + "context" + "errors" + "log/slog" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" +) + +// Options for the KVM failover reservation consolidation weigher. +type KVMFailoverReservationConsolidationOpts struct { + // Weight multiplier for the total failover reservation count per host (consolidation signal). + // Higher values more aggressively pack failover reservations onto fewer hosts. + // Default: 1.0 + TotalCountWeight *float64 `json:"totalCountWeight,omitempty"` + // Penalty multiplier for same-spec reservation count per host (diversity signal). + // Higher values more aggressively avoid clustering reservations of the same size on one host. + // Should be less than TotalCountWeight to ensure consolidation is the primary goal. + // Default: 0.1 + SameSpecPenalty *float64 `json:"sameSpecPenalty,omitempty"` +} + +func (o KVMFailoverReservationConsolidationOpts) Validate() error { + w := o.GetTotalCountWeight() + p := o.GetSameSpecPenalty() + if w < 0 { + return errors.New("totalCountWeight must be non-negative") + } + if p < 0 { + return errors.New("sameSpecPenalty must be non-negative") + } + if w == 0 && p > 0 { + return errors.New("sameSpecPenalty must be zero when totalCountWeight is zero") + } + if w > 0 && p >= w { + return errors.New("sameSpecPenalty must be less than totalCountWeight") + } + return nil +} + +func (o KVMFailoverReservationConsolidationOpts) GetTotalCountWeight() float64 { + if o.TotalCountWeight == nil { + return 1.0 + } + return *o.TotalCountWeight +} + +func (o KVMFailoverReservationConsolidationOpts) GetSameSpecPenalty() float64 { + if o.SameSpecPenalty == nil { + return 0.1 + } + return *o.SameSpecPenalty +} + +// KVMFailoverReservationConsolidationStep weighs hosts for failover reservation placement. +// It encourages consolidating failover reservations onto as few hosts as possible (primary goal), +// while preferring hosts with fewer reservations of the same ResourceGroup (secondary tiebreaker). +// +// The ResourceGroup is passed via the scheduler hint "_cortex_resource_group" and compared against +// each existing reservation's Spec.FailoverReservation.ResourceGroup. This groups reservations +// by flavor group (or individual flavor name when no group exists). +// +// Score formula (normalized by total reservation count T): +// +// score = (totalCountWeight / T) × hostCount - (sameSpecPenalty / T) × sameGroupCount +// +// This produces bounded output (~0 to 1) that plays nicely with other weighers. +type KVMFailoverReservationConsolidationStep struct { + lib.BaseWeigher[api.ExternalSchedulerRequest, KVMFailoverReservationConsolidationOpts] +} + +// Run the weigher step. +// For reserve_for_failover requests, hosts are scored based on existing failover reservation density +// and same-spec diversity. For all other request types, this weigher has no effect. +func (s *KVMFailoverReservationConsolidationStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { + result := s.IncludeAllHostsFromRequest(request) + + intent, err := request.GetIntent() + if err != nil || intent != api.ReserveForFailoverIntent { + traceLog.Info("skipping failover reservation consolidation weigher for non-failover-reservation request") + return result, nil //nolint:nilerr // intentionally skip weigher on error + } + + // Extract the resource group from the scheduler hint. + // This identifies which "spec group" the incoming reservation belongs to. + // If the hint is missing, requestResourceGroup will be empty and the same-group penalty is skipped. + requestResourceGroup, _ := request.Spec.Data.GetSchedulerHintStr(api.HintKeyResourceGroup) //nolint:errcheck // missing hint is fine, same-group penalty is simply skipped + + // Fetch all reservations. + var reservations v1alpha1.ReservationList + if err := s.Client.List(context.Background(), &reservations); err != nil { + return nil, err + } + + // Count failover reservations per host, and same-group reservations per host. + totalPerHost := make(map[string]float64) + sameGroupPerHost := make(map[string]float64) + totalReservations := 0 + + for _, reservation := range reservations.Items { + // Only consider active failover reservations (Ready condition is True). + if !reservation.IsReady() { + continue + } + if reservation.Spec.Type != v1alpha1.ReservationTypeFailover { + continue + } + + host := reservation.Status.Host + if host == "" { + continue + } + + totalReservations++ + totalPerHost[host]++ + + // Check if this reservation belongs to the same resource group as the request. + if requestResourceGroup != "" && reservation.Spec.FailoverReservation != nil && + reservation.Spec.FailoverReservation.ResourceGroup == requestResourceGroup { + sameGroupPerHost[host]++ + } + } + + // If there are no failover reservations, the weigher has no information to act on. + if totalReservations == 0 { + traceLog.Info("no active failover reservations found, skipping consolidation weigher") + return result, nil + } + + totalCountWeight := s.Options.GetTotalCountWeight() + sameSpecPenalty := s.Options.GetSameSpecPenalty() + t := float64(totalReservations) + + for _, host := range request.Hosts { + hostTotal := totalPerHost[host.ComputeHost] + hostSameGroup := sameGroupPerHost[host.ComputeHost] + + // Normalized score: bounded output for compatibility with other weighers. + score := (totalCountWeight/t)*hostTotal - (sameSpecPenalty/t)*hostSameGroup + + result.Activations[host.ComputeHost] = score + traceLog.Info("calculated failover consolidation score for host", + "host", host.ComputeHost, + "totalOnHost", hostTotal, + "sameGroupOnHost", hostSameGroup, + "resourceGroup", requestResourceGroup, + "totalReservations", totalReservations, + "score", score) + } + + return result, nil +} + +func init() { + Index["kvm_failover_reservation_consolidation"] = func() NovaWeigher { + return &KVMFailoverReservationConsolidationStep{} + } +} diff --git a/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go new file mode 100644 index 000000000..62d69d319 --- /dev/null +++ b/internal/scheduling/nova/plugins/weighers/kvm_failover_reservation_consolidation_test.go @@ -0,0 +1,368 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package weighers + +import ( + "log/slog" + "math" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + testlib "github.com/cobaltcore-dev/cortex/pkg/testing" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// newFailoverReservationWithGroup creates a failover reservation with a specific resource group. +func newFailoverReservationWithGroup(name, targetHost, resourceGroup string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: targetHost, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: *resource.NewQuantity(4, resource.DecimalSI), + hv1.ResourceMemory: *resource.NewQuantity(8192*1_000_000, resource.DecimalSI), + }, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ + ResourceGroup: resourceGroup, + }, + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []metav1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + }, + }, + Host: targetHost, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: map[string]string{"some-vm": "some-host"}, + }, + }, + } +} + +func newFailoverReservationRequest(resourceGroup string, hosts []string) api.ExternalSchedulerRequest { + hostList := make([]api.ExternalSchedulerHost, len(hosts)) + for i, h := range hosts { + hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + } + + spec := api.NovaSpec{ + ProjectID: "project-A", + InstanceUUID: "test-instance", + NumInstances: 1, + SchedulerHints: map[string]any{ + "_nova_check_type": string(api.ReserveForFailoverIntent), + api.HintKeyResourceGroup: resourceGroup, + }, + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + Name: "m1.large", + VCPUs: 4, + MemoryMB: 8192, + ExtraSpecs: map[string]string{ + "capabilities:hypervisor_type": "qemu", + }, + }, + }, + } + + weights := make(map[string]float64) + for _, h := range hosts { + weights[h] = 1.0 + } + + return api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{Data: spec}, + Hosts: hostList, + Weights: weights, + } +} + +func approxEqual(a, b, epsilon float64) bool { + return math.Abs(a-b) < epsilon +} + +func TestKVMFailoverReservationConsolidationStep_Run(t *testing.T) { + scheme := buildTestScheme(t) + + tests := []struct { + name string + reservations []*v1alpha1.Reservation + request api.ExternalSchedulerRequest + opts KVMFailoverReservationConsolidationOpts + expectedWeights map[string]float64 + }{ + { + name: "consolidation: prefer host with existing failover reservations", + reservations: []*v1alpha1.Reservation{ + // host1 has 3 reservations (different groups) + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + newFailoverReservationWithGroup("res-2", "host1", "group-B"), + newFailoverReservationWithGroup("res-3", "host1", "group-C"), + // host2 has 1 reservation + newFailoverReservationWithGroup("res-4", "host2", "group-B"), + }, + // Request for group-D - no same-group on any host + request: newFailoverReservationRequest("group-D", []string{"host1", "host2", "host3"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + // T=4, host1: (1/4)*3=0.75, host2: (1/4)*1=0.25, host3: 0 + expectedWeights: map[string]float64{"host1": 0.75, "host2": 0.25, "host3": 0}, + }, + { + name: "same-group penalty: prefer host with fewer same-group reservations", + reservations: []*v1alpha1.Reservation{ + // host1 has 5 reservations, 0 same-group (group-A) + newFailoverReservationWithGroup("res-1", "host1", "group-B"), + newFailoverReservationWithGroup("res-2", "host1", "group-B"), + newFailoverReservationWithGroup("res-3", "host1", "group-C"), + newFailoverReservationWithGroup("res-4", "host1", "group-C"), + newFailoverReservationWithGroup("res-5", "host1", "group-D"), + // host2 has 5 reservations, 3 same-group (group-A) + newFailoverReservationWithGroup("res-6", "host2", "group-A"), + newFailoverReservationWithGroup("res-7", "host2", "group-A"), + newFailoverReservationWithGroup("res-8", "host2", "group-A"), + newFailoverReservationWithGroup("res-9", "host2", "group-C"), + newFailoverReservationWithGroup("res-10", "host2", "group-D"), + }, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2", "host3"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + // T=10 + // host1: (1/10)*5 - (0.1/10)*0 = 0.5 + // host2: (1/10)*5 - (0.1/10)*3 = 0.5 - 0.03 = 0.47 + // host3: 0 + expectedWeights: map[string]float64{"host1": 0.5, "host2": 0.47, "host3": 0}, + }, + { + name: "consolidation dominates: host with reservations preferred over empty host even with same-group", + reservations: []*v1alpha1.Reservation{ + // host2 has 5 reservations, 3 same-group (group-A) + newFailoverReservationWithGroup("res-1", "host2", "group-A"), + newFailoverReservationWithGroup("res-2", "host2", "group-A"), + newFailoverReservationWithGroup("res-3", "host2", "group-A"), + newFailoverReservationWithGroup("res-4", "host2", "group-C"), + newFailoverReservationWithGroup("res-5", "host2", "group-D"), + }, + request: newFailoverReservationRequest("group-A", []string{"host2", "host3"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + // T=5 + // host2: (1/5)*5 - (0.1/5)*3 = 1.0 - 0.06 = 0.94 + // host3: 0 + expectedWeights: map[string]float64{"host2": 0.94, "host3": 0}, + }, + { + name: "no reservations: all hosts get default weight (no effect)", + reservations: []*v1alpha1.Reservation{}, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + expectedWeights: map[string]float64{"host1": 0, "host2": 0}, + }, + { + name: "non-failover request: weigher has no effect", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + }, + // Use a non-failover request (evacuation) + request: newNovaRequest("instance-123", true, []string{"host1", "host2"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + expectedWeights: map[string]float64{"host1": 0, "host2": 0}, + }, + { + name: "non-failover request without hints: weigher has no effect", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + }, + // Use a non-failover request (no hints = create intent) + request: newNovaRequest("instance-123", false, []string{"host1", "host2"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + expectedWeights: map[string]float64{"host1": 0, "host2": 0}, + }, + { + name: "default options work correctly", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-B"), + newFailoverReservationWithGroup("res-2", "host1", "group-A"), // same group + newFailoverReservationWithGroup("res-3", "host2", "group-B"), + }, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2", "host3"}), + opts: KVMFailoverReservationConsolidationOpts{}, // nil = use defaults + // Defaults: TotalCountWeight=1.0, SameSpecPenalty=0.1, T=3 + // host1: (1/3)*2 - (0.1/3)*1 ≈ 0.6667 - 0.0333 = 0.6333 + // host2: (1/3)*1 - (0.1/3)*0 ≈ 0.3333 + // host3: 0 + expectedWeights: map[string]float64{"host1": 2.0/3.0 - 0.1/3.0, "host2": 1.0 / 3.0, "host3": 0}, + }, + { + name: "committed resource reservations are ignored", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + newCommittedReservation("committed-1", "host2"), + }, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2", "host3"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + // T=1 (only 1 failover reservation), committed reservation ignored + // host1: (1/1)*1 - (0.1/1)*1 = 0.9 + // host2: 0 (committed reservation not counted) + // host3: 0 + expectedWeights: map[string]float64{"host1": 0.9, "host2": 0, "host3": 0}, + }, + { + name: "failed reservations are ignored", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + newFailoverReservation("failed-res", "host2", true, map[string]string{"vm-1": "h-1"}), + }, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(1.0), SameSpecPenalty: testlib.Ptr(0.1)}, + // T=1 (failed reservation ignored) + // host1: (1/1)*1 - (0.1/1)*1 = 0.9 + // host2: 0 + expectedWeights: map[string]float64{"host1": 0.9, "host2": 0}, + }, + { + name: "custom weights adjust scoring", + reservations: []*v1alpha1.Reservation{ + newFailoverReservationWithGroup("res-1", "host1", "group-A"), + newFailoverReservationWithGroup("res-2", "host1", "group-A"), + newFailoverReservationWithGroup("res-3", "host2", "group-B"), + }, + request: newFailoverReservationRequest("group-A", []string{"host1", "host2"}), + opts: KVMFailoverReservationConsolidationOpts{TotalCountWeight: testlib.Ptr(2.0), SameSpecPenalty: testlib.Ptr(0.5)}, + // T=3, W=2.0, P=0.5 + // host1: (2/3)*2 - (0.5/3)*2 = 1.3333 - 0.3333 = 1.0 + // host2: (2/3)*1 - (0.5/3)*0 = 0.6667 + expectedWeights: map[string]float64{"host1": 1.0, "host2": 2.0 / 3.0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := make([]client.Object, 0, len(tt.reservations)) + for _, r := range tt.reservations { + objects = append(objects, r) + } + + step := &KVMFailoverReservationConsolidationStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = tt.opts + + result, err := step.Run(slog.Default(), tt.request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + for host, expectedWeight := range tt.expectedWeights { + actualWeight, ok := result.Activations[host] + if !ok { + t.Errorf("expected host %s to be in activations", host) + continue + } + if !approxEqual(actualWeight, expectedWeight, 0.0001) { + t.Errorf("host %s: expected weight %v, got %v", host, expectedWeight, actualWeight) + } + } + }) + } +} + +func TestKVMFailoverReservationConsolidationOpts_Defaults(t *testing.T) { + opts := KVMFailoverReservationConsolidationOpts{} + if opts.GetTotalCountWeight() != 1.0 { + t.Errorf("expected default TotalCountWeight 1.0, got %v", opts.GetTotalCountWeight()) + } + if opts.GetSameSpecPenalty() != 0.1 { + t.Errorf("expected default SameSpecPenalty 0.1, got %v", opts.GetSameSpecPenalty()) + } +} + +func TestKVMFailoverReservationConsolidationOpts_Validate(t *testing.T) { + tests := []struct { + name string + opts KVMFailoverReservationConsolidationOpts + wantErr string + }{ + { + name: "valid: both set, p < w", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(2.0), + SameSpecPenalty: testlib.Ptr(0.5), + }, + }, + { + name: "valid: defaults (nil)", + opts: KVMFailoverReservationConsolidationOpts{}, + }, + { + name: "valid: both zero", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(0.0), + SameSpecPenalty: testlib.Ptr(0.0), + }, + }, + { + name: "invalid: negative totalCountWeight", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(-1.0), + }, + wantErr: "totalCountWeight must be non-negative", + }, + { + name: "invalid: negative sameSpecPenalty", + opts: KVMFailoverReservationConsolidationOpts{ + SameSpecPenalty: testlib.Ptr(-0.1), + }, + wantErr: "sameSpecPenalty must be non-negative", + }, + { + name: "invalid: p >= w", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(1.0), + SameSpecPenalty: testlib.Ptr(1.0), + }, + wantErr: "sameSpecPenalty must be less than totalCountWeight", + }, + { + name: "invalid: w=0 with p>0 (default penalty with zero weight)", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(0.0), + // SameSpecPenalty defaults to 0.1 + }, + wantErr: "sameSpecPenalty must be zero when totalCountWeight is zero", + }, + { + name: "invalid: w=0 with explicit p>0", + opts: KVMFailoverReservationConsolidationOpts{ + TotalCountWeight: testlib.Ptr(0.0), + SameSpecPenalty: testlib.Ptr(0.5), + }, + wantErr: "sameSpecPenalty must be zero when totalCountWeight is zero", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.opts.Validate() + if tt.wantErr == "" { + if err != nil { + t.Errorf("expected no error, got %v", err) + } + } else { + if err == nil { + t.Errorf("expected error %q, got nil", tt.wantErr) + } else if err.Error() != tt.wantErr { + t.Errorf("expected error %q, got %q", tt.wantErr, err.Error()) + } + } + }) + } +} diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index e076e41c9..9849075b9 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -19,10 +19,13 @@ import ( "github.com/go-logr/logr" "github.com/google/uuid" "github.com/sapcc/go-api-declarations/liquid" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // sortedKeys returns map keys sorted alphabetically for deterministic iteration. @@ -37,42 +40,47 @@ func sortedKeys[K ~string, V any](m map[K]V) []K { return keys } -// implements POST /commitments/v1/change-commitments from Limes LIQUID API: +// crSnapshot captures a CommittedResource CRD's prior state for batch rollback. +// prevSpec is nil when the CRD was newly created (i.e. did not exist before the batch). +// wasDeleted is true when the batch operation deleted the CRD; rollback must re-create it. +type crSnapshot struct { + crName string + prevSpec *v1alpha1.CommittedResourceSpec + wasDeleted bool +} + +// HandleChangeCommitments implements POST /commitments/v1/change-commitments from the Limes LIQUID API. +// It writes CommittedResource CRDs (one per commitment) and polls their status conditions until +// the controller confirms or rejects each one. On any failure the whole batch is rolled back. +// // See: https://github.com/sapcc/go-api-declarations/blob/main/liquid/commitment.go // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid -// -// This endpoint handles commitment changes by creating/updating/deleting Reservation CRDs based on the commitment lifecycle. -// A request may contain multiple commitment changes which are processed in a single transaction. If any change fails, all changes are rolled back. func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Request) { startTime := time.Now() - // Initialize resp := liquid.CommitmentChangeResponse{} req := liquid.CommitmentChangeRequest{} statusCode := http.StatusOK - // Extract or generate request ID for tracing - always set in response header requestID := r.Header.Get("X-Request-ID") if requestID == "" { requestID = uuid.New().String() } w.Header().Set("X-Request-ID", requestID) - // Check if API is enabled - if !api.config.EnableChangeCommitmentsAPI { + if !api.config.EnableChangeCommitments { statusCode = http.StatusServiceUnavailable http.Error(w, "change-commitments API is disabled", statusCode) api.recordMetrics(req, resp, statusCode, startTime) return } - // Serialize all change-commitments requests + // Serialize all change-commitments requests so the controller sees a consistent world. api.changeMutex.Lock() defer api.changeMutex.Unlock() ctx := reservations.WithGlobalRequestID(context.Background(), "committed-resource-"+requestID) logger := commitments.LoggerFromContext(ctx).WithValues("component", "api", "endpoint", "/commitments/v1/change-commitments") - // Only accept POST method if r.Method != http.MethodPost { statusCode = http.StatusMethodNotAllowed http.Error(w, "Method not allowed", statusCode) @@ -80,7 +88,6 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque return } - // Parse request body if err := json.NewDecoder(r.Body).Decode(&req); err != nil { logger.Error(err, "invalid request body") statusCode = http.StatusBadRequest @@ -91,7 +98,6 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque logger.Info("received change commitments request", "affectedProjects", len(req.ByProject), "dryRun", req.DryRun, "availabilityZone", req.AZ) - // Check for dry run -> early reject, not supported yet if req.DryRun { resp.RejectionReason = "Dry run not supported yet" api.recordMetrics(req, resp, statusCode, startTime) @@ -104,26 +110,17 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque return } - // Process commitment changes - // For now, we'll implement a simplified path that checks capacity for immediate start CRs - if err := api.processCommitmentChanges(ctx, w, logger, req, &resp); err != nil { - // Error already written to response by processCommitmentChanges - // Determine status code from error context (409 or 503) if strings.Contains(err.Error(), "version mismatch") { statusCode = http.StatusConflict } else if strings.Contains(err.Error(), "caches not ready") { statusCode = http.StatusServiceUnavailable } - // Record metrics for error cases api.recordMetrics(req, resp, statusCode, startTime) return } - // Record metrics api.recordMetrics(req, resp, statusCode, startTime) - - // Return response w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) if err := json.NewEncoder(w).Encode(resp); err != nil { @@ -132,11 +129,6 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.ResponseWriter, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) error { - manager := commitments.NewReservationManager(api.client) - requireRollback := false - failedCommitments := make(map[string]string) // commitmentUUID to reason for failure, for better response messages in case of rollback - creatorRequestID := reservations.GlobalRequestIDFromContext(ctx) - knowledge := &reservations.FlavorGroupKnowledgeClient{Client: api.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) if err != nil { @@ -145,12 +137,10 @@ func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.Respons return errors.New("caches not ready") } - // Validate InfoVersion from request matches current version (= last content change of flavor group knowledge) var currentVersion int64 = -1 if knowledgeCRD, err := knowledge.Get(ctx); err == nil && knowledgeCRD != nil && !knowledgeCRD.Status.LastContentChange.IsZero() { currentVersion = knowledgeCRD.Status.LastContentChange.Unix() } - if req.InfoVersion != currentVersion { logger.Info("version mismatch in commitment change request", "requestVersion", req.InfoVersion, @@ -160,160 +150,166 @@ func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.Respons return errors.New("version mismatch") } - statesBefore := make(map[string]*commitments.CommitmentState) // map of commitmentID to existing state for rollback - var reservationsToWatch []v1alpha1.Reservation + // If Limes does not require confirmation for this batch (e.g. deletions, status-only transitions), + // the controller must not reject — it must retry until it succeeds (AllowRejection=false). + // Conversely, when Limes requires confirmation, the controller may reject and report back. + allowRejection := req.RequiresConfirmation() - if req.DryRun { - resp.RejectionReason = "Dry run not supported yet" - return nil - } + var ( + toWatch []string // CRD names to poll for terminal conditions (upserts only) + snapshots []crSnapshot // ordered list for deterministic rollback + failedReason string + rollback bool + ) ProcessLoop: for _, projectID := range sortedKeys(req.ByProject) { projectChanges := req.ByProject[projectID] + + // Extract domain ID from Keystone project metadata if Limes provided it. + domainID := "" + if pm := projectChanges.ProjectMetadata.UnwrapOr(liquid.ProjectMetadata{}); pm.Domain.UUID != "" { + domainID = pm.Domain.UUID + } + for _, resourceName := range sortedKeys(projectChanges.ByResource) { resourceChanges := projectChanges.ByResource[resourceName] - // Validate resource name pattern (instances_group_*) + flavorGroupName, err := commitments.GetFlavorGroupNameFromResource(string(resourceName)) if err != nil { - resp.RejectionReason = fmt.Sprintf("project with unknown resource name %s: %v", projectID, err) - requireRollback = true + failedReason = fmt.Sprintf("project with unknown resource name %s: %v", projectID, err) + rollback = true break ProcessLoop } - // Verify flavor group exists in Knowledge CRDs - flavorGroup, flavorGroupExists := flavorGroups[flavorGroupName] - if !flavorGroupExists { - resp.RejectionReason = "flavor group not found: " + flavorGroupName - requireRollback = true + flavorGroup, ok := flavorGroups[flavorGroupName] + if !ok { + failedReason = "flavor group not found: " + flavorGroupName + rollback = true break ProcessLoop } - // Reject commitments for flavor groups that don't accept CRs if !commitments.FlavorGroupAcceptsCommitments(&flavorGroup) { - resp.RejectionReason = commitments.FlavorGroupCommitmentRejectionReason(&flavorGroup) - requireRollback = true + failedReason = commitments.FlavorGroupCommitmentRejectionReason(&flavorGroup) + rollback = true break ProcessLoop } for _, commitment := range resourceChanges.Commitments { - logger.V(1).Info("processing commitment", "commitmentUUID", commitment.UUID, "oldStatus", commitment.OldStatus.UnwrapOr("none"), "newStatus", commitment.NewStatus.UnwrapOr("none")) - - // TODO add configurable upper limit validation for commitment size (number of instances) to prevent excessive reservation creation - // TODO add domain - - // List all committed resource reservations, then filter by name prefix - var all_reservations v1alpha1.ReservationList - if err := api.client.List(ctx, &all_reservations, client.MatchingLabels{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }); err != nil { - failedCommitments[string(commitment.UUID)] = "failed to list reservations" - logger.Info("failed to list reservations for commitment", "commitmentUUID", commitment.UUID, "error", err) - requireRollback = true - break ProcessLoop - } - - // Filter by name prefix to find reservations for this commitment - namePrefix := fmt.Sprintf("commitment-%s-", string(commitment.UUID)) - var existing_reservations v1alpha1.ReservationList - for _, res := range all_reservations.Items { - if len(res.Name) >= len(namePrefix) && res.Name[:len(namePrefix)] == namePrefix { - existing_reservations.Items = append(existing_reservations.Items, res) + isDelete := commitment.NewStatus.IsNone() + crName := "commitment-" + string(commitment.UUID) + + logger.V(1).Info("processing commitment", + "commitmentUUID", commitment.UUID, + "oldStatus", commitment.OldStatus.UnwrapOr("none"), + "newStatus", commitment.NewStatus.UnwrapOr("none"), + "delete", isDelete) + + // Snapshot the current spec before mutation so we can restore it on rollback. + snap := crSnapshot{crName: crName} + existing := &v1alpha1.CommittedResource{} + if err := api.client.Get(ctx, types.NamespacedName{Name: crName}, existing); err != nil { + if !apierrors.IsNotFound(err) { + failedReason = fmt.Sprintf("commitment %s: failed to read pre-update snapshot: %v", commitment.UUID, err) + rollback = true + break ProcessLoop } + // Not found: CR is new (or already absent for deletes), prevSpec stays nil. + } else { + specCopy := existing.Spec + snap.prevSpec = &specCopy } - var stateBefore *commitments.CommitmentState - if len(existing_reservations.Items) == 0 { - stateBefore = &commitments.CommitmentState{ - CommitmentUUID: string(commitment.UUID), - ProjectID: string(projectID), - FlavorGroupName: flavorGroupName, - TotalMemoryBytes: 0, - } - } else { - stateBefore, err = commitments.FromReservations(existing_reservations.Items) - if err != nil { - failedCommitments[string(commitment.UUID)] = "failed to parse existing commitment reservations" - logger.Info("failed to get existing state for commitment", "commitmentUUID", commitment.UUID, "error", err) - requireRollback = true - break ProcessLoop + if isDelete { + // Limes is removing this commitment; delete the CRD if it exists. + snap.wasDeleted = true + if snap.prevSpec != nil { + if err := api.client.Delete(ctx, existing); err != nil && !apierrors.IsNotFound(err) { + failedReason = fmt.Sprintf("commitment %s: failed to delete CommittedResource CRD: %v", commitment.UUID, err) + rollback = true + break ProcessLoop + } + logger.V(1).Info("deleted CommittedResource CRD", "name", crName) } + snapshots = append(snapshots, snap) + continue } - statesBefore[string(commitment.UUID)] = stateBefore - // get desired state - stateDesired, err := commitments.FromChangeCommitmentTargetState(commitment, string(projectID), flavorGroupName, flavorGroup, string(req.AZ)) + stateDesired, err := commitments.FromChangeCommitmentTargetState( + commitment, string(projectID), domainID, flavorGroupName, flavorGroup, string(req.AZ)) if err != nil { - failedCommitments[string(commitment.UUID)] = err.Error() - logger.Info("failed to get desired state for commitment", "commitmentUUID", commitment.UUID, "error", err) - requireRollback = true + failedReason = fmt.Sprintf("commitment %s: %s", commitment.UUID, err) + rollback = true break ProcessLoop } - // Set creator request ID for traceability across controller reconciles - stateDesired.CreatorRequestID = creatorRequestID - - logger.V(1).Info("applying commitment state change", "commitmentUUID", commitment.UUID, "oldMemory", stateBefore.TotalMemoryBytes, "desiredMemory", stateDesired.TotalMemoryBytes) - applyResult, err := manager.ApplyCommitmentState(ctx, logger, stateDesired, flavorGroups, "changeCommitmentsApi") - if err != nil { - failedCommitments[string(commitment.UUID)] = "failed to apply commitment state" - logger.Info("failed to apply commitment state for commitment", "commitmentUUID", commitment.UUID, "error", err) - requireRollback = true + cr := &v1alpha1.CommittedResource{} + cr.Name = crName + if _, err := controllerutil.CreateOrUpdate(ctx, api.client, cr, func() error { + applyCRSpec(cr, stateDesired, allowRejection) + return nil + }); err != nil { + failedReason = fmt.Sprintf("commitment %s: failed to write CommittedResource CRD: %v", commitment.UUID, err) + rollback = true break ProcessLoop } - logger.V(1).Info("applied commitment state change", "commitmentUUID", commitment.UUID, "touchedReservations", len(applyResult.TouchedReservations), "deletedReservations", len(applyResult.RemovedReservations)) - reservationsToWatch = append(reservationsToWatch, applyResult.TouchedReservations...) + + toWatch = append(toWatch, crName) + snapshots = append(snapshots, snap) + logger.V(1).Info("upserted CommittedResource CRD", "name", crName) } } } - // TODO make the rollback defer safe - if !requireRollback { - logger.Info("applied commitment changes, now watching for reservation readiness", "reservationsToWatch", len(reservationsToWatch)) + if !rollback { + // Non-confirming changes (RequiresConfirmation=false): Limes ignores our RejectionReason, + // so there is no point blocking on the controller outcome. The CRDs are written with + // AllowRejection=false, meaning the controller will retry indefinitely in the background. + if !allowRejection { + logger.Info("non-confirming changes applied, returning without polling", "count", len(toWatch)) + return nil + } + + logger.Info("CommittedResource CRDs written, polling for controller outcome", "count", len(toWatch)) + watchStart := time.Now() - time_start := time.Now() + rejected, watchErrs := watchCRsUntilReady( + ctx, logger, api.client, toWatch, + api.config.WatchTimeout.Duration, + api.config.WatchPollInterval.Duration, + ) - if failedReservations, errors := watchReservationsUntilReady(ctx, logger, api.client, reservationsToWatch, api.config.ChangeAPIWatchReservationsTimeout, api.config.ChangeAPIWatchReservationsPollInterval); len(failedReservations) > 0 || len(errors) > 0 { - logger.Info("reservations failed to become ready, initiating rollback", - "failedReservations", len(failedReservations), - "errors", errors) + logger.Info("polling complete", "duration", time.Since(watchStart).Round(time.Millisecond)) - for _, res := range failedReservations { - failedCommitments[res.Spec.CommittedResourceReservation.CommitmentUUID] = "not sufficient capacity" + switch { + case len(rejected) > 0: + var b strings.Builder + fmt.Fprintf(&b, "%d commitment(s) failed to apply:", len(rejected)) + for _, crName := range toWatch { // iterate toWatch for deterministic order + if reason, ok := rejected[crName]; ok { + fmt.Fprintf(&b, "\n- commitment %s: %s", strings.TrimPrefix(crName, "commitment-"), reason) + } } - if len(failedReservations) == 0 { - resp.RejectionReason += "timeout reached while processing commitment changes" - api.monitor.timeouts.Inc() + failedReason = b.String() + rollback = true + case len(watchErrs) > 0: + msgs := make([]string, len(watchErrs)) + for i, e := range watchErrs { + msgs[i] = e.Error() } - requireRollback = true + failedReason = "timeout reached while processing commitment changes: " + strings.Join(msgs, "; ") + api.monitor.timeouts.Inc() + rollback = true } - - logger.Info("finished watching reservation", "totalSchedulingTimeSeconds", time.Since(time_start).Seconds()) } - if requireRollback { - // Build rejection reason from failed commitments - if len(failedCommitments) > 0 { - var reasonBuilder strings.Builder - fmt.Fprintf(&reasonBuilder, "%d commitment(s) failed to apply: ", len(failedCommitments)) - for commitmentUUID, reason := range failedCommitments { - fmt.Fprintf(&reasonBuilder, "\n- commitment %s: %s", commitmentUUID, reason) - } - resp.RejectionReason = reasonBuilder.String() + if rollback { + resp.RejectionReason = failedReason + logger.Info("rolling back CommittedResource CRDs", "reason", failedReason, "count", len(snapshots)) + for i := len(snapshots) - 1; i >= 0; i-- { + rollbackCR(ctx, logger, api.client, snapshots[i]) } - - logger.Info("rollback of commitment changes") - for commitmentUUID, state := range statesBefore { - // Rollback to statesBefore for this commitment - logger.Info("applying rollback for commitment", "commitmentUUID", commitmentUUID, "stateBefore", state) - _, err := manager.ApplyCommitmentState(ctx, logger, state, flavorGroups, "changeCommitmentsApiRollback") - if err != nil { - logger.Info("failed to apply rollback state for commitment", "commitmentUUID", commitmentUUID, "error", err) - // continue with best effort rollback for other projects - } - } - - logger.Info("finished applying rollbacks for commitment changes", "reasonOfRollback", resp.RejectionReason) + logger.Info("rollback complete") return nil } @@ -321,111 +317,139 @@ ProcessLoop: return nil } -// watchReservationsUntilReady polls until all reservations reach Ready=True or timeout. -// Returns failed reservations and any errors encountered. -func watchReservationsUntilReady( +// watchCRsUntilReady polls CommittedResource conditions until each CRD reaches a terminal state: +// - Ready=True (Accepted) — success +// - Ready=False, Reason=Planned — success; controller reserves capacity at activation time +// - Ready=False, Reason=Rejected — failure; reason reported to caller +// +// Returns a map of crName → rejection reason for failed CRDs, and any polling errors (e.g. timeout). +func watchCRsUntilReady( ctx context.Context, logger logr.Logger, k8sClient client.Client, - reservations []v1alpha1.Reservation, + crNames []string, timeout time.Duration, pollInterval time.Duration, -) (failedReservations []v1alpha1.Reservation, errors []error) { +) (rejected map[string]string, errs []error) { - if len(reservations) == 0 { - return failedReservations, nil + if len(crNames) == 0 { + return nil, nil } + rejected = make(map[string]string) deadline := time.Now().Add(timeout) - startTime := time.Now() - totalReservations := len(reservations) - reservationsToWatch := make([]v1alpha1.Reservation, len(reservations)) - copy(reservationsToWatch, reservations) - - // Track successful reservations for summary - var successfulReservations []string - pollCount := 0 + pending := make(map[string]struct{}, len(crNames)) + for _, name := range crNames { + pending[name] = struct{}{} + } for { - pollCount++ - var stillWaiting []v1alpha1.Reservation if time.Now().After(deadline) { - errors = append(errors, fmt.Errorf("timeout after %v waiting for reservations to become ready", timeout)) - // Log summary on timeout - logger.Info("reservation watch completed (timeout)", - "total", totalReservations, - "ready", len(successfulReservations), - "failed", len(failedReservations), - "timedOut", len(reservationsToWatch), - "duration", time.Since(startTime).Round(time.Millisecond), - "polls", pollCount) - return failedReservations, errors + errs = append(errs, fmt.Errorf("timeout after %v waiting for %d CommittedResource CRD(s)", timeout, len(pending))) + return rejected, errs } - for _, res := range reservationsToWatch { - // Fetch current state - var current v1alpha1.Reservation - nn := types.NamespacedName{ - Name: res.Name, - Namespace: res.Namespace, + for name := range pending { + cr := &v1alpha1.CommittedResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, cr); err != nil { + continue // transient; keep waiting } - if err := k8sClient.Get(ctx, nn, ¤t); err != nil { - // Reservation is still in process of being created, or there is a transient error - stillWaiting = append(stillWaiting, res) - continue + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + continue // controller hasn't reconciled yet } - // Check Ready condition - readyCond := meta.FindStatusCondition( - current.Status.Conditions, - v1alpha1.ReservationConditionReady, - ) - - if readyCond == nil { - // Condition not set yet, keep waiting - stillWaiting = append(stillWaiting, res) - continue - } - - switch readyCond.Status { - case metav1.ConditionTrue: - // Only consider truly ready if Status.Host is populated - if current.Spec.TargetHost == "" || current.Status.Host == "" { - stillWaiting = append(stillWaiting, res) - continue - } - // Reservation is successfully scheduled - track for summary - successfulReservations = append(successfulReservations, current.Name) - - case metav1.ConditionFalse: - // Any failure reason counts as failed - failedReservations = append(failedReservations, current) - case metav1.ConditionUnknown: - stillWaiting = append(stillWaiting, res) + switch { + case cond.Status == metav1.ConditionTrue: + delete(pending, name) + case cond.Status == metav1.ConditionFalse && cond.Reason == v1alpha1.CommittedResourceReasonPlanned: + delete(pending, name) // planned = accepted; controller will reserve at activation + case cond.Status == metav1.ConditionFalse && cond.Reason == v1alpha1.CommittedResourceReasonRejected: + delete(pending, name) + rejected[name] = cond.Message + // Reason=Reserving: controller is placing slots; keep waiting. } } - if len(stillWaiting) == 0 { - // All reservations have reached a terminal state - log summary - logger.Info("reservation watch completed", - "total", totalReservations, - "ready", len(successfulReservations), - "failed", len(failedReservations), - "duration", time.Since(startTime).Round(time.Millisecond), - "polls", pollCount) - return failedReservations, errors + if len(pending) == 0 { + return rejected, nil } - reservationsToWatch = stillWaiting - - // Wait before next poll select { case <-time.After(pollInterval): - // Continue polling case <-ctx.Done(): - return failedReservations, append(errors, fmt.Errorf("context cancelled while waiting for reservations: %w", ctx.Err())) + return rejected, append(errs, fmt.Errorf("context cancelled: %w", ctx.Err())) } + logger.V(1).Info("polling CommittedResource CRDs", "pending", len(pending)) + } +} + +// rollbackCR reverses the batch-local change to a single CommittedResource CRD. +// - wasDeleted=true, prevSpec!=nil: CRD was deleted; re-create it from the snapshot. +// - wasDeleted=true, prevSpec==nil: CRD was absent before and after; nothing to do. +// - wasDeleted=false, prevSpec==nil: CRD was newly created; delete it. +// - wasDeleted=false, prevSpec!=nil: CRD was updated; restore its spec. +func rollbackCR(ctx context.Context, logger logr.Logger, k8sClient client.Client, snap crSnapshot) { + if snap.wasDeleted { + if snap.prevSpec == nil { + return // was absent before deletion attempt; nothing to undo + } + cr := &v1alpha1.CommittedResource{} + cr.Name = snap.crName + cr.Spec = *snap.prevSpec + if err := k8sClient.Create(ctx, cr); client.IgnoreAlreadyExists(err) != nil { + logger.Error(err, "failed to re-create CommittedResource CRD during rollback", "name", snap.crName) + } + return + } + + if snap.prevSpec == nil { + cr := &v1alpha1.CommittedResource{} + cr.Name = snap.crName + if err := k8sClient.Delete(ctx, cr); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to delete CommittedResource CRD during rollback", "name", snap.crName) + } + return + } + + cr := &v1alpha1.CommittedResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: snap.crName}, cr); err != nil { + logger.Error(err, "failed to fetch CommittedResource CRD for rollback", "name", snap.crName) + return + } + cr.Spec = *snap.prevSpec + if err := k8sClient.Update(ctx, cr); err != nil { + logger.Error(err, "failed to restore CommittedResource CRD spec during rollback", "name", snap.crName) + } +} + +// applyCRSpec writes CommitmentState fields into a CommittedResource CRD spec. +// allowRejection=true for the change-commitments API path: the controller may reject +// on failure and the API reports the outcome to Limes. +func applyCRSpec(cr *v1alpha1.CommittedResource, state *commitments.CommitmentState, allowRejection bool) { + cr.Spec.CommitmentUUID = state.CommitmentUUID + cr.Spec.SchedulingDomain = v1alpha1.SchedulingDomainNova + cr.Spec.FlavorGroupName = state.FlavorGroupName + cr.Spec.ResourceType = v1alpha1.CommittedResourceTypeMemory + cr.Spec.Amount = *resource.NewQuantity(state.TotalMemoryBytes, resource.BinarySI) + cr.Spec.AvailabilityZone = state.AvailabilityZone + cr.Spec.ProjectID = state.ProjectID + cr.Spec.DomainID = state.DomainID + cr.Spec.State = state.State + cr.Spec.AllowRejection = allowRejection + + if state.StartTime != nil { + t := metav1.NewTime(*state.StartTime) + cr.Spec.StartTime = &t + } else { + cr.Spec.StartTime = nil + } + if state.EndTime != nil { + t := metav1.NewTime(*state.EndTime) + cr.Spec.EndTime = &t + } else { + cr.Spec.EndTime = nil } } diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go new file mode 100644 index 000000000..ee546655b --- /dev/null +++ b/internal/scheduling/reservations/commitments/api/change_commitments_e2e_test.go @@ -0,0 +1,411 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package api + +// End-to-end tests: HTTP → CommittedResource CRD → Reservation CRDs → scheduler → controllers → HTTP response. +// +// Unlike change_commitments_test.go which uses fakeControllerClient (which immediately sets +// conditions), these tests wire real CommittedResourceController and CommitmentReservationController +// against a fake k8s client. A background goroutine drives reconcile loops so the API polling +// loop can observe terminal conditions within its timeout window. + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/prometheus/client_golang/prometheus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// Field index paths for the fake client — must match the unexported constants in the commitments package. +const ( + e2eIdxCommittedResourceByUUID = "spec.commitmentUUID" + e2eIdxReservationByCommitmentUUID = "spec.committedResourceReservation.commitmentUUID" +) + +// e2eEnv is a full end-to-end test environment: real controllers, fake k8s client, +// mock scheduler, and a background reconcile driver goroutine. +type e2eEnv struct { + t *testing.T + k8sClient client.Client + httpServer *httptest.Server + schedulerSrv *httptest.Server + crCtrl *commitments.CommittedResourceController + resCtrl *commitments.CommitmentReservationController + cancelBg context.CancelFunc + bgDone chan struct{} +} + +// newE2EEnv creates an e2eEnv with the given flavors and scheduler handler. +// The scheduler handler controls what the mock Nova scheduler returns. +func newE2EEnv(t *testing.T, flavors []*TestFlavor, infoVersion int64, schedulerHandler http.HandlerFunc) *e2eEnv { + t.Helper() + log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) + + // Scheme: v1alpha1 for CR/Reservation/Knowledge types; hv1 for Hypervisor. + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("add hv1 scheme: %v", err) + } + + // One hypervisor so the reservation controller can build a non-empty eligible-hosts list. + hypervisor := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}} + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(createKnowledgeCRD(buildFlavorGroupsKnowledge(flavors, infoVersion)), hypervisor). + WithStatusSubresource( + &v1alpha1.CommittedResource{}, + &v1alpha1.Reservation{}, + &v1alpha1.Knowledge{}, + ). + WithIndex(&v1alpha1.Reservation{}, e2eIdxReservationByCommitmentUUID, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.CommitmentUUID == "" { + return nil + } + return []string{res.Spec.CommittedResourceReservation.CommitmentUUID} + }). + WithIndex(&v1alpha1.CommittedResource{}, e2eIdxCommittedResourceByUUID, func(obj client.Object) []string { + cr, ok := obj.(*v1alpha1.CommittedResource) + if !ok || cr.Spec.CommitmentUUID == "" { + return nil + } + return []string{cr.Spec.CommitmentUUID} + }). + Build() + + schedulerSrv := httptest.NewServer(schedulerHandler) + + crCtrl := &commitments.CommittedResourceController{ + Client: k8sClient, + Scheme: scheme, + Conf: commitments.CommittedResourceControllerConfig{RequeueIntervalRetry: metav1.Duration{Duration: 100 * time.Millisecond}}, + } + + resCtrl := &commitments.CommitmentReservationController{ + Client: k8sClient, + Scheme: scheme, + Conf: commitments.ReservationControllerConfig{ + SchedulerURL: schedulerSrv.URL, + AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, + RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, + RequeueIntervalRetry: metav1.Duration{Duration: 100 * time.Millisecond}, + }, + } + if err := resCtrl.Init(context.Background(), resCtrl.Conf); err != nil { + t.Fatalf("resCtrl.Init: %v", err) + } + + // HTTPAPI wired directly to the real k8s client (no fakeControllerClient wrapper). + cfg := commitments.DefaultAPIConfig() + cfg.WatchTimeout = metav1.Duration{Duration: 5 * time.Second} + cfg.WatchPollInterval = metav1.Duration{Duration: 100 * time.Millisecond} + api := NewAPIWithConfig(k8sClient, cfg, nil) + mux := http.NewServeMux() + api.Init(mux, prometheus.NewRegistry(), log.Log) + httpServer := httptest.NewServer(mux) + + ctx, cancel := context.WithCancel(context.Background()) + env := &e2eEnv{ + t: t, + k8sClient: k8sClient, + httpServer: httpServer, + schedulerSrv: schedulerSrv, + crCtrl: crCtrl, + resCtrl: resCtrl, + cancelBg: cancel, + bgDone: make(chan struct{}), + } + go env.driveReconciles(ctx) + return env +} + +func (e *e2eEnv) close() { + e.cancelBg() + <-e.bgDone + e.httpServer.Close() + e.schedulerSrv.Close() +} + +// asCRTestEnv wraps e2eEnv as a CRTestEnv to reuse its HTTP-call and assertion helpers. +func (e *e2eEnv) asCRTestEnv() *CRTestEnv { + return &CRTestEnv{T: e.t, K8sClient: e.k8sClient, HTTPServer: e.httpServer} +} + +// driveReconciles runs in the background, reconciling pending CRs and Reservations until ctx is cancelled. +func (e *e2eEnv) driveReconciles(ctx context.Context) { + defer close(e.bgDone) + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + e.reconcileAll(ctx) + } + } +} + +// reconcileAll drives one round of reconciles: +// 1. CR pass 1 — adds finalizer and creates Reservation CRDs. +// 2. Reservation pass — calls the scheduler, sets TargetHost (first reconcile) then Ready=True (second). +// 3. CR pass 2 — re-fetches each CR and picks up Reservation outcomes (placed or rejected). +// +// CRs and Reservations that have already reached a terminal state are skipped to avoid +// overwriting the rejection signal the API polling loop needs to read. +func (e *e2eEnv) reconcileAll(ctx context.Context) { + var crList v1alpha1.CommittedResourceList + if err := e.k8sClient.List(ctx, &crList); err != nil { + return + } + + // CR pass 1. + for _, cr := range crList.Items { + if e2eIsTerminalCR(cr) { + continue + } + e.crCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: cr.Name}}) //nolint:errcheck + } + + // Reservation pass (two reconciles per slot: first sets TargetHost, second sets Ready=True). + var resList v1alpha1.ReservationList + if err := e.k8sClient.List(ctx, &resList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + return + } + for _, res := range resList.Items { + if e2eIsTerminalReservation(res) { + continue + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} + e.resCtrl.Reconcile(ctx, req) //nolint:errcheck + e.resCtrl.Reconcile(ctx, req) //nolint:errcheck + } + + // CR pass 2: re-fetch so we see any condition changes made during the Reservation pass. + for _, cr := range crList.Items { + var latest v1alpha1.CommittedResource + if err := e.k8sClient.Get(ctx, types.NamespacedName{Name: cr.Name}, &latest); err != nil { + continue // deleted or transient + } + if e2eIsTerminalCR(latest) { + continue + } + e.crCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: latest.Name}}) //nolint:errcheck + } +} + +// e2eIsTerminalCR returns true for states the API polling loop treats as final: +// Accepted (Ready=True), Rejected, or Planned. +// CRs with DeletionTimestamp are never terminal here: they need one more reconcile to remove +// their finalizer (set by the controller on first reconcile) so the fake client can delete them. +func e2eIsTerminalCR(cr v1alpha1.CommittedResource) bool { + if !cr.DeletionTimestamp.IsZero() { + return false + } + cond := apimeta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + return false + } + if cond.Status == metav1.ConditionTrue { + return true + } + return cond.Reason == v1alpha1.CommittedResourceReasonRejected || + cond.Reason == v1alpha1.CommittedResourceReasonPlanned +} + +// waitForCRAbsent polls until the named CommittedResource no longer exists or the 1s deadline passes. +// Used after rollback calls because the finalizer removal happens asynchronously in the background reconcile loop. +func (e *e2eEnv) waitForCRAbsent(t *testing.T, crName string) { + t.Helper() + deadline := time.Now().Add(1 * time.Second) + for { + cr := &v1alpha1.CommittedResource{} + err := e.k8sClient.Get(context.Background(), types.NamespacedName{Name: crName}, cr) + if apierrors.IsNotFound(err) { + return + } + if time.Now().After(deadline) { + t.Errorf("expected CommittedResource %q to be absent after rollback, but it still exists", crName) + return + } + time.Sleep(50 * time.Millisecond) + } +} + +// e2eIsTerminalReservation returns true when a Reservation is fully placed (Ready=True). +func e2eIsTerminalReservation(res v1alpha1.Reservation) bool { + cond := apimeta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + return cond != nil && cond.Status == metav1.ConditionTrue +} + +// ============================================================================ +// Scheduler handlers +// ============================================================================ + +func e2eAcceptScheduler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Errorf("scheduler encode: %v", err) + } + } +} + +func e2eRejectScheduler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + // Return an empty hosts list — the reservation controller treats this as NoHostsFound. + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Errorf("scheduler encode: %v", err) + } + } +} + +// ============================================================================ +// E2E test cases +// ============================================================================ + +const e2eInfoVersion = int64(1234) + +var e2eFlavor = &TestFlavor{Name: "m1.small", Group: "hana_1", MemoryMB: 1024, VCPUs: 4} + +// TestE2EChangeCommitments is the full end-to-end suite: HTTP → CRD → controller → scheduler → HTTP response. +func TestE2EChangeCommitments(t *testing.T) { + testCases := []struct { + Name string + Scheduler func(*testing.T) http.HandlerFunc + ReqJSON string + WantResp APIResponseExpectation + WantAbsent []string + Verify func(*testing.T, *e2eEnv) + }{ + { + Name: "scheduler accepts: CR placed, Reservation on host-1", + Scheduler: e2eAcceptScheduler, + ReqJSON: buildRequestJSON(newCommitmentRequest("az-a", false, e2eInfoVersion, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-e2e-ok", "confirmed", 1))), + WantResp: newAPIResponse(), + Verify: func(t *testing.T, env *e2eEnv) { + t.Helper() + env.asCRTestEnv().VerifyCRsExist([]string{"commitment-uuid-e2e-ok"}) + + var cr v1alpha1.CommittedResource + if err := env.k8sClient.Get(context.Background(), types.NamespacedName{Name: "commitment-uuid-e2e-ok"}, &cr); err != nil { + t.Fatalf("get CR: %v", err) + } + if !apimeta.IsStatusConditionTrue(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Errorf("expected CR Ready=True") + } + + var resList v1alpha1.ReservationList + if err := env.k8sClient.List(context.Background(), &resList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + if len(resList.Items) != 1 { + t.Fatalf("expected 1 Reservation, got %d", len(resList.Items)) + } + res := resList.Items[0] + if !apimeta.IsStatusConditionTrue(res.Status.Conditions, v1alpha1.ReservationConditionReady) { + t.Errorf("expected Reservation Ready=True") + } + if res.Status.Host != "host-1" { + t.Errorf("Reservation Status.Host: want host-1, got %q", res.Status.Host) + } + }, + }, + { + Name: "scheduler rejects: rejection propagates to API response, CR rolled back", + Scheduler: e2eRejectScheduler, + ReqJSON: buildRequestJSON(newCommitmentRequest("az-a", false, e2eInfoVersion, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-e2e-rej", "confirmed", 2))), + WantResp: newAPIResponse("no hosts found"), + WantAbsent: []string{"commitment-uuid-e2e-rej"}, + }, + { + Name: "batch with one rejection: entire batch rolled back", + Scheduler: e2eRejectScheduler, + ReqJSON: buildRequestJSON(newCommitmentRequest("az-a", false, e2eInfoVersion, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-e2e-batch-a", "confirmed", 2), + createCommitment("hw_version_hana_1_ram", "project-B", "uuid-e2e-batch-b", "confirmed", 2), + )), + WantResp: newAPIResponse("no hosts found"), + WantAbsent: []string{"commitment-uuid-e2e-batch-a", "commitment-uuid-e2e-batch-b"}, + }, + { + Name: "lifecycle: create then delete, CR and child Reservations cleaned up", + Scheduler: e2eAcceptScheduler, + ReqJSON: buildRequestJSON(newCommitmentRequest("az-a", false, e2eInfoVersion, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-e2e-lifecycle", "confirmed", 1))), + WantResp: newAPIResponse(), + Verify: func(t *testing.T, env *e2eEnv) { + t.Helper() + env.asCRTestEnv().VerifyCRsExist([]string{"commitment-uuid-e2e-lifecycle"}) + + te := env.asCRTestEnv() + deleteJSON := buildRequestJSON(newCommitmentRequest("az-a", false, e2eInfoVersion, + deleteCommitment("hw_version_hana_1_ram", "project-A", "uuid-e2e-lifecycle", "confirmed", 1))) + resp, _, statusCode := te.CallChangeCommitmentsAPI(deleteJSON) + te.VerifyAPIResponse(newAPIResponse(), resp, statusCode) + + env.waitForCRAbsent(t, "commitment-uuid-e2e-lifecycle") + + var resList v1alpha1.ReservationList + if err := env.k8sClient.List(context.Background(), &resList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + if len(resList.Items) != 0 { + t.Errorf("expected 0 Reservations after delete, got %d", len(resList.Items)) + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + env := newE2EEnv(t, []*TestFlavor{e2eFlavor}, e2eInfoVersion, tc.Scheduler(t)) + defer env.close() + + te := env.asCRTestEnv() + resp, _, statusCode := te.CallChangeCommitmentsAPI(tc.ReqJSON) + te.VerifyAPIResponse(tc.WantResp, resp, statusCode) + for _, name := range tc.WantAbsent { + env.waitForCRAbsent(t, name) + } + if tc.Verify != nil { + tc.Verify(t, env) + } + }) + } +} diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_metrics.go b/internal/scheduling/reservations/commitments/api/change_commitments_metrics.go index 2c9562ee8..1afeea5f5 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_metrics.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_metrics.go @@ -23,7 +23,7 @@ func (api *HTTPAPI) recordMetrics(req liquid.CommitmentChangeRequest, resp liqui commitmentCount := countCommitments(req) // Determine result based on response - result := "success" + result := "accepted" if resp.RejectionReason != "" { result = "rejected" } diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_test.go index deffc91c3..579173460 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_test.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_test.go @@ -1,14 +1,13 @@ // Copyright SAP SE // SPDX-License-Identifier: Apache-2.0 -//nolint:unparam,unused // test helper functions have fixed parameters for simplicity +//nolint:unparam // test helper functions have fixed parameters for simplicity package api import ( "bytes" "context" "encoding/json" - "fmt" "io" "net/http" "net/http/httptest" @@ -23,10 +22,11 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" commitments "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" - hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + . "github.com/majewsky/gg/option" "github.com/prometheus/client_golang/prometheus" "github.com/sapcc/go-api-declarations/liquid" - corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -40,657 +40,420 @@ import ( // Integration Tests // ============================================================================ -func TestCommitmentChangeIntegration(t *testing.T) { - m1Tiny := &TestFlavor{Name: "m1.tiny", Group: "gp_1", MemoryMB: 256, VCPUs: 1} +func TestHandleChangeCommitments(t *testing.T) { m1Small := &TestFlavor{Name: "m1.small", Group: "hana_1", MemoryMB: 1024, VCPUs: 4} - m1Large := &TestFlavor{Name: "m1.large", Group: "hana_1", MemoryMB: 4096, VCPUs: 16} - m1XL := &TestFlavor{Name: "m1.xl", Group: "hana_1", MemoryMB: 8192, VCPUs: 32} testCases := []CommitmentChangeTestCase{ + // --- Basic flow --- { - Name: "Shrinking CR - unused reservations removed, used reservations untouched", - VMs: []*TestVM{{UUID: "vm-a1", Flavor: m1Large, ProjectID: "project-A", Host: "host-1", AZ: "az-a"}}, - Flavors: []*TestFlavor{m1Small, m1Large}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-123", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-a1"}}, - {CommitmentID: "uuid-123", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-123", Host: "host-3", Flavor: m1Small, ProjectID: "project-A"}, - }, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, createCommitment("hw_version_hana_1_ram", "project-A", "uuid-123", "confirmed", 2)), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-123", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-a1"}}, - {CommitmentID: "uuid-123", Host: "host-3", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse(), - }, - { - Name: "Insufficient capacity when increasing CR", - VMs: []*TestVM{}, - Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{{CommitmentID: "uuid-456", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, createCommitment("hw_version_hana_1_ram", "project-A", "uuid-456", "confirmed", 3)), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 1024, "host-2": 0}}, - ExpectedReservations: []*TestReservation{{CommitmentID: "uuid-456", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}}, - ExpectedAPIResponse: newAPIResponse("1 commitment(s) failed", "commitment uuid-456: not sufficient capacity"), - }, - { - Name: "Invalid CR name - too long", - VMs: []*TestVM{}, - Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", strings.Repeat("long-", 13), "confirmed", 3), - ), - AvailableResources: &AvailableResources{}, - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse("1 commitment(s) failed", "commitment long-long-long-long-long-long-long-long-long-long-long-long-long-: unexpected commitment format"), - }, - { - Name: "Planned CR is ignored in validation, no scheduling or capacity reservation", - VMs: []*TestVM{}, + Name: "New CR: controller accepts → API returns accepted", Flavors: []*TestFlavor{m1Small}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-new", "planned", 200), - ), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-new", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-new"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-new": true}, }, { - Name: "Invalid CR name - spaces", - VMs: []*TestVM{}, - Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{}, + Name: "New CR: controller rejects → API returns rejection reason", + Flavors: []*TestFlavor{m1Small}, + CROutcomes: map[string]string{ + "commitment-uuid-rej": "not sufficient capacity", + }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid with space", "confirmed", 3), - ), - AvailableResources: &AvailableResources{}, - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse("1 commitment(s) failed", "commitment uuid with space: unexpected commitment format"), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-rej", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse("commitment uuid-rej: not sufficient capacity"), }, + // --- Planned state --- { - Name: "Swap capacity between CRs - order dependent - delete-first succeeds", + Name: "Planned CR: controller sets Ready=False/Planned → API accepts", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-456", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-456", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}}, + CROutcomes: map[string]string{ + "commitment-uuid-plan": v1alpha1.CommittedResourceReasonPlanned, + }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-456", "confirmed", 0), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-123", "confirmed", 2), - ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 0, "host-2": 0}}, - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-123", Host: "host-1", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-123", Host: "host-2", Flavor: m1Small, ProjectID: "project-B"}}, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-plan", "planned", 2)), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-plan"}, }, + // --- Update existing CR --- { - Name: "Swap capacity between CRs - order dependent - create-first fails", + Name: "Resize up: existing CR updated with new amount, accepted", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-123", Host: "host-1", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-123", Host: "host-2", Flavor: m1Small, ProjectID: "project-B"}}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-resize", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, + }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-456", "confirmed", 2), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-123", "confirmed", 0), - ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 0, "host-2": 0}}, - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-123", Host: "host-1", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-123", Host: "host-2", Flavor: m1Small, ProjectID: "project-B"}}, - ExpectedAPIResponse: newAPIResponse("1 commitment(s) failed", "commitment uuid-456: not sufficient capacity"), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-resize", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-resize"}, }, + // --- Rollback: new CR deleted on batch failure --- { - Name: "Flavor bin-packing - mixed sizes when largest doesn't fit", - // Greedy selection: 10GB request with 8/4/1GB flavors → picks 1×8GB + 2×1GB - Flavors: []*TestFlavor{m1XL, m1Large, m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-binpack", "confirmed", 10), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-binpack", Flavor: m1XL, ProjectID: "project-A"}, - {CommitmentID: "uuid-binpack", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-binpack", Flavor: m1Small, ProjectID: "project-A"}, + Name: "Rollback new CR: newly created CRD deleted on rejection", + Flavors: []*TestFlavor{m1Small}, + CROutcomes: map[string]string{ + "commitment-uuid-rollback": "not sufficient capacity", }, - ExpectedAPIResponse: newAPIResponse(), + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-rollback", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse("uuid-rollback: not sufficient capacity"), + ExpectedDeletedCRs: []string{"commitment-uuid-rollback"}, }, + // --- Rollback: updated CR spec restored on batch failure --- { - Name: "Version mismatch - request rejected with 409 Conflict", - // InfoVersion validation prevents stale requests (1233 vs 1234) + Name: "Rollback updated CR: spec restored on rejection", Flavors: []*TestFlavor{m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1233, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-version", "confirmed", 2), - ), - EnvInfoVersion: 1234, - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: APIResponseExpectation{StatusCode: 409}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-restore", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, + }, + CROutcomes: map[string]string{ + "commitment-uuid-restore": "not sufficient capacity", + }, + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-restore", "confirmed", 4)), + ExpectedAPIResponse: newAPIResponse("uuid-restore: not sufficient capacity"), + // CRD still exists but amount restored to 1024 MiB + ExpectedCRSpecs: map[string]int64{"commitment-uuid-restore": 1024 * 1024 * 1024}, }, + // --- Batch rollback: one failure rolls back all --- { - Name: "Multi-project rollback - one failure rolls back all", - // Transactional: project-B fails (insufficient capacity) → both projects rollback + Name: "Batch rollback: project-B fails → project-A new CR also rolled back", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-project-a", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, + CROutcomes: map[string]string{ + "commitment-uuid-b": "not sufficient capacity", }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-project-a", "confirmed", 2), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-project-b", "confirmed", 2), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-a", "confirmed", 2), + createCommitment("hw_version_hana_1_ram", "project-B", "uuid-b", "confirmed", 2), ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 1024, "host-2": 0}}, - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-project-a", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse("uuid-project-b", "not sufficient capacity"), + ExpectedAPIResponse: newAPIResponse("uuid-b: not sufficient capacity"), + ExpectedDeletedCRs: []string{"commitment-uuid-a", "commitment-uuid-b"}, }, + // --- Timeout --- { - Name: "Rollback with VMs allocated - limitation: VM allocations not rolled back", - // Controller will eventually clean up and repair inconsistent state - VMs: []*TestVM{{UUID: "vm-rollback", Flavor: m1Small, ProjectID: "project-A", Host: "host-1", AZ: "az-a"}}, + Name: "Timeout: no condition set → rollback and timeout error", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "commitment-A", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-rollback"}}, - {CommitmentID: "commitment-A", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, + CROutcomes: map[string]string{ + "commitment-uuid-timeout": "", // empty string = no condition set (controller not responding) }, + NoCondition: []string{"commitment-uuid-timeout"}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "commitment-A", "confirmed", 0), - createCommitment("hw_version_hana_1_ram", "project-B", "commitment-B", "confirmed", 6), - ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 0}}, - ExpectedReservations: []*TestReservation{ - // Rollback creates unscheduled reservations (empty Host accepts any in matching) - {CommitmentID: "commitment-A", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "commitment-A", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse("commitment-B", "not sufficient capacity"), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-timeout", "confirmed", 2)), + CustomConfig: func() *commitments.APIConfig { + cfg := commitments.DefaultAPIConfig() + cfg.WatchTimeout = metav1.Duration{} + cfg.WatchPollInterval = metav1.Duration{Duration: 100 * time.Millisecond} + return &cfg + }(), + ExpectedAPIResponse: newAPIResponse("timeout reached while processing commitment changes"), + ExpectedDeletedCRs: []string{"commitment-uuid-timeout"}, }, + // --- Input validation --- { - Name: "New commitment creation - from zero to N reservations", + Name: "Invalid commitment UUID: rejected before CRD write", Flavors: []*TestFlavor{m1Small}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-new", "confirmed", 3), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-new", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-new", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-new", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", strings.Repeat("x", 50), "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse("unexpected commitment format"), + ExpectedDeletedCRs: []string{"commitment-" + strings.Repeat("x", 50)}, }, { - Name: "New commitment creation - large batch", + Name: "Unknown flavor group: rejected without CRD write", Flavors: []*TestFlavor{m1Small}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-new", "confirmed", 200), - ), - ExpectedReservations: func() []*TestReservation { - var reservations []*TestReservation - for range 200 { - reservations = append(reservations, &TestReservation{ - CommitmentID: "uuid-new", - Flavor: m1Small, - ProjectID: "project-A", - }) - } - return reservations - }(), - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_nonexistent_ram", "project-A", "uuid-unk", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse("flavor group not found"), }, + // --- Infrastructure --- { - Name: "With reservations of custom size - total unchanged", - // Preserves custom-sized reservations when total matches (2×2GB = 4GB) + Name: "Version mismatch: 409 Conflict", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Host: "host-2", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - }, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-custom", "confirmed", 4), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Host: "host-2", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - }, - ExpectedAPIResponse: newAPIResponse(), + CommitmentRequest: newCommitmentRequest("az-a", false, 9999, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-v", "confirmed", 2)), + EnvInfoVersion: 1234, // env is at 1234, request claims 9999 → mismatch + ExpectedAPIResponse: APIResponseExpectation{StatusCode: 409}, }, { - Name: "With reservations of custom size - increase total", - // 4GB (2×2GB custom) → 6GB: preserves custom sizes, adds standard-sized reservations + Name: "API disabled: 503 Service Unavailable", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Host: "host-2", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-custom", "confirmed", 6), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Host: "host-2", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-custom", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-dis", "confirmed", 2)), + CustomConfig: func() *commitments.APIConfig { + cfg := commitments.DefaultAPIConfig() + cfg.EnableChangeCommitments = false + return &cfg + }(), + ExpectedAPIResponse: APIResponseExpectation{StatusCode: 503}, }, { - Name: "With reservations of custom size - decrease total", - // 4GB (2×2GB custom) → 3GB: removes 1×2GB custom, adds 1×1GB standard + Name: "Knowledge not ready: 503 Service Unavailable", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Host: "host-2", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-custom", "confirmed", 3), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-custom", Flavor: m1Small, ProjectID: "project-A", MemoryMB: 2048}, - {CommitmentID: "uuid-custom", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-kr", "confirmed", 2)), + EnvInfoVersion: -1, // skip Knowledge CRD creation + ExpectedAPIResponse: APIResponseExpectation{StatusCode: 503}, }, { - Name: "Complete commitment deletion - N to zero reservations", + Name: "Dry run: not supported yet", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-delete", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-delete", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-delete", Host: "host-3", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-b-1", Host: "host-3", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-a-1", Host: "host-3", Flavor: m1Small, ProjectID: "project-A"}, - }, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-delete", "confirmed", 0), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-b-1", Host: "host-3", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-a-1", Host: "host-3", Flavor: m1Small, ProjectID: "project-A"}, - }, + CommitmentRequest: newCommitmentRequest("az-a", true, 1234, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-dry", "confirmed", 2)), + ExpectedAPIResponse: newAPIResponse("Dry run not supported"), + }, + { + Name: "Empty request: no CRDs created", + Flavors: []*TestFlavor{m1Small}, + CommitmentRequest: newCommitmentRequest("az-a", false, 1234), ExpectedAPIResponse: newAPIResponse(), }, + // --- Deletion --- { - Name: "VM allocation preservation - keep VMs during growth", - VMs: []*TestVM{{UUID: "vm-existing", Flavor: m1Small, ProjectID: "project-A", Host: "host-1", AZ: "az-a"}}, + Name: "Deletion: existing CRD is deleted", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-growth", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-existing"}}, - {CommitmentID: "uuid-growth", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-del", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-growth", "confirmed", 3), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-growth", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-existing"}}, - {CommitmentID: "uuid-growth", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-growth", Flavor: m1Small, ProjectID: "project-A"}, - }, + deleteCommitment("hw_version_hana_1_ram", "project-A", "uuid-del", "confirmed", 2)), ExpectedAPIResponse: newAPIResponse(), + ExpectedDeletedCRs: []string{"commitment-uuid-del"}, }, { - Name: "Multi-project success - both projects succeed", + Name: "Deletion: non-existing CRD is a no-op", Flavors: []*TestFlavor{m1Small}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-a", "confirmed", 2), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-b", "confirmed", 2), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-a", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-a", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-b", Flavor: m1Small, ProjectID: "project-B"}, - {CommitmentID: "uuid-b", Flavor: m1Small, ProjectID: "project-B"}, - }, + deleteCommitment("hw_version_hana_1_ram", "project-A", "uuid-absent", "confirmed", 2)), ExpectedAPIResponse: newAPIResponse(), }, { - Name: "Multiple flavor groups - hw_version_hana_1_ram and hw_version_hana_2_ram", - // Amount in multiples of smallest flavor: hana_1 (2×1GB), hana_2 (2×2GB) - Flavors: []*TestFlavor{ - m1Small, - {Name: "m2.small", Group: "hana_2", MemoryMB: 2048, VCPUs: 8}, + Name: "Deletion rollback: delete succeeds but later commitment fails → CRD re-created", + Flavors: []*TestFlavor{m1Small}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-del-rb", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-hana1", "confirmed", 2), - createCommitment("hw_version_hana_2_ram", "project-A", "uuid-hana2", "confirmed", 2), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-hana1", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-hana1", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-hana2", Flavor: &TestFlavor{Name: "m2.small", Group: "hana_2", MemoryMB: 2048, VCPUs: 8}, ProjectID: "project-A"}, - {CommitmentID: "uuid-hana2", Flavor: &TestFlavor{Name: "m2.small", Group: "hana_2", MemoryMB: 2048, VCPUs: 8}, ProjectID: "project-A"}, + CROutcomes: map[string]string{ + "commitment-uuid-new-rb": "not enough capacity", }, - ExpectedAPIResponse: newAPIResponse(), - }, - { - Name: "Unknown flavor group - clear rejection message", - Flavors: []*TestFlavor{m1Small}, + // project-A deletion sorts before project-B creation; deletion succeeds then creation fails. CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_nonexistent_ram", "project-A", "uuid-unknown", "confirmed", 2), + deleteCommitment("hw_version_hana_1_ram", "project-A", "uuid-del-rb", "confirmed", 2), + createCommitment("hw_version_hana_1_ram", "project-B", "uuid-new-rb", "confirmed", 2), ), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse("flavor group not found"), + ExpectedAPIResponse: newAPIResponse("not enough capacity"), + ExpectedCreatedCRNames: []string{"commitment-uuid-del-rb"}, // re-created during rollback }, + // --- Non-confirming changes (RequiresConfirmation=false → AllowRejection=false, no watch) --- { - Name: "Three-way capacity swap - complex reallocation", - // A:2→0, B:1→0, C:0→3 in single transaction + Name: "Non-confirming: guaranteed→confirmed, AllowRejection=false, watch skipped", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-a", Host: "host-1", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-a", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, - {CommitmentID: "uuid-b", Host: "host-3", Flavor: m1Small, ProjectID: "project-B"}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-guar", State: v1alpha1.CommitmentStatusGuaranteed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, + // Controller would reject, but we skip watching for non-confirming changes. + CROutcomes: map[string]string{"commitment-uuid-guar": "not enough capacity"}, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-a", "confirmed", 0), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-b", "confirmed", 0), - createCommitment("hw_version_hana_1_ram", "project-C", "uuid-c", "confirmed", 3), - ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 0, "host-2": 0, "host-3": 0}}, - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-c", Host: "host-1", Flavor: m1Small, ProjectID: "project-C"}, - {CommitmentID: "uuid-c", Host: "host-2", Flavor: m1Small, ProjectID: "project-C"}, - {CommitmentID: "uuid-c", Host: "host-3", Flavor: m1Small, ProjectID: "project-C"}, - }, - ExpectedAPIResponse: newAPIResponse(), + TestCommitment{ + ResourceName: "hw_version_hana_1_ram", + ProjectID: "project-A", + ConfirmationID: "uuid-guar", + OldState: "guaranteed", + State: "confirmed", + Amount: 2, + }), + ExpectedAPIResponse: newAPIResponse(), // no rejection even though controller would reject + ExpectedCreatedCRNames: []string{"commitment-uuid-guar"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-guar": false}, }, { - Name: "Reservation repair - existing reservations with wrong metadata", - Flavors: []*TestFlavor{m1Small, m1Large}, - ExistingReservations: []*TestReservation{ - {CommitmentID: "uuid-repair", Host: "host-preserved", Flavor: m1Small, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Host: "host-1", Flavor: m1Small, ProjectID: "wrong-project", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Host: "host-2", Flavor: &TestFlavor{Name: "m1.small", Group: "hana_13", MemoryMB: 1024, VCPUs: 4}, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Host: "host-4", Flavor: m1Small, ProjectID: "project-A", AZ: "wrong-az"}, - }, + Name: "Non-confirming: planned, AllowRejection=false", + Flavors: []*TestFlavor{m1Small}, + // CROutcomes not set: controller accepts (irrelevant since watch is skipped). CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-repair", "confirmed", 8, "az-a"), - ), - ExpectedReservations: []*TestReservation{ - {CommitmentID: "uuid-repair", Host: "host-preserved", Flavor: m1Small, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Flavor: m1Small, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Flavor: m1Small, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Flavor: m1Small, ProjectID: "project-A", AZ: "az-a"}, - {CommitmentID: "uuid-repair", Flavor: m1Large, ProjectID: "project-A", AZ: "az-a"}, - }, - ExpectedAPIResponse: newAPIResponse(), - }, - { - Name: "Empty request - no commitment changes", - Flavors: []*TestFlavor{m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse(), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-plan-nc", "planned", 2)), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-plan-nc"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-plan-nc": false}, }, + // --- Pending state --- { - Name: "Dry run request - feature not yet implemented", + Name: "None→pending: non-confirming, AllowRejection=false, watch skipped", Flavors: []*TestFlavor{m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", true, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-dryrun", "confirmed", 2), - ), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse("Dry run not supported"), + // pending creates Reservation slots (like confirmed) but RequiresConfirmation=false. + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-pend", "pending", 2)), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-pend"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-pend": false}, + ExpectedCRSpecs: map[string]int64{"commitment-uuid-pend": 2 * 1024 * 1024 * 1024}, }, + // --- Inactive state transitions via upsert --- { - Name: "Knowledge not ready - clear rejection with RetryAt", + Name: "confirmed→expired: non-confirming upsert, AllowRejection=false, watch skipped", Flavors: []*TestFlavor{m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-knowledge", "confirmed", 2), - ), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: APIResponseExpectation{ - StatusCode: 503, - RetryAtPresent: false, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-to-exp", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, - EnvInfoVersion: -1, // Skip Knowledge CRD creation + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + TestCommitment{ + ResourceName: "hw_version_hana_1_ram", + ProjectID: "project-A", + ConfirmationID: "uuid-to-exp", + OldState: "confirmed", + State: "expired", + Amount: 1, + }), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-to-exp"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-to-exp": false}, + ExpectedCRSpecs: map[string]int64{"commitment-uuid-to-exp": 0}, }, { - Name: "API disabled - returns 503 Service Unavailable", + Name: "confirmed→superseded: confirming upsert, AllowRejection=true, controller accepts", Flavors: []*TestFlavor{m1Small}, - CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-disabled", "confirmed", 2), - ), - CustomConfig: func() *commitments.Config { - cfg := commitments.DefaultConfig() - cfg.EnableChangeCommitmentsAPI = false - return &cfg - }(), - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: APIResponseExpectation{ - StatusCode: 503, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-to-sup", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, + // confirmed→superseded is a confirming change (not in the liquid API's free-transition list). + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + TestCommitment{ + ResourceName: "hw_version_hana_1_ram", + ProjectID: "project-A", + ConfirmationID: "uuid-to-sup", + OldState: "confirmed", + State: "superseded", + Amount: 1, + }), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-to-sup"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-to-sup": true}, + ExpectedCRSpecs: map[string]int64{"commitment-uuid-to-sup": 0}, }, + // --- Resize --- { - Name: "Multiple commitments insufficient capacity - all listed in error", - // Tests that multiple failed commitments are all mentioned in the rejection reason - Flavors: []*TestFlavor{m1Small, m1Tiny}, + Name: "Resize down: confirmed→confirmed with less capacity, RequiresConfirmation=true", + Flavors: []*TestFlavor{m1Small}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-dn", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 4 * 1024, ProjectID: "project-A", AZ: "az-a"}, + }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-multi-fail-1", "confirmed", 3), - createCommitment("hw_version_hana_1_ram", "project-B", "uuid-multi-fail-2", "confirmed", 3), - createCommitment("hw_version_gp_1_ram", "project-C", "uuid-would-not-fail", "confirmed", 1), // would be rolled back, but not part of the reject reason - ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 256}}, - ExpectedReservations: []*TestReservation{}, - ExpectedAPIResponse: newAPIResponse("2 commitment(s) failed", "commitment uuid-multi-fail-1: not sufficient capacity", "commitment uuid-multi-fail-2: not sufficient capacity"), + TestCommitment{ + ResourceName: "hw_version_hana_1_ram", + ProjectID: "project-A", + ConfirmationID: "uuid-dn", + OldState: "confirmed", + OldAmount: 4, + State: "confirmed", + Amount: 2, + }), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-dn"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-dn": true}, + ExpectedCRSpecs: map[string]int64{"commitment-uuid-dn": 2 * 1024 * 1024 * 1024}, }, + // --- Mixed batch success --- { - Name: "Deletion priority during rollback - unscheduled removed first", - // Tests that during rollback, unscheduled reservations (no TargetHost) are deleted first, - // preserving scheduled reservations (with TargetHost), especially those with VM allocations - VMs: []*TestVM{{UUID: "vm-priority", Flavor: m1Small, ProjectID: "project-A", Host: "host-1", AZ: "az-a"}}, + Name: "Mixed batch: delete + create both succeed without rollback", Flavors: []*TestFlavor{m1Small}, - ExistingReservations: []*TestReservation{ - // Reservation with VM allocation - should be preserved (lowest deletion priority) - {CommitmentID: "commitment-1", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-priority"}}, - // Scheduled but unused - medium deletion priority - {CommitmentID: "commitment-1", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-mbdel", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, }, CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "commitment-1", "confirmed", 4), + deleteCommitment("hw_version_hana_1_ram", "project-A", "uuid-mbdel", "confirmed", 1), + createCommitment("hw_version_hana_1_ram", "project-B", "uuid-mbnew", "confirmed", 2), ), - AvailableResources: &AvailableResources{PerHost: map[string]int64{"host-1": 0, "host-2": 1024}}, - ExpectedReservations: []*TestReservation{ - // After rollback, should preserve the scheduled reservations (especially with VMs) - // and remove unscheduled ones first - {CommitmentID: "commitment-1", Host: "host-1", Flavor: m1Small, ProjectID: "project-A", VMs: []string{"vm-priority"}}, - {CommitmentID: "commitment-1", Host: "host-2", Flavor: m1Small, ProjectID: "project-A"}, - }, - ExpectedAPIResponse: newAPIResponse("commitment commitment-1: not sufficient capacity"), + ExpectedAPIResponse: newAPIResponse(), + ExpectedDeletedCRs: []string{"commitment-uuid-mbdel"}, + ExpectedCreatedCRNames: []string{"commitment-uuid-mbnew"}, + ExpectedAllowRejection: map[string]bool{"commitment-uuid-mbnew": true}, }, + // --- Pre-write validation failure rollback --- { - Name: "Watch timeout with custom config - triggers rollback with timeout error", + Name: "Pre-write validation failure: first CR written then rolled back on second CR's unknown flavor group", Flavors: []*TestFlavor{m1Small}, + // project-A (valid) sorts before project-B (invalid): A's CR is written, then B's + // unknown flavor group triggers a pre-watch rollback that deletes A's CR. CommitmentRequest: newCommitmentRequest("az-a", false, 1234, - createCommitment("hw_version_hana_1_ram", "project-A", "uuid-timeout", "confirmed", 2), + createCommitment("hw_version_hana_1_ram", "project-A", "uuid-pva", "confirmed", 2), + createCommitment("hw_version_nonexistent_ram", "project-B", "uuid-pvb", "confirmed", 2), ), - // With 0ms timeout, the watch will timeout immediately before reservations become ready - CustomConfig: func() *commitments.Config { - cfg := commitments.DefaultConfig() - cfg.ChangeAPIWatchReservationsTimeout = 0 * time.Millisecond - cfg.ChangeAPIWatchReservationsPollInterval = 100 * time.Millisecond - return &cfg - }(), - ExpectedReservations: []*TestReservation{}, // Rollback removes all reservations - ExpectedAPIResponse: newAPIResponse("timeout reached while processing commitment changes"), + ExpectedAPIResponse: newAPIResponse("flavor group not found"), + ExpectedDeletedCRs: []string{"commitment-uuid-pva"}, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { - runCommitmentChangeTest(t, tc) + runChangeCommitmentsTest(t, tc) }) } } -// runCommitmentChangeTest executes a single commitment change integration test case. -func runCommitmentChangeTest(t *testing.T, tc CommitmentChangeTestCase) { +func runChangeCommitmentsTest(t *testing.T, tc CommitmentChangeTestCase) { t.Helper() - // Convert test types to actual types - var vms []VM - for _, testVM := range tc.VMs { - vms = append(vms, testVM.ToVM()) - } - - var flavorInGroups []compute.FlavorInGroup - for _, testFlavor := range tc.Flavors { - flavorInGroups = append(flavorInGroups, testFlavor.ToFlavorInGroup()) - } - - // Use EnvInfoVersion if specified (non-zero), otherwise default to CommitmentRequest.InfoVersion - envInfoVersion := tc.CommitmentRequest.InfoVersion - if tc.EnvInfoVersion != 0 { - envInfoVersion = tc.EnvInfoVersion - } - - flavorGroups := TestFlavorGroup{ - infoVersion: envInfoVersion, - flavors: flavorInGroups, - }.ToFlavorGroupsKnowledge() - - // Convert existing reservations with auto-numbering per commitment - var existingReservations []*v1alpha1.Reservation - numberCounters := make(map[string]int) - for _, testRes := range tc.ExistingReservations { - number := numberCounters[testRes.CommitmentID] - numberCounters[testRes.CommitmentID]++ - existingReservations = append(existingReservations, testRes.toReservation(number)) - } - - // Create test environment with available resources and custom config if provided - env := newCommitmentTestEnv(t, vms, nil, existingReservations, flavorGroups, tc.AvailableResources, tc.CustomConfig) + env := newCRTestEnv(t, tc) defer env.Close() - t.Log("Initial state:") - env.LogStateSummary() - - // Call commitment change API reqJSON := buildRequestJSON(tc.CommitmentRequest) - resp, respJSON, statusCode := env.CallChangeCommitmentsAPI(reqJSON) + resp, _, statusCode := env.CallChangeCommitmentsAPI(reqJSON) - t.Log("After API call:") - env.LogStateSummary() + env.VerifyAPIResponse(tc.ExpectedAPIResponse, resp, statusCode) - // Verify API response - env.VerifyAPIResponse(tc.ExpectedAPIResponse, resp, respJSON, statusCode) - - // Verify reservations using content-based matching - env.VerifyReservationsMatch(tc.ExpectedReservations) - - // Log final test result - if t.Failed() { - t.Log("❌ Test FAILED") - } else { - t.Log("✅ Test PASSED") + if len(tc.ExpectedCreatedCRNames) > 0 { + env.VerifyCRsExist(tc.ExpectedCreatedCRNames) + } + if tc.ExpectedAllowRejection != nil { + env.VerifyAllowRejection(tc.ExpectedAllowRejection) + } + for crName, expectedAmountBytes := range tc.ExpectedCRSpecs { + env.VerifyCRAmountBytes(crName, expectedAmountBytes) + } + for _, crName := range tc.ExpectedDeletedCRs { + env.VerifyCRAbsent(crName) } } // ============================================================================ -// Test Types & Constants +// Test Types // ============================================================================ const ( - defaultFlavorDiskGB = 40 - flavorGroupsKnowledgeName = "flavor-groups" - knowledgeRecencyDuration = 60 * time.Second - defaultCommitmentExpiryYears = 1 + defaultFlavorDiskGB = 40 + flavorGroupsKnowledgeName = "flavor-groups" + knowledgeRecencyDuration = 60 * time.Second ) type CommitmentChangeTestCase struct { - Name string - VMs []*TestVM - Flavors []*TestFlavor - ExistingReservations []*TestReservation - CommitmentRequest CommitmentChangeRequest - ExpectedReservations []*TestReservation - ExpectedAPIResponse APIResponseExpectation - AvailableResources *AvailableResources // If nil, all reservations accepted without checks - EnvInfoVersion int64 // Override InfoVersion for version mismatch tests - CustomConfig *commitments.Config // Override default config for testing timeout behavior -} - -// AvailableResources defines available memory per host (MB). -// Scheduler uses first-come-first-serve. CPU is ignored. -type AvailableResources struct { - PerHost map[string]int64 // host -> available memory MB -} - -type TestFlavorGroup struct { - infoVersion int64 - flavors []compute.FlavorInGroup -} - -func (tfg TestFlavorGroup) ToFlavorGroupsKnowledge() FlavorGroupsKnowledge { - groupMap := make(map[string][]compute.FlavorInGroup) - - for _, flavor := range tfg.flavors { - groupName := flavor.ExtraSpecs["quota:hw_version"] - if groupName == "" { - panic("Flavor " + flavor.Name + " is missing quota:hw_version in extra specs") - } - groupMap[groupName] = append(groupMap[groupName], flavor) - } - - // Sort group names for deterministic iteration - sortedGroupNames := make([]string, 0, len(groupMap)) - for groupName := range groupMap { - sortedGroupNames = append(sortedGroupNames, groupName) - } - sort.Strings(sortedGroupNames) - - var groups []compute.FlavorGroupFeature - for _, groupName := range sortedGroupNames { - groupFlavors := groupMap[groupName] - if len(groupFlavors) == 0 { - continue - } - - // Sort descending: required by reservation manager's flavor selection - sort.Slice(groupFlavors, func(i, j int) bool { - return groupFlavors[i].MemoryMB > groupFlavors[j].MemoryMB - }) - - smallest := groupFlavors[len(groupFlavors)-1] - largest := groupFlavors[0] - - // Compute RAM/core ratio (MiB per vCPU) - var minRatio, maxRatio uint64 = ^uint64(0), 0 - for _, f := range groupFlavors { - if f.VCPUs == 0 { - continue - } - ratio := f.MemoryMB / f.VCPUs - if ratio < minRatio { - minRatio = ratio - } - if ratio > maxRatio { - maxRatio = ratio - } - } - - var ramCoreRatio, ramCoreRatioMin, ramCoreRatioMax *uint64 - if minRatio == maxRatio && maxRatio != 0 { - ramCoreRatio = &minRatio - } else if maxRatio != 0 { - ramCoreRatioMin = &minRatio - ramCoreRatioMax = &maxRatio - } - - groups = append(groups, compute.FlavorGroupFeature{ - Name: groupName, - Flavors: groupFlavors, - SmallestFlavor: smallest, - LargestFlavor: largest, - RamCoreRatio: ramCoreRatio, - RamCoreRatioMin: ramCoreRatioMin, - RamCoreRatioMax: ramCoreRatioMax, - }) - } - - return FlavorGroupsKnowledge{ - InfoVersion: tfg.infoVersion, - Groups: groups, - } + Name string + Flavors []*TestFlavor + // ExistingCRs: CommittedResource CRDs present before the API call. + ExistingCRs []*TestCR + // CROutcomes: what condition the fake controller sets per crName. + // Value = rejection reason if non-empty and not a named reason constant. + // Value = CommittedResourceReasonPlanned to simulate a planned outcome. + // Absent entry = controller accepts (Ready=True). + CROutcomes map[string]string + // NoCondition: crNames for which the fake controller sets no condition (simulate stall/timeout). + NoCondition []string + CommitmentRequest CommitmentChangeRequest + ExpectedAPIResponse APIResponseExpectation + // Post-call assertions. + ExpectedCreatedCRNames []string + ExpectedAllowRejection map[string]bool // crName → expected AllowRejection value + ExpectedCRSpecs map[string]int64 // crName → expected Amount.Value() in bytes + ExpectedDeletedCRs []string + CustomConfig *commitments.APIConfig + EnvInfoVersion int64 } -type FlavorGroupsKnowledge struct { - InfoVersion int64 - Groups []compute.FlavorGroupFeature +// TestCR defines a pre-existing CommittedResource CRD. +type TestCR struct { + CommitmentUUID string + State v1alpha1.CommitmentStatus + AmountMiB int64 + ProjectID string + AZ string } type CommitmentChangeRequest struct { @@ -704,29 +467,15 @@ type TestCommitment struct { ResourceName liquid.ResourceName ProjectID string ConfirmationID string - State string + OldState string // empty = None (no prior status) + State string // empty = None (deletion) Amount uint64 + OldAmount uint64 // if non-zero, used for TotalBefore totals instead of Amount (for resize-down) } type APIResponseExpectation struct { StatusCode int RejectReasonSubstrings []string - RetryAtPresent bool -} - -type ReservationVerification struct { - Host string - Allocations map[string]string -} - -type VM struct { - UUID string - FlavorName string - ProjectID string - CurrentHypervisor string - AvailabilityZone string - Resources map[string]int64 - FlavorExtraSpecs map[string]string } type TestFlavor struct { @@ -735,7 +484,7 @@ type TestFlavor struct { MemoryMB int64 VCPUs int64 DiskGB uint64 - VideoRAMMiB *uint64 // optional, from flavor extra_specs hw_video:ram_max_mb + VideoRAMMiB *uint64 } func (f *TestFlavor) ToFlavorInGroup() compute.FlavorInGroup { @@ -758,1098 +507,576 @@ func (f *TestFlavor) ToFlavorInGroup() compute.FlavorInGroup { } } -type TestVM struct { - UUID string - Flavor *TestFlavor - ProjectID string - Host string - AZ string -} - -func (vm *TestVM) ToVM() VM { - return VM{ - UUID: vm.UUID, - FlavorName: vm.Flavor.Name, - ProjectID: vm.ProjectID, - CurrentHypervisor: vm.Host, - AvailabilityZone: vm.AZ, - Resources: map[string]int64{ - "memory": vm.Flavor.MemoryMB, - "vcpus": vm.Flavor.VCPUs, - }, - FlavorExtraSpecs: map[string]string{ - "quota:hw_version": vm.Flavor.Group, - }, - } +type FlavorGroupsKnowledge struct { + InfoVersion int64 + Groups []compute.FlavorGroupFeature } -type TestReservation struct { - CommitmentID string - Host string // Empty = any host accepted in matching - Flavor *TestFlavor - ProjectID string - VMs []string // VM UUIDs - MemoryMB int64 // If 0, uses Flavor.MemoryMB; else custom size - AZ string +// TestFlavorGroup groups a flat list of FlavorInGroup by hw_version extra spec +// and builds a FlavorGroupsKnowledge. Used by usage_test.go and report_usage_test.go. +type TestFlavorGroup struct { + infoVersion int64 + flavors []compute.FlavorInGroup } -func (tr *TestReservation) toReservation(number int) *v1alpha1.Reservation { - name := fmt.Sprintf("commitment-%s-%d", tr.CommitmentID, number) - - memoryMB := tr.MemoryMB - if memoryMB == 0 { - memoryMB = tr.Flavor.MemoryMB - } - - specAllocations := make(map[string]v1alpha1.CommittedResourceAllocation) - statusAllocations := make(map[string]string) - for _, vmUUID := range tr.VMs { - specAllocations[vmUUID] = v1alpha1.CommittedResourceAllocation{ - CreationTimestamp: metav1.Now(), - Resources: map[hv1.ResourceName]resource.Quantity{ - "memory": resource.MustParse(strconv.FormatInt(memoryMB, 10) + "Mi"), - "cpu": resource.MustParse(strconv.FormatInt(tr.Flavor.VCPUs, 10)), - }, - } - statusAllocations[vmUUID] = tr.Host - } - - spec := v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - TargetHost: tr.Host, - Resources: map[hv1.ResourceName]resource.Quantity{ - "memory": resource.MustParse(strconv.FormatInt(memoryMB, 10) + "Mi"), - "cpu": resource.MustParse(strconv.FormatInt(tr.Flavor.VCPUs, 10)), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - CommitmentUUID: tr.CommitmentID, - ProjectID: tr.ProjectID, - ResourceName: tr.Flavor.Name, - ResourceGroup: tr.Flavor.Group, - Allocations: specAllocations, - }, +func (tg TestFlavorGroup) ToFlavorGroupsKnowledge() FlavorGroupsKnowledge { + groupMap := make(map[string][]compute.FlavorInGroup) + for _, f := range tg.flavors { + name := f.ExtraSpecs["quota:hw_version"] + groupMap[name] = append(groupMap[name], f) } - if tr.AZ != "" { - spec.AvailabilityZone = tr.AZ + sortedNames := make([]string, 0, len(groupMap)) + for n := range groupMap { + sortedNames = append(sortedNames, n) } + sort.Strings(sortedNames) - return &v1alpha1.Reservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: spec, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - }, - }, - Host: tr.Host, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ - Allocations: statusAllocations, - }, - }, + var groups []compute.FlavorGroupFeature + for _, name := range sortedNames { + gFlavors := groupMap[name] + sort.Slice(gFlavors, func(i, j int) bool { return gFlavors[i].MemoryMB > gFlavors[j].MemoryMB }) + smallest := gFlavors[len(gFlavors)-1] + largest := gFlavors[0] + + var minR, maxR uint64 = ^uint64(0), 0 + for _, f := range gFlavors { + if f.VCPUs == 0 { + continue + } + r := f.MemoryMB / f.VCPUs + if r < minR { + minR = r + } + if r > maxR { + maxR = r + } + } + var ratio, ratioMin, ratioMax *uint64 + if minR == maxR && maxR != 0 { + ratio = &minR + } else if maxR != 0 { + ratioMin = &minR + ratioMax = &maxR + } + groups = append(groups, compute.FlavorGroupFeature{ + Name: name, + Flavors: gFlavors, + SmallestFlavor: smallest, + LargestFlavor: largest, + RamCoreRatio: ratio, + RamCoreRatioMin: ratioMin, + RamCoreRatioMax: ratioMax, + }) } + return FlavorGroupsKnowledge{InfoVersion: tg.infoVersion, Groups: groups} } // ============================================================================ -// Test Environment +// Fake Controller Client // ============================================================================ -type CommitmentTestEnv struct { - T *testing.T - Scheme *runtime.Scheme - K8sClient client.Client - VMSource *MockVMSource - FlavorGroups FlavorGroupsKnowledge - HTTPServer *httptest.Server - API *HTTPAPI - availableResources map[string]int64 // host -> available memory MB - processedReserv map[string]bool // track processed reservations - mu sync.Mutex // protects availableResources and processedReserv +// fakeControllerClient wraps a client.Client and simulates the CommittedResource +// controller by immediately setting conditions after any Create or Update of a +// CommittedResource CRD. Entries in noCondition suppress condition-setting to +// simulate a stalled controller (used for timeout tests). +type fakeControllerClient struct { + client.Client + outcomes map[string]string // crName → rejection reason (or reason constant); absent = accept + noCondition map[string]struct{} + mu sync.Mutex } -// FakeReservationController simulates synchronous reservation controller. -type FakeReservationController struct { - env *CommitmentTestEnv +func (c *fakeControllerClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.Client.Create(ctx, obj, opts...); err != nil { + return err + } + if cr, ok := obj.(*v1alpha1.CommittedResource); ok { + c.setConditionFor(ctx, cr.Name) + } + return nil } -func (c *FakeReservationController) OnReservationCreated(res *v1alpha1.Reservation) { - c.env.processNewReservation(res) +func (c *fakeControllerClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if err := c.Client.Update(ctx, obj, opts...); err != nil { + return err + } + if cr, ok := obj.(*v1alpha1.CommittedResource); ok { + c.setConditionFor(ctx, cr.Name) + } + return nil } -func (c *FakeReservationController) OnReservationDeleted(res *v1alpha1.Reservation) { - c.env.mu.Lock() - defer c.env.mu.Unlock() +func (c *fakeControllerClient) setConditionFor(ctx context.Context, crName string) { + c.mu.Lock() + _, skip := c.noCondition[crName] + outcome, hasOutcome := c.outcomes[crName] + c.mu.Unlock() - // Return memory when Delete() is called directly (before deletion timestamp is set) - if c.env.availableResources != nil && res.Status.Host != "" { - memoryQuantity := res.Spec.Resources["memory"] - memoryBytes := memoryQuantity.Value() - memoryMB := memoryBytes / (1024 * 1024) + if skip { + return + } - if _, exists := c.env.availableResources[res.Status.Host]; exists { - c.env.availableResources[res.Status.Host] += memoryMB - c.env.T.Logf("↩ Returned %d MB to %s (now %d MB available) via OnReservationDeleted for %s", - memoryMB, res.Status.Host, c.env.availableResources[res.Status.Host], res.Name) + var cond metav1.Condition + switch { + case !hasOutcome || outcome == "": + // Default: controller accepts. + cond = metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionTrue, + Reason: v1alpha1.CommittedResourceReasonAccepted, + Message: "accepted", + } + case outcome == v1alpha1.CommittedResourceReasonPlanned: + cond = metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionFalse, + Reason: v1alpha1.CommittedResourceReasonPlanned, + Message: "commitment is not yet active", + } + default: + cond = metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionFalse, + Reason: v1alpha1.CommittedResourceReasonRejected, + Message: outcome, } } - // Clear tracking so recreated reservations with same name are processed - delete(c.env.processedReserv, res.Name) + cr := &v1alpha1.CommittedResource{} + if err := c.Get(ctx, client.ObjectKey{Name: crName}, cr); err != nil { + return + } + meta.SetStatusCondition(&cr.Status.Conditions, cond) + if err := c.Client.Status().Update(ctx, cr); err != nil { + return // best-effort: if the update races with another write, the polling loop retries + } } -// operationInterceptorClient routes reservation events to FakeReservationController. -type operationInterceptorClient struct { - client.Client - controller *FakeReservationController +// ============================================================================ +// Test Environment +// ============================================================================ + +type CRTestEnv struct { + T *testing.T + K8sClient client.Client + HTTPServer *httptest.Server } -func (d *operationInterceptorClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - err := d.Client.Create(ctx, obj, opts...) - if err != nil { - return err - } - - if res, ok := obj.(*v1alpha1.Reservation); ok { - d.controller.OnReservationCreated(res) - } - - return nil -} - -func (d *operationInterceptorClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - if res, ok := obj.(*v1alpha1.Reservation); ok { - d.controller.OnReservationDeleted(res) - } - - return d.Client.Delete(ctx, obj, opts...) -} - -func (env *CommitmentTestEnv) Close() { - if env.HTTPServer != nil { - env.HTTPServer.Close() - } -} - -func newCommitmentTestEnv( - t *testing.T, - vms []VM, - hypervisors []*hv1.Hypervisor, - reservations []*v1alpha1.Reservation, - flavorGroups FlavorGroupsKnowledge, - resources *AvailableResources, - customConfig *commitments.Config, -) *CommitmentTestEnv { - +func newCRTestEnv(t *testing.T, tc CommitmentChangeTestCase) *CRTestEnv { t.Helper() - log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) - objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) - for _, hv := range hypervisors { - objects = append(objects, hv) - } - for _, res := range reservations { - objects = append(objects, res) - } - scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + t.Fatalf("failed to add v1alpha1 scheme: %v", err) } - if err := hv1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add hv1 scheme: %v", err) + + objects := make([]client.Object, 0) + + // Knowledge CRD (InfoVersion=-1 simulates "not ready"). + envInfoVersion := tc.CommitmentRequest.InfoVersion + if tc.EnvInfoVersion != 0 { + envInfoVersion = tc.EnvInfoVersion + } + if envInfoVersion != -1 { + objects = append(objects, createKnowledgeCRD(buildFlavorGroupsKnowledge(tc.Flavors, envInfoVersion))) } - // InfoVersion of -1 skips Knowledge CRD creation (tests "not ready" scenario) - if flavorGroups.InfoVersion != -1 { - knowledgeCRD := createKnowledgeCRD(flavorGroups) - objects = append(objects, knowledgeCRD) + // Pre-existing CommittedResource CRDs. + for _, tcr := range tc.ExistingCRs { + objects = append(objects, tcr.toCommittedResource()) } - baseK8sClient := fake.NewClientBuilder(). + baseClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Reservation{}). - WithStatusSubresource(&v1alpha1.Knowledge{}). - WithIndex(&v1alpha1.Reservation{}, "spec.type", func(obj client.Object) []string { - res := obj.(*v1alpha1.Reservation) - return []string{string(res.Spec.Type)} - }). + WithStatusSubresource(&v1alpha1.CommittedResource{}, &v1alpha1.Knowledge{}). Build() - var availableResources map[string]int64 - if resources != nil && resources.PerHost != nil { - availableResources = make(map[string]int64) - for host, memMB := range resources.PerHost { - availableResources[host] = memMB - } + noCondition := make(map[string]struct{}) + for _, name := range tc.NoCondition { + noCondition[name] = struct{}{} } - env := &CommitmentTestEnv{ - T: t, - Scheme: scheme, - K8sClient: nil, // Will be set below - VMSource: NewMockVMSource(vms), - FlavorGroups: flavorGroups, - HTTPServer: nil, // Will be set below - API: nil, // Will be set below - availableResources: availableResources, - processedReserv: make(map[string]bool), + wrapped := &fakeControllerClient{ + Client: baseClient, + outcomes: tc.CROutcomes, + noCondition: noCondition, } - controller := &FakeReservationController{env: env} - wrappedClient := &operationInterceptorClient{ - Client: baseK8sClient, - controller: controller, - } - env.K8sClient = wrappedClient - - // Use custom config if provided, otherwise use default var api *HTTPAPI - if customConfig != nil { - api = NewAPIWithConfig(wrappedClient, *customConfig, nil) + if tc.CustomConfig != nil { + api = NewAPIWithConfig(wrapped, *tc.CustomConfig, nil) } else { - api = NewAPI(wrappedClient) + api = NewAPI(wrapped) } mux := http.NewServeMux() registry := prometheus.NewRegistry() api.Init(mux, registry, log.Log) - httpServer := httptest.NewServer(mux) - - env.HTTPServer = httpServer - env.API = api - - return env -} - -// ============================================================================ -// Environment Helper Methods -// ============================================================================ - -// ListVMs returns all VMs from the VMSource. -func (env *CommitmentTestEnv) ListVMs() []VM { - vms, err := env.VMSource.ListVMs(context.Background()) - if err != nil { - env.T.Fatalf("Failed to list VMs: %v", err) - } - return vms -} - -// ListReservations returns all reservations. -func (env *CommitmentTestEnv) ListReservations() []v1alpha1.Reservation { - var list v1alpha1.ReservationList - if err := env.K8sClient.List(context.Background(), &list); err != nil { - env.T.Fatalf("Failed to list reservations: %v", err) - } - return list.Items -} -// ListHypervisors returns all hypervisors. -func (env *CommitmentTestEnv) ListHypervisors() []hv1.Hypervisor { - var list hv1.HypervisorList - if err := env.K8sClient.List(context.Background(), &list); err != nil { - env.T.Fatalf("Failed to list hypervisors: %v", err) + return &CRTestEnv{ + T: t, + K8sClient: wrapped, + HTTPServer: httptest.NewServer(mux), } - return list.Items } -// LogStateSummary logs a summary of the current state. -func (env *CommitmentTestEnv) LogStateSummary() { - env.T.Helper() - - hypervisors := env.ListHypervisors() - vms := env.ListVMs() - reservations := env.ListReservations() - - env.T.Log("=== State Summary ===") - env.T.Logf("Hypervisors: %d", len(hypervisors)) - env.T.Logf("VMs: %d", len(vms)) - env.T.Logf("Reservations: %d", len(reservations)) - - for _, res := range reservations { - allocCount := 0 - if res.Status.CommittedResourceReservation != nil { - allocCount = len(res.Status.CommittedResourceReservation.Allocations) - } - env.T.Logf(" - %s (host: %s, allocations: %d)", res.Name, res.Status.Host, allocCount) +func (env *CRTestEnv) Close() { + if env.HTTPServer != nil { + env.HTTPServer.Close() } - env.T.Log("=====================") } -// CallChangeCommitmentsAPI calls the change commitments API endpoint with JSON. -// Reservation processing is fully synchronous via operationInterceptorClient hooks. -func (env *CommitmentTestEnv) CallChangeCommitmentsAPI(reqJSON string) (resp liquid.CommitmentChangeResponse, respJSON string, statusCode int) { +func (env *CRTestEnv) CallChangeCommitmentsAPI(reqJSON string) (resp liquid.CommitmentChangeResponse, respBody string, statusCode int) { env.T.Helper() - - // Make HTTP request - reservation processing happens synchronously via Create/Delete hooks url := env.HTTPServer.URL + "/commitments/v1/change-commitments" - httpResp, err := http.Post(url, "application/json", bytes.NewReader([]byte(reqJSON))) //nolint:gosec,noctx // test server URL, not user input + httpResp, err := http.Post(url, "application/json", bytes.NewReader([]byte(reqJSON))) //nolint:gosec,noctx if err != nil { - env.T.Fatalf("Failed to make HTTP request: %v", err) + env.T.Fatalf("HTTP request failed: %v", err) } defer httpResp.Body.Close() - - // Read response body - respBytes, err := io.ReadAll(httpResp.Body) + raw, err := io.ReadAll(httpResp.Body) if err != nil { - env.T.Fatalf("Failed to read response body: %v", err) + env.T.Fatalf("failed to read response: %v", err) } - - respJSON = string(respBytes) - - // Parse response - only for 200 OK responses - // Non-200 responses (like 409 Conflict for version mismatch) use plain text via http.Error() if httpResp.StatusCode == http.StatusOK { - if err := json.Unmarshal(respBytes, &resp); err != nil { - env.T.Fatalf("Failed to unmarshal response: %v", err) + if err := json.Unmarshal(raw, &resp); err != nil { + env.T.Fatalf("failed to unmarshal response: %v", err) } } - - // Final pass to handle any deletions (finalizer removal) - env.processReservations() - - statusCode = httpResp.StatusCode - return resp, respJSON, statusCode + return resp, string(raw), httpResp.StatusCode } -// processReservations handles all reservation lifecycle events synchronously. -// This includes marking reservations as Ready/Failed and removing finalizers from deleted reservations. -func (env *CommitmentTestEnv) processReservations() { - ctx := context.Background() - reservations := env.ListReservations() - - for _, res := range reservations { - // Handle deletion - return memory to host and remove finalizers - if !res.DeletionTimestamp.IsZero() { - env.T.Logf("Processing deletion for reservation %s (host: %s)", res.Name, res.Status.Host) - - env.mu.Lock() - // Return memory to host if resource tracking is enabled - if env.availableResources != nil { - env.T.Logf("Resource tracking enabled, returning memory for %s", res.Name) - memoryQuantity := res.Spec.Resources["memory"] - memoryBytes := memoryQuantity.Value() - memoryMB := memoryBytes / (1024 * 1024) - - env.T.Logf("Reservation %s has host=%s, memory=%d MB", res.Name, res.Status.Host, memoryMB) - - // Check if host exists in our tracking - if _, exists := env.availableResources[res.Status.Host]; !exists { - env.mu.Unlock() - env.T.Fatalf("Host %s not found in available resources for reservation %s - this indicates an inconsistency", - res.Status.Host, res.Name) - } - - // Return memory to host - env.availableResources[res.Status.Host] += memoryMB - env.T.Logf("↩ Returned %d MB to %s (now %d MB available) from deleted reservation %s", - memoryMB, res.Status.Host, env.availableResources[res.Status.Host], res.Name) - } else { - env.T.Logf("Resource tracking NOT enabled for %s", res.Name) - } - - // Clear tracking so recreated reservations with same name are processed - delete(env.processedReserv, res.Name) - env.mu.Unlock() - - // Remove finalizers to allow deletion - if len(res.Finalizers) > 0 { - res.Finalizers = []string{} - if err := env.K8sClient.Update(ctx, &res); err != nil { - // Ignore errors - might be already deleted - continue - } - } - continue - } - - // Skip if already processed (has a condition set) - if env.hasCondition(&res) { - continue - } - - env.mu.Lock() - alreadyProcessed := env.processedReserv[res.Name] - env.mu.Unlock() - - // Skip if already tracked as processed - if alreadyProcessed { - continue - } - - // Process new reservation with resource-based scheduling - env.processNewReservation(&res) - } -} - -// hasCondition checks if a reservation has any Ready condition set. -func (env *CommitmentTestEnv) hasCondition(res *v1alpha1.Reservation) bool { - for _, cond := range res.Status.Conditions { - if cond.Type == v1alpha1.ReservationConditionReady { - return true - } - } - return false -} - -// processNewReservation implements first-come-first-serve scheduling based on available resources. -// It tries to find a host with enough memory capacity and assigns the reservation to that host. -func (env *CommitmentTestEnv) processNewReservation(res *v1alpha1.Reservation) { - env.mu.Lock() - defer env.mu.Unlock() - - env.processedReserv[res.Name] = true - - if res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.ResourceGroup == "" || res.Spec.Resources == nil || res.Spec.Resources["memory"] == (resource.Quantity{}) { - env.markReservationFailedStatus(res, "invalid reservation spec") - env.T.Logf("✗ Invalid reservation spec for %s: marking as failed (resource group: %s, resources: %v)", res.Name, res.Spec.CommittedResourceReservation.ResourceGroup, res.Spec.Resources) - return - } - - // If no available resources configured, accept all reservations without host assignment - if env.availableResources == nil { - env.T.Logf("✓ Scheduled reservation %s - no resource tracking, simply accept", res.Name) - env.markReservationSchedulerProcessedStatus(res, "some-host") - return - } - - // Get required memory from reservation spec - memoryQuantity := res.Spec.Resources["memory"] - memoryBytes := memoryQuantity.Value() - memoryMB := memoryBytes / (1024 * 1024) - - // First-come-first-serve: find first host with enough capacity - // Sort hosts to ensure deterministic behavior (Go map iteration is random) - hosts := make([]string, 0, len(env.availableResources)) - for host := range env.availableResources { - hosts = append(hosts, host) +func (env *CRTestEnv) VerifyAPIResponse(expected APIResponseExpectation, resp liquid.CommitmentChangeResponse, statusCode int) { + env.T.Helper() + expectedCode := expected.StatusCode + if expectedCode == 0 { + expectedCode = http.StatusOK } - sort.Strings(hosts) - - var selectedHost string - for _, host := range hosts { - if env.availableResources[host] >= memoryMB { - selectedHost = host - break - } + if statusCode != expectedCode { + env.T.Errorf("expected status %d, got %d", expectedCode, statusCode) } - - if selectedHost != "" { - // SUCCESS: Schedule on this host - env.availableResources[selectedHost] -= memoryMB - - // Update reservation with selected host - ctx := context.Background() - - // Update spec (TargetHost) - res.Spec.TargetHost = selectedHost - if err := env.K8sClient.Update(ctx, res); err != nil { - env.T.Logf("Warning: Failed to update reservation spec: %v", err) - } - - // Update status (Host) - requires Status().Update - res.Status.Host = selectedHost - if err := env.K8sClient.Status().Update(ctx, res); err != nil { - env.T.Logf("Warning: Failed to update reservation status host: %v", err) + for _, sub := range expected.RejectReasonSubstrings { + if !strings.Contains(resp.RejectionReason, sub) { + env.T.Errorf("rejection reason %q does not contain %q", resp.RejectionReason, sub) } - - env.markReservationSchedulerProcessedStatus(res, selectedHost) - env.T.Logf("✓ Scheduled reservation %s on %s (%d MB used, %d MB remaining)", - res.Name, selectedHost, memoryMB, env.availableResources[selectedHost]) - } else { - env.markReservationSchedulerProcessedStatus(res, "") - env.T.Logf("✗ Failed to schedule reservation %s (needs %d MB, no host has capacity)", - res.Name, memoryMB) } } -// markReservationSchedulerProcessedStatus updates a reservation status based on scheduling result. -// If host is non-empty, sets Ready=True (success). If host is empty, sets Ready=False with NoHostsFound (failure). -func (env *CommitmentTestEnv) markReservationSchedulerProcessedStatus(res *v1alpha1.Reservation, host string) { - ctx := context.Background() - - // Update spec first - res.Spec.TargetHost = host - if err := env.K8sClient.Update(ctx, res); err != nil { - env.T.Logf("Warning: Failed to update reservation spec: %v", err) - return - } - - // Then update status - Ready=True only if host was found, Ready=False otherwise - res.Status.Host = host - if host != "" { - res.Status.Conditions = []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - Message: "Reservation is ready (set by test controller)", - LastTransitionTime: metav1.Now(), - }, - } - } else { - res.Status.Conditions = []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "NoHostsFound", - Message: "No hosts with sufficient capacity (set by test controller)", - LastTransitionTime: metav1.Now(), - }, +func (env *CRTestEnv) VerifyCRsExist(names []string) { + env.T.Helper() + for _, name := range names { + cr := &v1alpha1.CommittedResource{} + if err := env.K8sClient.Get(context.Background(), client.ObjectKey{Name: name}, cr); err != nil { + env.T.Errorf("expected CommittedResource %q to exist, but got: %v", name, err) } } - if err := env.K8sClient.Status().Update(ctx, res); err != nil { - env.T.Logf("Warning: Failed to update reservation status: %v", err) - } } -// markReservationFailedStatus updates a reservation to have Ready=False status -func (env *CommitmentTestEnv) markReservationFailedStatus(res *v1alpha1.Reservation, reason string) { - res.Status.Conditions = []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "Reservation invalid", - Message: reason, - LastTransitionTime: metav1.Now(), - }, - } - - if err := env.K8sClient.Status().Update(context.Background(), res); err != nil { - // Ignore errors - might be deleted during update - return +func (env *CRTestEnv) VerifyCRAbsent(name string) { + env.T.Helper() + cr := &v1alpha1.CommittedResource{} + err := env.K8sClient.Get(context.Background(), client.ObjectKey{Name: name}, cr) + if err == nil { + env.T.Errorf("expected CommittedResource %q to be absent after rollback, but it still exists", name) + } else if !apierrors.IsNotFound(err) { + env.T.Errorf("unexpected error checking if CommittedResource %q is absent: %v", name, err) } } -// VerifyAPIResponse verifies the API response matches expectations. -// For rejection reasons, it checks if ALL expected substrings are present in the actual rejection reason. -func (env *CommitmentTestEnv) VerifyAPIResponse(expected APIResponseExpectation, actual liquid.CommitmentChangeResponse, respJSON string, statusCode int) { +func (env *CRTestEnv) VerifyAllowRejection(expected map[string]bool) { env.T.Helper() - - if statusCode != expected.StatusCode { - env.T.Errorf("Expected status code %d, got %d", expected.StatusCode, statusCode) - } - - if len(expected.RejectReasonSubstrings) > 0 { - if actual.RejectionReason == "" { - env.T.Errorf("Expected rejection reason containing substrings %v, got none", expected.RejectReasonSubstrings) - } else { - // Check that ALL expected substrings are present - for _, substring := range expected.RejectReasonSubstrings { - if !strings.Contains(actual.RejectionReason, substring) { - env.T.Errorf("Expected rejection reason to contain %q, but got %q", substring, actual.RejectionReason) - } - } - } - } else { - if actual.RejectionReason != "" { - env.T.Errorf("Expected no rejection reason, got %q", actual.RejectionReason) - } - } - - // Check RetryAt field presence in JSON (avoids dealing with option.Option type) - retryAtPresent := strings.Contains(respJSON, `"retryAt"`) - if expected.RetryAtPresent { - if !retryAtPresent { - env.T.Error("Expected retryAt field to be present in JSON response, but it was not found") + for crName, want := range expected { + cr := &v1alpha1.CommittedResource{} + if err := env.K8sClient.Get(context.Background(), client.ObjectKey{Name: crName}, cr); err != nil { + env.T.Errorf("CommittedResource %q not found: %v", crName, err) + continue } - } else { - if retryAtPresent { - env.T.Error("Expected retryAt field to be absent from JSON response, but it was found") + if cr.Spec.AllowRejection != want { + env.T.Errorf("CommittedResource %q: AllowRejection=%v, want %v", crName, cr.Spec.AllowRejection, want) } } } -// VerifyReservationsMatch verifies that actual reservations match expected reservations by content. -func (env *CommitmentTestEnv) VerifyReservationsMatch(expected []*TestReservation) { +func (env *CRTestEnv) VerifyCRAmountBytes(crName string, wantBytes int64) { env.T.Helper() - - actualReservations := env.ListReservations() - - // Make copies of both lists so we can remove matched items - expectedCopy := make([]*TestReservation, len(expected)) - copy(expectedCopy, expected) - - actualCopy := make([]v1alpha1.Reservation, len(actualReservations)) - copy(actualCopy, actualReservations) - - // Track unmatched items for detailed reporting - var unmatchedExpected []*TestReservation - var unmatchedActual []v1alpha1.Reservation - - // Greedy matching: while there are expected items, find matches and remove - for len(expectedCopy) > 0 { - exp := expectedCopy[0] - found := false - - // Find first actual that matches this expected - for i, actual := range actualCopy { - if env.reservationMatches(exp, &actual) { - expectedCopy = expectedCopy[1:] - actualCopy = append(actualCopy[:i], actualCopy[i+1:]...) - found = true - break - } - } - - if !found { - unmatchedExpected = append(unmatchedExpected, exp) - expectedCopy = expectedCopy[1:] - } + cr := &v1alpha1.CommittedResource{} + if err := env.K8sClient.Get(context.Background(), client.ObjectKey{Name: crName}, cr); err != nil { + env.T.Errorf("CommittedResource %q not found: %v", crName, err) + return } - - unmatchedActual = actualCopy - - // If there are any mismatches, print detailed comparison - if len(unmatchedExpected) > 0 || len(unmatchedActual) > 0 { - env.T.Error("❌ Reservation mismatch detected!") - env.T.Log("") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.T.Log("EXPECTED RESERVATIONS:") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.printExpectedReservations(expected, unmatchedExpected) - - env.T.Log("") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.T.Log("ACTUAL RESERVATIONS:") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.printActualReservations(actualReservations, unmatchedActual) - - env.T.Log("") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.T.Log("DIFF SUMMARY:") - env.T.Log("═══════════════════════════════════════════════════════════════") - env.printDiffSummary(unmatchedExpected, unmatchedActual) - env.T.Log("═══════════════════════════════════════════════════════════════") + got := cr.Spec.Amount.Value() + if got != wantBytes { + env.T.Errorf("CommittedResource %q: Amount=%d bytes, want %d bytes", crName, got, wantBytes) } } -// String returns a compact string representation of a TestReservation. -func (tr *TestReservation) String() string { - flavorName := "" - flavorGroup := "" - if tr.Flavor != nil { - flavorName = tr.Flavor.Name - flavorGroup = tr.Flavor.Group - } - - host := tr.Host - if host == "" { - host = "<any>" - } - - az := tr.AZ - if az == "" { - az = "<any>" - } +// ============================================================================ +// TestCR → v1alpha1.CommittedResource +// ============================================================================ - vmInfo := "" - if len(tr.VMs) > 0 { - vmInfo = fmt.Sprintf(" VMs=%v", tr.VMs) +func (tc *TestCR) toCommittedResource() *v1alpha1.CommittedResource { + amount := resource.NewQuantity(tc.AmountMiB*1024*1024, resource.BinarySI) + return &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "commitment-" + tc.CommitmentUUID, + }, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: tc.CommitmentUUID, + FlavorGroupName: "hana_1", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: *amount, + AvailabilityZone: tc.AZ, + ProjectID: tc.ProjectID, + State: tc.State, + }, } - - return fmt.Sprintf("%s/%s/%s(%s)/%s/az=%s%s", tr.CommitmentID, tr.ProjectID, flavorName, flavorGroup, host, az, vmInfo) } -// compactReservationString returns a compact string representation of an actual Reservation. -func compactReservationString(res *v1alpha1.Reservation) string { - commitmentID := "<none>" - projectID := "<none>" - flavorName := "<none>" - flavorGroup := "<none>" - vmCount := 0 - - if res.Spec.CommittedResourceReservation != nil { - commitmentID = res.Spec.CommittedResourceReservation.CommitmentUUID - projectID = res.Spec.CommittedResourceReservation.ProjectID - flavorName = res.Spec.CommittedResourceReservation.ResourceName - flavorGroup = res.Spec.CommittedResourceReservation.ResourceGroup - if res.Status.CommittedResourceReservation != nil { - vmCount = len(res.Status.CommittedResourceReservation.Allocations) - } - } - - host := res.Status.Host - if host == "" { - host = "<unscheduled>" - } - - az := res.Spec.AvailabilityZone - if az == "" { - az = "<none>" - } +// ============================================================================ +// Request / Response helpers +// ============================================================================ - vmInfo := "" - if vmCount > 0 { - vmInfo = fmt.Sprintf(" VMs=%d", vmCount) +func newAPIResponse(rejectSubstrings ...string) APIResponseExpectation { + return APIResponseExpectation{ + StatusCode: http.StatusOK, + RejectReasonSubstrings: rejectSubstrings, } - - return fmt.Sprintf("%s/%s/%s(%s)/%s/az=%s%s", commitmentID, projectID, flavorName, flavorGroup, host, az, vmInfo) } -// printExpectedReservations prints all expected reservations with markers for unmatched ones. -func (env *CommitmentTestEnv) printExpectedReservations(all, unmatched []*TestReservation) { - env.T.Helper() - - unmatchedMap := make(map[*TestReservation]bool) - for _, res := range unmatched { - unmatchedMap[res] = true - } - - if len(all) == 0 { - env.T.Log(" (none)") - return - } - - for i, res := range all { - marker := "✓" - if unmatchedMap[res] { - marker = "✗" - } - env.T.Logf(" %s [%d] %s", marker, i+1, res.String()) +func newCommitmentRequest(az string, dryRun bool, infoVersion int64, commitments ...TestCommitment) CommitmentChangeRequest { + return CommitmentChangeRequest{ + AZ: az, + DryRun: dryRun, + InfoVersion: infoVersion, + Commitments: commitments, } - - env.T.Logf(" Total: %d (%d matched, %d missing)", - len(all), len(all)-len(unmatched), len(unmatched)) } -// printActualReservations prints all actual reservations with markers for unmatched ones. -func (env *CommitmentTestEnv) printActualReservations(all, unmatched []v1alpha1.Reservation) { - env.T.Helper() - - unmatchedMap := make(map[string]bool) - for _, res := range unmatched { - unmatchedMap[res.Name] = true - } - - if len(all) == 0 { - env.T.Log(" (none)") - return +func createCommitment(resourceName, projectID, uuid, state string, amount uint64, _ ...string) TestCommitment { + return TestCommitment{ + ResourceName: liquid.ResourceName(resourceName), + ProjectID: projectID, + ConfirmationID: uuid, + State: state, + Amount: amount, } +} - for i, res := range all { - marker := "✓" - if unmatchedMap[res.Name] { - marker = "⊕" - } - env.T.Logf(" %s [%d] %s", marker, i+1, compactReservationString(&res)) +// deleteCommitment builds a TestCommitment representing a removal (OldStatus=oldState, NewStatus=None). +func deleteCommitment(resourceName, projectID, uuid, oldState string, amount uint64) TestCommitment { + return TestCommitment{ + ResourceName: liquid.ResourceName(resourceName), + ProjectID: projectID, + ConfirmationID: uuid, + OldState: oldState, + State: "", // NewStatus = None + Amount: amount, } - - env.T.Logf(" Total: %d (%d matched, %d unexpected)", - len(all), len(all)-len(unmatched), len(unmatched)) } -// printDiffSummary prints a summary of differences between expected and actual. -func (env *CommitmentTestEnv) printDiffSummary(unmatchedExpected []*TestReservation, unmatchedActual []v1alpha1.Reservation) { - env.T.Helper() - - if len(unmatchedExpected) > 0 { - env.T.Logf(" MISSING (%d expected, not found):", len(unmatchedExpected)) - for _, res := range unmatchedExpected { - env.T.Logf(" • %s", res.String()) +func buildRequestJSON(req CommitmentChangeRequest) string { + byProject := make(map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset) + for _, tc := range req.Commitments { + pid := liquid.ProjectUUID(tc.ProjectID) + if byProject[pid].ByResource == nil { + byProject[pid] = liquid.ProjectCommitmentChangeset{ + ByResource: make(map[liquid.ResourceName]liquid.ResourceCommitmentChangeset), + } } - } - - if len(unmatchedActual) > 0 { - env.T.Logf(" UNEXPECTED (%d found, not expected):", len(unmatchedActual)) - for _, res := range unmatchedActual { - env.T.Logf(" • %s", compactReservationString(&res)) + var oldStatus Option[liquid.CommitmentStatus] + if tc.OldState != "" { + oldStatus = Some(liquid.CommitmentStatus(tc.OldState)) + } else { + oldStatus = None[liquid.CommitmentStatus]() } - } - - if len(unmatchedExpected) == 0 && len(unmatchedActual) == 0 { - env.T.Log(" ✓ All match!") - } -} - -// reservationMatches checks if an actual reservation matches an expected one. -// All fields are checked comprehensively for complete validation. -func (env *CommitmentTestEnv) reservationMatches(expected *TestReservation, actual *v1alpha1.Reservation) bool { - // Check CommitmentID (from reservation name prefix) - if !strings.HasPrefix(actual.Name, "commitment-"+expected.CommitmentID+"-") { - return false - } - - // Check that CommittedResourceReservation spec exists - if actual.Spec.CommittedResourceReservation == nil { - return false - } - - // Check CommitmentUUID in spec matches - if actual.Spec.CommittedResourceReservation.CommitmentUUID != expected.CommitmentID { - return false - } - - // Check ProjectID - if actual.Spec.CommittedResourceReservation.ProjectID != expected.ProjectID { - return false - } - - // Check ResourceName (flavor name) - if expected.Flavor != nil { - if actual.Spec.CommittedResourceReservation.ResourceName != expected.Flavor.Name { - return false + var newStatus Option[liquid.CommitmentStatus] + if tc.State != "" { + newStatus = Some(liquid.CommitmentStatus(tc.State)) + } else { + newStatus = None[liquid.CommitmentStatus]() } - } - - // Check ResourceGroup (flavor group) - if expected.Flavor != nil { - if actual.Spec.CommittedResourceReservation.ResourceGroup != expected.Flavor.Group { - return false + commitment := liquid.Commitment{ + UUID: liquid.CommitmentUUID(tc.ConfirmationID), + Amount: tc.Amount, + OldStatus: oldStatus, + NewStatus: newStatus, + ExpiresAt: time.Now().Add(365 * 24 * time.Hour), + } + byResource := byProject[pid].ByResource[tc.ResourceName] + byResource.Commitments = append(byResource.Commitments, commitment) + + // Compute per-resource totals so RequiresConfirmation() behaves correctly. + // OldAmount overrides Amount for TotalBefore (resize-down: old amount != new amount). + oldAmt := tc.Amount + if tc.OldAmount != 0 { + oldAmt = tc.OldAmount + } + if oldStatus == Some(liquid.CommitmentStatusConfirmed) { + byResource.TotalConfirmedBefore += oldAmt + } + if newStatus == Some(liquid.CommitmentStatusConfirmed) { + byResource.TotalConfirmedAfter += tc.Amount + } + if oldStatus == Some(liquid.CommitmentStatusGuaranteed) { + byResource.TotalGuaranteedBefore += oldAmt + } + if newStatus == Some(liquid.CommitmentStatusGuaranteed) { + byResource.TotalGuaranteedAfter += tc.Amount } - } - - // Check Host (if specified in expected) - if expected.Host != "" && actual.Status.Host != expected.Host { - return false - } - // Check AZ (if specified in expected) - if expected.AZ != "" && actual.Spec.AvailabilityZone != expected.AZ { - return false + byProject[pid].ByResource[tc.ResourceName] = byResource } - // Check Memory (use custom MemoryMB if non-zero, otherwise use flavor size) - expectedMemoryMB := expected.MemoryMB - if expectedMemoryMB == 0 && expected.Flavor != nil { - expectedMemoryMB = expected.Flavor.MemoryMB + request := liquid.CommitmentChangeRequest{ + InfoVersion: req.InfoVersion, + AZ: liquid.AvailabilityZone(req.AZ), + DryRun: req.DryRun, + ByProject: byProject, } - memoryQuantity := actual.Spec.Resources["memory"] - actualMemoryBytes := memoryQuantity.Value() - actualMemoryMB := actualMemoryBytes / (1024 * 1024) - if actualMemoryMB != expectedMemoryMB { - return false + raw, err := json.Marshal(request) + if err != nil { + panic("failed to marshal request: " + err.Error()) } + return string(raw) +} - // Check CPU (from flavor if available) - if expected.Flavor != nil { - cpuQuantity := actual.Spec.Resources["cpu"] - actualCPU := cpuQuantity.Value() - if actualCPU != expected.Flavor.VCPUs { - return false - } - } +// ============================================================================ +// FlavorGroup Knowledge helpers +// ============================================================================ - // Check VM allocations (set comparison - order doesn't matter) - if !env.vmAllocationsMatch(expected.VMs, actual) { - return false +func buildFlavorGroupsKnowledge(flavors []*TestFlavor, infoVersion int64) FlavorGroupsKnowledge { + groupMap := make(map[string][]compute.FlavorInGroup) + for _, f := range flavors { + groupMap[f.Group] = append(groupMap[f.Group], f.ToFlavorInGroup()) } - // Check reservation type - if actual.Spec.Type != v1alpha1.ReservationTypeCommittedResource { - return false + sortedNames := make([]string, 0, len(groupMap)) + for n := range groupMap { + sortedNames = append(sortedNames, n) } + sort.Strings(sortedNames) - return true -} + var groups []compute.FlavorGroupFeature + for _, name := range sortedNames { + gFlavors := groupMap[name] + sort.Slice(gFlavors, func(i, j int) bool { return gFlavors[i].MemoryMB > gFlavors[j].MemoryMB }) -// vmAllocationsMatch checks if VM allocations match (set comparison). -func (env *CommitmentTestEnv) vmAllocationsMatch(expectedVMs []string, actual *v1alpha1.Reservation) bool { - if actual.Status.CommittedResourceReservation == nil { - return len(expectedVMs) == 0 - } + smallest := gFlavors[len(gFlavors)-1] + largest := gFlavors[0] - actualVMs := make(map[string]bool) - for vmUUID := range actual.Status.CommittedResourceReservation.Allocations { - actualVMs[vmUUID] = true + var minR, maxR uint64 = ^uint64(0), 0 + for _, f := range gFlavors { + if f.VCPUs == 0 { + continue + } + r := f.MemoryMB / f.VCPUs + if r < minR { + minR = r + } + if r > maxR { + maxR = r + } + } + var ratio, ratioMin, ratioMax *uint64 + if minR == maxR && maxR != 0 { + ratio = &minR + } else if maxR != 0 { + ratioMin = &minR + ratioMax = &maxR + } + groups = append(groups, compute.FlavorGroupFeature{ + Name: name, + Flavors: gFlavors, + SmallestFlavor: smallest, + LargestFlavor: largest, + RamCoreRatio: ratio, + RamCoreRatioMin: ratioMin, + RamCoreRatioMax: ratioMax, + }) } + return FlavorGroupsKnowledge{InfoVersion: infoVersion, Groups: groups} +} - // Check counts match - if len(expectedVMs) != len(actualVMs) { - return false +func createKnowledgeCRD(fgk FlavorGroupsKnowledge) *v1alpha1.Knowledge { + raw, err := v1alpha1.BoxFeatureList(fgk.Groups) + if err != nil { + panic("failed to box flavor group features: " + err.Error()) } - // Check all expected VMs are in actual - for _, vmUUID := range expectedVMs { - if !actualVMs[vmUUID] { - return false - } + lastChange := metav1.NewTime(time.Unix(fgk.InfoVersion, 0)) + return &v1alpha1.Knowledge{ + ObjectMeta: metav1.ObjectMeta{ + Name: flavorGroupsKnowledgeName, + }, + Spec: v1alpha1.KnowledgeSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + }, + Status: v1alpha1.KnowledgeStatus{ + Conditions: []metav1.Condition{{Type: v1alpha1.KnowledgeConditionReady, Status: metav1.ConditionTrue, Reason: "Extracted"}}, + Raw: raw, + LastContentChange: lastChange, + }, } - - return true } // ============================================================================ -// Mock VM Source +// MockVMSource (kept for compatibility with handler.go / report_usage tests) // ============================================================================ -// MockVMSource implements VMSource for testing. type MockVMSource struct { - VMs []VM + vms []VM + mu sync.Mutex +} + +type VM struct { + UUID string + FlavorName string + ProjectID string + CurrentHypervisor string + AvailabilityZone string + Resources map[string]int64 + FlavorExtraSpecs map[string]string } -// NewMockVMSource creates a new MockVMSource with the given VMs. func NewMockVMSource(vms []VM) *MockVMSource { - return &MockVMSource{VMs: vms} + return &MockVMSource{vms: vms} } -// ListVMs returns the configured VMs. -func (s *MockVMSource) ListVMs(_ context.Context) ([]VM, error) { - return s.VMs, nil +func (m *MockVMSource) ListVMs(_ context.Context) ([]VM, error) { + m.mu.Lock() + defer m.mu.Unlock() + result := make([]VM, len(m.vms)) + copy(result, m.vms) + return result, nil } // ============================================================================ -// Helper Functions +// TestVM (kept for tests in other files that still use it) // ============================================================================ -// newHypervisorWithAZ creates a Hypervisor CRD with the given parameters including availability zone. -func newHypervisorWithAZ(name string, cpuCap, memoryGi, cpuAlloc, memoryGiAlloc int, instances []hv1.Instance, traits []string, az string) *hv1.Hypervisor { - labels := make(map[string]string) - if az != "" { - labels[corev1.LabelTopologyZone] = az - } - return &hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: labels, - }, - Status: hv1.HypervisorStatus{ - Capacity: map[hv1.ResourceName]resource.Quantity{ - "cpu": resource.MustParse(strconv.Itoa(cpuCap)), - "memory": resource.MustParse(strconv.Itoa(memoryGi) + "Gi"), - }, - Allocation: map[hv1.ResourceName]resource.Quantity{ - "cpu": resource.MustParse(strconv.Itoa(cpuAlloc)), - "memory": resource.MustParse(strconv.Itoa(memoryGiAlloc) + "Gi"), - }, - NumInstances: len(instances), - Instances: instances, - Traits: traits, - }, - } -} - -// createCommitment creates a TestCommitment for use in test cases. -// The az parameter is optional - if empty string, no AZ constraint is set. -func createCommitment(resourceName, projectID, confirmationID, state string, amount uint64, az ...string) TestCommitment { - return TestCommitment{ - ResourceName: liquid.ResourceName(resourceName), - ProjectID: projectID, - ConfirmationID: confirmationID, - State: state, - Amount: amount, - } -} - -// newCommitmentRequest creates a CommitmentChangeRequest with the given commitments. -func newCommitmentRequest(az string, dryRun bool, infoVersion int64, commitments ...TestCommitment) CommitmentChangeRequest { - return CommitmentChangeRequest{ - AZ: az, - DryRun: dryRun, - InfoVersion: infoVersion, - Commitments: commitments, - } -} - -// newAPIResponse creates an APIResponseExpectation with 200 OK status. -func newAPIResponse(rejectReasonSubstrings ...string) APIResponseExpectation { - return APIResponseExpectation{ - StatusCode: 200, - RejectReasonSubstrings: rejectReasonSubstrings, - } -} - -// buildRequestJSON converts a test CommitmentChangeRequest to JSON string. -// Builds the nested JSON structure directly for simplicity. -// Uses sorted iteration to ensure deterministic JSON output. -func buildRequestJSON(req CommitmentChangeRequest) string { - // Group commitments by project and resource for nested structure - type projectResources map[liquid.ResourceName][]TestCommitment - byProject := make(map[string]projectResources) - - for _, commit := range req.Commitments { - if byProject[commit.ProjectID] == nil { - byProject[commit.ProjectID] = make(projectResources) - } - byProject[commit.ProjectID][commit.ResourceName] = append( - byProject[commit.ProjectID][commit.ResourceName], - commit, - ) - } - - // Sort projects for deterministic iteration - sortedProjects := make([]string, 0, len(byProject)) - for projectID := range byProject { - sortedProjects = append(sortedProjects, projectID) - } - sort.Strings(sortedProjects) - - // Build nested JSON structure with sorted iteration - var projectParts []string - for _, projectID := range sortedProjects { - resources := byProject[projectID] - - // Sort resource names for deterministic iteration - sortedResources := make([]liquid.ResourceName, 0, len(resources)) - for resourceName := range resources { - sortedResources = append(sortedResources, resourceName) - } - sort.Slice(sortedResources, func(i, j int) bool { - return string(sortedResources[i]) < string(sortedResources[j]) - }) - - var resourceParts []string - for _, resourceName := range sortedResources { - commits := resources[resourceName] - var commitParts []string - for _, c := range commits { - expiryTime := time.Now().Add(time.Duration(defaultCommitmentExpiryYears) * 365 * 24 * time.Hour) - commitParts = append(commitParts, fmt.Sprintf(`{"uuid":"%s","newStatus":"%s","amount":%d,"expiresAt":"%s"}`, - c.ConfirmationID, c.State, c.Amount, expiryTime.Format(time.RFC3339))) - } - resourceParts = append(resourceParts, fmt.Sprintf(`"%s":{"commitments":[%s]}`, - resourceName, strings.Join(commitParts, ","))) - } - projectParts = append(projectParts, fmt.Sprintf(`"%s":{"byResource":{%s}}`, - projectID, strings.Join(resourceParts, ","))) - } - - return fmt.Sprintf(`{"az":"%s","dryRun":%t,"infoVersion":%d,"byProject":{%s}}`, - req.AZ, req.DryRun, req.InfoVersion, strings.Join(projectParts, ",")) +type TestVM struct { + UUID string + Flavor *TestFlavor + ProjectID string + Host string + AZ string } -// createKnowledgeCRD creates a Knowledge CRD populated with flavor groups. -func createKnowledgeCRD(flavorGroups FlavorGroupsKnowledge) *v1alpha1.Knowledge { - rawExt, err := v1alpha1.BoxFeatureList(flavorGroups.Groups) - if err != nil { - panic("Failed to box flavor groups: " + err.Error()) - } - - lastContentChange := time.Unix(flavorGroups.InfoVersion, 0) - - return &v1alpha1.Knowledge{ - ObjectMeta: metav1.ObjectMeta{ - Name: flavorGroupsKnowledgeName, - }, - Spec: v1alpha1.KnowledgeSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Extractor: v1alpha1.KnowledgeExtractorSpec{ - Name: flavorGroupsKnowledgeName, - }, - Recency: metav1.Duration{Duration: knowledgeRecencyDuration}, +func (vm *TestVM) ToVM() VM { + return VM{ + UUID: vm.UUID, + FlavorName: vm.Flavor.Name, + ProjectID: vm.ProjectID, + CurrentHypervisor: vm.Host, + AvailabilityZone: vm.AZ, + Resources: map[string]int64{ + "memory": vm.Flavor.MemoryMB, + "vcpus": vm.Flavor.VCPUs, }, - Status: v1alpha1.KnowledgeStatus{ - LastExtracted: metav1.Time{Time: lastContentChange}, - LastContentChange: metav1.Time{Time: lastContentChange}, - Raw: rawExt, - RawLength: len(flavorGroups.Groups), - Conditions: []metav1.Condition{ - { - Type: v1alpha1.KnowledgeConditionReady, - Status: metav1.ConditionTrue, - Reason: "KnowledgeReady", - Message: "Flavor groups knowledge is ready", - LastTransitionTime: metav1.Time{Time: lastContentChange}, - }, - }, + FlavorExtraSpecs: map[string]string{ + "quota:hw_version": vm.Flavor.Group, }, } } diff --git a/internal/scheduling/reservations/commitments/api/handler.go b/internal/scheduling/reservations/commitments/api/handler.go index f0eb24110..051a82fa2 100644 --- a/internal/scheduling/reservations/commitments/api/handler.go +++ b/internal/scheduling/reservations/commitments/api/handler.go @@ -20,7 +20,7 @@ var apiLog = ctrl.Log.WithName("committed-resource") // HTTPAPI implements Limes LIQUID commitment validation endpoints. type HTTPAPI struct { client client.Client - config commitments.Config + config commitments.APIConfig usageDB commitments.UsageDBClient monitor ChangeCommitmentsAPIMonitor usageMonitor ReportUsageAPIMonitor @@ -31,11 +31,11 @@ type HTTPAPI struct { } func NewAPI(client client.Client) *HTTPAPI { - return NewAPIWithConfig(client, commitments.DefaultConfig(), nil) + return NewAPIWithConfig(client, commitments.DefaultAPIConfig(), nil) } // NewAPIWithConfig creates an HTTPAPI with the given config and optional usageDB client. -func NewAPIWithConfig(k8sClient client.Client, config commitments.Config, usageDB commitments.UsageDBClient) *HTTPAPI { +func NewAPIWithConfig(k8sClient client.Client, config commitments.APIConfig, usageDB commitments.UsageDBClient) *HTTPAPI { return &HTTPAPI{ client: k8sClient, config: config, @@ -58,9 +58,9 @@ func (api *HTTPAPI) Init(mux *http.ServeMux, registry prometheus.Registerer, log mux.HandleFunc("/commitments/v1/projects/", api.handleProjectEndpoint) // routes to report-usage or quota log.Info("commitments API initialized", - "changeCommitmentsEnabled", api.config.EnableChangeCommitmentsAPI, - "reportUsageEnabled", api.config.EnableReportUsageAPI, - "reportCapacityEnabled", api.config.EnableReportCapacityAPI) + "changeCommitmentsEnabled", api.config.EnableChangeCommitments, + "reportUsageEnabled", api.config.EnableReportUsage, + "reportCapacityEnabled", api.config.EnableReportCapacity) } // handleProjectEndpoint routes /commitments/v1/projects/:project_id/... requests to the appropriate handler. diff --git a/internal/scheduling/reservations/commitments/api/info.go b/internal/scheduling/reservations/commitments/api/info.go index 6999b38d6..2e8ddc8a8 100644 --- a/internal/scheduling/reservations/commitments/api/info.go +++ b/internal/scheduling/reservations/commitments/api/info.go @@ -219,7 +219,8 @@ func (api *HTTPAPI) buildServiceInfo(ctx context.Context, logger logr.Logger) (l "version", version) return liquid.ServiceInfo{ - Version: version, - Resources: resources, + Version: version, + Resources: resources, + CommitmentHandlingNeedsProjectMetadata: true, }, nil } diff --git a/internal/scheduling/reservations/commitments/api/report_capacity.go b/internal/scheduling/reservations/commitments/api/report_capacity.go index f846fea8e..9f0966cce 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity.go @@ -31,7 +31,7 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) w.Header().Set("X-Request-ID", requestID) // Check if API is enabled - if !api.config.EnableReportCapacityAPI { + if !api.config.EnableReportCapacity { statusCode = http.StatusServiceUnavailable http.Error(w, "report-capacity API is disabled", statusCode) api.recordCapacityMetrics(statusCode, startTime) diff --git a/internal/scheduling/reservations/commitments/api/report_usage.go b/internal/scheduling/reservations/commitments/api/report_usage.go index d87f7c24a..bf48dfe00 100644 --- a/internal/scheduling/reservations/commitments/api/report_usage.go +++ b/internal/scheduling/reservations/commitments/api/report_usage.go @@ -36,7 +36,7 @@ func (api *HTTPAPI) HandleReportUsage(w http.ResponseWriter, r *http.Request) { log := apiLog.WithValues("requestID", requestID, "endpoint", "report-usage") // Check if API is enabled - if !api.config.EnableReportUsageAPI { + if !api.config.EnableReportUsage { statusCode = http.StatusServiceUnavailable log.Info("report-usage API is disabled, rejecting request") http.Error(w, "report-usage API is disabled", statusCode) diff --git a/internal/scheduling/reservations/commitments/api/report_usage_test.go b/internal/scheduling/reservations/commitments/api/report_usage_test.go index 4cafdc213..719a7bbb1 100644 --- a/internal/scheduling/reservations/commitments/api/report_usage_test.go +++ b/internal/scheduling/reservations/commitments/api/report_usage_test.go @@ -580,7 +580,7 @@ func newUsageTestEnv( } // Create API with mock DB client - api := NewAPIWithConfig(k8sClient, commitments.DefaultConfig(), dbClient) + api := NewAPIWithConfig(k8sClient, commitments.DefaultAPIConfig(), dbClient) mux := http.NewServeMux() registry := prometheus.NewRegistry() api.Init(mux, registry, log.Log) diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go new file mode 100644 index 000000000..0481395fc --- /dev/null +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -0,0 +1,376 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" +) + +const crFinalizer = "committed-resource.reservations.cortex.cloud/cleanup" + +// CommittedResourceController reconciles CommittedResource CRDs and owns all child Reservation CRUD. +type CommittedResourceController struct { + client.Client + Scheme *runtime.Scheme + Conf CommittedResourceControllerConfig +} + +func (r *CommittedResourceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, req.NamespacedName, &cr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + ctx = WithNewGlobalRequestID(ctx) + logger := LoggerFromContext(ctx).WithValues( + "component", "committed-resource-controller", + "committedResource", req.Name, + ) + + if !cr.DeletionTimestamp.IsZero() { + return r.reconcileDeletion(ctx, logger, &cr) + } + + if !controllerutil.ContainsFinalizer(&cr, crFinalizer) { + controllerutil.AddFinalizer(&cr, crFinalizer) + if err := r.Update(ctx, &cr); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to add finalizer: %w", err) + } + return ctrl.Result{}, nil + } + + switch cr.Spec.State { + case v1alpha1.CommitmentStatusPlanned: + return ctrl.Result{}, r.setNotReady(ctx, &cr, v1alpha1.CommittedResourceReasonPlanned, "commitment is not yet active") + case v1alpha1.CommitmentStatusPending: + return r.reconcilePending(ctx, logger, &cr) + case v1alpha1.CommitmentStatusGuaranteed, v1alpha1.CommitmentStatusConfirmed: + return r.reconcileCommitted(ctx, logger, &cr) + case v1alpha1.CommitmentStatusSuperseded, v1alpha1.CommitmentStatusExpired: + return r.reconcileInactive(ctx, logger, &cr) + default: + logger.Info("unknown commitment state, skipping", "state", cr.Spec.State) + return ctrl.Result{}, nil + } +} + +// reconcilePending handles a confirmation attempt (Limes state: pending). +// If AllowRejection=true (API path), placement failure marks the CR Rejected so the HTTP API +// can report the outcome back to Limes. If AllowRejection=false (syncer path), the controller +// retries indefinitely — Limes does not require confirmation for these transitions. +func (r *CommittedResourceController) reconcilePending(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + result, applyErr := r.applyReservationState(ctx, logger, cr) + if applyErr != nil { + if cr.Spec.AllowRejection { + logger.Error(applyErr, "pending commitment placement failed, rejecting") + if rollbackErr := r.deleteChildReservations(ctx, cr); rollbackErr != nil { + return ctrl.Result{}, rollbackErr + } + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonRejected, applyErr.Error()) + } + logger.Error(applyErr, "pending commitment placement failed, will retry", "requeueAfter", r.Conf.RequeueIntervalRetry.Duration) + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, applyErr.Error()) + } + allReady, anyFailed, failReason, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) + if err != nil { + return ctrl.Result{}, err + } + if anyFailed { + if cr.Spec.AllowRejection { + logger.Info("pending commitment rejected: reservation placement failed", "reason", failReason) + if rollbackErr := r.deleteChildReservations(ctx, cr); rollbackErr != nil { + return ctrl.Result{}, rollbackErr + } + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonRejected, failReason) + } + logger.Info("pending commitment placement failed, will retry", "reason", failReason, "requeueAfter", r.Conf.RequeueIntervalRetry.Duration) + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, failReason) + } + if !allReady { + // Reservation controller hasn't processed all slots yet; Reservation watch will re-enqueue. + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement") + } + return ctrl.Result{}, r.setAccepted(ctx, cr) +} + +func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + // Spec errors are permanent regardless of AllowRejection — a bad spec won't fix itself. + if _, err := FromCommittedResource(*cr); err != nil { + logger.Error(err, "invalid commitment spec, rejecting") + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonRejected, err.Error()) + } + result, applyErr := r.applyReservationState(ctx, logger, cr) + if applyErr != nil { + if cr.Spec.AllowRejection { + logger.Error(applyErr, "committed placement failed, rolling back to accepted amount") + if rollbackErr := r.rollbackToAccepted(ctx, logger, cr); rollbackErr != nil { + return ctrl.Result{}, rollbackErr + } + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonRejected, applyErr.Error()) + } + logger.Error(applyErr, "committed placement incomplete, will retry", "requeueAfter", r.Conf.RequeueIntervalRetry.Duration) + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, applyErr.Error()) + } + allReady, anyFailed, failReason, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) + if err != nil { + return ctrl.Result{}, err + } + if anyFailed { + if cr.Spec.AllowRejection { + logger.Info("committed placement failed, rolling back to accepted amount", "reason", failReason) + if rollbackErr := r.rollbackToAccepted(ctx, logger, cr); rollbackErr != nil { + return ctrl.Result{}, rollbackErr + } + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonRejected, failReason) + } + logger.Info("committed placement failed, will retry", "reason", failReason, "requeueAfter", r.Conf.RequeueIntervalRetry.Duration) + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, failReason) + } + if !allReady { + // Reservation controller hasn't processed all slots yet; Reservation watch will re-enqueue. + return ctrl.Result{}, r.setNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement") + } + return ctrl.Result{}, r.setAccepted(ctx, cr) +} + +func (r *CommittedResourceController) applyReservationState(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (*ApplyResult, error) { + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) + if err != nil { + return nil, fmt.Errorf("flavor knowledge not ready: %w", err) + } + + state, err := FromCommittedResource(*cr) + if err != nil { + return nil, fmt.Errorf("invalid commitment spec: %w", err) + } + state.NamePrefix = cr.Name + "-" + state.CreatorRequestID = reservations.GlobalRequestIDFromContext(ctx) + state.ParentGeneration = cr.Generation + + result, err := NewReservationManager(r.Client).ApplyCommitmentState(ctx, logger, state, flavorGroups, "committed-resource-controller") + if err != nil { + return nil, err + } + logger.Info("commitment state applied", "created", result.Created, "deleted", result.Deleted, "repaired", result.Repaired) + return result, nil +} + +// checkChildReservationStatus inspects the Ready conditions of all child Reservations for cr. +// Returns allReady=true when every child has Ready=True. +// Returns anyFailed=true (and the first failure message) when any child has Ready=False. +// Returns allReady=false, anyFailed=false when some children have no condition yet (placement pending). +func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int) (allReady, anyFailed bool, failReason string, err error) { + var list v1alpha1.ReservationList + if err := r.List(ctx, &list, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + client.MatchingFields{idxReservationByCommitmentUUID: cr.Spec.CommitmentUUID}, + ); err != nil { + return false, false, "", fmt.Errorf("failed to list reservations: %w", err) + } + + // Cache hasn't caught up yet; Reservation watch will re-enqueue. + if len(list.Items) < expectedSlots { + return false, false, "", nil + } + + if len(list.Items) == 0 { + return true, false, "", nil + } + + // First pass: failures take priority over pending — but only for the current generation. + // A Ready=False condition from a previous generation means the reservation controller + // hasn't reprocessed this slot yet; treat it as still-pending, not as a current failure. + for _, res := range list.Items { + if res.Status.CommittedResourceReservation == nil || + res.Status.CommittedResourceReservation.ObservedParentGeneration != cr.Generation { + continue + } + cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond != nil && cond.Status == metav1.ConditionFalse { + return false, true, cond.Message, nil + } + } + // Second pass: check generation and readiness for all slots. + for _, res := range list.Items { + // ObservedParentGeneration must match cr.Generation before we trust the Ready condition. + if res.Status.CommittedResourceReservation == nil || + res.Status.CommittedResourceReservation.ObservedParentGeneration != cr.Generation { + return false, false, "", nil + } + cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil || cond.Status != metav1.ConditionTrue { + return false, false, "", nil + } + } + return true, false, "", nil +} + +func (r *CommittedResourceController) setAccepted(ctx context.Context, cr *v1alpha1.CommittedResource) error { + now := metav1.Now() + old := cr.DeepCopy() + acceptedAmount := cr.Spec.Amount.DeepCopy() + cr.Status.AcceptedAmount = &acceptedAmount + cr.Status.AcceptedAt = &now + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionTrue, + Reason: v1alpha1.CommittedResourceReasonAccepted, + Message: "commitment successfully reserved", + LastTransitionTime: now, + }) + if err := r.Status().Patch(ctx, cr, client.MergeFrom(old)); err != nil { + return client.IgnoreNotFound(err) + } + return nil +} + +func (r *CommittedResourceController) reconcileInactive(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + if err := r.deleteChildReservations(ctx, cr); err != nil { + return ctrl.Result{}, err + } + logger.Info("commitment inactive, child reservations removed", "state", cr.Spec.State) + return ctrl.Result{}, r.setNotReady(ctx, cr, string(cr.Spec.State), "commitment is no longer active") +} + +func (r *CommittedResourceController) reconcileDeletion(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + if err := r.deleteChildReservations(ctx, cr); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(cr, crFinalizer) + if err := r.Update(ctx, cr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + logger.Info("committed resource deleted, child reservations cleaned up") + return ctrl.Result{}, nil +} + +// deleteChildReservations deletes all Reservation CRDs owned by this CommittedResource, +// identified by matching CommitmentUUID in the reservation spec. +func (r *CommittedResourceController) deleteChildReservations(ctx context.Context, cr *v1alpha1.CommittedResource) error { + var list v1alpha1.ReservationList + if err := r.List(ctx, &list, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + client.MatchingFields{idxReservationByCommitmentUUID: cr.Spec.CommitmentUUID}, + ); err != nil { + return fmt.Errorf("failed to list reservations: %w", err) + } + for i := range list.Items { + res := &list.Items[i] + if err := r.Delete(ctx, res); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to delete reservation %s: %w", res.Name, err) + } + } + return nil +} + +// rollbackToAccepted restores child Reservations to match Status.AcceptedAmount. +// If AcceptedAmount is nil (new CR that was never accepted), all child Reservations are deleted. +func (r *CommittedResourceController) rollbackToAccepted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) error { + if cr.Status.AcceptedAmount == nil { + return r.deleteChildReservations(ctx, cr) + } + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: r.Client} + flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) + if err != nil { + // Can't compute the rollback target — fall back to full delete rather than leaving + // a partial state that's inconsistent with the unknown AcceptedAmount. + logger.Error(err, "flavor knowledge unavailable during rollback, deleting all child reservations") + return r.deleteChildReservations(ctx, cr) + } + state, err := FromCommittedResource(*cr) + if err != nil { + logger.Error(err, "invalid spec during rollback, deleting all child reservations") + return r.deleteChildReservations(ctx, cr) + } + state.TotalMemoryBytes = cr.Status.AcceptedAmount.Value() + state.NamePrefix = cr.Name + "-" + state.CreatorRequestID = reservations.GlobalRequestIDFromContext(ctx) + state.ParentGeneration = cr.Generation + if _, err := NewReservationManager(r.Client).ApplyCommitmentState(ctx, logger, state, flavorGroups, "committed-resource-controller-rollback"); err != nil { + return fmt.Errorf("rollback apply failed: %w", err) + } + return nil +} + +// setNotReady patches Ready=False on CommittedResource status. +func (r *CommittedResourceController) setNotReady(ctx context.Context, cr *v1alpha1.CommittedResource, reason, message string) error { + old := cr.DeepCopy() + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + LastTransitionTime: metav1.Now(), + }) + if err := r.Status().Patch(ctx, cr, client.MergeFrom(old)); err != nil { + return client.IgnoreNotFound(err) + } + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CommittedResourceController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + ctx := context.Background() + if err := IndexFields(ctx, mcl); err != nil { + return fmt.Errorf("failed to set up field indexes: %w", err) + } + + bldr := multicluster.BuildController(mcl, mgr) + var err error + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.CommittedResource{}, + &handler.EnqueueRequestForObject{}, + ) + if err != nil { + return err + } + // Re-enqueue the parent CommittedResource when a child Reservation changes (e.g. external deletion). + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.Reservation{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.CommittedResourceReservation == nil { + return nil + } + uuid := res.Spec.CommittedResourceReservation.CommitmentUUID + var crList v1alpha1.CommittedResourceList + if err := r.List(ctx, &crList, client.MatchingFields{idxCommittedResourceByUUID: uuid}); err != nil { + LoggerFromContext(ctx).Error(err, "failed to list CommittedResources by UUID", "uuid", uuid) + return nil + } + if len(crList.Items) == 0 { + return nil + } + return []ctrl.Request{{NamespacedName: types.NamespacedName{Name: crList.Items[0].Name}}} + }), + ) + if err != nil { + return err + } + // MaxConcurrentReconciles=1: the change-commitments API handler snapshots each CR's spec + // before writing and restores it on rollback. Concurrent reconciles across overlapping + // batch requests could interleave those snapshots and produce incorrect rollback state. + return bldr.Named("committed-resource"). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + }). + Complete(r) +} diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go new file mode 100644 index 000000000..471c013e3 --- /dev/null +++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go @@ -0,0 +1,670 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "encoding/json" + "testing" + "time" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// ============================================================================ +// Helpers +// ============================================================================ + +// newTestCommittedResource returns a CommittedResource with sensible defaults. +// The finalizer is pre-populated so tests can call Reconcile once without a +// separate finalizer-add round-trip. +func newTestCommittedResource(name string, state v1alpha1.CommitmentStatus) *v1alpha1.CommittedResource { + return &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Finalizers: []string{crFinalizer}, + }, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "test-uuid-1234", + FlavorGroupName: "test-group", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: resource.MustParse("4Gi"), + AvailabilityZone: "test-az", + ProjectID: "test-project", + DomainID: "test-domain", + State: state, + }, + } +} + +// newTestFlavorKnowledge returns a Knowledge CRD with a single 4 GiB flavor so +// a 4 GiB commitment produces exactly one slot. +func newTestFlavorKnowledge() *v1alpha1.Knowledge { + raw, err := json.Marshal(map[string]any{ + "features": []map[string]any{ + { + "name": "test-group", + "flavors": []map[string]any{ + { + "name": "test-flavor", + "memoryMB": 4096, + "vcpus": 2, + "extraSpecs": map[string]string{}, + }, + }, + }, + }, + }) + if err != nil { + panic(err) + } + return &v1alpha1.Knowledge{ + ObjectMeta: metav1.ObjectMeta{Name: "flavor-groups"}, + Spec: v1alpha1.KnowledgeSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + Extractor: v1alpha1.KnowledgeExtractorSpec{Name: "flavor_groups"}, + }, + Status: v1alpha1.KnowledgeStatus{ + Raw: runtime.RawExtension{Raw: raw}, + RawLength: 1, + Conditions: []metav1.Condition{ + { + Type: v1alpha1.KnowledgeConditionReady, + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + }, + } +} + +func newCRTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 scheme: %v", err) + } + return scheme +} + +func newCRTestClient(scheme *runtime.Scheme, objects ...client.Object) client.Client { + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&v1alpha1.CommittedResource{}, &v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, idxReservationByCommitmentUUID, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.CommitmentUUID == "" { + return nil + } + return []string{res.Spec.CommittedResourceReservation.CommitmentUUID} + }). + WithIndex(&v1alpha1.CommittedResource{}, idxCommittedResourceByUUID, func(obj client.Object) []string { + cr, ok := obj.(*v1alpha1.CommittedResource) + if !ok || cr.Spec.CommitmentUUID == "" { + return nil + } + return []string{cr.Spec.CommitmentUUID} + }). + Build() +} + +func reconcileReq(name string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Name: name}} +} + +// assertCondition checks the Ready condition status and reason on a CommittedResource. +func assertCondition(t *testing.T, k8sClient client.Client, crName string, expectedStatus metav1.ConditionStatus, expectedReason string) { + t.Helper() + var cr v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: crName}, &cr); err != nil { + t.Fatalf("failed to get CommittedResource %s: %v", crName, err) + } + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + t.Errorf("Ready condition not set on %s", crName) + return + } + if cond.Status != expectedStatus { + t.Errorf("%s: expected Ready=%s, got %s", crName, expectedStatus, cond.Status) + } + if cond.Reason != expectedReason { + t.Errorf("%s: expected Reason=%s, got %s", crName, expectedReason, cond.Reason) + } +} + +// countChildReservations counts Reservation CRDs owned by the given CommitmentUUID, +// using the same identity predicate as the controller. +func countChildReservations(t *testing.T, k8sClient client.Client, commitmentUUID string) int { + t.Helper() + var list v1alpha1.ReservationList + if err := k8sClient.List(context.Background(), &list, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("failed to list reservations: %v", err) + } + count := 0 + for _, r := range list.Items { + if r.Spec.CommittedResourceReservation != nil && + r.Spec.CommittedResourceReservation.CommitmentUUID == commitmentUUID { + count++ + } + } + return count +} + +// setChildReservationsReady simulates the reservation controller by marking all child +// Reservations for the given commitmentUUID as Ready=True and echoing ParentGeneration +// into ObservedParentGeneration (matching what echoParentGeneration does in production). +func setChildReservationsReady(t *testing.T, k8sClient client.Client, commitmentUUID string) { + t.Helper() + var list v1alpha1.ReservationList + if err := k8sClient.List(context.Background(), &list, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + for i := range list.Items { + res := &list.Items[i] + if res.Spec.CommittedResourceReservation == nil || + res.Spec.CommittedResourceReservation.CommitmentUUID != commitmentUUID { + continue + } + res.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + LastTransitionTime: metav1.Now(), + }} + if res.Status.CommittedResourceReservation == nil { + res.Status.CommittedResourceReservation = &v1alpha1.CommittedResourceReservationStatus{} + } + res.Status.CommittedResourceReservation.ObservedParentGeneration = res.Spec.CommittedResourceReservation.ParentGeneration + if err := k8sClient.Status().Update(context.Background(), res); err != nil { + t.Fatalf("set reservation Ready=True: %v", err) + } + } +} + +// ============================================================================ +// Tests: per-state reconcile paths +// ============================================================================ + +func TestCommittedResourceController_Reconcile(t *testing.T) { + tests := []struct { + name string + state v1alpha1.CommitmentStatus + expectedStatus metav1.ConditionStatus + expectedReason string + expectedSlots int + needsKnowledge bool + }{ + { + name: "planned: no Reservations created, Ready=False/Planned", + state: v1alpha1.CommitmentStatusPlanned, + expectedStatus: metav1.ConditionFalse, + expectedReason: "Planned", + expectedSlots: 0, + }, + { + name: "pending: Reservations created, Ready=True", + state: v1alpha1.CommitmentStatusPending, + expectedStatus: metav1.ConditionTrue, + expectedReason: "Accepted", + expectedSlots: 1, + needsKnowledge: true, + }, + { + name: "guaranteed: Reservations created, Ready=True", + state: v1alpha1.CommitmentStatusGuaranteed, + expectedStatus: metav1.ConditionTrue, + expectedReason: "Accepted", + expectedSlots: 1, + needsKnowledge: true, + }, + { + name: "confirmed: Reservations created, Ready=True", + state: v1alpha1.CommitmentStatusConfirmed, + expectedStatus: metav1.ConditionTrue, + expectedReason: "Accepted", + expectedSlots: 1, + needsKnowledge: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", tt.state) + objects := []client.Object{cr} + if tt.needsKnowledge { + objects = append(objects, newTestFlavorKnowledge()) + } + k8sClient := newCRTestClient(scheme, objects...) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + // First reconcile: creates Reservation CRDs; if slots are expected, controller + // waits for the reservation controller to set Ready=True before accepting. + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile 1: %v", err) + } + + if tt.expectedSlots > 0 { + // Simulate reservation controller: mark all child reservations as Ready=True. + setChildReservationsReady(t, k8sClient, cr.Spec.CommitmentUUID) + // Second reconcile: sees all Ready=True and accepts. + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile 2: %v", err) + } + } + + assertCondition(t, k8sClient, cr.Name, tt.expectedStatus, tt.expectedReason) + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != tt.expectedSlots { + t.Errorf("expected %d child reservations, got %d", tt.expectedSlots, got) + } + + if tt.expectedSlots > 0 { + var updated v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &updated); err != nil { + t.Fatalf("get CR: %v", err) + } + if updated.Status.AcceptedAmount == nil { + t.Errorf("expected AcceptedAmount to be set on acceptance") + } + } + }) + } +} + +func TestCommittedResourceController_InactiveStates(t *testing.T) { + tests := []struct { + name string + state v1alpha1.CommitmentStatus + }{ + {name: "superseded: child Reservations deleted, Ready=False", state: v1alpha1.CommitmentStatusSuperseded}, + {name: "expired: child Reservations deleted, Ready=False", state: v1alpha1.CommitmentStatusExpired}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", tt.state) + existing := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr-0", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: "test-uuid-1234", + }, + }, + } + k8sClient := newCRTestClient(scheme, cr, existing) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile: %v", err) + } + + assertCondition(t, k8sClient, cr.Name, metav1.ConditionFalse, string(tt.state)) + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != 0 { + t.Errorf("expected 0 child reservations after %s, got %d", tt.state, got) + } + }) + } +} + +// ============================================================================ +// Tests: placement failure paths +// ============================================================================ + +func TestCommittedResourceController_PlacementFailure(t *testing.T) { + // Knowledge absent → placement fails. Tests diverging behavior by state and AllowRejection. + tests := []struct { + name string + state v1alpha1.CommitmentStatus + allowRejection bool + expectedReason string + expectRequeue bool + }{ + { + name: "pending AllowRejection=true: rejects on failure, no retry", + state: v1alpha1.CommitmentStatusPending, + allowRejection: true, + expectedReason: "Rejected", + expectRequeue: false, + }, + { + name: "pending AllowRejection=false: retries on failure", + state: v1alpha1.CommitmentStatusPending, + allowRejection: false, + expectedReason: "Reserving", + expectRequeue: true, + }, + { + name: "guaranteed AllowRejection=true: rejects on failure, no retry", + state: v1alpha1.CommitmentStatusGuaranteed, + allowRejection: true, + expectedReason: "Rejected", + expectRequeue: false, + }, + { + name: "confirmed AllowRejection=true: rejects on failure, no retry", + state: v1alpha1.CommitmentStatusConfirmed, + allowRejection: true, + expectedReason: "Rejected", + expectRequeue: false, + }, + { + name: "guaranteed AllowRejection=false: retries on failure", + state: v1alpha1.CommitmentStatusGuaranteed, + allowRejection: false, + expectedReason: "Reserving", + expectRequeue: true, + }, + { + name: "confirmed AllowRejection=false: retries on failure", + state: v1alpha1.CommitmentStatusConfirmed, + allowRejection: false, + expectedReason: "Reserving", + expectRequeue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", tt.state) + cr.Spec.AllowRejection = tt.allowRejection + k8sClient := newCRTestClient(scheme, cr) // no Knowledge → placement fails + controller := &CommittedResourceController{ + Client: k8sClient, + Scheme: scheme, + Conf: CommittedResourceControllerConfig{RequeueIntervalRetry: metav1.Duration{Duration: 1 * time.Minute}}, + } + + result, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + + assertCondition(t, k8sClient, cr.Name, metav1.ConditionFalse, tt.expectedReason) + if tt.expectRequeue && result.RequeueAfter == 0 { + t.Errorf("expected requeue after failure, got none") + } + if !tt.expectRequeue && result.RequeueAfter != 0 { + t.Errorf("expected no requeue after rejection, got RequeueAfter=%v", result.RequeueAfter) + } + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != 0 { + t.Errorf("expected 0 child reservations after failure, got %d", got) + } + }) + } +} + +func TestCommittedResourceController_Rollback(t *testing.T) { + scheme := newCRTestScheme(t) + + // CR at generation 2; AcceptedAmount reflects what was accepted at generation 1. + cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusConfirmed) + cr.Generation = 2 + accepted := resource.MustParse("4Gi") + cr.Status.AcceptedAmount = &accepted + + // Existing reservation with stale ParentGeneration from the previous generation. + existing := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr-0", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + AvailabilityZone: "test-az", + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("4Gi"), + hv1.ResourceCPU: resource.MustParse("2"), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: "test-uuid-1234", + ProjectID: "test-project", + DomainID: "test-domain", + ResourceGroup: "test-group", + ParentGeneration: 1, // stale + }, + }, + } + + k8sClient := newCRTestClient(scheme, cr, existing, newTestFlavorKnowledge()) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + if err := controller.rollbackToAccepted(context.Background(), logr.Discard(), cr); err != nil { + t.Fatalf("rollbackToAccepted: %v", err) + } + + var res v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "test-cr-0"}, &res); err != nil { + t.Fatalf("get reservation: %v", err) + } + if got := res.Spec.CommittedResourceReservation.ParentGeneration; got != cr.Generation { + t.Errorf("ParentGeneration: want %d, got %d", cr.Generation, got) + } +} + +func TestCommittedResourceController_BadSpec(t *testing.T) { + scheme := newCRTestScheme(t) + cr := &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr", + Finalizers: []string{crFinalizer}, + }, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "x", // too short, fails commitmentUUIDPattern + FlavorGroupName: "test-group", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: resource.MustParse("4Gi"), + AvailabilityZone: "test-az", + ProjectID: "test-project", + DomainID: "test-domain", + State: v1alpha1.CommitmentStatusConfirmed, + }, + } + k8sClient := newCRTestClient(scheme, cr, newTestFlavorKnowledge()) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile: %v", err) + } + + assertCondition(t, k8sClient, cr.Name, metav1.ConditionFalse, "Rejected") + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != 0 { + t.Errorf("expected 0 child reservations after bad-spec rejection, got %d", got) + } +} + +func TestCommittedResourceController_Idempotent(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusConfirmed) + k8sClient := newCRTestClient(scheme, cr, newTestFlavorKnowledge()) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + // Round 1: creates reservation, waits for placement. + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile 1: %v", err) + } + // Simulate reservation controller setting Ready=True. + setChildReservationsReady(t, k8sClient, cr.Spec.CommitmentUUID) + // Rounds 2 and 3: accepts, then stays accepted. + for i := 2; i <= 3; i++ { + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile %d: %v", i, err) + } + } + + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != 1 { + t.Errorf("expected 1 child reservation after 3 reconciles (idempotency), got %d", got) + } + assertCondition(t, k8sClient, cr.Name, metav1.ConditionTrue, "Accepted") +} + +// ============================================================================ +// Tests: checkChildReservationStatus generation guard +// ============================================================================ + +// TestCheckChildReservationStatus_GenerationGuard verifies the two-pass logic that +// distinguishes a stale Ready=False (previous generation) from a current failure. +func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { + tests := []struct { + name string + obsGen int64 + condStatus metav1.ConditionStatus // "" = no condition set + condMessage string + wantAllReady bool + wantAnyFailed bool + wantReason string + }{ + { + name: "Ready=False at stale generation: treated as pending", + obsGen: 1, + condStatus: metav1.ConditionFalse, + condMessage: "no hosts available", + wantAllReady: false, + wantAnyFailed: false, + }, + { + name: "Ready=False at current generation: is a current failure", + obsGen: 2, + condStatus: metav1.ConditionFalse, + condMessage: "no hosts available", + wantAllReady: false, + wantAnyFailed: true, + wantReason: "no hosts available", + }, + { + name: "Ready=True at current generation: allReady", + obsGen: 2, + condStatus: metav1.ConditionTrue, + wantAllReady: true, + }, + { + name: "no condition yet at current generation: still pending", + obsGen: 2, + condStatus: "", // no condition + wantAllReady: false, + wantAnyFailed: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusConfirmed) + cr.Generation = 2 + + child := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr-0", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: cr.Spec.CommitmentUUID, + ParentGeneration: cr.Generation, + }, + }, + } + k8sClient := newCRTestClient(scheme, child) + + child.Status.CommittedResourceReservation = &v1alpha1.CommittedResourceReservationStatus{ + ObservedParentGeneration: tt.obsGen, + } + if tt.condStatus != "" { + child.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.ReservationConditionReady, + Status: tt.condStatus, + Reason: "Test", + Message: tt.condMessage, + LastTransitionTime: metav1.Now(), + }} + } + if err := k8sClient.Status().Update(context.Background(), child); err != nil { + t.Fatalf("set reservation status: %v", err) + } + + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme} + allReady, anyFailed, reason, err := controller.checkChildReservationStatus(context.Background(), cr, 1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if allReady != tt.wantAllReady { + t.Errorf("allReady: want %v, got %v", tt.wantAllReady, allReady) + } + if anyFailed != tt.wantAnyFailed { + t.Errorf("anyFailed: want %v, got %v", tt.wantAnyFailed, anyFailed) + } + if reason != tt.wantReason { + t.Errorf("reason: want %q, got %q", tt.wantReason, reason) + } + }) + } +} + +func TestCommittedResourceController_Deletion(t *testing.T) { + scheme := newCRTestScheme(t) + cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusConfirmed) + child := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr-0", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: "test-uuid-1234", + }, + }, + } + k8sClient := newCRTestClient(scheme, cr, child) + controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme, Conf: CommittedResourceControllerConfig{}} + + if err := k8sClient.Delete(context.Background(), cr); err != nil { + t.Fatalf("delete CR: %v", err) + } + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile: %v", err) + } + + if got := countChildReservations(t, k8sClient, cr.Spec.CommitmentUUID); got != 0 { + t.Errorf("expected 0 child reservations after deletion, got %d", got) + } + var deleted v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &deleted); err == nil { + t.Errorf("expected CR to be gone after deletion, but it still exists with finalizers=%v", deleted.Finalizers) + } +} diff --git a/internal/scheduling/reservations/commitments/committed_resource_integration_test.go b/internal/scheduling/reservations/commitments/committed_resource_integration_test.go new file mode 100644 index 000000000..0090e45f5 --- /dev/null +++ b/internal/scheduling/reservations/commitments/committed_resource_integration_test.go @@ -0,0 +1,633 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +// Integration tests for the CR lifecycle spanning CommittedResourceController and +// CommitmentReservationController. These tests drive both controllers against a shared +// fake client and verify the end-to-end state transitions without mocking internal logic. +// +// Scope: +// - State transition: planned → confirmed produces child Reservations +// - State transition: confirmed → expired cleans up child Reservations +// - Reservation controller places a child Reservation created by the CR controller +// - CR deletion removes all child Reservations + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// crIntegrationEnv holds shared state for integration tests. +type crIntegrationEnv struct { + k8sClient client.Client + crController *CommittedResourceController + resController *CommitmentReservationController + schedulerServer *httptest.Server +} + +func newCRIntegrationEnv(t *testing.T) *crIntegrationEnv { + t.Helper() + scheme := newCRTestScheme(t) + + hypervisor := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}} + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(newTestFlavorKnowledge(), hypervisor). + WithStatusSubresource( + &v1alpha1.CommittedResource{}, + &v1alpha1.Reservation{}, + &v1alpha1.Knowledge{}, + ). + WithIndex(&v1alpha1.Reservation{}, idxReservationByCommitmentUUID, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.CommitmentUUID == "" { + return nil + } + return []string{res.Spec.CommittedResourceReservation.CommitmentUUID} + }). + WithIndex(&v1alpha1.CommittedResource{}, idxCommittedResourceByUUID, func(obj client.Object) []string { + cr, ok := obj.(*v1alpha1.CommittedResource) + if !ok || cr.Spec.CommitmentUUID == "" { + return nil + } + return []string{cr.Spec.CommitmentUUID} + }). + Build() + + schedulerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Errorf("scheduler encode: %v", err) + } + })) + + crCtrl := &CommittedResourceController{ + Client: k8sClient, + Scheme: scheme, + Conf: CommittedResourceControllerConfig{RequeueIntervalRetry: metav1.Duration{Duration: 5 * time.Minute}}, + } + + resCtrl := &CommitmentReservationController{ + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{ + SchedulerURL: schedulerServer.URL, + AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, + RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, + }, + } + if err := resCtrl.Init(context.Background(), resCtrl.Conf); err != nil { + t.Fatalf("resCtrl.Init: %v", err) + } + + return &crIntegrationEnv{ + k8sClient: k8sClient, + crController: crCtrl, + resController: resCtrl, + schedulerServer: schedulerServer, + } +} + +func (e *crIntegrationEnv) close() { e.schedulerServer.Close() } + +func (e *crIntegrationEnv) reconcileCR(t *testing.T, crName string) { + t.Helper() + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: crName}} + if _, err := e.crController.Reconcile(context.Background(), req); err != nil { + t.Fatalf("CR reconcile: %v", err) + } +} + +func (e *crIntegrationEnv) reconcileReservation(t *testing.T, resName string) { + t.Helper() + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: resName}} + if _, err := e.resController.Reconcile(context.Background(), req); err != nil { + t.Fatalf("reservation reconcile %s: %v", resName, err) + } +} + +func (e *crIntegrationEnv) listChildReservations(t *testing.T, crName string) []v1alpha1.Reservation { + t.Helper() + var list v1alpha1.ReservationList + if err := e.k8sClient.List(context.Background(), &list, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + prefix := crName + "-" + var children []v1alpha1.Reservation + for _, r := range list.Items { + if strings.HasPrefix(r.Name, prefix) { + children = append(children, r) + } + } + return children +} + +func (e *crIntegrationEnv) getCR(t *testing.T, name string) v1alpha1.CommittedResource { + t.Helper() + var cr v1alpha1.CommittedResource + if err := e.k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &cr); err != nil { + t.Fatalf("get CR %s: %v", name, err) + } + return cr +} + +// reconcileChildReservations runs the reservation controller twice on every child Reservation +// for crName (first reconcile sets TargetHost, second sets Ready=True), then re-reconciles +// the CR so it can observe the placement outcomes. +func (e *crIntegrationEnv) reconcileChildReservations(t *testing.T, crName string) { + t.Helper() + for _, res := range e.listChildReservations(t, crName) { + e.reconcileReservation(t, res.Name) // calls scheduler → sets TargetHost + e.reconcileReservation(t, res.Name) // syncs TargetHost to Status → Ready=True + } + e.reconcileCR(t, crName) +} + +// ============================================================================ +// Integration tests +// ============================================================================ + +// TestCRLifecycle covers the multi-step state transitions that require imperative +// mid-test patches and cannot be expressed as a purely declarative table. +func TestCRLifecycle(t *testing.T) { + t.Run("planned→confirmed: child Reservations created and placed", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusPlanned) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + // Reconcile as planned: finalizer added, no Reservations. + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + if got := env.listChildReservations(t, cr.Name); len(got) != 0 { + t.Fatalf("planned: expected 0 reservations, got %d", len(got)) + } + crState := env.getCR(t, cr.Name) + cond := meta.FindStatusCondition(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.Reason != "Planned" { + t.Errorf("planned: expected Reason=Planned, got %v", cond) + } + + // Transition to confirmed. + patch := client.MergeFrom(crState.DeepCopy()) + crState.Spec.State = v1alpha1.CommitmentStatusConfirmed + if err := env.k8sClient.Patch(context.Background(), &crState, patch); err != nil { + t.Fatalf("patch state to confirmed: %v", err) + } + env.reconcileCR(t, cr.Name) + + children := env.listChildReservations(t, cr.Name) + if len(children) != 1 { + t.Fatalf("confirmed: expected 1 reservation, got %d", len(children)) + } + env.reconcileChildReservations(t, cr.Name) + + crState = env.getCR(t, cr.Name) + if !meta.IsStatusConditionTrue(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Errorf("confirmed: expected Ready=True") + } + }) + + t.Run("confirmed→expired: child Reservations deleted, CR marked inactive", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + // Bring to confirmed+Ready=True. + env.reconcileCR(t, cr.Name) // adds finalizer + env.reconcileCR(t, cr.Name) // creates Reservations + env.reconcileChildReservations(t, cr.Name) // places slots → Ready=True + + if got := env.listChildReservations(t, cr.Name); len(got) != 1 { + t.Fatalf("pre-expire: expected 1 reservation, got %d", len(got)) + } + + // Transition to expired. + crState := env.getCR(t, cr.Name) + patch := client.MergeFrom(crState.DeepCopy()) + crState.Spec.State = v1alpha1.CommitmentStatusExpired + if err := env.k8sClient.Patch(context.Background(), &crState, patch); err != nil { + t.Fatalf("patch state to expired: %v", err) + } + env.reconcileCR(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 0 { + t.Errorf("expired: expected 0 reservations, got %d", len(got)) + } + crState = env.getCR(t, cr.Name) + cond := meta.FindStatusCondition(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.Status != metav1.ConditionFalse { + t.Errorf("expired: expected Ready=False, got %v", cond) + } + if cond != nil && cond.Reason != string(v1alpha1.CommitmentStatusExpired) { + t.Errorf("expired: expected Reason=%s, got %s", v1alpha1.CommitmentStatusExpired, cond.Reason) + } + }) + + t.Run("reservation placement: two reconciles set TargetHost then Ready=True", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + + children := env.listChildReservations(t, cr.Name) + if len(children) != 1 { + t.Fatalf("expected 1 child reservation, got %d", len(children)) + } + child := children[0] + + // First reconcile: scheduler call → TargetHost written to Spec. + env.reconcileReservation(t, child.Name) + var afterFirst v1alpha1.Reservation + if err := env.k8sClient.Get(context.Background(), types.NamespacedName{Name: child.Name}, &afterFirst); err != nil { + t.Fatalf("get reservation after first reconcile: %v", err) + } + if afterFirst.Spec.TargetHost == "" { + t.Fatalf("expected TargetHost set after first reservation reconcile") + } + + // Second reconcile: TargetHost synced to Status, Ready=True. + env.reconcileReservation(t, child.Name) + var afterSecond v1alpha1.Reservation + if err := env.k8sClient.Get(context.Background(), types.NamespacedName{Name: child.Name}, &afterSecond); err != nil { + t.Fatalf("get reservation after second reconcile: %v", err) + } + if !meta.IsStatusConditionTrue(afterSecond.Status.Conditions, v1alpha1.ReservationConditionReady) { + t.Errorf("expected reservation Ready=True after placement, got %v", afterSecond.Status.Conditions) + } + if afterSecond.Status.Host != "host-1" { + t.Errorf("expected Status.Host=host-1, got %q", afterSecond.Status.Host) + } + }) + + t.Run("deletion: finalizer removed, child Reservations cleaned up", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + // Pre-create a child Reservation to verify it gets cleaned up on deletion. + // newTestCommittedResource pre-populates the finalizer, so Delete() immediately sets DeletionTimestamp. + child := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cr-0", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: "test-uuid-1234", + }, + }, + } + if err := env.k8sClient.Create(context.Background(), child); err != nil { + t.Fatalf("create child reservation: %v", err) + } + + crState := env.getCR(t, cr.Name) + if err := env.k8sClient.Delete(context.Background(), &crState); err != nil { + t.Fatalf("delete CR: %v", err) + } + env.reconcileCR(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 0 { + t.Errorf("post-deletion: expected 0 reservations, got %d", len(got)) + } + var final v1alpha1.CommittedResource + err := env.k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &final) + if client.IgnoreNotFound(err) != nil { + t.Fatalf("unexpected error after deletion: %v", err) + } + if err == nil { + for _, f := range final.Finalizers { + if f == crFinalizer { + t.Errorf("finalizer not removed after deletion reconcile") + } + } + } + }) + + t.Run("confirmed→superseded: child Reservations deleted, CR marked inactive", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 1 { + t.Fatalf("pre-supersede: expected 1 reservation, got %d", len(got)) + } + + crState := env.getCR(t, cr.Name) + patch := client.MergeFrom(crState.DeepCopy()) + crState.Spec.State = v1alpha1.CommitmentStatusSuperseded + if err := env.k8sClient.Patch(context.Background(), &crState, patch); err != nil { + t.Fatalf("patch state to superseded: %v", err) + } + env.reconcileCR(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 0 { + t.Errorf("superseded: expected 0 reservations, got %d", len(got)) + } + crState = env.getCR(t, cr.Name) + cond := meta.FindStatusCondition(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.Status != metav1.ConditionFalse { + t.Errorf("superseded: expected Ready=False, got %v", cond) + } + if cond != nil && cond.Reason != string(v1alpha1.CommitmentStatusSuperseded) { + t.Errorf("superseded: expected Reason=%s, got %s", v1alpha1.CommitmentStatusSuperseded, cond.Reason) + } + }) + + t.Run("idempotency: extra reconciles after Accepted do not create extra slots", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 1 { + t.Fatalf("pre-idempotency check: expected 1 reservation, got %d", len(got)) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 1 { + t.Errorf("idempotency: expected 1 reservation after extra reconciles, got %d", len(got)) + } + crState := env.getCR(t, cr.Name) + if !meta.IsStatusConditionTrue(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Errorf("idempotency: expected CR to remain Ready=True after extra reconciles") + } + }) + + t.Run("AllowRejection=false: stays Reserving when scheduler rejects", func(t *testing.T) { + hypervisor := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}} + env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), hypervisor}, intgRejectScheduler) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + // AllowRejection stays false (the default), so placement failure must requeue, not reject. + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + ctx := context.Background() + crReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: cr.Name}} + for range 3 { + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + var resList v1alpha1.ReservationList + env.k8sClient.List(ctx, &resList, client.MatchingLabels{ //nolint:errcheck + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }) + for _, res := range resList.Items { + resReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + } + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + } + + var final v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: cr.Name}, &final); err != nil { + t.Fatalf("get CR: %v", err) + } + cond := meta.FindStatusCondition(final.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + t.Fatalf("no Ready condition") + } + if cond.Reason == v1alpha1.CommittedResourceReasonRejected { + t.Errorf("AllowRejection=false: CR must not transition to Rejected, got Reason=%s", cond.Reason) + } + if cond.Reason != v1alpha1.CommittedResourceReasonReserving { + t.Errorf("AllowRejection=false: expected Reason=Reserving, got %s", cond.Reason) + } + }) + + t.Run("externally deleted child Reservation is recreated by CR controller", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + children := env.listChildReservations(t, cr.Name) + if len(children) != 1 { + t.Fatalf("expected 1 child reservation before deletion, got %d", len(children)) + } + + // Simulate out-of-band deletion of the slot. + child := children[0] + if err := env.k8sClient.Delete(context.Background(), &child); err != nil { + t.Fatalf("delete child reservation: %v", err) + } + + // CR controller detects the missing slot and recreates it. + env.reconcileCR(t, cr.Name) + // Place the new slot. + env.reconcileChildReservations(t, cr.Name) + // CR controller observes Ready=True on the recreated slot. + env.reconcileCR(t, cr.Name) + + if got := env.listChildReservations(t, cr.Name); len(got) != 1 { + t.Errorf("expected 1 reservation after recreation, got %d", len(got)) + } + crState := env.getCR(t, cr.Name) + if !meta.IsStatusConditionTrue(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Errorf("expected CR to be Ready=True after slot recreation") + } + }) + + t.Run("AcceptedAt: set when CR accepted", func(t *testing.T) { + env := newCRIntegrationEnv(t) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + env.reconcileCR(t, cr.Name) + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + crState := env.getCR(t, cr.Name) + if !meta.IsStatusConditionTrue(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Fatalf("expected CR to be Ready=True") + } + if crState.Status.AcceptedAt == nil { + t.Errorf("expected AcceptedAt to be set on acceptance") + } + if crState.Status.AcceptedAmount == nil { + t.Errorf("expected AcceptedAmount to be set on acceptance") + } else if crState.Status.AcceptedAmount.Cmp(resource.MustParse("4Gi")) != 0 { + t.Errorf("AcceptedAmount: want 4Gi, got %s", crState.Status.AcceptedAmount.String()) + } + }) + + t.Run("resize failure: rolls back to AcceptedAmount, prior slot preserved", func(t *testing.T) { + // Scheduler: accepts the first placement call (initial 4 GiB slot), rejects all subsequent. + objects := []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")} + env := newIntgEnv(t, objects, intgAcceptFirstScheduler(1)) + defer env.close() + + cr := intgCRAllowRejection("my-cr", "uuid-resize-0001", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + // Phase 1: accept at 4 GiB (1 slot). Uses 1 scheduler call. + intgDriveToTerminal(t, env, []string{cr.Name}) + var crState v1alpha1.CommittedResource + if err := env.k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &crState); err != nil { + t.Fatalf("get CR: %v", err) + } + if !meta.IsStatusConditionTrue(crState.Status.Conditions, v1alpha1.CommittedResourceConditionReady) { + t.Fatalf("phase 1: expected CR to be Ready=True after initial placement") + } + if crState.Status.AcceptedAmount == nil || crState.Status.AcceptedAmount.Cmp(resource.MustParse("4Gi")) != 0 { + t.Fatalf("phase 1: AcceptedAmount must be 4Gi, got %v", crState.Status.AcceptedAmount) + } + + // Phase 2: resize to 8 GiB (needs 2 slots). Scheduler has no more accepts. + patch := client.MergeFrom(crState.DeepCopy()) + crState.Spec.Amount = resource.MustParse("8Gi") + if err := env.k8sClient.Patch(context.Background(), &crState, patch); err != nil { + t.Fatalf("patch CR to 8Gi: %v", err) + } + + ctx := context.Background() + crReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: cr.Name}} + + // CR controller: applyReservationState bumps gen on existing slot, creates 2nd slot. + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + // Reservation controller: existing slot echoes new ParentGeneration (no scheduler call); + // new slot calls scheduler → rejected. + var resList v1alpha1.ReservationList + env.k8sClient.List(ctx, &resList, client.MatchingLabels{ //nolint:errcheck + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }) + for _, res := range resList.Items { + resReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + } + // CR controller: detects 2nd slot Ready=False → rollbackToAccepted (keeps 1 slot) → Rejected. + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + + // Rollback must preserve 1 slot (matching AcceptedAmount=4Gi), not delete all. + var finalList v1alpha1.ReservationList + if err := env.k8sClient.List(ctx, &finalList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + if len(finalList.Items) != 1 { + t.Errorf("resize rollback: want 1 slot (AcceptedAmount), got %d", len(finalList.Items)) + } + intgAssertCRCondition(t, env.k8sClient, []string{cr.Name}, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected) + }) + + t.Run("AllowRejection=false: eventually accepted after scheduler starts accepting", func(t *testing.T) { + // Scheduler rejects the first 2 calls (one per reservation controller reconcile pair), + // then accepts all subsequent. AllowRejection=false means the CR controller retries rather + // than rejecting, so the CR must eventually reach Accepted once the scheduler cooperates. + objects := []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")} + env := newIntgEnv(t, objects, intgRejectFirstScheduler(2)) + defer env.close() + + cr := newTestCommittedResource("my-cr", v1alpha1.CommitmentStatusConfirmed) + // AllowRejection stays false (default), so placement failure must requeue, not reject. + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + + ctx := context.Background() + crReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: cr.Name}} + for range 3 { + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + var resList v1alpha1.ReservationList + env.k8sClient.List(ctx, &resList, client.MatchingLabels{ //nolint:errcheck + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }) + for _, res := range resList.Items { + resReq := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + env.resController.Reconcile(ctx, resReq) //nolint:errcheck + } + env.crController.Reconcile(ctx, crReq) //nolint:errcheck + } + + var final v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: cr.Name}, &final); err != nil { + t.Fatalf("get CR: %v", err) + } + cond := meta.FindStatusCondition(final.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + t.Fatalf("no Ready condition after retries") + } + if cond.Reason == v1alpha1.CommittedResourceReasonRejected { + t.Errorf("AllowRejection=false: CR must not be Rejected, got Reason=%s", cond.Reason) + } + if cond.Status != metav1.ConditionTrue || cond.Reason != v1alpha1.CommittedResourceReasonAccepted { + t.Errorf("AllowRejection=false: expected Ready=True/Accepted after retries, got Ready=%s/Reason=%s", cond.Status, cond.Reason) + } + }) +} diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index 888d37018..fe05fcc20 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -5,103 +5,73 @@ package commitments import ( "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Config aggregates configuration for all commitments components. +// Each controller and the API have their own sub-struct so that unrelated +// fields are never visible to the wrong component. type Config struct { + ReservationController ReservationControllerConfig `json:"committedResourceReservationController"` + CommittedResourceController CommittedResourceControllerConfig `json:"committedResourceController"` + API APIConfig `json:"committedResourceAPI"` - // RequeueIntervalActive is the interval for requeueing active reservations for periodic verification. - RequeueIntervalActive time.Duration `json:"committedResourceRequeueIntervalActive"` - // RequeueIntervalRetry is the interval for requeueing when retrying after knowledge is not ready. - RequeueIntervalRetry time.Duration `json:"committedResourceRequeueIntervalRetry"` - // AllocationGracePeriod is the time window after a VM is allocated to a reservation - // during which it's expected to appear on the target host. VMs not confirmed within - // this period are considered stale and removed from the reservation. - AllocationGracePeriod time.Duration `json:"committedResourceAllocationGracePeriod"` - // RequeueIntervalGracePeriod is the interval for requeueing when VMs are in grace period. - // Shorter than RequeueIntervalActive for faster verification of new allocations. - RequeueIntervalGracePeriod time.Duration `json:"committedResourceRequeueIntervalGracePeriod"` - // PipelineDefault is the default pipeline used for scheduling committed resource reservations. - PipelineDefault string `json:"committedResourcePipelineDefault"` - - // SchedulerURL is the endpoint of the nova external scheduler - SchedulerURL string `json:"schedulerURL"` - - // DatasourceName is the name of the Datasource CRD that provides database connection info. - // Used to query VM state for report-usage. If empty, report-usage returns an error. + // DatasourceName is the name of the Datasource CRD that provides database + // connection info. Used to construct the UsageDBClient for report-usage. DatasourceName string `json:"datasourceName,omitempty"` +} - // FlavorGroupPipelines maps flavor group names to pipeline names. - // Example: {"2152": "kvm-hana-bin-packing", "2101": "kvm-general-purpose-load-balancing", "*": "kvm-general-purpose-load-balancing"} - // Used to select different scheduling pipelines based on flavor group characteristics. - FlavorGroupPipelines map[string]string `json:"committedResourceFlavorGroupPipelines,omitempty"` - - // API configuration - - // ChangeAPIWatchReservationsTimeout defines how long to wait for reservations to become ready before timing out and rolling back. - ChangeAPIWatchReservationsTimeout time.Duration `json:"committedResourceChangeAPIWatchReservationsTimeout"` - - // ChangeAPIWatchReservationsPollInterval defines how frequently to poll reservation status during watch. - ChangeAPIWatchReservationsPollInterval time.Duration `json:"committedResourceChangeAPIWatchReservationsPollInterval"` - - // EnableChangeCommitmentsAPI controls whether the change-commitments API endpoint is active. - // When false, the endpoint will return HTTP 503 Service Unavailable. - // The info endpoint remains available for health checks. - EnableChangeCommitmentsAPI bool `json:"committedResourceEnableChangeCommitmentsAPI"` - - // EnableReportUsageAPI controls whether the report-usage API endpoint is active. - // When false, the endpoint will return HTTP 503 Service Unavailable. - // This can be used as an emergency switch if the usage reporting is causing issues. - EnableReportUsageAPI bool `json:"committedResourceEnableReportUsageAPI"` +// ReservationControllerConfig holds tuning knobs for the Reservation CRD controller. +type ReservationControllerConfig struct { + // RequeueIntervalActive is how often to re-verify a healthy Reservation CRD. + RequeueIntervalActive metav1.Duration `json:"requeueIntervalActive"` + // RequeueIntervalRetry is the back-off interval when knowledge is unavailable. + RequeueIntervalRetry metav1.Duration `json:"requeueIntervalRetry"` + // RequeueIntervalGracePeriod is how often to re-check while a VM allocation + // is still within AllocationGracePeriod. Shorter than RequeueIntervalActive. + RequeueIntervalGracePeriod metav1.Duration `json:"requeueIntervalGracePeriod"` + // AllocationGracePeriod is the time window after a VM is allocated to a + // reservation during which it's expected to appear on the target host. + // VMs not confirmed within this period are considered stale and removed. + AllocationGracePeriod metav1.Duration `json:"allocationGracePeriod"` + // SchedulerURL is the endpoint of the nova external scheduler. + SchedulerURL string `json:"schedulerURL"` + // PipelineDefault is the fallback pipeline when no FlavorGroupPipelines entry matches. + PipelineDefault string `json:"pipelineDefault"` + // FlavorGroupPipelines maps flavor group IDs to pipeline names; "*" acts as catch-all. + FlavorGroupPipelines map[string]string `json:"flavorGroupPipelines,omitempty"` +} - // EnableReportCapacityAPI controls whether the report-capacity API endpoint is active. - // When false, the endpoint will return HTTP 503 Service Unavailable. - // This can be used as an emergency switch if the capacity reporting is causing issues. - EnableReportCapacityAPI bool `json:"committedResourceEnableReportCapacityAPI"` +// CommittedResourceControllerConfig holds tuning knobs for the CommittedResource CRD controller. +type CommittedResourceControllerConfig struct { + // RequeueIntervalRetry is the back-off interval when placement is pending or failed. + RequeueIntervalRetry metav1.Duration `json:"requeueIntervalRetry"` } -// ApplyDefaults fills in any unset values with defaults. -func (c *Config) ApplyDefaults() { - defaults := DefaultConfig() - if c.RequeueIntervalActive == 0 { - c.RequeueIntervalActive = defaults.RequeueIntervalActive - } - if c.RequeueIntervalRetry == 0 { - c.RequeueIntervalRetry = defaults.RequeueIntervalRetry - } - if c.RequeueIntervalGracePeriod == 0 { - c.RequeueIntervalGracePeriod = defaults.RequeueIntervalGracePeriod - } - if c.AllocationGracePeriod == 0 { - c.AllocationGracePeriod = defaults.AllocationGracePeriod - } - if c.PipelineDefault == "" { - c.PipelineDefault = defaults.PipelineDefault - } - if c.SchedulerURL == "" { - c.SchedulerURL = defaults.SchedulerURL - } - if c.ChangeAPIWatchReservationsTimeout == 0 { - c.ChangeAPIWatchReservationsTimeout = defaults.ChangeAPIWatchReservationsTimeout - } - if c.ChangeAPIWatchReservationsPollInterval == 0 { - c.ChangeAPIWatchReservationsPollInterval = defaults.ChangeAPIWatchReservationsPollInterval - } - // Note: EnableChangeCommitmentsAPI, EnableReportUsageAPI, EnableReportCapacityAPI - // are booleans where false is a valid value, so we don't apply defaults for them +// APIConfig holds configuration for the LIQUID commitment HTTP endpoints. +type APIConfig struct { + // EnableChangeCommitments controls whether the change-commitments endpoint is active. + // When false the endpoint returns HTTP 503; the info endpoint remains available. + EnableChangeCommitments bool `json:"enableChangeCommitments"` + // EnableReportUsage controls whether the report-usage endpoint is active. + EnableReportUsage bool `json:"enableReportUsage"` + // EnableReportCapacity controls whether the report-capacity endpoint is active. + EnableReportCapacity bool `json:"enableReportCapacity"` + // WatchTimeout is how long the change-commitments handler polls CommittedResource + // CRD conditions before giving up and rolling back. + WatchTimeout metav1.Duration `json:"watchTimeout"` + // WatchPollInterval is how frequently the change-commitments handler polls + // CommittedResource CRD conditions while waiting for the controller outcome. + WatchPollInterval metav1.Duration `json:"watchPollInterval"` } -func DefaultConfig() Config { - return Config{ - RequeueIntervalActive: 5 * time.Minute, - RequeueIntervalRetry: 1 * time.Minute, - RequeueIntervalGracePeriod: 1 * time.Minute, - AllocationGracePeriod: 15 * time.Minute, - PipelineDefault: "kvm-general-purpose-load-balancing", - SchedulerURL: "http://localhost:8080/scheduler/nova/external", - ChangeAPIWatchReservationsTimeout: 10 * time.Second, - ChangeAPIWatchReservationsPollInterval: 500 * time.Millisecond, - EnableChangeCommitmentsAPI: true, - EnableReportUsageAPI: true, - EnableReportCapacityAPI: true, +func DefaultAPIConfig() APIConfig { + return APIConfig{ + EnableChangeCommitments: true, + EnableReportUsage: true, + EnableReportCapacity: true, + WatchTimeout: metav1.Duration{Duration: 10 * time.Second}, + WatchPollInterval: metav1.Duration{Duration: 500 * time.Millisecond}, } } diff --git a/internal/scheduling/reservations/commitments/e2e_checks.go b/internal/scheduling/reservations/commitments/e2e_checks.go index 2292bcaa1..cd4b15d05 100644 --- a/internal/scheduling/reservations/commitments/e2e_checks.go +++ b/internal/scheduling/reservations/commitments/e2e_checks.go @@ -4,13 +4,17 @@ package commitments import ( + "bytes" "context" "encoding/json" "fmt" "io" "log/slog" "net/http" + "strings" + "time" + . "github.com/majewsky/gg/option" liquid "github.com/sapcc/go-api-declarations/liquid" "github.com/sapcc/go-bits/must" ) @@ -19,38 +23,46 @@ const ( // Default URL for the commitments API endpoint. // This should match the service name in the helm chart. defaultCommitmentsAPIURL = "http://cortex-nova-scheduler:8080" + + // defaultE2EProjectUUID is a well-known fake project UUID used when no TestProjectID is configured. + // It is intentionally not a real OpenStack project — commitments created under it self-expire. + defaultE2EProjectUUID = "00000000-0000-0000-0000-000000000e2e" ) // E2EChecksConfig holds the configuration for CR e2e checks. type E2EChecksConfig struct { - // Base URL for the commitments API. If empty, defaults to defaultCommitmentsAPIURL. + // BaseURL for the commitments API. If empty, defaults to defaultCommitmentsAPIURL. BaseURL string `json:"baseURL"` + // RoundTripCheck holds optional overrides for the round-trip check. + // If nil, defaults are used: testProjectID = defaultE2EProjectUUID, az = "". + RoundTripCheck *E2ERoundTripConfig `json:"roundTripCheck,omitempty"` +} + +// E2ERoundTripConfig holds optional overrides for the create→delete round-trip e2e check. +type E2ERoundTripConfig struct { + // AZ is the availability zone to use (e.g. "qa-de-1d"). Defaults to "" if not set. + AZ string `json:"az"` + // TestProjectID is the OpenStack project UUID to create test commitments under. + // Defaults to defaultE2EProjectUUID if not set. + TestProjectID string `json:"testProjectID"` } -// CheckCommitmentsInfoEndpoint sends a GET request to the /commitments/v1/info endpoint -// and verifies that it returns HTTP 200 with a valid ServiceInfo response. +// CheckCommitmentsInfoEndpoint verifies that GET /commitments/v1/info returns 200 with a valid ServiceInfo. func CheckCommitmentsInfoEndpoint(ctx context.Context, config E2EChecksConfig) { - baseURL := config.BaseURL - if baseURL == "" { - baseURL = defaultCommitmentsAPIURL - } + baseURL := e2eBaseURL(config) apiURL := baseURL + "/commitments/v1/info" slog.Info("checking commitments info endpoint", "apiURL", apiURL) httpReq := must.Return(http.NewRequestWithContext(ctx, http.MethodGet, apiURL, http.NoBody)) httpReq.Header.Set("Accept", "application/json") - //nolint:bodyclose // Body is closed in the deferred function below. + //nolint:bodyclose resp := must.Return(http.DefaultClient.Do(httpReq)) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { bodyBytes := must.Return(io.ReadAll(resp.Body)) - slog.Error("commitments info API returned non-200 status code", - "statusCode", resp.StatusCode, - "responseBody", string(bodyBytes), - ) - panic(fmt.Sprintf("commitments info API returned status %d, expected 200", resp.StatusCode)) + panic(fmt.Sprintf("commitments info API returned status %d: %s", resp.StatusCode, bodyBytes)) } var serviceInfo liquid.ServiceInfo @@ -58,20 +70,208 @@ func CheckCommitmentsInfoEndpoint(ctx context.Context, config E2EChecksConfig) { panic(fmt.Sprintf("failed to decode ServiceInfo response: %v", err)) } - // Basic validation of the response if serviceInfo.Version < 0 { slog.Warn("commitments info returned version -1, knowledge may not be ready yet") } - slog.Info("commitments info endpoint check passed", "version", serviceInfo.Version, "resourceCount", len(serviceInfo.Resources), ) } +// CheckCommitmentsRoundTrip iterates all HandlesCommitments resources from /info and for each one: +// 1. Creates a confirmed test commitment (amount=2, expires in 5 minutes) +// 2. If accepted: calls the usage API to verify it returns 200, then deletes the commitment +// 3. If rejected: logs the reason and continues — capacity rejection is not an error +// +// Panics on infrastructure failures (non-200 from the API, deletion failure after acceptance). +func CheckCommitmentsRoundTrip(ctx context.Context, config E2EChecksConfig) { + baseURL := e2eBaseURL(config) + az := liquid.AvailabilityZone("") + projectID := liquid.ProjectUUID(defaultE2EProjectUUID) + if rt := config.RoundTripCheck; rt != nil { + if rt.AZ != "" { + az = liquid.AvailabilityZone(rt.AZ) + } + if rt.TestProjectID != "" { + projectID = liquid.ProjectUUID(rt.TestProjectID) + } + } + + serviceInfo := e2eFetchServiceInfo(ctx, baseURL) + + checked := 0 + for resourceName, resInfo := range serviceInfo.Resources { + if !resInfo.HandlesCommitments { + continue + } + e2eRoundTripResource(ctx, baseURL, serviceInfo.Version, az, projectID, resourceName) + checked++ + } + + if checked == 0 { + slog.Warn("round-trip check: no HandlesCommitments resources found in /info — nothing checked") + } +} + +// e2eRoundTripResource runs the create→usageCheck→delete cycle for one resource. +func e2eRoundTripResource( + ctx context.Context, + baseURL string, + infoVersion int64, + az liquid.AvailabilityZone, + projectID liquid.ProjectUUID, + resourceName liquid.ResourceName, +) { + + testUUID := liquid.CommitmentUUID(fmt.Sprintf("e2e-%d", time.Now().UnixMilli())) + expiresAt := time.Now().Add(5 * time.Minute) + const amount = uint64(2) + + createReq := liquid.CommitmentChangeRequest{ + InfoVersion: infoVersion, + AZ: az, + ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ + projectID: { + ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ + resourceName: { + TotalConfirmedAfter: amount, + Commitments: []liquid.Commitment{{ + UUID: testUUID, + Amount: amount, + NewStatus: Some(liquid.CommitmentStatusConfirmed), + ExpiresAt: expiresAt, + }}, + }, + }, + }, + }, + } + + slog.Info("round-trip check: creating test commitment", + "resource", resourceName, "uuid", testUUID, "project", projectID, "az", az) + + rejectionReason := e2eSendChangeCommitments(ctx, baseURL, createReq) + if rejectionReason != "" { + // Only capacity rejections (no hosts available) are expected in production clusters. + // Any other reason (flavor group ineligible, config error, timeout) indicates a + // regression and should surface as a failure. + if !strings.Contains(rejectionReason, "no hosts found") { + panic(fmt.Sprintf("round-trip check: commitment rejected with unexpected reason for resource %s: %s", resourceName, rejectionReason)) + } + slog.Info("round-trip check: commitment rejected — no capacity, continuing", + "resource", resourceName, "reason", rejectionReason) + return + } + slog.Info("round-trip check: commitment accepted", "resource", resourceName, "uuid", testUUID) + + // Register cleanup immediately so it runs even if the usage check panics. + defer func() { + deleteReq := liquid.CommitmentChangeRequest{ + InfoVersion: infoVersion, + AZ: az, + ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ + projectID: { + ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ + resourceName: { + TotalConfirmedBefore: amount, + Commitments: []liquid.Commitment{{ + UUID: testUUID, + Amount: amount, + OldStatus: Some(liquid.CommitmentStatusConfirmed), + NewStatus: None[liquid.CommitmentStatus](), + ExpiresAt: expiresAt, + }}, + }, + }, + }, + }, + } + slog.Info("round-trip check: deleting test commitment", "resource", resourceName, "uuid", testUUID) + if reason := e2eSendChangeCommitments(ctx, baseURL, deleteReq); reason != "" { + panic(fmt.Sprintf("round-trip check: delete of test commitment %s was rejected: %s", testUUID, reason)) + } + slog.Info("round-trip check: commitment deleted", "resource", resourceName, "uuid", testUUID) + }() + + // Smoke-check the usage API: verifies the usage calculation pipeline works for this project. + e2eCheckUsageAPI(ctx, baseURL, az, projectID) +} + +// e2eCheckUsageAPI calls POST /commitments/v1/projects/:id/report-usage and verifies 200. +// The usage report for a project with no VMs will show zero usage — we only verify the endpoint works. +func e2eCheckUsageAPI(ctx context.Context, baseURL string, az liquid.AvailabilityZone, projectID liquid.ProjectUUID) { + usageReq := liquid.ServiceUsageRequest{AllAZs: []liquid.AvailabilityZone{az}} + body := must.Return(json.Marshal(usageReq)) + url := fmt.Sprintf("%s/commitments/v1/projects/%s/report-usage", baseURL, projectID) + httpReq := must.Return(http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))) + httpReq.Header.Set("Content-Type", "application/json") + + //nolint:bodyclose + resp := must.Return(http.DefaultClient.Do(httpReq)) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bodyBytes := must.Return(io.ReadAll(resp.Body)) + panic(fmt.Sprintf("usage API returned %d: %s", resp.StatusCode, bodyBytes)) + } + slog.Info("round-trip check: usage API returned 200", "project", projectID) +} + +// e2eSendChangeCommitments sends a change-commitments request. +// Panics on HTTP non-200 (infrastructure error). +// Returns the rejection reason on 200+rejection (expected for capacity-constrained clusters). +// Returns "" on success. +func e2eSendChangeCommitments(ctx context.Context, baseURL string, req liquid.CommitmentChangeRequest) string { + body := must.Return(json.Marshal(req)) + httpReq := must.Return(http.NewRequestWithContext(ctx, http.MethodPost, + baseURL+"/commitments/v1/change-commitments", bytes.NewReader(body))) + httpReq.Header.Set("Content-Type", "application/json") + + //nolint:bodyclose + resp := must.Return(http.DefaultClient.Do(httpReq)) + defer resp.Body.Close() + respBody := must.Return(io.ReadAll(resp.Body)) + + if resp.StatusCode != http.StatusOK { + panic(fmt.Sprintf("change-commitments returned %d: %s", resp.StatusCode, respBody)) + } + var result liquid.CommitmentChangeResponse + if err := json.Unmarshal(respBody, &result); err != nil { + panic(fmt.Sprintf("failed to decode change-commitments response: %v", err)) + } + return result.RejectionReason +} + +// e2eFetchServiceInfo fetches and decodes /info. Panics on failure. +func e2eFetchServiceInfo(ctx context.Context, baseURL string) liquid.ServiceInfo { + httpReq := must.Return(http.NewRequestWithContext(ctx, http.MethodGet, + baseURL+"/commitments/v1/info", http.NoBody)) + httpReq.Header.Set("Accept", "application/json") + //nolint:bodyclose + resp := must.Return(http.DefaultClient.Do(httpReq)) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bodyBytes := must.Return(io.ReadAll(resp.Body)) + panic(fmt.Sprintf("info endpoint returned %d: %s", resp.StatusCode, bodyBytes)) + } + var info liquid.ServiceInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + panic(fmt.Sprintf("failed to decode ServiceInfo: %v", err)) + } + return info +} + +func e2eBaseURL(config E2EChecksConfig) string { + if config.BaseURL != "" { + return config.BaseURL + } + return defaultCommitmentsAPIURL +} + // RunCommitmentsE2EChecks runs all e2e checks for the commitments API. func RunCommitmentsE2EChecks(ctx context.Context, config E2EChecksConfig) { slog.Info("running commitments e2e checks") CheckCommitmentsInfoEndpoint(ctx, config) + CheckCommitmentsRoundTrip(ctx, config) slog.Info("all commitments e2e checks passed") } diff --git a/internal/scheduling/reservations/commitments/field_index.go b/internal/scheduling/reservations/commitments/field_index.go new file mode 100644 index 000000000..40760655d --- /dev/null +++ b/internal/scheduling/reservations/commitments/field_index.go @@ -0,0 +1,63 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "errors" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +const idxCommittedResourceByUUID = "spec.commitmentUUID" +const idxReservationByCommitmentUUID = "spec.committedResourceReservation.commitmentUUID" + +// IndexFields registers field indexes required by the CommittedResource controller. +func IndexFields(ctx context.Context, mcl *multicluster.Client) error { + log := logf.FromContext(ctx) + log.Info("Setting up field indexes for the CommittedResource controller") + if err := mcl.IndexField(ctx, + &v1alpha1.CommittedResource{}, + &v1alpha1.CommittedResourceList{}, + idxCommittedResourceByUUID, + func(obj client.Object) []string { + cr, ok := obj.(*v1alpha1.CommittedResource) + if !ok { + log.Error(errors.New("unexpected type"), "expected CommittedResource", "object", obj) + return nil + } + if cr.Spec.CommitmentUUID == "" { + return nil + } + return []string{cr.Spec.CommitmentUUID} + }, + ); err != nil { + log.Error(err, "failed to set up index for commitmentUUID") + return err + } + if err := mcl.IndexField(ctx, + &v1alpha1.Reservation{}, + &v1alpha1.ReservationList{}, + idxReservationByCommitmentUUID, + func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok { + log.Error(errors.New("unexpected type"), "expected Reservation", "object", obj) + return nil + } + if res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.CommitmentUUID == "" { + return nil + } + return []string{res.Spec.CommittedResourceReservation.CommitmentUUID} + }, + ); err != nil { + log.Error(err, "failed to set up index for reservation commitmentUUID") + return err + } + log.Info("Successfully set up field indexes") + return nil +} diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go new file mode 100644 index 000000000..138f3c74c --- /dev/null +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -0,0 +1,618 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +// Table-driven integration tests for the committed-resource lifecycle. +// +// Each test case wires CommittedResourceController and CommitmentReservationController +// against a shared fake k8s client and a mock Nova scheduler, then drives both +// controllers synchronously until every CR reaches a terminal condition. +// +// Terminal conditions (no further reconcile expected without external input): +// - Ready=True / Accepted +// - Ready=False / Rejected +// - Ready=False / Planned (controller waits for StartTime) +// - Ready=False / Expired (controller has cleaned up children) +// - Ready=False / Superseded + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// ============================================================================ +// Test cases +// ============================================================================ + +// CRIntegrationTestCase defines one end-to-end scenario for the committed-resource +// lifecycle spanning both controllers and the mock scheduler. +type CRIntegrationTestCase struct { + Name string + + // Initial cluster state. + Hypervisors []*hv1.Hypervisor + ExistingReservations []*v1alpha1.Reservation // pre-placed slots (for expiry/supersede scenarios) + + // CRs to create and drive to terminal state. + CommittedResources []*v1alpha1.CommittedResource + + // When true the mock scheduler returns an empty hosts list (NoHostsFound). + SchedulerRejects bool + // SchedulerAcceptFirst, when > 0, makes the mock scheduler accept only the first N + // placement calls and reject all subsequent ones. Used to test partial placement + // (e.g. first slot placed, second slot rejected). Takes precedence over SchedulerRejects. + SchedulerAcceptFirst int + + // Expected state after all CRs reach a terminal condition. + ExpectedSlots int // total Reservation CRDs remaining in the store + AcceptedCRs []string // CRs expected Ready=True / Accepted + RejectedCRs []string // CRs expected Ready=False / Rejected + PlannedCRs []string // CRs expected Ready=False / Planned + ExpiredCRs []string // CRs expected Ready=False / Expired + SupersededCRs []string // CRs expected Ready=False / Superseded +} + +func TestCRIntegration(t *testing.T) { + testCases := []CRIntegrationTestCase{ + // ------------------------------------------------------------------ + // Acceptance: slot count from commitment amount + // ------------------------------------------------------------------ + { + Name: "single confirmed CR: one slot placed, CR accepted", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-1", "uuid-intg-0001", v1alpha1.CommitmentStatusConfirmed), + }, + ExpectedSlots: 1, + AcceptedCRs: []string{"cr-1"}, + }, + { + // 8 GiB commitment with the default 4 GiB test flavor → 2 slots + Name: "large CR: commitment amount spans multiple flavors, two slots placed", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCRAmount("cr-large", "uuid-intg-0002", v1alpha1.CommitmentStatusConfirmed, "8Gi"), + }, + ExpectedSlots: 2, + AcceptedCRs: []string{"cr-large"}, + }, + // ------------------------------------------------------------------ + // Pending / guaranteed: same placement path as confirmed + // ------------------------------------------------------------------ + { + Name: "pending CR: slot placed, CR accepted", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-pending", "uuid-intg-0003", v1alpha1.CommitmentStatusPending), + }, + ExpectedSlots: 1, + AcceptedCRs: []string{"cr-pending"}, + }, + { + Name: "guaranteed CR: slot placed, CR accepted", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-guaranteed", "uuid-intg-0004", v1alpha1.CommitmentStatusGuaranteed), + }, + ExpectedSlots: 1, + AcceptedCRs: []string{"cr-guaranteed"}, + }, + // ------------------------------------------------------------------ + // Planned: no slots, condition stays Planned + // ------------------------------------------------------------------ + { + Name: "planned CR: no slots created, condition stays Planned", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-planned", "uuid-intg-0005", v1alpha1.CommitmentStatusPlanned), + }, + ExpectedSlots: 0, + PlannedCRs: []string{"cr-planned"}, + }, + // ------------------------------------------------------------------ + // Rejection paths + // ------------------------------------------------------------------ + { + Name: "scheduler returns no hosts: CR rejected and slots cleaned up", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCRAllowRejection("cr-rej", "uuid-intg-0006", v1alpha1.CommitmentStatusConfirmed), + }, + SchedulerRejects: true, + ExpectedSlots: 0, + RejectedCRs: []string{"cr-rej"}, + }, + { + // Reservation controller detects the empty hosts list before calling the scheduler. + Name: "no hypervisors in cluster: CR rejected with NoHostsAvailable", + Hypervisors: []*hv1.Hypervisor{}, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCRAllowRejection("cr-nohosts", "uuid-intg-0007", v1alpha1.CommitmentStatusConfirmed), + }, + ExpectedSlots: 0, + RejectedCRs: []string{"cr-nohosts"}, + }, + // ------------------------------------------------------------------ + // Multiple independent CRs + // ------------------------------------------------------------------ + { + Name: "two CRs with different UUIDs: each gets its own slot, both accepted", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + intgHypervisor("host-2"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-a", "uuid-intg-0008", v1alpha1.CommitmentStatusConfirmed), + intgCR("cr-b", "uuid-intg-0009", v1alpha1.CommitmentStatusConfirmed), + }, + ExpectedSlots: 2, + AcceptedCRs: []string{"cr-a", "cr-b"}, + }, + { + // One CR in planned state should not block the other from being accepted. + Name: "one planned CR and one confirmed CR: only confirmed CR gets a slot", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-plan", "uuid-intg-0010", v1alpha1.CommitmentStatusPlanned), + intgCR("cr-conf", "uuid-intg-0011", v1alpha1.CommitmentStatusConfirmed), + }, + ExpectedSlots: 1, + PlannedCRs: []string{"cr-plan"}, + AcceptedCRs: []string{"cr-conf"}, + }, + // ------------------------------------------------------------------ + // Inactive states: existing slots must be cleaned up + // ------------------------------------------------------------------ + { + Name: "expired CR with existing slot: slot deleted, CR marked inactive", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + ExistingReservations: []*v1alpha1.Reservation{ + intgExistingReservation("cr-expire-0", "uuid-intg-0012"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-expire", "uuid-intg-0012", v1alpha1.CommitmentStatusExpired), + }, + ExpectedSlots: 0, + ExpiredCRs: []string{"cr-expire"}, + }, + { + Name: "superseded CR with existing slot: slot deleted, CR marked inactive", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + ExistingReservations: []*v1alpha1.Reservation{ + intgExistingReservation("cr-supersede-0", "uuid-intg-0013"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCR("cr-supersede", "uuid-intg-0013", v1alpha1.CommitmentStatusSuperseded), + }, + ExpectedSlots: 0, + SupersededCRs: []string{"cr-supersede"}, + }, + // ------------------------------------------------------------------ + // Spec validation: unknown flavor group + // ------------------------------------------------------------------ + { + // ApplyCommitmentState returns "flavor group not found" which triggers + // rollback+Rejected (AllowRejection=true); no child slots are ever created. + Name: "unknown flavor group: CR rejected, no slots created", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCRUnknownFlavorGroup("cr-unk", "uuid-intg-0014", v1alpha1.CommitmentStatusConfirmed), + }, + ExpectedSlots: 0, + RejectedCRs: []string{"cr-unk"}, + }, + // ------------------------------------------------------------------ + // Partial placement: first slot placed, second slot rejected + // ------------------------------------------------------------------ + { + // 8 GiB CR needs 2 slots. Scheduler accepts the first call (slot 0 placed) + // then rejects the second (slot 1 gets NoHostsFound). With AllowRejection=true + // the CR controller rolls back: deletes both slots and sets Rejected. + Name: "partial placement: first slot placed, second slot rejected, CR rolled back", + Hypervisors: []*hv1.Hypervisor{ + intgHypervisor("host-1"), + }, + CommittedResources: []*v1alpha1.CommittedResource{ + intgCRAmountAllowRejection("cr-partial", "uuid-intg-0015", v1alpha1.CommitmentStatusConfirmed, "8Gi"), + }, + SchedulerAcceptFirst: 1, + ExpectedSlots: 0, + RejectedCRs: []string{"cr-partial"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + runCRIntegrationTestCase(t, tc) + }) + } +} + +// ============================================================================ +// Runner +// ============================================================================ + +func runCRIntegrationTestCase(t *testing.T, tc CRIntegrationTestCase) { + t.Helper() + + schedulerFn := intgAcceptScheduler + switch { + case tc.SchedulerAcceptFirst > 0: + schedulerFn = intgAcceptFirstScheduler(tc.SchedulerAcceptFirst) + case tc.SchedulerRejects: + schedulerFn = intgRejectScheduler + } + + objects := []client.Object{newTestFlavorKnowledge()} + for _, hv := range tc.Hypervisors { + objects = append(objects, hv) + } + for _, res := range tc.ExistingReservations { + objects = append(objects, res) + } + + env := newIntgEnv(t, objects, schedulerFn) + defer env.close() + + crNames := make([]string, len(tc.CommittedResources)) + for i, cr := range tc.CommittedResources { + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR %s: %v", cr.Name, err) + } + crNames[i] = cr.Name + } + + intgDriveToTerminal(t, env, crNames) + + // Assert total reservation slot count. + var resList v1alpha1.ReservationList + if err := env.k8sClient.List(context.Background(), &resList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + t.Fatalf("list reservations: %v", err) + } + if len(resList.Items) != tc.ExpectedSlots { + t.Errorf("reservation slots: want %d, got %d", tc.ExpectedSlots, len(resList.Items)) + } + + // Assert CR conditions. + intgAssertCRCondition(t, env.k8sClient, tc.AcceptedCRs, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted) + intgAssertCRCondition(t, env.k8sClient, tc.RejectedCRs, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected) + intgAssertCRCondition(t, env.k8sClient, tc.PlannedCRs, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonPlanned) + intgAssertCRCondition(t, env.k8sClient, tc.ExpiredCRs, metav1.ConditionFalse, string(v1alpha1.CommitmentStatusExpired)) + intgAssertCRCondition(t, env.k8sClient, tc.SupersededCRs, metav1.ConditionFalse, string(v1alpha1.CommitmentStatusSuperseded)) +} + +// ============================================================================ +// Integration environment +// ============================================================================ + +type intgEnv struct { + k8sClient client.Client + crController *CommittedResourceController + resController *CommitmentReservationController + schedulerSrv *httptest.Server +} + +func newIntgEnv(t *testing.T, initialObjects []client.Object, schedulerFn http.HandlerFunc) *intgEnv { + t.Helper() + scheme := newCRTestScheme(t) + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(initialObjects...). + WithStatusSubresource( + &v1alpha1.CommittedResource{}, + &v1alpha1.Reservation{}, + &v1alpha1.Knowledge{}, + ). + WithIndex(&v1alpha1.Reservation{}, idxReservationByCommitmentUUID, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation.CommitmentUUID == "" { + return nil + } + return []string{res.Spec.CommittedResourceReservation.CommitmentUUID} + }). + WithIndex(&v1alpha1.CommittedResource{}, idxCommittedResourceByUUID, func(obj client.Object) []string { + cr, ok := obj.(*v1alpha1.CommittedResource) + if !ok || cr.Spec.CommitmentUUID == "" { + return nil + } + return []string{cr.Spec.CommitmentUUID} + }). + Build() + + schedulerSrv := httptest.NewServer(schedulerFn) + + crCtrl := &CommittedResourceController{ + Client: k8sClient, + Scheme: scheme, + Conf: CommittedResourceControllerConfig{RequeueIntervalRetry: metav1.Duration{Duration: 5 * time.Minute}}, + } + resCtrl := &CommitmentReservationController{ + Client: k8sClient, + Scheme: scheme, + Conf: ReservationControllerConfig{ + SchedulerURL: schedulerSrv.URL, + AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}, + RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, + }, + } + if err := resCtrl.Init(context.Background(), resCtrl.Conf); err != nil { + t.Fatalf("resCtrl.Init: %v", err) + } + return &intgEnv{k8sClient: k8sClient, crController: crCtrl, resController: resCtrl, schedulerSrv: schedulerSrv} +} + +func (e *intgEnv) close() { e.schedulerSrv.Close() } + +// ============================================================================ +// Reconcile driver +// ============================================================================ + +// intgDriveToTerminal runs reconcile passes until every named CR has a terminal +// condition or the 5 s deadline is reached. +// +// One pass: +// 1. CR controller (adds finalizer / creates Reservation CRDs / handles inactive states) +// 2. Reservation controller ×2 per slot (first call sets TargetHost, second sets Ready=True) +// 3. CR controller again (picks up placement outcomes: Accepted or Rejected) +func intgDriveToTerminal(t *testing.T, env *intgEnv, crNames []string) { + t.Helper() + ctx := context.Background() + deadline := time.Now().Add(5 * time.Second) + + for { + if time.Now().After(deadline) { + for _, name := range crNames { + var cr v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: name}, &cr); err == nil { + t.Logf("CR %s: conditions=%v", name, cr.Status.Conditions) + } + } + t.Fatal("timed out waiting for CRs to reach terminal state") + } + + allDone := true + for _, name := range crNames { + var cr v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: name}, &cr); err != nil { + continue // deleted = done + } + if !intgIsTerminalCR(cr) { + allDone = false + } + } + if allDone { + return + } + + // Pass 1: CR controller. + for _, name := range crNames { + var cr v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: name}, &cr); err != nil { + continue + } + if intgIsTerminalCR(cr) { + continue + } + env.crController.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: name}}) //nolint:errcheck + } + + // Pass 2: Reservation controller (two reconciles per slot). + var resList v1alpha1.ReservationList + env.k8sClient.List(ctx, &resList, client.MatchingLabels{ //nolint:errcheck + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }) + for _, res := range resList.Items { + if intgIsTerminalReservation(res) { + continue + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: res.Name}} + env.resController.Reconcile(ctx, req) //nolint:errcheck + env.resController.Reconcile(ctx, req) //nolint:errcheck + } + + // Pass 3: CR controller picks up Reservation outcomes. + for _, name := range crNames { + var cr v1alpha1.CommittedResource + if err := env.k8sClient.Get(ctx, types.NamespacedName{Name: name}, &cr); err != nil { + continue + } + if intgIsTerminalCR(cr) { + continue + } + env.crController.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: name}}) //nolint:errcheck + } + } +} + +func intgIsTerminalCR(cr v1alpha1.CommittedResource) bool { + if !cr.DeletionTimestamp.IsZero() { + return false // needs one more reconcile to remove its finalizer + } + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + return false + } + if cond.Status == metav1.ConditionTrue { + return true + } + return cond.Reason == v1alpha1.CommittedResourceReasonRejected || + cond.Reason == v1alpha1.CommittedResourceReasonPlanned || + cond.Reason == string(v1alpha1.CommitmentStatusExpired) || + cond.Reason == string(v1alpha1.CommitmentStatusSuperseded) +} + +// intgIsTerminalReservation returns true once the Reservation controller has set any +// condition (Ready=True after placement, or Ready=False after rejection). +func intgIsTerminalReservation(res v1alpha1.Reservation) bool { + return meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) != nil +} + +// ============================================================================ +// Assertion helpers +// ============================================================================ + +func intgAssertCRCondition(t *testing.T, k8sClient client.Client, crNames []string, wantStatus metav1.ConditionStatus, wantReason string) { + t.Helper() + for _, name := range crNames { + var cr v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &cr); err != nil { + t.Errorf("CR %s not found: %v", name, err) + continue + } + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + t.Errorf("CR %s: no Ready condition", name) + continue + } + if cond.Status != wantStatus || cond.Reason != wantReason { + t.Errorf("CR %s: want Ready=%s/Reason=%s, got Ready=%s/Reason=%s", name, wantStatus, wantReason, cond.Status, cond.Reason) + } + } +} + +// ============================================================================ +// Scheduler handlers +// ============================================================================ + +func intgAcceptScheduler(w http.ResponseWriter, r *http.Request) { + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + json.NewEncoder(w).Encode(resp) //nolint:errcheck +} + +func intgRejectScheduler(w http.ResponseWriter, r *http.Request) { + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{}} + json.NewEncoder(w).Encode(resp) //nolint:errcheck +} + +// intgAcceptFirstScheduler returns a handler that accepts the first count placement calls +// and rejects all subsequent ones. Uses an atomic counter so concurrent calls are safe. +func intgAcceptFirstScheduler(count int) http.HandlerFunc { + var calls atomic.Int32 + return func(w http.ResponseWriter, r *http.Request) { + if int(calls.Add(1)) <= count { + intgAcceptScheduler(w, r) + } else { + intgRejectScheduler(w, r) + } + } +} + +// intgRejectFirstScheduler returns a handler that rejects the first count placement calls +// and accepts all subsequent ones. Used to test AllowRejection=false retry-until-success paths. +func intgRejectFirstScheduler(count int) http.HandlerFunc { + var calls atomic.Int32 + return func(w http.ResponseWriter, r *http.Request) { + if int(calls.Add(1)) <= count { + intgRejectScheduler(w, r) + } else { + intgAcceptScheduler(w, r) + } + } +} + +// ============================================================================ +// Test object builders +// ============================================================================ + +// intgHypervisor returns a minimal Hypervisor with the given name. +func intgHypervisor(name string) *hv1.Hypervisor { + return &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: name}} +} + +// intgCR returns a CommittedResource with the default 4 GiB amount. +// commitmentUUID must be unique per test case to avoid field-index collisions. +func intgCR(name, commitmentUUID string, state v1alpha1.CommitmentStatus) *v1alpha1.CommittedResource { + cr := newTestCommittedResource(name, state) + cr.Spec.CommitmentUUID = commitmentUUID + return cr +} + +// intgCRAmount returns a CommittedResource with a custom amount string (e.g. "8Gi"). +func intgCRAmount(name, commitmentUUID string, state v1alpha1.CommitmentStatus, amount string) *v1alpha1.CommittedResource { + cr := intgCR(name, commitmentUUID, state) + cr.Spec.Amount = resource.MustParse(amount) + return cr +} + +// intgCRAllowRejection returns a CommittedResource with AllowRejection=true so the +// controller rolls back and sets Rejected (rather than retrying indefinitely). +func intgCRAllowRejection(name, commitmentUUID string, state v1alpha1.CommitmentStatus) *v1alpha1.CommittedResource { + cr := intgCR(name, commitmentUUID, state) + cr.Spec.AllowRejection = true + return cr +} + +// intgCRAmountAllowRejection returns a CommittedResource with a custom amount and AllowRejection=true. +func intgCRAmountAllowRejection(name, commitmentUUID string, state v1alpha1.CommitmentStatus, amount string) *v1alpha1.CommittedResource { + cr := intgCRAmount(name, commitmentUUID, state, amount) + cr.Spec.AllowRejection = true + return cr +} + +// intgCRUnknownFlavorGroup returns a CommittedResource referencing a flavor group +// that does not exist in the Knowledge CRD, with AllowRejection=true so the +// controller reaches Rejected rather than retrying indefinitely. +func intgCRUnknownFlavorGroup(name, commitmentUUID string, state v1alpha1.CommitmentStatus) *v1alpha1.CommittedResource { + cr := intgCRAllowRejection(name, commitmentUUID, state) + cr.Spec.FlavorGroupName = "nonexistent-group" + return cr +} + +// intgExistingReservation returns a pre-placed Reservation tied to the given commitment UUID, +// used to verify that expiry/supersede paths delete children. +func intgExistingReservation(name, commitmentUUID string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: commitmentUUID, + }, + }, + } +} diff --git a/internal/scheduling/reservations/commitments/controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go similarity index 89% rename from internal/scheduling/reservations/commitments/controller.go rename to internal/scheduling/reservations/commitments/reservation_controller.go index b6078daf1..96d86aeb2 100644 --- a/internal/scheduling/reservations/commitments/controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -5,7 +5,6 @@ package commitments import ( "context" - "errors" "fmt" "time" @@ -25,7 +24,6 @@ import ( schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -39,11 +37,24 @@ type CommitmentReservationController struct { // Kubernetes scheme to use for the reservations. Scheme *runtime.Scheme // Configuration for the controller. - Conf Config + Conf ReservationControllerConfig // SchedulerClient for making scheduler API calls. SchedulerClient *reservations.SchedulerClient } +// echoParentGeneration copies Spec.CommittedResourceReservation.ParentGeneration to +// Status.CommittedResourceReservation.ObservedParentGeneration so the CommittedResource +// controller can confirm this reservation was processed for the current CR generation. +func echoParentGeneration(res *v1alpha1.Reservation) { + if res.Spec.CommittedResourceReservation == nil { + return + } + if res.Status.CommittedResourceReservation == nil { + res.Status.CommittedResourceReservation = &v1alpha1.CommittedResourceReservationStatus{} + } + res.Status.CommittedResourceReservation.ObservedParentGeneration = res.Spec.CommittedResourceReservation.ParentGeneration +} + // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. // Note: This controller only handles commitment reservations, as filtered by the predicate. @@ -78,6 +89,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Reason: "MissingResourceName", Message: "reservation has no resource name", }) + echoParentGeneration(&res) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { // Ignore not-found errors during background deletion @@ -91,9 +103,22 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, nil // Don't need to requeue. } - if meta.IsStatusConditionTrue(res.Status.Conditions, v1alpha1.ReservationConditionReady) { + if res.IsReady() { logger.V(1).Info("reservation is active, verifying allocations") + // Sync ObservedParentGeneration if the CR controller bumped ParentGeneration since + // the last time this reservation was processed (e.g. after a spec update). Without + // this patch the CR controller would spin in Reserving forever for already-ready slots. + if res.Spec.CommittedResourceReservation != nil && + (res.Status.CommittedResourceReservation == nil || + res.Status.CommittedResourceReservation.ObservedParentGeneration != res.Spec.CommittedResourceReservation.ParentGeneration) { + old := res.DeepCopy() + echoParentGeneration(&res) + if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + } + // Verify all allocations in Spec against actual VM state result, err := r.reconcileAllocations(ctx, &res) if err != nil { @@ -104,9 +129,9 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // Requeue with appropriate interval based on allocation state // Use shorter interval if there are allocations in grace period for faster verification if result.HasAllocationsInGracePeriod { - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalGracePeriod}, nil + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalGracePeriod.Duration}, nil } - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive}, nil + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive.Duration}, nil } // TODO trigger re-placement of unused reservations over time @@ -128,6 +153,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Reason: "PreAllocated", Message: "reservation pre-allocated with VM allocations", }) + echoParentGeneration(&res) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { // Ignore not-found errors during background deletion @@ -157,6 +183,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Reason: "ReservationActive", Message: "reservation is successfully scheduled", }) + echoParentGeneration(&res) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { // Ignore not-found errors during background deletion @@ -191,29 +218,13 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr logger.Info("flavor knowledge not ready, requeueing", "resourceName", resourceName, "error", err) - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry}, nil + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, nil } // Search for the flavor across all flavor groups - // Also capture the flavor group name for pipeline selection - var flavorDetails *compute.FlavorInGroup - var flavorGroupName string - for groupName, fg := range flavorGroups { - for _, flavor := range fg.Flavors { - if flavor.Name == resourceName { - flavorDetails = &flavor - flavorGroupName = groupName - break - } - } - if flavorDetails != nil { - break - } - } - - // Check if flavor was found - if flavorDetails == nil { - logger.Error(errors.New("flavor not found"), "flavor not found in any flavor group", + flavorGroupName, flavorDetails, err := reservations.FindFlavorInGroups(resourceName, flavorGroups) + if err != nil { + logger.Error(err, "flavor not found in any flavor group", "resourceName", resourceName) return ctrl.Result{RequeueAfter: 5 * time.Minute}, nil } @@ -242,11 +253,12 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Reason: "NoHostsAvailable", Message: "no hypervisors available for scheduling", }) + echoParentGeneration(&res) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry}, nil + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, nil } // Select appropriate pipeline based on flavor group @@ -289,6 +301,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr Reason: "NoHostsFound", Message: "no hosts found for reservation", }) + echoParentGeneration(&res) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { // Ignore not-found errors during background deletion @@ -388,7 +401,7 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte for vmUUID, allocation := range res.Spec.CommittedResourceReservation.Allocations { allocationAge := now.Sub(allocation.CreationTimestamp.Time) - isInGracePeriod := allocationAge < r.Conf.AllocationGracePeriod + isInGracePeriod := allocationAge < r.Conf.AllocationGracePeriod.Duration if isInGracePeriod { // New allocation: VM may not yet appear in the HV CRD (still spawning). @@ -413,7 +426,7 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte "reservation", res.Name, "expectedHost", expectedHost, "allocationAge", allocationAge, - "gracePeriod", r.Conf.AllocationGracePeriod) + "gracePeriod", r.Conf.AllocationGracePeriod.Duration) } } @@ -516,11 +529,9 @@ func (r *CommitmentReservationController) hypervisorToReservations(ctx context.C } // Init initializes the reconciler with required clients and DB connection. -func (r *CommitmentReservationController) Init(ctx context.Context, client client.Client, conf Config) error { - // Initialize scheduler client +func (r *CommitmentReservationController) Init(ctx context.Context, conf ReservationControllerConfig) error { r.SchedulerClient = reservations.NewSchedulerClient(conf.SchedulerURL) logf.FromContext(ctx).Info("scheduler client initialized for commitment reservation controller", "url", conf.SchedulerURL) - return nil } @@ -561,7 +572,7 @@ var commitmentReservationPredicate = predicate.Funcs{ // SetupWithManager sets up the controller with the Manager. func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - if err := r.Init(ctx, mgr.GetClient(), r.Conf); err != nil { + if err := r.Init(ctx, r.Conf); err != nil { return err } return nil @@ -598,7 +609,10 @@ func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl return bldr.Named("commitment-reservation"). WithOptions(controller.Options{ - // We want to process reservations one at a time to avoid overbooking. + // MaxConcurrentReconciles=1: conservative default. Note that this does NOT prevent + // the cache-staleness race where two back-to-back reconciles both pick the same host + // before the first write is visible to the capacity filter — that requires pessimistic + // blocking at the scheduler level. MaxConcurrentReconciles: 1, }). Complete(r) diff --git a/internal/scheduling/reservations/commitments/controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go similarity index 74% rename from internal/scheduling/reservations/commitments/controller_test.go rename to internal/scheduling/reservations/commitments/reservation_controller_test.go index afb8ebcfc..df6316d46 100644 --- a/internal/scheduling/reservations/commitments/controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -15,24 +15,16 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) func TestCommitmentReservationController_Reconcile(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := hv1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add hypervisor scheme: %v", err) - } + scheme := newCRTestScheme(t) tests := []struct { name string @@ -83,17 +75,13 @@ func TestCommitmentReservationController_Reconcile(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(tt.reservation). - WithStatusSubresource(&v1alpha1.Reservation{}). - Build() + k8sClient := newCRTestClient(scheme, tt.reservation) reconciler := &CommitmentReservationController{ - Client: client, + Client: k8sClient, Scheme: scheme, - Conf: Config{ - RequeueIntervalActive: 5 * time.Minute, + Conf: ReservationControllerConfig{ + RequeueIntervalActive: metav1.Duration{Duration: 5 * time.Minute}, }, } @@ -118,9 +106,8 @@ func TestCommitmentReservationController_Reconcile(t *testing.T) { t.Errorf("Expected no requeue but got %v", result.RequeueAfter) } - // Verify the reservation status var updated v1alpha1.Reservation - err = client.Get(context.Background(), req.NamespacedName, &updated) + err = k8sClient.Get(context.Background(), req.NamespacedName, &updated) if err != nil { t.Errorf("Failed to get updated reservation: %v", err) return @@ -146,23 +133,18 @@ func TestCommitmentReservationController_Reconcile(t *testing.T) { // ============================================================================ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := hv1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add hypervisor scheme: %v", err) - } + scheme := newCRTestScheme(t) now := time.Now() recentTime := metav1.NewTime(now.Add(-5 * time.Minute)) // 5 minutes ago (within grace period) oldTime := metav1.NewTime(now.Add(-30 * time.Minute)) // 30 minutes ago (past grace period) + config := ReservationControllerConfig{AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}} + tests := []struct { name string reservation *v1alpha1.Reservation hypervisor *hv1.Hypervisor - config Config expectedStatusAllocations map[string]string expectedSpecAllocations []string // VM UUIDs expected to remain in spec; nil means no check expectedHasGracePeriodAllocs bool @@ -175,7 +157,6 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { hypervisor: newTestHypervisorCRD("host-1", []hv1.Instance{ {ID: "vm-1", Name: "vm-1", Active: true}, }), - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{"vm-1": "host-1"}, expectedSpecAllocations: []string{"vm-1"}, expectedHasGracePeriodAllocs: false, @@ -186,9 +167,8 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { "vm-stopped": oldTime, }), hypervisor: newTestHypervisorCRD("host-1", []hv1.Instance{ - {ID: "vm-stopped", Name: "vm-stopped", Active: false}, // Inactive VM should still be found + {ID: "vm-stopped", Name: "vm-stopped", Active: false}, }), - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{"vm-stopped": "host-1"}, expectedSpecAllocations: []string{"vm-stopped"}, expectedHasGracePeriodAllocs: false, @@ -198,10 +178,9 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { reservation: newTestCRReservation(map[string]metav1.Time{ "vm-1": oldTime, }), - hypervisor: newTestHypervisorCRD("host-1", []hv1.Instance{}), // Empty - config: Config{AllocationGracePeriod: 15 * time.Minute}, + hypervisor: newTestHypervisorCRD("host-1", []hv1.Instance{}), expectedStatusAllocations: map[string]string{}, - expectedSpecAllocations: []string{}, // Removed from spec + expectedSpecAllocations: []string{}, expectedHasGracePeriodAllocs: false, }, { @@ -209,31 +188,26 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { reservation: newTestCRReservation(map[string]metav1.Time{ "vm-1": recentTime, }), - hypervisor: nil, - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{}, - expectedSpecAllocations: []string{"vm-1"}, // Kept in spec during grace period + expectedSpecAllocations: []string{"vm-1"}, expectedHasGracePeriodAllocs: true, }, { name: "mixed allocations - old verified via CRD, new in grace period", reservation: newTestCRReservation(map[string]metav1.Time{ - "vm-new": recentTime, // In grace period - "vm-old": oldTime, // Past grace period + "vm-new": recentTime, + "vm-old": oldTime, }), hypervisor: newTestHypervisorCRD("host-1", []hv1.Instance{ {ID: "vm-old", Name: "vm-old", Active: true}, }), - config: Config{AllocationGracePeriod: 15 * time.Minute}, - expectedStatusAllocations: map[string]string{"vm-old": "host-1"}, // Only old one confirmed via CRD + expectedStatusAllocations: map[string]string{"vm-old": "host-1"}, expectedSpecAllocations: []string{"vm-new", "vm-old"}, expectedHasGracePeriodAllocs: true, }, { name: "empty allocations - no work to do", reservation: newTestCRReservation(map[string]metav1.Time{}), - hypervisor: nil, - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{}, expectedHasGracePeriodAllocs: false, }, @@ -242,10 +216,8 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { reservation: newTestCRReservation(map[string]metav1.Time{ "vm-1": oldTime, }), - hypervisor: nil, // HV CRD does not exist (e.g. host deleted) - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{}, - expectedSpecAllocations: []string{}, // Removed from spec + expectedSpecAllocations: []string{}, expectedHasGracePeriodAllocs: false, }, { @@ -253,32 +225,25 @@ func TestReconcileAllocations_HypervisorCRDPath(t *testing.T) { reservation: newTestCRReservation(map[string]metav1.Time{ "vm-1": recentTime, }), - hypervisor: nil, // HV CRD does not exist - config: Config{AllocationGracePeriod: 15 * time.Minute}, expectedStatusAllocations: map[string]string{}, - expectedSpecAllocations: []string{"vm-1"}, // Kept during grace period + expectedSpecAllocations: []string{"vm-1"}, expectedHasGracePeriodAllocs: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Build fake client with objects objects := []client.Object{tt.reservation} if tt.hypervisor != nil { objects = append(objects, tt.hypervisor) } - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Reservation{}). - Build() + k8sClient := newCRTestClient(scheme, objects...) controller := &CommitmentReservationController{ Client: k8sClient, Scheme: scheme, - Conf: tt.config, + Conf: config, } ctx := WithNewGlobalRequestID(context.Background()) @@ -406,10 +371,7 @@ func newTestHypervisorCRD(name string, instances []hv1.Instance) *hv1.Hypervisor // This covers the mapper logic; the watch wiring itself (informer → mapper → enqueue) // is controller-runtime's responsibility and is not unit-testable without envtest. func TestHypervisorToReservations(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("failed to add scheme: %v", err) - } + scheme := newCRTestScheme(t) res1 := &v1alpha1.Reservation{ ObjectMeta: metav1.ObjectMeta{Name: "res-host-1"}, @@ -436,11 +398,7 @@ func TestHypervisorToReservations(t *testing.T) { Status: v1alpha1.ReservationStatus{Host: "host-1"}, } - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(res1, res2, resOtherHost, resNoHost, resFailover). - WithStatusSubresource(&v1alpha1.Reservation{}). - Build() + k8sClient := newCRTestClient(scheme, res1, res2, resOtherHost, resNoHost, resFailover) controller := &CommitmentReservationController{Client: k8sClient} @@ -467,13 +425,7 @@ func TestHypervisorToReservations(t *testing.T) { // ============================================================================ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := hv1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add hypervisor scheme: %v", err) - } + scheme := newCRTestScheme(t) reservation := &v1alpha1.Reservation{ ObjectMeta: ctrl.ObjectMeta{ @@ -486,91 +438,16 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t ResourceName: "test-flavor", }, Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("1Gi"), + hv1.ResourceMemory: resource.MustParse("4Gi"), hv1.ResourceCPU: resource.MustParse("2"), }, }, } - // Create flavor group knowledge CRD for the test - flavorGroups := []struct { - Name string `json:"name"` - Flavors []struct { - Name string `json:"name"` - MemoryMB uint64 `json:"memoryMB"` - VCPUs uint64 `json:"vcpus"` - ExtraSpecs map[string]string `json:"extraSpecs"` - } `json:"flavors"` - }{ - { - Name: "test-group", - Flavors: []struct { - Name string `json:"name"` - MemoryMB uint64 `json:"memoryMB"` - VCPUs uint64 `json:"vcpus"` - ExtraSpecs map[string]string `json:"extraSpecs"` - }{ - { - Name: "test-flavor", - MemoryMB: 1024, - VCPUs: 2, - ExtraSpecs: map[string]string{}, - }, - }, - }, - } - - // Marshal flavor groups into runtime.RawExtension - flavorGroupsJSON, err := json.Marshal(map[string]interface{}{ - "features": flavorGroups, - }) - if err != nil { - t.Fatalf("Failed to marshal flavor groups: %v", err) - } - - flavorGroupKnowledge := &v1alpha1.Knowledge{ - ObjectMeta: metav1.ObjectMeta{ - Name: "flavor-groups", - }, - Spec: v1alpha1.KnowledgeSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - Extractor: v1alpha1.KnowledgeExtractorSpec{ - Name: "flavor_groups", - }, - Recency: metav1.Duration{Duration: 0}, - }, - Status: v1alpha1.KnowledgeStatus{ - Raw: runtime.RawExtension{Raw: flavorGroupsJSON}, - RawLength: 1, - Conditions: []metav1.Condition{ - { - Type: v1alpha1.KnowledgeConditionReady, - Status: metav1.ConditionTrue, - Reason: "TestReady", - }, - }, - }, - } - - // Create mock hypervisors - hypervisor1 := &hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-host-1", - }, - Spec: hv1.HypervisorSpec{}, - } - hypervisor2 := &hv1.Hypervisor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-host-2", - }, - Spec: hv1.HypervisorSpec{}, - } + hypervisor1 := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "test-host-1"}} + hypervisor2 := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "test-host-2"}} - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(reservation, flavorGroupKnowledge, hypervisor1, hypervisor2). - WithStatusSubresource(&v1alpha1.Reservation{}, &v1alpha1.Knowledge{}). - Build() + k8sClient := newCRTestClient(scheme, reservation, newTestFlavorKnowledge(), hypervisor1, hypervisor2) // Create a mock server that returns a successful response mockResponse := &schedulerdelegationapi.ExternalSchedulerResponse{ @@ -597,18 +474,18 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t })) defer server.Close() - config := Config{ + config := ReservationControllerConfig{ SchedulerURL: server.URL, } reconciler := &CommitmentReservationController{ - Client: client, + Client: k8sClient, Scheme: scheme, Conf: config, } // Initialize the reconciler (this sets up SchedulerClient) - if err := reconciler.Init(context.Background(), client, config); err != nil { + if err := reconciler.Init(context.Background(), config); err != nil { t.Fatalf("Failed to initialize reconciler: %v", err) } @@ -630,7 +507,7 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t // Verify Spec.TargetHost is set after first reconcile var afterFirstReconcile v1alpha1.Reservation - if err = client.Get(context.Background(), req.NamespacedName, &afterFirstReconcile); err != nil { + if err = k8sClient.Get(context.Background(), req.NamespacedName, &afterFirstReconcile); err != nil { t.Errorf("Failed to get reservation after first reconcile: %v", err) return } @@ -650,7 +527,7 @@ func TestCommitmentReservationController_reconcileInstanceReservation_Success(t // Verify the reservation status after second reconcile var updated v1alpha1.Reservation - if err = client.Get(context.Background(), req.NamespacedName, &updated); err != nil { + if err = k8sClient.Get(context.Background(), req.NamespacedName, &updated); err != nil { t.Errorf("Failed to get updated reservation: %v", err) return } diff --git a/internal/scheduling/reservations/commitments/reservation_manager.go b/internal/scheduling/reservations/commitments/reservation_manager.go index 0cdbc9f12..d1fa28fda 100644 --- a/internal/scheduling/reservations/commitments/reservation_manager.go +++ b/internal/scheduling/reservations/commitments/reservation_manager.go @@ -25,6 +25,9 @@ type ApplyResult struct { Deleted int // Repaired is the number of reservations repaired (metadata sync or recreated due to wrong config) Repaired int + // TotalSlots is the total number of reservation slots that should exist after the apply. + // Used by the CR controller to wait for the correct number of children in the cache. + TotalSlots int // TouchedReservations are reservations that were created or updated TouchedReservations []v1alpha1.Reservation // RemovedReservations are reservations that were deleted @@ -77,11 +80,11 @@ func (m *ReservationManager) ApplyCommitmentState( return nil, fmt.Errorf("failed to list reservations: %w", err) } - // Filter by name prefix to find reservations for this commitment - namePrefix := fmt.Sprintf("commitment-%s-", desiredState.CommitmentUUID) + // Filter by CommitmentUUID to find reservations for this commitment var existing []v1alpha1.Reservation for _, res := range allReservations.Items { - if len(res.Name) >= len(namePrefix) && res.Name[:len(namePrefix)] == namePrefix { + if res.Spec.CommittedResourceReservation != nil && + res.Spec.CommittedResourceReservation.CommitmentUUID == desiredState.CommitmentUUID { existing = append(existing, res) } } @@ -92,6 +95,9 @@ func (m *ReservationManager) ApplyCommitmentState( if !exists { return nil, fmt.Errorf("flavor group not found: %s", desiredState.FlavorGroupName) } + if len(flavorGroup.Flavors) == 0 { + return nil, fmt.Errorf("flavor group %s has no flavors", desiredState.FlavorGroupName) + } deltaMemoryBytes := desiredState.TotalMemoryBytes for _, res := range existing { memoryQuantity := res.Spec.Resources[hv1.ResourceMemory] @@ -210,6 +216,7 @@ func (m *ReservationManager) ApplyCommitmentState( "total", len(existing)+result.Created) } + result.TotalSlots = len(existing) + result.Created return result, nil } @@ -225,7 +232,8 @@ func (m *ReservationManager) syncReservationMetadata( if (state.CommitmentUUID != "" && reservation.Spec.CommittedResourceReservation.CommitmentUUID != state.CommitmentUUID) || (state.AvailabilityZone != "" && reservation.Spec.AvailabilityZone != state.AvailabilityZone) || (state.StartTime != nil && (reservation.Spec.StartTime == nil || !reservation.Spec.StartTime.Time.Equal(*state.StartTime))) || - (state.EndTime != nil && (reservation.Spec.EndTime == nil || !reservation.Spec.EndTime.Time.Equal(*state.EndTime))) { + (state.EndTime != nil && (reservation.Spec.EndTime == nil || !reservation.Spec.EndTime.Time.Equal(*state.EndTime))) || + (state.ParentGeneration != 0 && reservation.Spec.CommittedResourceReservation.ParentGeneration != state.ParentGeneration) { // Apply patch logger.V(1).Info("syncing reservation metadata", "reservation", reservation.Name, @@ -236,6 +244,9 @@ func (m *ReservationManager) syncReservationMetadata( if state.CommitmentUUID != "" { reservation.Spec.CommittedResourceReservation.CommitmentUUID = state.CommitmentUUID } + if state.ParentGeneration != 0 { + reservation.Spec.CommittedResourceReservation.ParentGeneration = state.ParentGeneration + } if state.AvailabilityZone != "" { reservation.Spec.AvailabilityZone = state.AvailabilityZone @@ -266,7 +277,11 @@ func (m *ReservationManager) newReservation( creator string, ) *v1alpha1.Reservation { - name := fmt.Sprintf("commitment-%s-%d", state.CommitmentUUID, slotIndex) + namePrefix := state.NamePrefix + if namePrefix == "" { + namePrefix = fmt.Sprintf("commitment-%s-", state.CommitmentUUID) + } + name := fmt.Sprintf("%s%d", namePrefix, slotIndex) // Select first flavor that fits remaining memory (flavors sorted descending by size) flavorInGroup := flavorGroup.Flavors[len(flavorGroup.Flavors)-1] // default to smallest @@ -297,13 +312,14 @@ func (m *ReservationManager) newReservation( ), }, CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: state.ProjectID, - CommitmentUUID: state.CommitmentUUID, - DomainID: state.DomainID, - ResourceGroup: state.FlavorGroupName, - ResourceName: flavorInGroup.Name, - Creator: creator, - Allocations: nil, + ProjectID: state.ProjectID, + CommitmentUUID: state.CommitmentUUID, + DomainID: state.DomainID, + ResourceGroup: state.FlavorGroupName, + ResourceName: flavorInGroup.Name, + Creator: creator, + ParentGeneration: state.ParentGeneration, + Allocations: nil, }, } diff --git a/internal/scheduling/reservations/commitments/reservation_manager_test.go b/internal/scheduling/reservations/commitments/reservation_manager_test.go index 7733cb6c2..b512fc9b5 100644 --- a/internal/scheduling/reservations/commitments/reservation_manager_test.go +++ b/internal/scheduling/reservations/commitments/reservation_manager_test.go @@ -13,691 +13,276 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestApplyCommitmentState_CreatesNewReservations(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) +// newTestCRSlot creates a Reservation slot for commitment "abc123" / project "project-1". +// Pass nil allocs for an empty allocation map. +func newTestCRSlot(name string, memGiB int64, targetHost, resourceGroup string, allocs map[string]v1alpha1.CommittedResourceAllocation) v1alpha1.Reservation { + if allocs == nil { + allocs = map[string]v1alpha1.CommittedResourceAllocation{} } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - Build() - - manager := NewReservationManager(client) - flavorGroup := testFlavorGroup() - flavorGroups := map[string]compute.FlavorGroupFeature{ - "test-group": flavorGroup, - } - - // Desired state: 3 multiples of smallest flavor (24 GiB) - desiredState := &CommitmentState{ - CommitmentUUID: "abc123", - ProjectID: "project-1", - FlavorGroupName: "test-group", - TotalMemoryBytes: 3 * 8192 * 1024 * 1024, - } - - applyResult, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", - ) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if len(applyResult.RemovedReservations) != 0 { - t.Errorf("expected 0 applyResult.RemovedReservations reservations, got %d", len(applyResult.RemovedReservations)) - } - - // Should create reservations to fulfill the commitment - if len(applyResult.TouchedReservations) == 0 { - t.Fatal("expected at least one reservation to be created") - } - - // Verify created reservations sum to desired state - totalMemory := int64(0) - for _, res := range applyResult.TouchedReservations { - memQuantity := res.Spec.Resources[hv1.ResourceMemory] - totalMemory += memQuantity.Value() + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + TargetHost: targetHost, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(memGiB*1024*1024*1024, resource.BinarySI), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: "abc123", + ProjectID: "project-1", + ResourceGroup: resourceGroup, + Creator: "syncer", + Allocations: allocs, + }, + }, } +} - if totalMemory != desiredState.TotalMemoryBytes { - t.Errorf("expected total memory %d, got %d", desiredState.TotalMemoryBytes, totalMemory) - } +// testFlavorGroups returns the default flavor groups map used across tests. +func testFlavorGroups() map[string]compute.FlavorGroupFeature { + return map[string]compute.FlavorGroupFeature{"test-group": testFlavorGroup()} } -func TestApplyCommitmentState_DeletesExcessReservations(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } +// ============================================================================ +// Tests: ApplyCommitmentState +// ============================================================================ - // Create existing reservations (32 GiB total) - existingReservations := []v1alpha1.Reservation{ +func TestApplyCommitmentState(t *testing.T) { + tests := []struct { + name string + existingSlots []v1alpha1.Reservation + desiredMemoryGiB int64 + flavorGroupOverride map[string]compute.FlavorGroupFeature // nil = testFlavorGroups() + wantError bool + wantRemovedCount int // exact count; -1 = at least one + validateRemoved func(t *testing.T, removed []v1alpha1.Reservation) + validateTouched func(t *testing.T, touched []v1alpha1.Reservation) + validateRemaining func(t *testing.T, remaining []v1alpha1.Reservation) + }{ { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, + name: "creates reservations to match desired memory", + desiredMemoryGiB: 24, // 3 × 8 GiB slots + validateTouched: func(t *testing.T, touched []v1alpha1.Reservation) { + if len(touched) == 0 { + t.Fatal("expected at least one reservation created") + } + var total int64 + for _, r := range touched { + q := r.Spec.Resources[hv1.ResourceMemory] + total += q.Value() + } + if want := int64(24 * 1024 * 1024 * 1024); total != want { + t.Errorf("expected total memory %d, got %d", want, total) + } }, - Spec: v1alpha1.ReservationSpec{ - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(16*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, + }, + { + // Algorithm removes both 16 GiB slots and creates a new 8 GiB one. + name: "removes excess reservations, remaining memory matches desired", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 16, "", "test-group", nil), + newTestCRSlot("commitment-abc123-1", 16, "", "test-group", nil), + }, + desiredMemoryGiB: 8, + wantRemovedCount: -1, + validateRemaining: func(t *testing.T, remaining []v1alpha1.Reservation) { + var total int64 + for _, r := range remaining { + q := r.Spec.Resources[hv1.ResourceMemory] + total += q.Value() + } + if want := int64(8 * 1024 * 1024 * 1024); total != want { + t.Errorf("expected remaining memory %d, got %d", want, total) + } }, }, { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-1", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, + name: "zero desired memory removes all reservations", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 8, "", "test-group", nil), }, - Spec: v1alpha1.ReservationSpec{ - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(16*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, + desiredMemoryGiB: 0, + wantRemovedCount: 1, + validateRemaining: func(t *testing.T, remaining []v1alpha1.Reservation) { + if len(remaining) != 0 { + t.Errorf("expected 0 remaining, got %d", len(remaining)) + } }, }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(&existingReservations[0], &existingReservations[1]). - Build() - - manager := NewReservationManager(client) - flavorGroup := testFlavorGroup() - flavorGroups := map[string]compute.FlavorGroupFeature{ - "test-group": flavorGroup, - } - - // Desired state: only 8 GiB (need to reduce) - desiredState := &CommitmentState{ - CommitmentUUID: "abc123", - ProjectID: "project-1", - FlavorGroupName: "test-group", - TotalMemoryBytes: 8 * 1024 * 1024 * 1024, - } - - applyResult, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", - ) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // Note: May create a new 8GiB reservation while removing the two 16GiB ones - // This is expected behavior based on the slot sizing algorithm - - // Should remove excess reservations - if len(applyResult.RemovedReservations) == 0 { - t.Fatal("expected reservations to be removed") - } - - // Verify remaining capacity matches desired state - var remainingList v1alpha1.ReservationList - if err := client.List(context.Background(), &remainingList); err != nil { - t.Fatal(err) - } - - totalMemory := int64(0) - for _, res := range remainingList.Items { - memQuantity := res.Spec.Resources[hv1.ResourceMemory] - totalMemory += memQuantity.Value() - } - - if totalMemory != desiredState.TotalMemoryBytes { - t.Errorf("expected remaining memory %d, got %d", desiredState.TotalMemoryBytes, totalMemory) - } -} - -func TestApplyCommitmentState_DeletionPriority(t *testing.T) { - tests := []struct { - name string - existingReservations []v1alpha1.Reservation - desiredMemoryBytes int64 - expectedRemovedCount int - validateRemoved func(t *testing.T, removed []v1alpha1.Reservation) - validateRemaining func(t *testing.T, remaining []v1alpha1.Reservation) - }{ { - name: "Priority 1: Unscheduled reservations (no TargetHost) deleted first", - existingReservations: []v1alpha1.Reservation{ - // Reservation 0: Has TargetHost and allocations - lowest priority (should remain) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "host-1", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{ - "vm-123": {}, - }, - }, - }, - }, - // Reservation 1: No TargetHost and no allocations - highest priority (should be deleted) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-1", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - }, + name: "replaces reservation with wrong flavor group", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 8, "", "wrong-group", nil), + }, + desiredMemoryGiB: 8, + wantRemovedCount: 1, + validateTouched: func(t *testing.T, touched []v1alpha1.Reservation) { + if len(touched) != 1 { + t.Fatalf("expected 1 new reservation, got %d", len(touched)) + } + if got := touched[0].Spec.CommittedResourceReservation.ResourceGroup; got != "test-group" { + t.Errorf("expected flavor group test-group, got %s", got) + } }, - desiredMemoryBytes: 8 * 1024 * 1024 * 1024, // Need to delete one - expectedRemovedCount: 1, + }, + { + name: "unknown flavor group returns error", + desiredMemoryGiB: 8, + flavorGroupOverride: map[string]compute.FlavorGroupFeature{}, + wantError: true, + }, + { + name: "deletion priority: unscheduled (no TargetHost) deleted before scheduled", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 8, "host-1", "test-group", map[string]v1alpha1.CommittedResourceAllocation{"vm-123": {}}), + newTestCRSlot("commitment-abc123-1", 8, "", "test-group", nil), + }, + desiredMemoryGiB: 8, + wantRemovedCount: 1, validateRemoved: func(t *testing.T, removed []v1alpha1.Reservation) { - // Should have removed the unscheduled one (no TargetHost) if removed[0].Spec.TargetHost != "" { - t.Errorf("expected unscheduled reservation to be removed, but removed %s with TargetHost %s", - removed[0].Name, removed[0].Spec.TargetHost) + t.Errorf("expected unscheduled reservation removed, got TargetHost=%q", removed[0].Spec.TargetHost) } }, validateRemaining: func(t *testing.T, remaining []v1alpha1.Reservation) { if len(remaining) != 1 { - t.Fatalf("expected 1 remaining reservation, got %d", len(remaining)) + t.Fatalf("expected 1 remaining, got %d", len(remaining)) } - // Should have kept the scheduled one with allocations - if remaining[0].Spec.TargetHost == "" { - t.Error("expected scheduled reservation to remain") - } - if len(remaining[0].Spec.CommittedResourceReservation.Allocations) == 0 { - t.Error("expected reservation with allocations to remain") + if remaining[0].Spec.TargetHost == "" || len(remaining[0].Spec.CommittedResourceReservation.Allocations) == 0 { + t.Error("expected scheduled reservation with allocations to remain") } }, }, { - name: "Priority 2: Unused scheduled reservations (no allocations) deleted next", - existingReservations: []v1alpha1.Reservation{ - // Has TargetHost AND allocations - lowest priority for deletion - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "host-1", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{ - "vm-123": {}, - }, - }, - }, - }, - // Has TargetHost but NO allocations - medium priority - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-1", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "host-2", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - }, + name: "deletion priority: unused scheduled (no allocations) deleted before allocated", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 8, "host-1", "test-group", map[string]v1alpha1.CommittedResourceAllocation{"vm-123": {}}), + newTestCRSlot("commitment-abc123-1", 8, "host-2", "test-group", nil), }, - desiredMemoryBytes: 8 * 1024 * 1024 * 1024, - expectedRemovedCount: 1, + desiredMemoryGiB: 8, + wantRemovedCount: 1, validateRemoved: func(t *testing.T, removed []v1alpha1.Reservation) { - // Should have removed the one without allocations if len(removed[0].Spec.CommittedResourceReservation.Allocations) != 0 { t.Error("expected reservation without allocations to be removed") } }, validateRemaining: func(t *testing.T, remaining []v1alpha1.Reservation) { if len(remaining) != 1 { - t.Fatalf("expected 1 remaining reservation, got %d", len(remaining)) + t.Fatalf("expected 1 remaining, got %d", len(remaining)) } - // Should have kept the one with allocations if len(remaining[0].Spec.CommittedResourceReservation.Allocations) == 0 { t.Error("expected reservation with allocations to remain") } }, }, { - name: "Mixed scenario: comprehensive deletion priority test", - existingReservations: []v1alpha1.Reservation{ - // Reservation 0: Has TargetHost + has allocations (lowest priority - should remain) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "host-1", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{ - "vm-allocated": {}, - }, - }, - }, - }, - // Reservation 1: Has TargetHost + no allocations (medium priority - should remain) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-1", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "host-2", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - }, - // Reservation 2: No TargetHost + no allocations (highest priority - should be deleted) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-2", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - }, - // Reservation 3: No TargetHost + no allocations (highest priority - should be deleted) - { - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-3", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - TargetHost: "", - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - }, + name: "deletion priority: unscheduled removed first across mixed set", + existingSlots: []v1alpha1.Reservation{ + newTestCRSlot("commitment-abc123-0", 8, "host-1", "test-group", map[string]v1alpha1.CommittedResourceAllocation{"vm-allocated": {}}), + newTestCRSlot("commitment-abc123-1", 8, "host-2", "test-group", nil), + newTestCRSlot("commitment-abc123-2", 8, "", "test-group", nil), + newTestCRSlot("commitment-abc123-3", 8, "", "test-group", nil), }, - desiredMemoryBytes: 16 * 1024 * 1024 * 1024, // Need to delete 2 out of 4 - expectedRemovedCount: 2, + desiredMemoryGiB: 16, + wantRemovedCount: 2, validateRemoved: func(t *testing.T, removed []v1alpha1.Reservation) { - // Both removed should have no TargetHost (highest priority for deletion) - for _, res := range removed { - if res.Spec.TargetHost != "" { - t.Errorf("expected unscheduled reservations to be removed first, but removed %s with TargetHost %s", - res.Name, res.Spec.TargetHost) + for _, r := range removed { + if r.Spec.TargetHost != "" { + t.Errorf("expected unscheduled reservations removed first, got TargetHost=%q on %s", r.Spec.TargetHost, r.Name) } } }, validateRemaining: func(t *testing.T, remaining []v1alpha1.Reservation) { if len(remaining) != 2 { - t.Fatalf("expected 2 remaining reservations, got %d", len(remaining)) - } - // Both remaining should have TargetHost - for _, res := range remaining { - if res.Spec.TargetHost == "" { - t.Errorf("expected scheduled reservations to remain, but %s has no TargetHost", res.Name) - } + t.Fatalf("expected 2 remaining, got %d", len(remaining)) } - // At least one should have allocations (the one with lowest deletion priority) - hasAllocations := false - for _, res := range remaining { - if len(res.Spec.CommittedResourceReservation.Allocations) > 0 { - hasAllocations = true - break + for _, r := range remaining { + if r.Spec.TargetHost == "" { + t.Errorf("expected scheduled reservations to remain, got empty TargetHost on %s", r.Name) } } - if !hasAllocations { - t.Error("expected at least one remaining reservation to have allocations") - } }, }, } + scheme := newCRTestScheme(t) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) + objects := make([]client.Object, len(tt.existingSlots)) + for i := range tt.existingSlots { + objects[i] = &tt.existingSlots[i] } + k8sClient := newCRTestClient(scheme, objects...) + manager := NewReservationManager(k8sClient) - // Convert slice to individual objects for WithObjects - objects := make([]client.Object, len(tt.existingReservations)) - for i := range tt.existingReservations { - objects[i] = &tt.existingReservations[i] + flavorGroups := testFlavorGroups() + if tt.flavorGroupOverride != nil { + flavorGroups = tt.flavorGroupOverride } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - - manager := NewReservationManager(client) - flavorGroup := testFlavorGroup() - flavorGroups := map[string]compute.FlavorGroupFeature{ - "test-group": flavorGroup, - } - desiredState := &CommitmentState{ CommitmentUUID: "abc123", ProjectID: "project-1", FlavorGroupName: "test-group", - TotalMemoryBytes: tt.desiredMemoryBytes, + TotalMemoryBytes: tt.desiredMemoryGiB * 1024 * 1024 * 1024, } applyResult, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", + context.Background(), logr.Discard(), desiredState, flavorGroups, "syncer", ) + if tt.wantError { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(applyResult.RemovedReservations) != tt.expectedRemovedCount { - t.Fatalf("expected %d removed reservations, got %d", tt.expectedRemovedCount, len(applyResult.RemovedReservations)) + switch { + case tt.wantRemovedCount > 0: + if len(applyResult.RemovedReservations) != tt.wantRemovedCount { + t.Fatalf("expected %d removed, got %d", tt.wantRemovedCount, len(applyResult.RemovedReservations)) + } + case tt.wantRemovedCount == 0: + if len(applyResult.RemovedReservations) != 0 { + t.Errorf("expected 0 removed, got %d", len(applyResult.RemovedReservations)) + } + case tt.wantRemovedCount == -1: + if len(applyResult.RemovedReservations) == 0 { + t.Fatal("expected at least one removed reservation") + } } if tt.validateRemoved != nil { tt.validateRemoved(t, applyResult.RemovedReservations) } - - // Get remaining reservations - var remainingList v1alpha1.ReservationList - if err := client.List(context.Background(), &remainingList); err != nil { - t.Fatal(err) + if tt.validateTouched != nil { + tt.validateTouched(t, applyResult.TouchedReservations) } - if tt.validateRemaining != nil { - tt.validateRemaining(t, remainingList.Items) + var remaining v1alpha1.ReservationList + if err := k8sClient.List(context.Background(), &remaining); err != nil { + t.Fatal(err) + } + tt.validateRemaining(t, remaining.Items) } }) } } -func TestApplyCommitmentState_HandlesZeroCapacity(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - // Create existing reservation - existingReservation := v1alpha1.Reservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "test-group", - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(&existingReservation). - Build() - - manager := NewReservationManager(client) - flavorGroup := testFlavorGroup() - flavorGroups := map[string]compute.FlavorGroupFeature{ - "test-group": flavorGroup, - } - - // Desired state: zero capacity (commitment expired or canceled) - desiredState := &CommitmentState{ - CommitmentUUID: "abc123", - ProjectID: "project-1", - FlavorGroupName: "test-group", - TotalMemoryBytes: 0, - } - - applyResult, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", - ) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if len(applyResult.TouchedReservations) != 0 { - t.Errorf("expected 0 new reservations, got %d", len(applyResult.TouchedReservations)) - } - - // Should remove all reservations - if len(applyResult.RemovedReservations) != 1 { - t.Fatalf("expected 1 removed reservation, got %d", len(applyResult.RemovedReservations)) - } - - // Verify no reservations remain - var remainingList v1alpha1.ReservationList - if err := client.List(context.Background(), &remainingList); err != nil { - t.Fatal(err) - } - - if len(remainingList.Items) != 0 { - t.Errorf("expected 0 remaining reservations, got %d", len(remainingList.Items)) - } -} - -func TestApplyCommitmentState_FixesWrongFlavorGroup(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - // Create reservation with wrong flavor group - existingReservation := v1alpha1.Reservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "commitment-abc123-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), - }, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "project-1", - ResourceGroup: "wrong-group", // Wrong flavor group - Creator: "syncer", - Allocations: map[string]v1alpha1.CommittedResourceAllocation{}, - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(&existingReservation). - Build() - - manager := NewReservationManager(client) - flavorGroup := testFlavorGroup() - flavorGroups := map[string]compute.FlavorGroupFeature{ - "test-group": flavorGroup, - } - - // Desired state with correct flavor group - desiredState := &CommitmentState{ - CommitmentUUID: "abc123", - ProjectID: "project-1", - FlavorGroupName: "test-group", - TotalMemoryBytes: 8 * 1024 * 1024 * 1024, - } - - applyResult, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", - ) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // Should remove wrong reservation and create new one - if len(applyResult.RemovedReservations) != 1 { - t.Fatalf("expected 1 removed reservation, got %d", len(applyResult.RemovedReservations)) - } - - if len(applyResult.TouchedReservations) != 1 { - t.Fatalf("expected 1 new reservation, got %d", len(applyResult.TouchedReservations)) - } - - // Verify new reservation has correct flavor group - if applyResult.TouchedReservations[0].Spec.CommittedResourceReservation.ResourceGroup != "test-group" { - t.Errorf("expected flavor group test-group, got %s", - applyResult.TouchedReservations[0].Spec.CommittedResourceReservation.ResourceGroup) - } -} - -func TestApplyCommitmentState_UnknownFlavorGroup(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - Build() - - manager := NewReservationManager(client) - flavorGroups := map[string]compute.FlavorGroupFeature{} // Empty - - desiredState := &CommitmentState{ - CommitmentUUID: "abc123", - ProjectID: "project-1", - FlavorGroupName: "unknown-group", - TotalMemoryBytes: 8 * 1024 * 1024 * 1024, - } - - _, err := manager.ApplyCommitmentState( - context.Background(), - logr.Discard(), - desiredState, - flavorGroups, - "syncer", - ) - - if err == nil { - t.Fatal("expected error for unknown flavor group, got nil") - } -} +// ============================================================================ +// Tests: newReservation flavor selection +// ============================================================================ func TestNewReservation_SelectsAppropriateFlavor(t *testing.T) { manager := &ReservationManager{} @@ -729,8 +314,8 @@ func TestNewReservation_SelectsAppropriateFlavor(t *testing.T) { }, { name: "oversized uses largest available flavor", - deltaMemory: 100 * 1024 * 1024 * 1024, // 100 GiB (larger than any flavor) - expectedName: "large", // Will use largest available + deltaMemory: 100 * 1024 * 1024 * 1024, // 100 GiB + expectedName: "large", expectedCores: 16, }, } @@ -744,26 +329,15 @@ func TestNewReservation_SelectsAppropriateFlavor(t *testing.T) { TotalMemoryBytes: tt.deltaMemory, } - reservation := manager.newReservation( - state, - 0, - tt.deltaMemory, - flavorGroup, - "syncer", - ) + reservation := manager.newReservation(state, 0, tt.deltaMemory, flavorGroup, "syncer") - // Verify flavor selection if reservation.Spec.CommittedResourceReservation.ResourceName != tt.expectedName { t.Errorf("expected flavor %s, got %s", - tt.expectedName, - reservation.Spec.CommittedResourceReservation.ResourceName) + tt.expectedName, reservation.Spec.CommittedResourceReservation.ResourceName) } - - // Verify CPU allocation cpuQuantity := reservation.Spec.Resources[hv1.ResourceCPU] if cpuQuantity.Value() != tt.expectedCores { - t.Errorf("expected %d cores, got %d", - tt.expectedCores, cpuQuantity.Value()) + t.Errorf("expected %d cores, got %d", tt.expectedCores, cpuQuantity.Value()) } }) } diff --git a/internal/scheduling/reservations/commitments/state.go b/internal/scheduling/reservations/commitments/state.go index 698aea428..149cdfc03 100644 --- a/internal/scheduling/reservations/commitments/state.go +++ b/internal/scheduling/reservations/commitments/state.go @@ -93,6 +93,17 @@ type CommitmentState struct { EndTime *time.Time // CreatorRequestID is the request ID that triggered this state change (for traceability) CreatorRequestID string + // NamePrefix overrides the default "commitment-<uuid>-" reservation naming convention. + // When set (e.g. "<cr-name>-"), Reservation CRDs are named "<NamePrefix><slot-index>". + // Used by the CommittedResource controller; leave empty for the legacy syncer path. + NamePrefix string + // ParentGeneration is the Generation of the parent CommittedResource CRD. Written into + // Reservation spec so the Reservation controller can echo it back in status, letting + // the CR controller detect when all children have been processed for the current spec. + // Zero for syncer-created reservations (no parent CR). + ParentGeneration int64 + // State is the lifecycle state from Limes (planned/pending/guaranteed/confirmed/superseded/expired). + State v1alpha1.CommitmentStatus } // FromCommitment converts Limes commitment to CommitmentState. @@ -140,6 +151,7 @@ func FromCommitment( AvailabilityZone: commitment.AvailabilityZone, StartTime: startTime, EndTime: endTime, + State: v1alpha1.CommitmentStatus(commitment.Status), }, nil } @@ -147,6 +159,7 @@ func FromCommitment( func FromChangeCommitmentTargetState( commitment liquid.Commitment, projectID string, + domainID string, flavorGroupName string, flavorGroup compute.FlavorGroupFeature, az string, @@ -162,8 +175,8 @@ func FromChangeCommitmentTargetState( var endTime *time.Time switch commitment.NewStatus.UnwrapOr("none") { - // guaranteed and confirmed commitments are honored with start time now - case liquid.CommitmentStatusGuaranteed, liquid.CommitmentStatusConfirmed: + // pending, guaranteed, confirmed commitments are honored with Reservation slots. + case liquid.CommitmentStatusPending, liquid.CommitmentStatusGuaranteed, liquid.CommitmentStatusConfirmed: amountMultiple = commitment.Amount // Set start time: use ConfirmBy if available (when the commitment was confirmed), // otherwise use time.Now() for immediate confirmation @@ -183,7 +196,7 @@ func FromChangeCommitmentTargetState( if !commitment.ExpiresAt.IsZero() { endTime = &commitment.ExpiresAt // check expiry time - if commitment.ExpiresAt.Before(time.Now()) || commitment.ExpiresAt.Equal(time.Now()) { + if !commitment.ExpiresAt.After(time.Now()) { // commitment is already expired, ignore capacity amountMultiple = 0 } @@ -199,11 +212,13 @@ func FromChangeCommitmentTargetState( return &CommitmentState{ CommitmentUUID: string(commitment.UUID), ProjectID: projectID, + DomainID: domainID, FlavorGroupName: flavorGroupName, TotalMemoryBytes: totalMemoryBytes, AvailabilityZone: az, StartTime: startTime, EndTime: endTime, + State: v1alpha1.CommitmentStatus(commitment.NewStatus.UnwrapOr("")), }, nil } diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 60c450b9a..8d3a43adf 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -13,7 +13,10 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) var ( @@ -30,21 +33,6 @@ type SyncerConfig struct { SyncInterval time.Duration `json:"committedResourceSyncInterval"` } -func DefaultSyncerConfig() SyncerConfig { - return SyncerConfig{ - SyncInterval: time.Hour, - } -} - -// ApplyDefaults fills in any unset values with defaults. -func (c *SyncerConfig) ApplyDefaults() { - defaults := DefaultSyncerConfig() - if c.SyncInterval == 0 { - c.SyncInterval = defaults.SyncInterval - } - // Note: KeystoneSecretRef and SSOSecretRef are not defaulted as they require explicit configuration -} - type Syncer struct { // Client to fetch commitments from Limes CommitmentsClient @@ -97,11 +85,6 @@ func (s *Syncer) getCommitmentStates(ctx context.Context, log logr.Logger, flavo skippedUUIDs: make(map[string]bool), } for id, commitment := range commitments { - // Record each commitment seen from Limes - if s.monitor != nil { - s.monitor.RecordCommitmentSeen() - } - if commitment.ServiceType != "compute" { log.Info("skipping non-compute commitment", "id", id, "serviceType", commitment.ServiceType) if s.monitor != nil { @@ -110,12 +93,19 @@ func (s *Syncer) getCommitmentStates(ctx context.Context, log logr.Logger, flavo continue } - // Only process commitments that are active (confirmed or guaranteed). - // planned/pending are not yet accepted by Cortex; superseded/expired are done. - if commitment.Status != "confirmed" && commitment.Status != "guaranteed" { - log.Info("skipping non-active commitment", "id", id, "status", commitment.Status) - if s.monitor != nil { - s.monitor.RecordCommitmentSkipped(SkipReasonNonActive) + // Validate that the commitment state is a known enum value. + switch v1alpha1.CommitmentStatus(commitment.Status) { + case v1alpha1.CommitmentStatusPlanned, + v1alpha1.CommitmentStatusPending, + v1alpha1.CommitmentStatusGuaranteed, + v1alpha1.CommitmentStatusConfirmed, + v1alpha1.CommitmentStatusSuperseded, + v1alpha1.CommitmentStatusExpired: + // valid, continue processing + default: + log.Info("skipping commitment with unknown status", "id", id, "status", commitment.Status) + if commitment.UUID != "" { + result.skippedUUIDs[commitment.UUID] = true } continue } @@ -194,11 +184,6 @@ func (s *Syncer) getCommitmentStates(ctx context.Context, log logr.Logger, flavo "totalMemoryBytes", state.TotalMemoryBytes) result.states = append(result.states, state) - - // Record successfully processed commitment - if s.monitor != nil { - s.monitor.RecordCommitmentProcessed() - } } return result, nil @@ -215,16 +200,21 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { logger.Info("starting commitment sync") - // Record sync run - if s.monitor != nil { - s.monitor.RecordSyncRun() - } + startTime := time.Now() + defer func() { + if s.monitor != nil { + s.monitor.RecordDuration(time.Since(startTime).Seconds()) + } + }() // Check if flavor group knowledge is ready knowledge := &reservations.FlavorGroupKnowledgeClient{Client: s.Client} knowledgeCRD, err := knowledge.Get(ctx) if err != nil { logger.Error(err, "failed to check flavor group knowledge readiness") + if s.monitor != nil { + s.monitor.RecordError() + } return err } if knowledgeCRD == nil { @@ -236,6 +226,9 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, knowledgeCRD) if err != nil { logger.Error(err, "failed to get flavor groups from knowledge") + if s.monitor != nil { + s.monitor.RecordError() + } return err } @@ -243,42 +236,48 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { commitmentResult, err := s.getCommitmentStates(ctx, logger, flavorGroups) if err != nil { logger.Error(err, "failed to get compute commitments") + if s.monitor != nil { + s.monitor.RecordError() + } return err } - // Create ReservationManager to handle state application - manager := NewReservationManager(s.Client) + if s.monitor != nil { + s.monitor.SetLimesCommitmentsActive(len(commitmentResult.states)) + } - // Apply each commitment state using the manager - var totalCreated, totalDeleted, totalRepaired int + // Upsert CommittedResource CRDs for each commitment + var totalCreated, totalUpdated int for _, state := range commitmentResult.states { - logger.Info("applying commitment state", + logger.Info("upserting committed resource CRD", "commitmentUUID", state.CommitmentUUID, "projectID", state.ProjectID, "flavorGroup", state.FlavorGroupName, - "totalMemoryBytes", state.TotalMemoryBytes) - - applyResult, err := manager.ApplyCommitmentState(ctx, logger, state, flavorGroups, CreatorValue) + "state", state.State) + + var ( + op controllerutil.OperationResult + err error + ) + if isTerminalCommitment(state) { + // Terminal commitments (superseded/expired state, or EndTime in the past): update + // existing CRD so the controller can clean up Reservations, but do not create a + // new one — if no CRD exists locally there are no Reservation slots to clean up. + op, err = s.updateCommittedResourceIfExists(ctx, logger, state) + } else { + op, err = s.upsertCommittedResource(ctx, logger, state) + } if err != nil { - logger.Error(err, "failed to apply commitment state", + logger.Error(err, "failed to upsert committed resource CRD", "commitmentUUID", state.CommitmentUUID) - // Continue with other commitments even if one fails continue } - - totalCreated += applyResult.Created - totalDeleted += applyResult.Deleted - totalRepaired += applyResult.Repaired - } - - // Delete reservations that are no longer in commitments - // Only query committed resource reservations using labels for efficiency - var existingReservations v1alpha1.ReservationList - if err := s.List(ctx, &existingReservations, client.MatchingLabels{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }); err != nil { - logger.Error(err, "failed to list existing committed resource reservations") - return err + switch op { + case controllerutil.OperationResultCreated: + totalCreated++ + case controllerutil.OperationResultUpdated: + totalUpdated++ + } } // Build set of commitment UUIDs we should have (processed + skipped) @@ -286,51 +285,173 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { for _, state := range commitmentResult.states { activeCommitments[state.CommitmentUUID] = true } - // Also include skipped commitments - don't delete their CRDs for uuid := range commitmentResult.skippedUUIDs { activeCommitments[uuid] = true } - // Delete reservations for commitments that no longer exist - for _, existing := range existingReservations.Items { - // Extract commitment UUID from reservation name - commitmentUUID := extractCommitmentUUID(existing.Name) - if commitmentUUID == "" { - logger.Info("skipping reservation with unparseable name", "name", existing.Name) + // Count CommittedResource CRDs present locally but absent from Limes (do not delete — Limes + // responses may be transient and deleting active CRDs would drop Reservation slots). + // Also GC CRDs whose EndTime has passed: the commitment is over, the controller's finalizer + // will clean up child Reservations on deletion. + var existingCRs v1alpha1.CommittedResourceList + if err := s.List(ctx, &existingCRs); err != nil { + logger.Error(err, "failed to list existing committed resource CRDs") + if s.monitor != nil { + s.monitor.RecordError() + } + return err + } + staleCRCount, gcDeleted := 0, 0 + for i := range existingCRs.Items { + cr := &existingCRs.Items[i] + if cr.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { continue } + isExpired := cr.Spec.EndTime != nil && !cr.Spec.EndTime.After(time.Now()) + if !activeCommitments[cr.Spec.CommitmentUUID] && !isExpired { + staleCRCount++ + } + if isExpired { + if err := s.Delete(ctx, cr); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to GC expired committed resource CRD", "name", cr.Name) + return err + } + logger.Info("GC'd expired committed resource CRD", + "name", cr.Name, "endTime", cr.Spec.EndTime) + gcDeleted++ + } + } + // Delete orphaned Reservation CRDs: type=committed-resource but commitment no longer active. + // These are left over from the pre-refactor path where the syncer wrote Reservations directly. + var existingReservations v1alpha1.ReservationList + if err := s.List(ctx, &existingReservations, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + logger.Error(err, "failed to list committed resource reservations") + return err + } + var totalReservationDeleted int + for i := range existingReservations.Items { + res := &existingReservations.Items[i] + if res.Spec.CommittedResourceReservation == nil { + logger.Info("skipping reservation without committed resource spec", "name", res.Name) + continue + } + commitmentUUID := res.Spec.CommittedResourceReservation.CommitmentUUID + if commitmentUUID == "" { + logger.Info("skipping reservation with empty commitment UUID", "name", res.Name) + continue + } if !activeCommitments[commitmentUUID] { - // This commitment no longer exists, delete the reservation - if err := s.Delete(ctx, &existing); err != nil { - logger.Error(err, "failed to delete reservation", "name", existing.Name) + if err := s.Delete(ctx, res); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to delete orphaned reservation", "name", res.Name) return err } - logger.Info("deleted reservation for expired commitment", - "name", existing.Name, - "commitmentUUID", commitmentUUID) - totalDeleted++ + logger.Info("deleted orphaned reservation", "name", res.Name, "commitmentUUID", commitmentUUID) + totalReservationDeleted++ } } - // Record reservation change metrics if s.monitor != nil { + s.monitor.RecordStaleCRs(staleCRCount) if totalCreated > 0 { - s.monitor.RecordReservationsCreated(totalCreated) + s.monitor.RecordCRCreates(totalCreated) } - if totalDeleted > 0 { - s.monitor.RecordReservationsDeleted(totalDeleted) + if totalUpdated > 0 { + s.monitor.RecordCRUpdates(totalUpdated) } - if totalRepaired > 0 { - s.monitor.RecordReservationsRepaired(totalRepaired) + if gcDeleted > 0 { + s.monitor.RecordCRDeletes(gcDeleted) } } - logger.Info("synced reservations", + if staleCRCount > 0 { + logger.Info("WARNING: committed resource CRDs present locally but absent from Limes — review for manual cleanup", + "staleCRs", staleCRCount) + } + + logger.Info("synced committed resource CRDs", "processedCount", len(commitmentResult.states), "skippedCount", len(commitmentResult.skippedUUIDs), "created", totalCreated, - "deleted", totalDeleted, - "repaired", totalRepaired) + "updated", totalUpdated, + "staleCRs", staleCRCount, + "expiredCRsGCd", gcDeleted, + "orphanReservationsDeleted", totalReservationDeleted) return nil } + +func (s *Syncer) applyCommittedResourceSpec(cr *v1alpha1.CommittedResource, state *CommitmentState) { + cr.Spec.CommitmentUUID = state.CommitmentUUID + cr.Spec.SchedulingDomain = v1alpha1.SchedulingDomainNova + cr.Spec.FlavorGroupName = state.FlavorGroupName + cr.Spec.ResourceType = v1alpha1.CommittedResourceTypeMemory + cr.Spec.Amount = *resource.NewQuantity(state.TotalMemoryBytes, resource.BinarySI) + cr.Spec.AvailabilityZone = state.AvailabilityZone + cr.Spec.ProjectID = state.ProjectID + cr.Spec.DomainID = state.DomainID + cr.Spec.State = state.State + cr.Spec.AllowRejection = false + + if state.StartTime != nil { + t := metav1.NewTime(*state.StartTime) + cr.Spec.StartTime = &t + } else { + cr.Spec.StartTime = nil + } + if state.EndTime != nil { + t := metav1.NewTime(*state.EndTime) + cr.Spec.EndTime = &t + } else { + cr.Spec.EndTime = nil + } +} + +func (s *Syncer) upsertCommittedResource(ctx context.Context, logger logr.Logger, state *CommitmentState) (controllerutil.OperationResult, error) { + cr := &v1alpha1.CommittedResource{} + cr.Name = "commitment-" + state.CommitmentUUID + + op, err := controllerutil.CreateOrUpdate(ctx, s.Client, cr, func() error { + s.applyCommittedResourceSpec(cr, state) + return nil + }) + if err != nil { + return op, err + } + logger.V(1).Info("upserted committed resource CRD", "name", cr.Name, "op", op) + return op, nil +} + +// updateCommittedResourceIfExists updates an existing CommittedResource CRD but does not +// create one if it is absent. Used for terminal states (superseded/expired): we want the +// controller to see the state transition and clean up child Reservations, but there is no +// point creating a CRD for a commitment Cortex has never tracked. +func (s *Syncer) updateCommittedResourceIfExists(ctx context.Context, logger logr.Logger, state *CommitmentState) (controllerutil.OperationResult, error) { + cr := &v1alpha1.CommittedResource{} + name := "commitment-" + state.CommitmentUUID + if err := s.Get(ctx, client.ObjectKey{Name: name}, cr); err != nil { + if client.IgnoreNotFound(err) == nil { + logger.V(1).Info("skipping terminal state — CRD does not exist locally", + "commitmentUUID", state.CommitmentUUID, "state", state.State) + return controllerutil.OperationResultNone, nil + } + return controllerutil.OperationResultNone, err + } + s.applyCommittedResourceSpec(cr, state) + if err := s.Update(ctx, cr); err != nil { + return controllerutil.OperationResultNone, err + } + logger.V(1).Info("updated committed resource CRD (terminal state)", "name", name, "state", state.State) + return controllerutil.OperationResultUpdated, nil +} + +// isTerminalCommitment returns true when a commitment should not result in new Reservation +// slots: either its Limes state is already terminal, or its EndTime has passed. +func isTerminalCommitment(state *CommitmentState) bool { + switch state.State { + case v1alpha1.CommitmentStatusSuperseded, v1alpha1.CommitmentStatusExpired: + return true + } + return state.EndTime != nil && !state.EndTime.After(time.Now()) +} diff --git a/internal/scheduling/reservations/commitments/syncer_monitor.go b/internal/scheduling/reservations/commitments/syncer_monitor.go index 853518f81..7e13478bb 100644 --- a/internal/scheduling/reservations/commitments/syncer_monitor.go +++ b/internal/scheduling/reservations/commitments/syncer_monitor.go @@ -14,60 +14,55 @@ const ( SkipReasonInvalidResource = "invalid_resource_name" SkipReasonEmptyUUID = "empty_uuid" SkipReasonNonCompute = "non_compute" - SkipReasonNonActive = "non_active" ) // SyncerMonitor provides metrics for the commitment syncer. type SyncerMonitor struct { - // Sync lifecycle - syncRuns prometheus.Counter - syncErrors prometheus.Counter - - // Commitment processing - commitmentsTotal prometheus.Counter // all commitments seen from Limes - commitmentsProcessed prometheus.Counter // successfully processed - commitmentsSkipped *prometheus.CounterVec // skipped with reason label - - // Reservation changes - reservationsCreated prometheus.Counter - reservationsDeleted prometheus.Counter - reservationsRepaired prometheus.Counter + syncErrors prometheus.Counter + syncDuration prometheus.Histogram + limesCommitmentsActive prometheus.Gauge + staleCRs prometheus.Gauge + commitmentsSkipped *prometheus.CounterVec + crCreates prometheus.Counter + crUpdates prometheus.Counter + crDeletes prometheus.Counter } // NewSyncerMonitor creates a new monitor with Prometheus metrics. func NewSyncerMonitor() *SyncerMonitor { m := &SyncerMonitor{ - syncRuns: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_runs_total", - Help: "Total number of commitment syncer runs", - }), syncErrors: prometheus.NewCounter(prometheus.CounterOpts{ Name: "cortex_committed_resource_syncer_errors_total", - Help: "Total number of commitment syncer errors", + Help: "Total number of commitment syncer runs that failed", + }), + syncDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_committed_resource_syncer_duration_seconds", + Help: "Duration of each commitment syncer run", + Buckets: []float64{0.5, 1, 5, 10, 30, 60, 120}, }), - commitmentsTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_commitments_total", - Help: "Total number of commitments seen from Limes", + limesCommitmentsActive: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_syncer_limes_commitments_active", + Help: "Number of commitments from Limes that passed filtering and should have CR CRDs", }), - commitmentsProcessed: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_commitments_processed_total", - Help: "Total number of commitments successfully processed", + staleCRs: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_syncer_crd_unmatched", + Help: "Number of CommittedResource CRDs present locally but absent from Limes", }), commitmentsSkipped: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "cortex_committed_resource_syncer_commitments_skipped_total", Help: "Total number of commitments skipped during sync", }, []string{"reason"}), - reservationsCreated: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_reservations_created_total", - Help: "Total number of reservations created during sync", + crCreates: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_committed_resource_syncer_cr_creates_total", + Help: "Total number of CommittedResource CRDs created by the syncer", }), - reservationsDeleted: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_reservations_deleted_total", - Help: "Total number of reservations deleted during sync", + crUpdates: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_committed_resource_syncer_cr_updates_total", + Help: "Total number of CommittedResource CRDs updated by the syncer", }), - reservationsRepaired: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "cortex_committed_resource_syncer_reservations_repaired_total", - Help: "Total number of reservations repaired during sync (wrong metadata)", + crDeletes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_committed_resource_syncer_cr_deletes_total", + Help: "Total number of CommittedResource CRDs deleted by the syncer (expired GC)", }), } @@ -78,7 +73,6 @@ func NewSyncerMonitor() *SyncerMonitor { SkipReasonInvalidResource, SkipReasonEmptyUUID, SkipReasonNonCompute, - SkipReasonNonActive, } { m.commitmentsSkipped.WithLabelValues(reason) } @@ -86,66 +80,58 @@ func NewSyncerMonitor() *SyncerMonitor { return m } -// RecordSyncRun records a syncer run. -func (m *SyncerMonitor) RecordSyncRun() { - m.syncRuns.Inc() +func (m *SyncerMonitor) RecordError() { + m.syncErrors.Inc() } -// RecordSyncError records a syncer error. -func (m *SyncerMonitor) RecordSyncError() { - m.syncErrors.Inc() +func (m *SyncerMonitor) RecordDuration(seconds float64) { + m.syncDuration.Observe(seconds) } -// RecordCommitmentSeen records a commitment seen from Limes. -func (m *SyncerMonitor) RecordCommitmentSeen() { - m.commitmentsTotal.Inc() +func (m *SyncerMonitor) SetLimesCommitmentsActive(count int) { + m.limesCommitmentsActive.Set(float64(count)) } -// RecordCommitmentProcessed records a commitment successfully processed. -func (m *SyncerMonitor) RecordCommitmentProcessed() { - m.commitmentsProcessed.Inc() +func (m *SyncerMonitor) RecordStaleCRs(count int) { + m.staleCRs.Set(float64(count)) } -// RecordCommitmentSkipped records a commitment skipped with a reason. func (m *SyncerMonitor) RecordCommitmentSkipped(reason string) { m.commitmentsSkipped.WithLabelValues(reason).Inc() } -// RecordReservationsCreated records reservations created. -func (m *SyncerMonitor) RecordReservationsCreated(count int) { - m.reservationsCreated.Add(float64(count)) +func (m *SyncerMonitor) RecordCRCreates(count int) { + m.crCreates.Add(float64(count)) } -// RecordReservationsDeleted records reservations deleted. -func (m *SyncerMonitor) RecordReservationsDeleted(count int) { - m.reservationsDeleted.Add(float64(count)) +func (m *SyncerMonitor) RecordCRUpdates(count int) { + m.crUpdates.Add(float64(count)) } -// RecordReservationsRepaired records reservations repaired. -func (m *SyncerMonitor) RecordReservationsRepaired(count int) { - m.reservationsRepaired.Add(float64(count)) +func (m *SyncerMonitor) RecordCRDeletes(count int) { + m.crDeletes.Add(float64(count)) } // Describe implements prometheus.Collector. func (m *SyncerMonitor) Describe(ch chan<- *prometheus.Desc) { - m.syncRuns.Describe(ch) m.syncErrors.Describe(ch) - m.commitmentsTotal.Describe(ch) - m.commitmentsProcessed.Describe(ch) + m.syncDuration.Describe(ch) + m.limesCommitmentsActive.Describe(ch) + m.staleCRs.Describe(ch) m.commitmentsSkipped.Describe(ch) - m.reservationsCreated.Describe(ch) - m.reservationsDeleted.Describe(ch) - m.reservationsRepaired.Describe(ch) + m.crCreates.Describe(ch) + m.crUpdates.Describe(ch) + m.crDeletes.Describe(ch) } // Collect implements prometheus.Collector. func (m *SyncerMonitor) Collect(ch chan<- prometheus.Metric) { - m.syncRuns.Collect(ch) m.syncErrors.Collect(ch) - m.commitmentsTotal.Collect(ch) - m.commitmentsProcessed.Collect(ch) + m.syncDuration.Collect(ch) + m.limesCommitmentsActive.Collect(ch) + m.staleCRs.Collect(ch) m.commitmentsSkipped.Collect(ch) - m.reservationsCreated.Collect(ch) - m.reservationsDeleted.Collect(ch) - m.reservationsRepaired.Collect(ch) + m.crCreates.Collect(ch) + m.crUpdates.Collect(ch) + m.crDeletes.Collect(ch) } diff --git a/internal/scheduling/reservations/commitments/syncer_monitor_test.go b/internal/scheduling/reservations/commitments/syncer_monitor_test.go index 853524a70..d973a95e9 100644 --- a/internal/scheduling/reservations/commitments/syncer_monitor_test.go +++ b/internal/scheduling/reservations/commitments/syncer_monitor_test.go @@ -36,14 +36,12 @@ func TestSyncerMonitor_MetricsRegistration(t *testing.T) { name string metricType dto.MetricType }{ - {"cortex_committed_resource_syncer_runs_total", dto.MetricType_COUNTER}, {"cortex_committed_resource_syncer_errors_total", dto.MetricType_COUNTER}, - {"cortex_committed_resource_syncer_commitments_total", dto.MetricType_COUNTER}, - {"cortex_committed_resource_syncer_commitments_processed_total", dto.MetricType_COUNTER}, {"cortex_committed_resource_syncer_commitments_skipped_total", dto.MetricType_COUNTER}, - {"cortex_committed_resource_syncer_reservations_created_total", dto.MetricType_COUNTER}, - {"cortex_committed_resource_syncer_reservations_deleted_total", dto.MetricType_COUNTER}, - {"cortex_committed_resource_syncer_reservations_repaired_total", dto.MetricType_COUNTER}, + {"cortex_committed_resource_syncer_cr_creates_total", dto.MetricType_COUNTER}, + {"cortex_committed_resource_syncer_cr_updates_total", dto.MetricType_COUNTER}, + {"cortex_committed_resource_syncer_cr_deletes_total", dto.MetricType_COUNTER}, + {"cortex_committed_resource_syncer_crd_unmatched", dto.MetricType_GAUGE}, } for _, tc := range cases { @@ -100,7 +98,6 @@ func TestSyncerMonitor_SkipReasonsPreInitialized(t *testing.T) { SkipReasonInvalidResource, SkipReasonEmptyUUID, SkipReasonNonCompute, - SkipReasonNonActive, } { if !presentReasons[reason] { t.Errorf("skip reason %q not pre-initialized in commitments_skipped_total", reason) diff --git a/internal/scheduling/reservations/commitments/syncer_test.go b/internal/scheduling/reservations/commitments/syncer_test.go index e4bf6e841..28a464d1e 100644 --- a/internal/scheduling/reservations/commitments/syncer_test.go +++ b/internal/scheduling/reservations/commitments/syncer_test.go @@ -7,14 +7,15 @@ import ( "context" "sort" "testing" + "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -273,44 +274,31 @@ func TestSyncer_SyncReservations_InstanceCommitments(t *testing.T) { return } - // Verify that reservations were created - var reservations v1alpha1.ReservationList - err = k8sClient.List(context.Background(), &reservations) - if err != nil { - t.Errorf("Failed to list reservations: %v", err) - return + // Verify one CommittedResource CRD was created with the correct spec + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - - // Should have 2 reservations (Amount = 2, each for smallest flavor) - if len(reservations.Items) != 2 { - t.Errorf("Expected 2 reservations, got %d", len(reservations.Items)) - return + if len(crList.Items) != 1 { + t.Fatalf("Expected 1 CommittedResource, got %d", len(crList.Items)) } - - // Verify the first reservation - res := reservations.Items[0] - if res.Spec.CommittedResourceReservation == nil { - t.Errorf("Expected CommittedResourceReservation to be set") - return + cr := crList.Items[0] + if cr.Name != "commitment-12345-67890-abcdef" { + t.Errorf("Expected name commitment-12345-67890-abcdef, got %s", cr.Name) } - if res.Spec.CommittedResourceReservation.ProjectID != "test-project-1" { - t.Errorf("Expected project ID test-project-1, got %v", res.Spec.CommittedResourceReservation.ProjectID) + if cr.Spec.ProjectID != "test-project-1" { + t.Errorf("Expected projectID test-project-1, got %s", cr.Spec.ProjectID) } - - if res.Spec.CommittedResourceReservation.ResourceGroup != "test_group_v1" { - t.Errorf("Expected resource group test_group_v1, got %v", res.Spec.CommittedResourceReservation.ResourceGroup) + if cr.Spec.FlavorGroupName != "test_group_v1" { + t.Errorf("Expected flavorGroupName test_group_v1, got %s", cr.Spec.FlavorGroupName) } - - // Check resource values - should be sized for the flavor that fits - // With 2048MB total capacity, we can fit 2x 1024MB flavors - expectedMemory := resource.MustParse("1073741824") // 1024MB in bytes - if !res.Spec.Resources[hv1.ResourceMemory].Equal(expectedMemory) { - t.Errorf("Expected memory %v, got %v", expectedMemory, res.Spec.Resources[hv1.ResourceMemory]) + if cr.Spec.State != v1alpha1.CommitmentStatusConfirmed { + t.Errorf("Expected state confirmed, got %s", cr.Spec.State) } - - expectedVCPUs := resource.MustParse("2") - if !res.Spec.Resources[hv1.ResourceCPU].Equal(expectedVCPUs) { - t.Errorf("Expected vCPUs %v, got %v", expectedVCPUs, res.Spec.Resources[hv1.ResourceCPU]) + // Amount = 2 slots × 1024 MiB = 2 GiB + expectedAmount := resource.NewQuantity(2*1024*1024*1024, resource.BinarySI) + if !cr.Spec.Amount.Equal(*expectedAmount) { + t.Errorf("Expected amount %v, got %v", expectedAmount, cr.Spec.Amount) } } @@ -320,7 +308,6 @@ func TestSyncer_SyncReservations_UpdateExisting(t *testing.T) { t.Fatalf("Failed to add scheme: %v", err) } - // Create flavor group knowledge CRD flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ "new_group_v1": { LargestFlavorName: "new-flavor", @@ -332,36 +319,26 @@ func TestSyncer_SyncReservations_UpdateExisting(t *testing.T) { }, }) - // Create an existing reservation with mismatched project/flavor group - // The ReservationManager will delete this and create a new one - existingReservation := &v1alpha1.Reservation{ - ObjectMeta: ctrl.ObjectMeta{ - Name: "commitment-12345-67890-abcdef-0", - Labels: map[string]string{ - v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, - }, - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ProjectID: "old-project", - ResourceName: "old-flavor", - ResourceGroup: "old_group", - Creator: CreatorValue, - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("512Mi"), - hv1.ResourceCPU: resource.MustParse("1"), - }, + // Pre-existing CommittedResource CRD with stale spec; syncer should update it. + existingCR := &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-12345-67890-abcdef"}, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "12345-67890-abcdef", + FlavorGroupName: "old_group", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: *resource.NewQuantity(512*1024*1024, resource.BinarySI), + ProjectID: "old-project", + DomainID: "old-domain", + AvailabilityZone: "az1", + State: v1alpha1.CommitmentStatusConfirmed, }, } k8sClient := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(existingReservation, flavorGroupsKnowledge). + WithObjects(existingCR, flavorGroupsKnowledge). Build() - // Create mock commitment that will replace the existing reservation mockCommitments := []Commitment{ { ID: 1, @@ -386,57 +363,29 @@ func TestSyncer_SyncReservations_UpdateExisting(t *testing.T) { return result, nil }, listProjectsFunc: func(ctx context.Context) ([]Project, error) { - return []Project{ - {ID: "new-project", DomainID: "new-domain", Name: "New Project"}, - }, nil - }, - listServersFunc: func(ctx context.Context, projects ...Project) (map[string][]Server, error) { - return map[string][]Server{}, nil // No active servers - }, - initFunc: func(ctx context.Context, client client.Client, conf SyncerConfig) error { - // No-op for init - return nil + return []Project{{ID: "new-project", DomainID: "new-domain"}}, nil }, } - syncer := &Syncer{ - CommitmentsClient: mockClient, - Client: k8sClient, - } - - err := syncer.SyncReservations(context.Background()) - if err != nil { - t.Errorf("SyncReservations() error = %v", err) - return - } + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient} - // Verify that reservations were updated (old one deleted, new one created) - // The new reservation will be at index 0 since the old one was deleted first - var reservations v1alpha1.ReservationList - err = k8sClient.List(context.Background(), &reservations) - if err != nil { - t.Errorf("Failed to list reservations: %v", err) - return + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) } - if len(reservations.Items) != 1 { - t.Errorf("Expected 1 reservation, got %d", len(reservations.Items)) - return + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - - newReservation := reservations.Items[0] - - // Verify the new reservation has correct values - if newReservation.Spec.CommittedResourceReservation == nil { - t.Errorf("Expected CommittedResourceReservation to be set") - return + if len(crList.Items) != 1 { + t.Fatalf("Expected 1 CommittedResource, got %d", len(crList.Items)) } - if newReservation.Spec.CommittedResourceReservation.ProjectID != "new-project" { - t.Errorf("Expected project ID new-project, got %v", newReservation.Spec.CommittedResourceReservation.ProjectID) + cr := crList.Items[0] + if cr.Spec.ProjectID != "new-project" { + t.Errorf("Expected projectID new-project, got %s", cr.Spec.ProjectID) } - - if newReservation.Spec.CommittedResourceReservation.ResourceGroup != "new_group_v1" { - t.Errorf("Expected resource group new_group_v1, got %v", newReservation.Spec.CommittedResourceReservation.ResourceGroup) + if cr.Spec.FlavorGroupName != "new_group_v1" { + t.Errorf("Expected flavorGroupName new_group_v1, got %s", cr.Spec.FlavorGroupName) } } @@ -510,19 +459,14 @@ func TestSyncer_SyncReservations_UnitMismatch(t *testing.T) { return } - // Verify that NO reservations were created due to unit mismatch - // The commitment is skipped and Cortex trusts existing CRDs - var reservations v1alpha1.ReservationList - err = k8sClient.List(context.Background(), &reservations) - if err != nil { - t.Errorf("Failed to list reservations: %v", err) - return + // Verify that NO CommittedResource CRDs were created due to unit mismatch. + // The commitment is skipped and Cortex trusts existing CRDs. + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - - // Should have 0 reservations - commitment is skipped due to unit mismatch - // Cortex waits for Limes to update the unit before processing - if len(reservations.Items) != 0 { - t.Errorf("Expected 0 reservations (commitment skipped due to unit mismatch), got %d", len(reservations.Items)) + if len(crList.Items) != 0 { + t.Errorf("Expected 0 CommittedResource CRDs (commitment skipped due to unit mismatch), got %d", len(crList.Items)) } } @@ -594,16 +538,13 @@ func TestSyncer_SyncReservations_UnitMatch(t *testing.T) { return } - // Verify that reservations were created - var reservations v1alpha1.ReservationList - err = k8sClient.List(context.Background(), &reservations) - if err != nil { - t.Errorf("Failed to list reservations: %v", err) - return + // Verify that one CommittedResource CRD was created + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - - if len(reservations.Items) != 2 { - t.Errorf("Expected 2 reservations, got %d", len(reservations.Items)) + if len(crList.Items) != 1 { + t.Errorf("Expected 1 CommittedResource CRD, got %d", len(crList.Items)) } } @@ -679,16 +620,13 @@ func TestSyncer_SyncReservations_EmptyUUID(t *testing.T) { return } - // Verify that no reservations were created due to empty UUID - var reservations v1alpha1.ReservationList - err = k8sClient.List(context.Background(), &reservations) - if err != nil { - t.Errorf("Failed to list reservations: %v", err) - return + // Verify that no CommittedResource CRDs were created due to empty UUID + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - - if len(reservations.Items) != 0 { - t.Errorf("Expected 0 reservations due to empty UUID, got %d", len(reservations.Items)) + if len(crList.Items) != 0 { + t.Errorf("Expected 0 CommittedResource CRDs due to empty UUID, got %d", len(crList.Items)) } } @@ -710,16 +648,16 @@ func TestSyncer_SyncReservations_StatusFilter(t *testing.T) { }) tests := []struct { - name string - status string - expectReservation bool + name string + status string + expectCR bool }{ - {"confirmed is processed", "confirmed", true}, - {"guaranteed is processed", "guaranteed", true}, - {"planned is skipped", "planned", false}, - {"pending is skipped", "pending", false}, - {"superseded is skipped", "superseded", false}, - {"expired is skipped", "expired", false}, + {"confirmed creates CR", "confirmed", true}, + {"guaranteed creates CR", "guaranteed", true}, + {"planned creates CR", "planned", true}, + {"pending creates CR", "pending", true}, + {"superseded does not create CR", "superseded", false}, + {"expired does not create CR", "expired", false}, {"empty status is skipped", "", false}, } @@ -768,17 +706,288 @@ func TestSyncer_SyncReservations_StatusFilter(t *testing.T) { t.Fatalf("SyncReservations() error = %v", err) } - var reservations v1alpha1.ReservationList - if err := k8sClient.List(context.Background(), &reservations); err != nil { - t.Fatalf("Failed to list reservations: %v", err) + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) } - if tc.expectReservation && len(reservations.Items) == 0 { - t.Errorf("status=%q: expected reservation to be created, got none", tc.status) + if tc.expectCR && len(crList.Items) == 0 { + t.Errorf("status=%q: expected CommittedResource CRD to be created, got none", tc.status) } - if !tc.expectReservation && len(reservations.Items) != 0 { - t.Errorf("status=%q: expected no reservation, got %d", tc.status, len(reservations.Items)) + if !tc.expectCR && len(crList.Items) != 0 { + t.Errorf("status=%q: expected no CommittedResource CRD, got %d", tc.status, len(crList.Items)) } }) } } + +func TestSyncer_SyncReservations_StaleCRCount(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ + "test_group_v1": { + LargestFlavorName: "test-flavor", + LargestFlavorVCPUs: 2, + LargestFlavorMemoryMB: 1024, + SmallestFlavorName: "test-flavor", + SmallestFlavorVCPUs: 2, + SmallestFlavorMemoryMB: 1024, + }, + }) + + // Pre-existing CRD whose commitment no longer appears in Limes + staleCR := &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-stale-uuid-1234"}, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "stale-uuid-1234", + FlavorGroupName: "test_group_v1", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: *resource.NewQuantity(1024*1024*1024, resource.BinarySI), + ProjectID: "test-project", + DomainID: "test-domain", + AvailabilityZone: "az1", + State: v1alpha1.CommitmentStatusConfirmed, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + }, + } + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(staleCR, flavorGroupsKnowledge). + Build() + + // Limes returns no commitments (stale-uuid-1234 is gone) + mockClient := &mockCommitmentsClient{ + listCommitmentsByIDFunc: func(ctx context.Context, projects ...Project) (map[string]Commitment, error) { + return map[string]Commitment{}, nil + }, + listProjectsFunc: func(ctx context.Context) ([]Project, error) { + return []Project{{ID: "test-project", DomainID: "test-domain"}}, nil + }, + } + + monitor := NewSyncerMonitor() + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient, monitor: monitor} + + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) + } + + // Stale CRD must still exist (syncer does not delete) + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list committed resources: %v", err) + } + if len(crList.Items) != 1 { + t.Errorf("Expected stale CRD to be preserved, got %d CRDs", len(crList.Items)) + } + + // Gauge must reflect the stale count + ch := make(chan prometheus.Metric, 10) + monitor.staleCRs.Collect(ch) + close(ch) + m := <-ch + var dto dto.Metric + if err := m.Write(&dto); err != nil { + t.Fatalf("failed to read metric: %v", err) + } + if got := dto.GetGauge().GetValue(); got != 1 { + t.Errorf("Expected staleCRs gauge=1, got %v", got) + } +} + +func TestSyncer_SyncReservations_TerminalState_NoCRDExists(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ + "test_group_v1": {SmallestFlavorName: "f", SmallestFlavorVCPUs: 2, SmallestFlavorMemoryMB: 1024, + LargestFlavorName: "f", LargestFlavorVCPUs: 2, LargestFlavorMemoryMB: 1024}, + }) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(flavorGroupsKnowledge).Build() + + for _, status := range []string{"superseded", "expired"} { + t.Run(status, func(t *testing.T) { + mockClient := &mockCommitmentsClient{ + listCommitmentsByIDFunc: func(ctx context.Context, projects ...Project) (map[string]Commitment, error) { + return map[string]Commitment{ + "term-uuid-1234": { + ID: 1, UUID: "term-uuid-1234", ServiceType: "compute", + ResourceName: "hw_version_test_group_v1_ram", AvailabilityZone: "az1", + Amount: 1, Status: status, ProjectID: "p", DomainID: "d", + }, + }, nil + }, + listProjectsFunc: func(ctx context.Context) ([]Project, error) { + return []Project{{ID: "p", DomainID: "d"}}, nil + }, + } + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient} + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) + } + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list: %v", err) + } + if len(crList.Items) != 0 { + t.Errorf("status=%q: expected no CRD to be created, got %d", status, len(crList.Items)) + } + }) + } +} + +func TestSyncer_SyncReservations_TerminalState_ExistingCRDUpdated(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ + "test_group_v1": {SmallestFlavorName: "f", SmallestFlavorVCPUs: 2, SmallestFlavorMemoryMB: 1024, + LargestFlavorName: "f", LargestFlavorVCPUs: 2, LargestFlavorMemoryMB: 1024}, + }) + + existingCR := &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-term-uuid-1234"}, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "term-uuid-1234", FlavorGroupName: "test_group_v1", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: *resource.NewQuantity(1024*1024*1024, resource.BinarySI), + ProjectID: "p", DomainID: "d", AvailabilityZone: "az1", + State: v1alpha1.CommitmentStatusConfirmed, + }, + } + + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCR, flavorGroupsKnowledge).Build() + + mockClient := &mockCommitmentsClient{ + listCommitmentsByIDFunc: func(ctx context.Context, projects ...Project) (map[string]Commitment, error) { + return map[string]Commitment{ + "term-uuid-1234": { + ID: 1, UUID: "term-uuid-1234", ServiceType: "compute", + ResourceName: "hw_version_test_group_v1_ram", AvailabilityZone: "az1", + Amount: 1, Status: "superseded", ProjectID: "p", DomainID: "d", + }, + }, nil + }, + listProjectsFunc: func(ctx context.Context) ([]Project, error) { + return []Project{{ID: "p", DomainID: "d"}}, nil + }, + } + + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient} + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) + } + + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list: %v", err) + } + if len(crList.Items) != 1 { + t.Fatalf("Expected CRD to be preserved, got %d", len(crList.Items)) + } + if crList.Items[0].Spec.State != v1alpha1.CommitmentStatusSuperseded { + t.Errorf("Expected state superseded, got %s", crList.Items[0].Spec.State) + } +} + +func TestSyncer_SyncReservations_ExpiredByTime_NoCRDCreated(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ + "test_group_v1": {SmallestFlavorName: "f", SmallestFlavorVCPUs: 2, SmallestFlavorMemoryMB: 1024, + LargestFlavorName: "f", LargestFlavorVCPUs: 2, LargestFlavorMemoryMB: 1024}, + }) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(flavorGroupsKnowledge).Build() + + pastTime := uint64(1) // Unix epoch — well in the past + mockClient := &mockCommitmentsClient{ + listCommitmentsByIDFunc: func(ctx context.Context, projects ...Project) (map[string]Commitment, error) { + return map[string]Commitment{ + "exp-uuid-1234": { + ID: 1, UUID: "exp-uuid-1234", ServiceType: "compute", + ResourceName: "hw_version_test_group_v1_ram", AvailabilityZone: "az1", + Amount: 1, Status: "confirmed", ExpiresAt: pastTime, + ProjectID: "p", DomainID: "d", + }, + }, nil + }, + listProjectsFunc: func(ctx context.Context) ([]Project, error) { + return []Project{{ID: "p", DomainID: "d"}}, nil + }, + } + + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient} + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) + } + + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list: %v", err) + } + if len(crList.Items) != 0 { + t.Errorf("Expected no CRD created for past-expiry confirmed commitment, got %d", len(crList.Items)) + } +} + +func TestSyncer_SyncReservations_GC_ExpiredEndTime(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + flavorGroupsKnowledge := createFlavorGroupKnowledge(t, map[string]FlavorGroupData{ + "test_group_v1": {SmallestFlavorName: "f", SmallestFlavorVCPUs: 2, SmallestFlavorMemoryMB: 1024, + LargestFlavorName: "f", LargestFlavorVCPUs: 2, LargestFlavorMemoryMB: 1024}, + }) + + pastTime := metav1.NewTime(time.Now().Add(-time.Hour)) + expiredCR := &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-gc-uuid-1234"}, + Spec: v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "gc-uuid-1234", FlavorGroupName: "test_group_v1", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: *resource.NewQuantity(1024*1024*1024, resource.BinarySI), + ProjectID: "p", DomainID: "d", AvailabilityZone: "az1", + State: v1alpha1.CommitmentStatusConfirmed, + EndTime: &pastTime, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + }, + } + + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(expiredCR, flavorGroupsKnowledge).Build() + + // Limes no longer returns this commitment + mockClient := &mockCommitmentsClient{ + listCommitmentsByIDFunc: func(ctx context.Context, projects ...Project) (map[string]Commitment, error) { + return map[string]Commitment{}, nil + }, + listProjectsFunc: func(ctx context.Context) ([]Project, error) { + return []Project{{ID: "p", DomainID: "d"}}, nil + }, + } + + syncer := &Syncer{CommitmentsClient: mockClient, Client: k8sClient} + if err := syncer.SyncReservations(context.Background()); err != nil { + t.Fatalf("SyncReservations() error = %v", err) + } + + var crList v1alpha1.CommittedResourceList + if err := k8sClient.List(context.Background(), &crList); err != nil { + t.Fatalf("Failed to list: %v", err) + } + if len(crList.Items) != 0 { + t.Errorf("Expected expired CRD to be GC'd, got %d CRDs", len(crList.Items)) + } +} diff --git a/internal/scheduling/reservations/failover/config.go b/internal/scheduling/reservations/failover/config.go index 79dc94480..b8bde49a6 100644 --- a/internal/scheduling/reservations/failover/config.go +++ b/internal/scheduling/reservations/failover/config.go @@ -77,6 +77,15 @@ type FailoverConfig struct { // rotates to process different VMs. This ensures all VMs eventually get processed. // Default: 4 (rotate every 4th reconcile cycle). Use 0 to disable rotation. VMSelectionRotationInterval *int `json:"vmSelectionRotationInterval"` + + // UseFlavorGroupResources when true, sizes failover reservation resources based on + // the LargestFlavor in the VM's flavor group instead of the VM's actual resources. + // This enables better sharing: a single reservation can accommodate any flavor in the + // group since it's sized for the largest one. + // When false (or when the flavor group lookup fails), falls back to using the VM's + // own reported resources (memory + vcpus). + // Default: false + UseFlavorGroupResources bool `json:"useFlavorGroupResources"` } // intPtr returns a pointer to the given int value. diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index 297ed09cc..c66295293 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -13,6 +13,7 @@ import ( "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -319,8 +320,22 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) ( } summary.reservationsDeleted = len(emptyReservationsToDelete) - // 6. Create and assign reservations for VMs that need them - assignSummary, hitMaxVMsLimit := c.reconcileCreateAndAssignReservations(ctx, vms, failoverReservations, allHypervisors) + // 6. Fetch flavor groups for reservation sizing (if configured) + var flavorGroups map[string]compute.FlavorGroupFeature + if c.Config.UseFlavorGroupResources { + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.Client} + fg, err := knowledge.GetAllFlavorGroups(ctx, nil) + if err != nil { + logger.Info("flavor group knowledge not available, will fall back to VM resources for sizing", + "error", err) + // flavorGroups remains nil — newFailoverReservation will fall back to VM resources + } else { + flavorGroups = fg + } + } + + // 7. Create and assign reservations for VMs that need them + assignSummary, hitMaxVMsLimit := c.reconcileCreateAndAssignReservations(ctx, vms, failoverReservations, allHypervisors, flavorGroups) summary.vmsMissingFailover = assignSummary.vmsMissingFailover summary.vmsProcessed = assignSummary.vmsProcessed summary.reservationsNeeded = assignSummary.reservationsNeeded @@ -572,6 +587,7 @@ func (c *FailoverReservationController) reconcileCreateAndAssignReservations( vms []VM, failoverReservations []v1alpha1.Reservation, allHypervisors []string, + flavorGroups map[string]compute.FlavorGroupFeature, // passed to resolveVMForScheduling per-VM ) (reconcileSummary, bool) { logger := LoggerFromContext(ctx) @@ -604,8 +620,11 @@ func (c *FailoverReservationController) reconcileCreateAndAssignReservations( vmLogger := LoggerFromContext(vmCtx).WithValues("vmUUID", need.VM.UUID) vmLogger.Info("processing VM for failover reservation") + // Resolve VM resources once per VM (may use LargestFlavor from flavor group) + resSpec := resolveVMSpecForScheduling(vmCtx, need.VM, c.Config.UseFlavorGroupResources, flavorGroups) + for i := range need.Count { - reusedRes := c.tryReuseExistingReservation(vmCtx, need.VM, failoverReservations, allHypervisors) + reusedRes := c.tryReuseExistingReservation(vmCtx, need.VM, failoverReservations, allHypervisors, resSpec) if reusedRes != nil { if err := c.patchReservationStatus(vmCtx, reusedRes); err != nil { @@ -628,7 +647,7 @@ func (c *FailoverReservationController) reconcileCreateAndAssignReservations( continue } - newRes, err := c.scheduleAndBuildNewFailoverReservation(vmCtx, need.VM, allHypervisors, failoverReservations, excludeHypervisors) + newRes, err := c.scheduleAndBuildNewFailoverReservation(vmCtx, need.VM, allHypervisors, failoverReservations, excludeHypervisors, resSpec) if err != nil { vmLogger.V(1).Info("failed to schedule failover reservation", "error", err, "iteration", i+1, "needed", need.Count) vmFailed++ diff --git a/internal/scheduling/reservations/failover/helpers.go b/internal/scheduling/reservations/failover/helpers.go index 0ec75456c..8623f117f 100644 --- a/internal/scheduling/reservations/failover/helpers.go +++ b/internal/scheduling/reservations/failover/helpers.go @@ -8,11 +8,91 @@ import ( "fmt" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// resolvedReservationSpec holds the resolved resource spec for scheduling and reservation sizing. +// When UseFlavorGroupResources is enabled and the VM's flavor is found in a group, +// resources are sized to the LargestFlavor. Otherwise, they come from the VM directly. +type resolvedReservationSpec struct { + FlavorName string // may be overridden to LargestFlavor.Name + FlavorGroupName string // flavor group name if found, empty otherwise + MemoryMB uint64 + VCPUs uint64 +} + +// ResourceGroup returns the flavor group name if available, otherwise falls back to the provided fallback value. +func (r resolvedReservationSpec) ResourceGroup(fallback string) string { + if r.FlavorGroupName != "" { + return r.FlavorGroupName + } + return fallback +} + +// HypervisorResources returns the reservation spec resources as a map suitable for the reservation CRD. +// We use "cpu" (not "vcpus") as the canonical key because the scheduling capacity logic +// (e.g., nova filter_has_enough_capacity) uses "cpu". +func (r resolvedReservationSpec) HypervisorResources() map[hv1.ResourceName]resource.Quantity { + return map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(int64(r.MemoryMB)*1024*1024, resource.BinarySI), //nolint:gosec // flavor memory (MiB) from specs, realistically bounded; use binary to match commitments/state.go and vm_source.go + hv1.ResourceCPU: *resource.NewQuantity(int64(r.VCPUs), resource.DecimalSI), //nolint:gosec // flavor vcpus from specs, realistically bounded + } +} + +// resolveVMSpecForScheduling resolves the VM's resources for scheduling. +// When useFlavorGroupResources is true and the flavor is found in a group, +// returns the LargestFlavor's name and size. Otherwise falls back to VM resources. +func resolveVMSpecForScheduling( + ctx context.Context, + vm VM, + useFlavorGroupResources bool, + flavorGroups map[string]compute.FlavorGroupFeature, +) resolvedReservationSpec { + + logger := LoggerFromContext(ctx) + + if useFlavorGroupResources && flavorGroups != nil { + groupName, _, err := reservations.FindFlavorInGroups(vm.FlavorName, flavorGroups) + if err == nil { + fg := flavorGroups[groupName] + largest := fg.LargestFlavor + logger.V(1).Info("resolved VM resources from flavor group LargestFlavor", + "vmFlavor", vm.FlavorName, + "flavorGroup", groupName, + "largestFlavor", largest.Name, + "memoryMB", largest.MemoryMB, + "vcpus", largest.VCPUs) + return resolvedReservationSpec{ + FlavorName: largest.Name, + FlavorGroupName: groupName, + MemoryMB: largest.MemoryMB, + VCPUs: largest.VCPUs, + } + } + logger.V(1).Info("flavor group lookup failed, falling back to VM resources", + "vmFlavor", vm.FlavorName, + "error", err) + } + + // Fallback: use VM's own resources + var memoryMB, vcpus uint64 + if memory, ok := vm.Resources["memory"]; ok { + memoryMB = uint64(memory.Value() / (1024 * 1024)) //nolint:gosec // memory values won't overflow; binary MiB matches commitments/state.go and vm_source.go + } + if v, ok := vm.Resources["vcpus"]; ok { + vcpus = uint64(v.Value()) //nolint:gosec // vcpus values won't overflow + } + return resolvedReservationSpec{ + FlavorName: vm.FlavorName, + MemoryMB: memoryMB, + VCPUs: vcpus, + } +} + // getFailoverAllocations safely returns the allocations map from a failover reservation. // Returns an empty map if the reservation has no failover status or allocations. func getFailoverAllocations(res *v1alpha1.Reservation) map[string]string { @@ -90,23 +170,20 @@ func ValidateFailoverReservationResources(res *v1alpha1.Reservation) error { // newFailoverReservation creates a new failover reservation for a VM on a specific hypervisor. // This does NOT persist the reservation to the cluster - it only creates the in-memory object. // The caller is responsible for persisting the reservation. -func newFailoverReservation(ctx context.Context, vm VM, hypervisor, creator string) *v1alpha1.Reservation { - logger := LoggerFromContext(ctx) +// +// The resolved parameter contains the pre-computed resources (from resolveVMForScheduling), +// which may come from the VM's flavor group LargestFlavor or from the VM's own resources. +// This ensures the same sizing is used for both the scheduler query and the reservation CRD. +func newFailoverReservation( + ctx context.Context, + vm VM, + hypervisor, creator string, + resSpec resolvedReservationSpec, +) *v1alpha1.Reservation { - // Build resources from VM's Resources map - // The VM struct uses "vcpus" and "memory" keys (see vm_source.go) - // We convert "vcpus" to "cpu" for the reservation because the scheduling capacity logic - // (e.g., nova filter_has_enough_capacity) uses "cpu" as the canonical key. + logger := LoggerFromContext(ctx) - // TODO we may want to use different resource (bigger) to enable better sharing - resources := make(map[hv1.ResourceName]resource.Quantity) - if memory, ok := vm.Resources["memory"]; ok { - resources["memory"] = memory - } - if vcpus, ok := vm.Resources["vcpus"]; ok { - // todo check if that is correct, i.e. that the cpu reported on e.g. hypervisors is vcpu and not pcpu - resources["cpu"] = vcpus - } + resources := resSpec.HypervisorResources() reservation := &v1alpha1.Reservation{ ObjectMeta: metav1.ObjectMeta{ @@ -123,7 +200,7 @@ func newFailoverReservation(ctx context.Context, vm VM, hypervisor, creator stri Resources: resources, TargetHost: hypervisor, // Set the desired hypervisor from scheduler response FailoverReservation: &v1alpha1.FailoverReservationSpec{ - ResourceGroup: vm.FlavorName, + ResourceGroup: resSpec.ResourceGroup(vm.FlavorName), }, }, } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 6859d2cc4..f482f3393 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -30,7 +30,7 @@ const ( PipelineAcknowledgeFailoverReservation = "kvm-acknowledge-failover-reservation" ) -func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm VM, allHypervisors []string, pipeline string) ([]string, error) { +func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec) ([]string, error) { logger := LoggerFromContext(ctx) // Build list of eligible hypervisors (excluding VM's current hypervisor) @@ -52,18 +52,6 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex ignoreHypervisors := []string{vm.CurrentHypervisor} - // Get memory and vcpus from VM resources - // The VM struct uses "vcpus" and "memory" keys (see vm_source.go) - var memoryMB uint64 - var vcpus uint64 - if memory, ok := vm.Resources["memory"]; ok { - // Convert from bytes to MB - memoryMB = uint64(memory.Value() / (1024 * 1024)) //nolint:gosec // memory values won't overflow - } - if vcpusRes, ok := vm.Resources["vcpus"]; ok { - vcpus = uint64(vcpusRes.Value()) //nolint:gosec // vcpus values won't overflow - } - // Build flavor extra specs from VM's extra specs // Start with the VM's actual extra specs, then ensure required defaults are set flavorExtraSpecs := make(map[string]string) @@ -78,18 +66,23 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex // Schedule the reservation using the SchedulerClient. // Note: We pass all hypervisors (from all AZs) in EligibleHosts. The scheduler pipeline's // filter_correct_az filter will exclude hosts that are not in the VM's availability zone. + // Use resSpec.FlavorName and reservation spec resources so the scheduler checks capacity for the + // correct flavor size (which may be the LargestFlavor from the flavor group). scheduleReq := reservations.ScheduleReservationRequest{ InstanceUUID: vm.UUID, ProjectID: vm.ProjectID, - FlavorName: vm.FlavorName, + FlavorName: resSpec.FlavorName, FlavorExtraSpecs: flavorExtraSpecs, - MemoryMB: memoryMB, - VCPUs: vcpus, + MemoryMB: resSpec.MemoryMB, + VCPUs: resSpec.VCPUs, EligibleHosts: eligibleHypervisors, IgnoreHosts: ignoreHypervisors, Pipeline: pipeline, AvailabilityZone: vm.AvailabilityZone, - SchedulerHints: map[string]any{"_nova_check_type": string(api.ReserveForFailoverIntent)}, + SchedulerHints: map[string]any{ + "_nova_check_type": string(api.ReserveForFailoverIntent), + api.HintKeyResourceGroup: resSpec.ResourceGroup(vm.FlavorName), + }, } logger.V(1).Info("scheduling failover reservation", @@ -123,11 +116,12 @@ func (c *FailoverReservationController) tryReuseExistingReservation( vm VM, failoverReservations []v1alpha1.Reservation, allHypervisors []string, + resSpec resolvedReservationSpec, ) *v1alpha1.Reservation { logger := LoggerFromContext(ctx) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation, resSpec) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -263,12 +257,14 @@ func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( allHypervisors []string, failoverReservations []v1alpha1.Reservation, excludeHypervisors map[string]bool, + resSpec resolvedReservationSpec, ) (*v1alpha1.Reservation, error) { logger := LoggerFromContext(ctx) - // Get potential hypervisors from scheduler - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation) + // Get potential hypervisors from scheduler using the reservation spec resources + // (which may be sized to the LargestFlavor from the flavor group) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec) if err != nil { return nil, fmt.Errorf("failed to get potential hypervisors for VM: %w", err) } @@ -307,8 +303,8 @@ func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( "selectedHypervisor", selectedHypervisor, "allReturnedHypervisors", validHypervisors) - // Build the failover reservation on the selected hypervisor (in-memory only) - reservation := newFailoverReservation(ctx, vm, selectedHypervisor, c.Config.Creator) + // Build the failover reservation using the same reservation spec resources + reservation := newFailoverReservation(ctx, vm, selectedHypervisor, c.Config.Creator, resSpec) return reservation, nil } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index 0ae69f8db..fa987d34b 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -138,7 +139,9 @@ func TestBuildNewFailoverReservation(t *testing.T) { ctx := context.Background() creator := "test-creator" - result := newFailoverReservation(ctx, tt.vm, tt.hypervisor, creator) + // Resolve using VM's own resources (no flavor groups) + resolved := resolveVMSpecForScheduling(ctx, tt.vm, false, nil) + result := newFailoverReservation(ctx, tt.vm, tt.hypervisor, creator, resolved) // Verify Status.Host if result.Status.Host != tt.wantHost { @@ -167,8 +170,10 @@ func TestBuildNewFailoverReservation(t *testing.T) { t.Errorf("allocated host = %v, want %v", allocatedHost, tt.vm.CurrentHypervisor) } - // Verify resources are copied from VM - // Note: VM uses "vcpus" but reservation uses "cpu" as the canonical key + // Verify resources are derived from VM + // Note: VM uses "vcpus" but reservation uses "cpu" as the canonical key. + // Memory uses binary MiB (bytes / 1024*1024 → MiB → MiB * 1024*1024), matching + // commitments/state.go and vm_source.go conventions. if tt.vm.Resources != nil { if memory, ok := tt.vm.Resources["memory"]; ok { if resMemory, ok := result.Spec.Resources[hv1.ResourceMemory]; !ok { @@ -221,6 +226,175 @@ func TestBuildNewFailoverReservation(t *testing.T) { } } +// ============================================================================ +// Test: resolveVMSpecForScheduling + newFailoverReservation with flavor group resources +// ============================================================================ + +func TestResolveVMForSchedulingAndNewFailoverReservation(t *testing.T) { + // Build a flavor group where the VM's flavor is "hana_c60_m960" (small) + // but the LargestFlavor is "hana_c120_m1920" (large). + // When UseFlavorGroupResources is true, the resolved resources should use + // the LargestFlavor's name and size. The reservation should then be sized accordingly. + flavorGroups := map[string]compute.FlavorGroupFeature{ + "hana_v2": { + Name: "hana_v2", + Flavors: []compute.FlavorInGroup{ + {Name: "hana_c120_m1920", VCPUs: 120, MemoryMB: 1966080}, + {Name: "hana_c60_m960", VCPUs: 60, MemoryMB: 983040}, + {Name: "hana_c30_m480", VCPUs: 30, MemoryMB: 491520}, + }, + LargestFlavor: compute.FlavorInGroup{Name: "hana_c120_m1920", VCPUs: 120, MemoryMB: 1966080}, + SmallestFlavor: compute.FlavorInGroup{Name: "hana_c30_m480", VCPUs: 30, MemoryMB: 491520}, + }, + } + + tests := []struct { + name string + vm VM + useFlavorGroupResources bool + flavorGroups map[string]compute.FlavorGroupFeature + wantFlavorName string + wantFlavorGroupName string + wantResourceGroup string + wantMemoryMB uint64 + wantVCPUs uint64 + }{ + { + name: "uses LargestFlavor resources when enabled and flavor found", + vm: VM{ + UUID: "vm-1", + CurrentHypervisor: "host1", + FlavorName: "hana_c60_m960", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(60, resource.DecimalSI), + "memory": *resource.NewQuantity(983040*1024*1024, resource.BinarySI), + }, + }, + useFlavorGroupResources: true, + flavorGroups: flavorGroups, + wantFlavorName: "hana_c120_m1920", // LargestFlavor name + wantFlavorGroupName: "hana_v2", // flavor group name + wantResourceGroup: "hana_v2", // ResourceGroup = flavor group name + wantMemoryMB: 1966080, // LargestFlavor memory + wantVCPUs: 120, // LargestFlavor vcpus + }, + { + name: "falls back to VM resources when disabled", + vm: VM{ + UUID: "vm-2", + CurrentHypervisor: "host1", + FlavorName: "hana_c60_m960", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(60, resource.DecimalSI), + "memory": *resource.NewQuantity(983040*1024*1024, resource.BinarySI), + }, + }, + useFlavorGroupResources: false, + flavorGroups: flavorGroups, + wantFlavorName: "hana_c60_m960", // VM's own flavor name + wantFlavorGroupName: "", // no flavor group (disabled) + wantResourceGroup: "hana_c60_m960", // ResourceGroup = fallback to flavor name + wantMemoryMB: 983040, // VM's own memory (MiB, binary) + wantVCPUs: 60, // VM's own vcpus + }, + { + name: "falls back to VM resources when flavor not in any group", + vm: VM{ + UUID: "vm-3", + CurrentHypervisor: "host1", + FlavorName: "unknown_flavor", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(8, resource.DecimalSI), + "memory": *resource.NewQuantity(16384*1024*1024, resource.BinarySI), + }, + }, + useFlavorGroupResources: true, + flavorGroups: flavorGroups, + wantFlavorName: "unknown_flavor", // VM's own flavor name (fallback) + wantFlavorGroupName: "", // no flavor group (not found) + wantResourceGroup: "unknown_flavor", // ResourceGroup = fallback to flavor name + wantMemoryMB: 16384, // VM's own memory (MiB, binary) + wantVCPUs: 8, // VM's own vcpus (fallback) + }, + { + name: "falls back to VM resources when flavorGroups is nil", + vm: VM{ + UUID: "vm-4", + CurrentHypervisor: "host1", + FlavorName: "hana_c60_m960", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(60, resource.DecimalSI), + "memory": *resource.NewQuantity(983040*1024*1024, resource.BinarySI), + }, + }, + useFlavorGroupResources: true, + flavorGroups: nil, // nil flavor groups + wantFlavorName: "hana_c60_m960", // VM's own flavor name (fallback) + wantFlavorGroupName: "", // no flavor group (nil groups) + wantResourceGroup: "hana_c60_m960", // ResourceGroup = fallback to flavor name + wantMemoryMB: 983040, // VM's own memory (MiB, binary) + wantVCPUs: 60, // VM's own vcpus (fallback) + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + creator := "test-creator" + + // Test resolveVMSpecForScheduling + resolved := resolveVMSpecForScheduling(ctx, tt.vm, tt.useFlavorGroupResources, tt.flavorGroups) + + if resolved.FlavorName != tt.wantFlavorName { + t.Errorf("resolved.FlavorName = %q, want %q", resolved.FlavorName, tt.wantFlavorName) + } + if resolved.MemoryMB != tt.wantMemoryMB { + t.Errorf("resolved.MemoryMB = %d, want %d", resolved.MemoryMB, tt.wantMemoryMB) + } + if resolved.VCPUs != tt.wantVCPUs { + t.Errorf("resolved.VCPUs = %d, want %d", resolved.VCPUs, tt.wantVCPUs) + } + if resolved.FlavorGroupName != tt.wantFlavorGroupName { + t.Errorf("resolved.FlavorGroupName = %q, want %q", resolved.FlavorGroupName, tt.wantFlavorGroupName) + } + + // Test that newFailoverReservation uses the resolved values correctly + result := newFailoverReservation(ctx, tt.vm, "target-host", creator, resolved) + + // Verify reservation memory matches resolved + resMemory, ok := result.Spec.Resources[hv1.ResourceMemory] + if !ok { + t.Fatal("reservation missing memory resource") + } + wantMemoryBytes := int64(tt.wantMemoryMB) * 1024 * 1024 //nolint:gosec // test values won't overflow; binary MiB matches OpenStack convention + if resMemory.Value() != wantMemoryBytes { + t.Errorf("reservation memory = %d bytes, want %d bytes", resMemory.Value(), wantMemoryBytes) + } + + // Verify reservation CPU matches resolved + resCPU, ok := result.Spec.Resources[hv1.ResourceCPU] + if !ok { + t.Fatal("reservation missing cpu resource") + } + if resCPU.Value() != int64(tt.wantVCPUs) { //nolint:gosec // test values won't overflow + t.Errorf("reservation cpu = %d, want %d", resCPU.Value(), tt.wantVCPUs) + } + + // Verify ResourceGroup on the reservation + if result.Spec.FailoverReservation == nil { + t.Fatal("reservation missing FailoverReservation spec") + } + if result.Spec.FailoverReservation.ResourceGroup != tt.wantResourceGroup { + t.Errorf("ResourceGroup = %q, want %q", result.Spec.FailoverReservation.ResourceGroup, tt.wantResourceGroup) + } + }) + } +} + // ============================================================================ // Test Helpers (local to this test file) // ============================================================================ diff --git a/internal/scheduling/reservations/flavor_groups.go b/internal/scheduling/reservations/flavor_groups.go index 197406eac..b6344630a 100644 --- a/internal/scheduling/reservations/flavor_groups.go +++ b/internal/scheduling/reservations/flavor_groups.go @@ -15,6 +15,20 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// FindFlavorInGroups searches all flavor groups for a flavor by name. +// Returns the flavor group name and flavor details, or an error if the flavor +// is not found in any group. +func FindFlavorInGroups(flavorName string, flavorGroups map[string]compute.FlavorGroupFeature) (groupName string, flavor *compute.FlavorInGroup, err error) { + for gName, fg := range flavorGroups { + for i, f := range fg.Flavors { + if f.Name == flavorName { + return gName, &fg.Flavors[i], nil + } + } + } + return "", nil, fmt.Errorf("flavor %q not found in any flavor group", flavorName) +} + // FlavorGroupKnowledgeClient accesses flavor group data from Knowledge CRDs. type FlavorGroupKnowledgeClient struct { client.Client diff --git a/internal/shim/placement/handle_allocation_candidates_e2e.go b/internal/shim/placement/handle_allocation_candidates_e2e.go index e90193224..c2dd4aacd 100644 --- a/internal/shim/placement/handle_allocation_candidates_e2e.go +++ b/internal/shim/placement/handle_allocation_candidates_e2e.go @@ -47,6 +47,15 @@ func e2eTestAllocationCandidates(ctx context.Context, _ client.Client) error { const testRC = "CUSTOM_CORTEX_E2E_CAND_RC" const apiVersion = "placement 1.26" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/allocation_candidates?resources=VCPU:1") + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete leftover test resources from a prior run. log.Info("Pre-cleanup: deleting leftover test resources") for _, cleanup := range []struct { @@ -296,5 +305,5 @@ func e2eTestAllocationCandidates(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "allocation_candidates", run: e2eTestAllocationCandidates}) + e2eTests = append(e2eTests, e2eTest{name: "allocation_candidates", run: e2eWrapWithModes(e2eTestAllocationCandidates)}) } diff --git a/internal/shim/placement/handle_allocations_e2e.go b/internal/shim/placement/handle_allocations_e2e.go index 7f09a507b..27887ca7f 100644 --- a/internal/shim/placement/handle_allocations_e2e.go +++ b/internal/shim/placement/handle_allocations_e2e.go @@ -56,6 +56,15 @@ func e2eTestAllocations(ctx context.Context, _ client.Client) error { const userID = "e2e50000-0000-0000-0000-000000000001" const apiVersion = "placement 1.28" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/allocations/"+consumerUUID1) + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete allocations, resource provider, and resource class. log.Info("Pre-cleanup: deleting leftover test resources") for _, cleanup := range []struct { @@ -476,5 +485,5 @@ func e2eTestAllocations(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "allocations", run: e2eTestAllocations}) + e2eTests = append(e2eTests, e2eTest{name: "allocations", run: e2eWrapWithModes(e2eTestAllocations)}) } diff --git a/internal/shim/placement/handle_reshaper_e2e.go b/internal/shim/placement/handle_reshaper_e2e.go index f43809de4..29f84be34 100644 --- a/internal/shim/placement/handle_reshaper_e2e.go +++ b/internal/shim/placement/handle_reshaper_e2e.go @@ -57,6 +57,15 @@ func e2eTestReshaper(ctx context.Context, _ client.Client) error { const userID = "e2e50000-0000-0000-0000-000000000001" const apiVersion = "placement 1.30" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/allocations/"+consumerUUID) + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete allocation, both RPs, and custom resource class. log.Info("Pre-cleanup: deleting leftover test resources") for _, cleanup := range []struct { @@ -571,5 +580,5 @@ func e2eTestReshaper(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "reshaper", run: e2eTestReshaper}) + e2eTests = append(e2eTests, e2eTest{name: "reshaper", run: e2eWrapWithModes(e2eTestReshaper)}) } diff --git a/internal/shim/placement/handle_resource_classes.go b/internal/shim/placement/handle_resource_classes.go index 9067079fd..f9f4f8b5d 100644 --- a/internal/shim/placement/handle_resource_classes.go +++ b/internal/shim/placement/handle_resource_classes.go @@ -4,68 +4,532 @@ package placement import ( + "context" + "encoding/json" + "fmt" + "io" "net/http" + "os" + "sort" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) // HandleListResourceClasses handles GET /resource_classes requests. // -// Returns the complete list of all resource classes, including both standard -// classes (e.g. VCPU, MEMORY_MB, DISK_GB, PCI_DEVICE, SRIOV_NET_VF) and -// deployer-defined custom classes prefixed with CUSTOM_. Resource classes -// categorize the types of resources that resource providers can offer as -// inventory. Available since microversion 1.2. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream placement. +// - crd: serves the resource class list from the local ConfigMap. +// +// See: https://docs.openstack.org/api-ref/placement/#list-resource-classes func (s *Shim) HandleListResourceClasses(w http.ResponseWriter, r *http.Request) { - s.dispatchPassthroughOnly(w, r, s.config.Features.ResourceClasses) + ctx := r.Context() + log := logf.FromContext(ctx) + + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceClasses, s.config.ResourceClasses != nil) { + case FeatureModePassthrough, FeatureModeHybrid: + s.forward(w, r) + return + case FeatureModeCRD: + // Serve from local ConfigMap. + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + return + } + + rcSet, err := s.getResourceClasses(ctx) + if err != nil { + log.Error(err, "failed to list resource classes from configmap") + http.Error(w, "failed to list resource classes", http.StatusInternalServerError) + return + } + + entries := make([]resourceClassEntry, 0, len(rcSet)) + for name := range rcSet { + entries = append(entries, resourceClassEntry{ + Name: name, + Links: []resourceClassLink{{Rel: "self", Href: "/resource_classes/" + name}}, + }) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + + log.Info("listing all resource classes", "count", len(entries)) + s.writeJSON(w, http.StatusOK, resourceClassesListResponse{ResourceClasses: entries}) } // HandleCreateResourceClass handles POST /resource_classes requests. // -// Creates a new custom resource class. The name must be prefixed with CUSTOM_ -// to distinguish it from standard resource classes. Returns 201 Created with -// a Location header on success. Returns 400 Bad Request if the CUSTOM_ prefix -// is missing, and 409 Conflict if a class with the same name already exists. -// Available since microversion 1.2. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream; on success, adds the class to the local ConfigMap. +// - crd: writes the class to the local ConfigMap (CUSTOM_ prefix required). +// +// See: https://docs.openstack.org/api-ref/placement/#create-resource-class func (s *Shim) HandleCreateResourceClass(w http.ResponseWriter, r *http.Request) { - s.dispatchPassthroughOnly(w, r, s.config.Features.ResourceClasses) + ctx := r.Context() + log := logf.FromContext(ctx) + + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceClasses, s.config.ResourceClasses != nil) + switch mode { + case FeatureModePassthrough: + s.forward(w, r) + return + case FeatureModeHybrid: + s.handleCreateResourceClassHybrid(w, r) + return + case FeatureModeCRD: + // Handle locally. + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + return + } + + var body struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Name == "" { + http.Error(w, "request body must contain a valid 'name' field", http.StatusBadRequest) + return + } + if !strings.HasPrefix(body.Name, "CUSTOM_") { + log.Info("rejected resource class without CUSTOM_ prefix", "class", body.Name) + http.Error(w, "resource class name must start with CUSTOM_", http.StatusBadRequest) + return + } + + exists, err := s.hasResourceClass(ctx, body.Name) + if err != nil { + log.Error(err, "failed to check resource class", "class", body.Name) + http.Error(w, "failed to check resource class", http.StatusInternalServerError) + return + } + if exists { + http.Error(w, "resource class already exists", http.StatusConflict) + return + } + + if _, err := s.addResourceClassToConfigMap(ctx, body.Name); err != nil { + log.Error(err, "failed to create resource class", "class", body.Name) + http.Error(w, "failed to create resource class", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusCreated) +} + +// handleCreateResourceClassHybrid forwards POST /resource_classes to upstream, +// then updates the local ConfigMap on success. +func (s *Shim) handleCreateResourceClassHybrid(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logf.FromContext(ctx) + + s.forwardWithHook(w, r, func(w http.ResponseWriter, resp *http.Response) { + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Error(err, "hybrid: failed to read upstream response body") + } + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := w.Write(body); err != nil { + log.Error(err, "hybrid: failed to write response body") + } + + if resp.StatusCode == http.StatusCreated { + var created struct { + Name string `json:"name"` + } + if err := json.Unmarshal(body, &created); err == nil && created.Name != "" { + if _, err := s.addResourceClassToConfigMap(ctx, created.Name); err != nil { + log.Error(err, "hybrid: failed to add resource class to local configmap", "class", created.Name) + } + } + } + }) } // HandleShowResourceClass handles GET /resource_classes/{name} requests. // -// Returns a representation of a single resource class identified by name. -// This can be used to verify the existence of a resource class. Returns 404 -// if the class does not exist. Available since microversion 1.2. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream placement. +// - crd: checks the local ConfigMap for the resource class. +// +// See: https://docs.openstack.org/api-ref/placement/#show-resource-class func (s *Shim) HandleShowResourceClass(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredPathParam(w, r, "name"); !ok { + ctx := r.Context() + log := logf.FromContext(ctx) + + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceClasses, s.config.ResourceClasses != nil) { + case FeatureModePassthrough, FeatureModeHybrid: + s.forward(w, r) + return + case FeatureModeCRD: + // Serve from local ConfigMap. + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + return + } + + name, ok := requiredPathParam(w, r, "name") + if !ok { + return + } + found, err := s.hasResourceClass(ctx, name) + if err != nil { + log.Error(err, "failed to check resource class", "class", name) + http.Error(w, "failed to check resource class", http.StatusInternalServerError) return } - s.dispatchPassthroughOnly(w, r, s.config.Features.ResourceClasses) + if !found { + log.Info("resource class not found", "class", name) + http.Error(w, "resource class not found", http.StatusNotFound) + return + } + log.Info("resource class found", "class", name) + s.writeJSON(w, http.StatusOK, resourceClassEntry{ + Name: name, + Links: []resourceClassLink{{Rel: "self", Href: "/resource_classes/" + name}}, + }) } // HandleUpdateResourceClass handles PUT /resource_classes/{name} requests. // -// Behavior differs by microversion. Since microversion 1.7, this endpoint -// creates or validates the existence of a single resource class: it returns -// 201 Created for a new class or 204 No Content if the class already exists. -// The name must carry the CUSTOM_ prefix. In earlier versions (1.2-1.6), the -// endpoint allowed renaming a class via a request body, but this usage is -// discouraged. Returns 400 Bad Request if the CUSTOM_ prefix is missing. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream; on success, adds the class to the local ConfigMap. +// - crd: writes the class to the local ConfigMap (CUSTOM_ prefix required). +// +// See: https://docs.openstack.org/api-ref/placement/#update-resource-class func (s *Shim) HandleUpdateResourceClass(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredPathParam(w, r, "name"); !ok { + ctx := r.Context() + log := logf.FromContext(ctx) + + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceClasses, s.config.ResourceClasses != nil) + switch mode { + case FeatureModePassthrough: + s.forward(w, r) + return + case FeatureModeHybrid: + s.handleUpdateResourceClassHybrid(w, r) + return + case FeatureModeCRD: + // Handle locally. + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + return + } + + name, ok := requiredPathParam(w, r, "name") + if !ok { + return + } + if !strings.HasPrefix(name, "CUSTOM_") { + log.Info("rejected resource class without CUSTOM_ prefix", "class", name) + http.Error(w, "resource class name must start with CUSTOM_", http.StatusBadRequest) + return + } + + created, err := s.addResourceClassToConfigMap(ctx, name) + if err != nil { + log.Error(err, "failed to create resource class", "class", name) + http.Error(w, "failed to create resource class", http.StatusInternalServerError) + return + } + if created { + w.WriteHeader(http.StatusCreated) + } else { + w.WriteHeader(http.StatusNoContent) + } +} + +// handleUpdateResourceClassHybrid forwards PUT /resource_classes/{name} to +// upstream, then updates the local ConfigMap on success. +func (s *Shim) handleUpdateResourceClassHybrid(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logf.FromContext(ctx) + + name, ok := requiredPathParam(w, r, "name") + if !ok { return } - s.dispatchPassthroughOnly(w, r, s.config.Features.ResourceClasses) + + s.forwardWithHook(w, r, func(w http.ResponseWriter, resp *http.Response) { + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + if resp.Body != nil { + if _, err := io.Copy(w, resp.Body); err != nil { + log.Error(err, "hybrid: failed to copy upstream response body") + } + } + + if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent { + if _, err := s.addResourceClassToConfigMap(ctx, name); err != nil { + log.Error(err, "hybrid: failed to add resource class to local configmap", "class", name) + } + } + }) } // HandleDeleteResourceClass handles DELETE /resource_classes/{name} requests. // -// Deletes a custom resource class. Only custom classes (prefixed with CUSTOM_) -// may be deleted; attempting to delete a standard class returns 400 Bad -// Request. Returns 409 Conflict if any resource provider has inventory of this -// class, and 404 if the class does not exist. Returns 204 No Content on -// success. Available since microversion 1.2. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream; on success, removes the class from the local ConfigMap. +// - crd: removes the class from the local ConfigMap (CUSTOM_ prefix required). +// +// See: https://docs.openstack.org/api-ref/placement/#delete-resource-class func (s *Shim) HandleDeleteResourceClass(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredPathParam(w, r, "name"); !ok { + ctx := r.Context() + log := logf.FromContext(ctx) + + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceClasses, s.config.ResourceClasses != nil) + switch mode { + case FeatureModePassthrough: + s.forward(w, r) + return + case FeatureModeHybrid: + s.handleDeleteResourceClassHybrid(w, r) + return + case FeatureModeCRD: + // Handle locally. + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + return + } + + name, ok := requiredPathParam(w, r, "name") + if !ok { + return + } + if !strings.HasPrefix(name, "CUSTOM_") { + log.Info("rejected deletion of standard resource class", "class", name) + http.Error(w, "cannot delete standard resource classes", http.StatusBadRequest) + return + } + + removed, err := s.removeResourceClassFromConfigMap(ctx, name) + if err != nil { + log.Error(err, "failed to delete resource class", "class", name) + http.Error(w, "failed to delete resource class", http.StatusInternalServerError) + return + } + if !removed { + log.Info("resource class not found in configmap", "class", name) + http.Error(w, "resource class not found", http.StatusNotFound) return } - s.dispatchPassthroughOnly(w, r, s.config.Features.ResourceClasses) + log.Info("deleted resource class from configmap", "class", name) + w.WriteHeader(http.StatusNoContent) +} + +// handleDeleteResourceClassHybrid forwards DELETE /resource_classes/{name} to +// upstream, then updates the local ConfigMap on success. +func (s *Shim) handleDeleteResourceClassHybrid(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logf.FromContext(ctx) + + name, ok := requiredPathParam(w, r, "name") + if !ok { + return + } + + s.forwardWithHook(w, r, func(w http.ResponseWriter, resp *http.Response) { + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + if resp.Body != nil { + if _, err := io.Copy(w, resp.Body); err != nil { + log.Error(err, "hybrid: failed to copy upstream response body") + } + } + + if resp.StatusCode == http.StatusNoContent { + if _, err := s.removeResourceClassFromConfigMap(ctx, name); err != nil { + log.Error(err, "hybrid: failed to remove resource class from local configmap", "class", name) + } + } + }) +} + +// getResourceClasses reads resource classes from the single ConfigMap. +func (s *Shim) getResourceClasses(ctx context.Context) (map[string]struct{}, error) { + cm := &corev1.ConfigMap{} + if err := s.Get(ctx, client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.ResourceClasses.ConfigMapName}, cm); err != nil { + return nil, fmt.Errorf("get resource classes configmap %s: %w", s.config.ResourceClasses.ConfigMapName, err) + } + return parseResourceClasses(cm) +} + +// parseResourceClasses extracts the resource class set from a ConfigMap. +func parseResourceClasses(cm *corev1.ConfigMap) (map[string]struct{}, error) { + raw, ok := cm.Data[configMapKeyResourceClasses] + if !ok || raw == "" { + return make(map[string]struct{}), nil + } + var classes []string + if err := json.Unmarshal([]byte(raw), &classes); err != nil { + return nil, fmt.Errorf("unmarshal resource classes from configmap: %w", err) + } + m := make(map[string]struct{}, len(classes)) + for _, c := range classes { + m[c] = struct{}{} + } + return m, nil +} + +func (s *Shim) hasResourceClass(ctx context.Context, name string) (bool, error) { + classes, err := s.getResourceClasses(ctx) + if err != nil { + return false, err + } + _, ok := classes[name] + return ok, nil +} + +// writeResourceClassesToConfigMap serializes the resource class set into the ConfigMap. +func writeResourceClassesToConfigMap(cm *corev1.ConfigMap, rcSet map[string]struct{}) error { + classes := make([]string, 0, len(rcSet)) + for c := range rcSet { + classes = append(classes, c) + } + sort.Strings(classes) + + data, err := json.Marshal(classes) + if err != nil { + return fmt.Errorf("marshal resource classes: %w", err) + } + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data[configMapKeyResourceClasses] = string(data) + return nil +} + +// addResourceClassToConfigMap adds a resource class to the ConfigMap under the +// resource lock. Returns true if the class was newly created, false if it +// already existed. +func (s *Shim) addResourceClassToConfigMap(ctx context.Context, name string) (bool, error) { + classes, err := s.getResourceClasses(ctx) + if err != nil { + return false, err + } + if _, exists := classes[name]; exists { + return false, nil + } + + host, err := os.Hostname() + if err != nil { + return false, fmt.Errorf("get hostname: %w", err) + } + lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) + if err := s.resourceLocker.AcquireLock(ctx, s.config.ResourceClasses.ConfigMapName+"-lock", lockerID); err != nil { + return false, fmt.Errorf("acquire resource classes lock: %w", err) + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.resourceLocker.ReleaseLock(releaseCtx, s.config.ResourceClasses.ConfigMapName+"-lock", lockerID); err != nil { + ctrl.Log.WithName("placement-shim").Error(err, "failed to release resource classes lock") + } + }() + + cm := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.ResourceClasses.ConfigMapName} + if err := s.Get(ctx, key, cm); err != nil { + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Data: map[string]string{configMapKeyResourceClasses: "[]"}, + } + current := map[string]struct{}{name: {}} + if err := writeResourceClassesToConfigMap(cm, current); err != nil { + return false, err + } + if err := s.Create(ctx, cm); err != nil { + return false, fmt.Errorf("create resource classes configmap: %w", err) + } + return true, nil + } + return false, fmt.Errorf("get resource classes configmap: %w", err) + } + + current, err := parseResourceClasses(cm) + if err != nil { + return false, err + } + if _, exists := current[name]; exists { + return false, nil + } + current[name] = struct{}{} + if err := writeResourceClassesToConfigMap(cm, current); err != nil { + return false, err + } + if err := s.Update(ctx, cm); err != nil { + return false, fmt.Errorf("update resource classes configmap: %w", err) + } + return true, nil +} + +// removeResourceClassFromConfigMap removes a resource class from the ConfigMap +// under the resource lock. Returns true if the class was found and removed. +func (s *Shim) removeResourceClassFromConfigMap(ctx context.Context, name string) (bool, error) { + host, err := os.Hostname() + if err != nil { + return false, fmt.Errorf("get hostname: %w", err) + } + lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) + if err := s.resourceLocker.AcquireLock(ctx, s.config.ResourceClasses.ConfigMapName+"-lock", lockerID); err != nil { + return false, fmt.Errorf("acquire resource classes lock: %w", err) + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.resourceLocker.ReleaseLock(releaseCtx, s.config.ResourceClasses.ConfigMapName+"-lock", lockerID); err != nil { + ctrl.Log.WithName("placement-shim").Error(err, "failed to release resource classes lock") + } + }() + + cm := &corev1.ConfigMap{} + if err := s.Get(ctx, client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.ResourceClasses.ConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("get resource classes configmap: %w", err) + } + current, err := parseResourceClasses(cm) + if err != nil { + return false, err + } + if _, exists := current[name]; !exists { + return false, nil + } + delete(current, name) + if err := writeResourceClassesToConfigMap(cm, current); err != nil { + return false, err + } + if err := s.Update(ctx, cm); err != nil { + return false, fmt.Errorf("update resource classes configmap: %w", err) + } + return true, nil } diff --git a/internal/shim/placement/handle_resource_classes_e2e.go b/internal/shim/placement/handle_resource_classes_e2e.go index 360e1ef80..6bc302cfe 100644 --- a/internal/shim/placement/handle_resource_classes_e2e.go +++ b/internal/shim/placement/handle_resource_classes_e2e.go @@ -6,6 +6,7 @@ package placement import ( "context" "encoding/json" + "errors" "fmt" "net/http" @@ -17,13 +18,24 @@ import ( // e2eTestResourceClasses tests the /resource_classes and // /resource_classes/{name} endpoints. // -// 1. Pre-cleanup: DELETE any leftover custom resource class (ignore 404). -// 2. GET /resource_classes — list all classes and verify the response. -// 3. GET /resource_classes/VCPU — confirm a standard class is retrievable. -// 4. PUT /resource_classes/{name} — create a custom test class. -// 5. GET /resource_classes/{name} — verify the custom class now exists. -// 6. DELETE /resource_classes/{name} — remove the custom class. -// 7. GET /resource_classes/{name} — confirm deletion returns 404. +// Phase 1 — read-only (always runs): +// +// 1. GET /resource_classes — list all resource classes; when mode is +// passthrough (forwarding to upstream) verify at least one class exists. +// 2. GET /resource_classes/VCPU — verify a standard class is retrievable +// (skipped when the list is empty). +// 3. GET /resource_classes/{name} — show a nonexistent class and verify 404. +// +// Phase 2 — CRUD (only when mode is non-passthrough): +// +// 1. Pre-cleanup: DELETE any leftover test class (ignore 404). +// 2. PUT /resource_classes/{name} — create a custom test class → 201. +// 3. PUT /resource_classes/{name} — idempotent create → 204. +// 4. GET /resource_classes/{name} — verify the custom class exists → 200. +// 5. DELETE /resource_classes/{name} — remove the custom class → 204. +// 6. GET /resource_classes/{name} — confirm deletion → 404. +// 7. PUT /resource_classes/{name} — bad prefix → 400. +// 8. DELETE /resource_classes/{name} — bad prefix → 400. func e2eTestResourceClasses(ctx context.Context, _ client.Client) error { log := logf.FromContext(ctx) log.Info("Running resource classes endpoint e2e test") @@ -40,191 +52,253 @@ func e2eTestResourceClasses(ctx context.Context, _ client.Client) error { } log.Info("Successfully created openstack client for resource classes e2e test") - const testRC = "CUSTOM_CORTEX_E2E_RC" + // ==================== Phase 1: read-only tests ==================== - // Pre-cleanup: delete any leftover test resource class from a prior run. - log.Info("Pre-cleanup: deleting leftover test resource class", "class", testRC) + log.Info("=== Phase 1: read-only resource class tests ===") + + rcMode := e2eCurrentMode(ctx) + + // Test GET /resource_classes + log.Info("Testing GET /resource_classes endpoint") req, err := http.NewRequestWithContext(ctx, - http.MethodDelete, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) + http.MethodGet, sc.Endpoint+"/resource_classes", http.NoBody) if err != nil { - log.Error(err, "failed to create pre-cleanup request") - return err + return fmt.Errorf("failed to create GET /resource_classes request: %w", err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") + req.Header.Set("Accept", "application/json") resp, err := sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send pre-cleanup request") - return err + return fmt.Errorf("failed to send GET /resource_classes request: %w", err) } defer resp.Body.Close() - // Ignore 404 (not found) — that's expected if no leftover exists. - if resp.StatusCode != http.StatusNotFound && - (resp.StatusCode < 200 || resp.StatusCode >= 300) { - err := fmt.Errorf("unexpected status code during pre-cleanup: %d", resp.StatusCode) - log.Error(err, "pre-cleanup failed") - return err + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET /resource_classes: expected 200, got %d", resp.StatusCode) } - log.Info("Pre-cleanup completed", "status", resp.StatusCode) + var listResp resourceClassesListResponse + if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return fmt.Errorf("failed to decode GET /resource_classes response: %w", err) + } + if rcMode == FeatureModePassthrough && len(listResp.ResourceClasses) == 0 { + return errors.New("GET /resource_classes: expected at least one class when forwarding to upstream, got 0") + } + log.Info("Successfully retrieved resource classes", "count", len(listResp.ResourceClasses)) - // Test GET /resource_classes - log.Info("Testing GET /resource_classes endpoint of placement shim") + // Test GET /resource_classes/{name} for a known class (skip when list is empty). + if len(listResp.ResourceClasses) > 0 { + knownClass := listResp.ResourceClasses[0].Name + log.Info("Testing GET /resource_classes/{name} for known class", "class", knownClass) + req, err = http.NewRequestWithContext(ctx, + http.MethodGet, sc.Endpoint+"/resource_classes/"+knownClass, http.NoBody) + if err != nil { + return fmt.Errorf("failed to create GET request for class %s: %w", knownClass, err) + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.7") + req.Header.Set("Accept", "application/json") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send GET request for class %s: %w", knownClass, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET /resource_classes/%s: expected 200, got %d", knownClass, resp.StatusCode) + } + log.Info("Successfully verified known class exists", "class", knownClass) + } else { + log.Info("Skipping GET /resource_classes/{name} for known class, list is empty") + } + + // Test GET /resource_classes/{name} for a nonexistent class. + log.Info("Testing GET /resource_classes/{name} for nonexistent class") req, err = http.NewRequestWithContext(ctx, - http.MethodGet, sc.Endpoint+"/resource_classes", http.NoBody) + http.MethodGet, sc.Endpoint+"/resource_classes/CUSTOM_CORTEX_E2E_NONEXISTENT", http.NoBody) if err != nil { - log.Error(err, "failed to create request for resource_classes endpoint") - return err + return fmt.Errorf("failed to create GET request for nonexistent class: %w", err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send request to /resource_classes endpoint") - return err + return fmt.Errorf("failed to send GET request for nonexistent class: %w", err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "/resource_classes endpoint returned an error") - return err + if resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("GET /resource_classes/CUSTOM_CORTEX_E2E_NONEXISTENT: expected 404, got %d", resp.StatusCode) } - var list struct { - ResourceClasses []struct { - Name string `json:"name"` - } `json:"resource_classes"` + log.Info("Correctly received 404 for nonexistent resource class") + + // ==================== Phase 2: CRUD tests ==================== + + log.Info("=== Phase 2: CRUD resource class tests ===") + + const testRC = "CUSTOM_CORTEX_E2E_RC" + + // Pre-cleanup: delete any leftover test class from a prior run. + log.Info("Pre-cleanup: deleting leftover test resource class", "class", testRC) + req, err = http.NewRequestWithContext(ctx, + http.MethodDelete, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) + if err != nil { + return fmt.Errorf("failed to create pre-cleanup request: %w", err) } - err = json.NewDecoder(resp.Body).Decode(&list) + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.7") + resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to decode response from /resource_classes endpoint") - return err + return fmt.Errorf("failed to send pre-cleanup request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("pre-cleanup DELETE /resource_classes/%s: unexpected status %d", testRC, resp.StatusCode) } - log.Info("Successfully retrieved resource classes from placement shim", - "count", len(list.ResourceClasses)) + log.Info("Pre-cleanup completed", "status", resp.StatusCode) - // Test GET /resource_classes/{name} for a standard class - log.Info("Testing GET /resource_classes/VCPU endpoint of placement shim") + // Test PUT /resource_classes/{name} — create → 201. + log.Info("Testing PUT /resource_classes/{name} to create custom class", "class", testRC) req, err = http.NewRequestWithContext(ctx, - http.MethodGet, sc.Endpoint+"/resource_classes/VCPU", http.NoBody) + http.MethodPut, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) if err != nil { - log.Error(err, "failed to create request for resource_classes/VCPU endpoint") - return err + return fmt.Errorf("failed to create PUT request for class %s: %w", testRC, err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") - req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send request to /resource_classes/VCPU endpoint") - return err + return fmt.Errorf("failed to send PUT request for class %s: %w", testRC, err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "/resource_classes/VCPU endpoint returned an error") - return err + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("PUT /resource_classes/%s (create): expected 201, got %d", testRC, resp.StatusCode) } - log.Info("Successfully retrieved standard resource class VCPU from placement shim") + log.Info("Successfully created custom resource class", "class", testRC) - // Test PUT /resource_classes/{name} (create custom class) - log.Info("Testing PUT /resource_classes/{name} to create custom class", "class", testRC) + // Test PUT /resource_classes/{name} — idempotent → 204. + log.Info("Testing PUT /resource_classes/{name} idempotent create", "class", testRC) req, err = http.NewRequestWithContext(ctx, http.MethodPut, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) if err != nil { - log.Error(err, "failed to create PUT request for resource_classes", "class", testRC) - return err + return fmt.Errorf("failed to create idempotent PUT request: %w", err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") - req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send PUT request to /resource_classes", "class", testRC) - return err + return fmt.Errorf("failed to send idempotent PUT request: %w", err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "PUT /resource_classes returned an error", "class", testRC) - return err + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("PUT /resource_classes/%s (idempotent): expected 204, got %d", testRC, resp.StatusCode) } - log.Info("Successfully created custom resource class", "class", testRC, - "status", resp.StatusCode) + log.Info("Successfully verified idempotent PUT", "class", testRC) - // Test GET /resource_classes/{name} for the custom class + // Test GET /resource_classes/{name} — verify exists → 200. log.Info("Testing GET /resource_classes/{name} for custom class", "class", testRC) req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for custom resource class", "class", testRC) - return err + return fmt.Errorf("failed to create GET request for class %s: %w", testRC, err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for custom resource class", "class", testRC) - return err + return fmt.Errorf("failed to send GET request for class %s: %w", testRC, err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET custom resource class returned an error", "class", testRC) - return err + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET /resource_classes/%s: expected 200, got %d", testRC, resp.StatusCode) } log.Info("Successfully verified custom resource class exists", "class", testRC) - // Cleanup: Test DELETE /resource_classes/{name} - log.Info("Cleaning up test resource class from placement shim", "class", testRC) + // Cleanup: DELETE /resource_classes/{name} → 204. + log.Info("Cleaning up test resource class", "class", testRC) req, err = http.NewRequestWithContext(ctx, http.MethodDelete, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) if err != nil { - log.Error(err, "failed to create DELETE request for resource class", "class", testRC) - return err + return fmt.Errorf("failed to create DELETE request for class %s: %w", testRC, err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send DELETE request for resource class", "class", testRC) - return err + return fmt.Errorf("failed to send DELETE request for class %s: %w", testRC, err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "DELETE resource class returned an error", "class", testRC) - return err + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /resource_classes/%s: expected 204, got %d", testRC, resp.StatusCode) } log.Info("Successfully deleted test resource class", "class", testRC) - // Verify deletion: GET should return 404 + // Verify deletion: GET → 404. log.Info("Verifying test resource class was deleted", "class", testRC) req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_classes/"+testRC, http.NoBody) if err != nil { - log.Error(err, "failed to create verification GET request", "class", testRC) - return err + return fmt.Errorf("failed to create verification GET request: %w", err) } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.7") req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send verification GET request", "class", testRC) - return err + return fmt.Errorf("failed to send verification GET request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusNotFound { - err := fmt.Errorf("expected 404 after deletion, got: %d", resp.StatusCode) - log.Error(err, "resource class still exists after deletion", "class", testRC) - return err + return fmt.Errorf("GET /resource_classes/%s after deletion: expected 404, got %d", + testRC, resp.StatusCode) } log.Info("Verified test resource class was deleted", "class", testRC) + // Bad-prefix validation is only enforced by the shim in crd mode. + if rcMode == FeatureModeCRD { + // Test PUT /resource_classes/{name} with bad prefix → 400. + log.Info("Testing PUT /resource_classes/{name} with non-CUSTOM_ prefix") + req, err = http.NewRequestWithContext(ctx, + http.MethodPut, sc.Endpoint+"/resource_classes/VCPU_CORTEX_E2E_BAD", http.NoBody) + if err != nil { + return fmt.Errorf("failed to create bad-prefix PUT request: %w", err) + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.7") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send bad-prefix PUT request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + return fmt.Errorf("PUT /resource_classes/VCPU_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) + } + log.Info("Correctly received 400 for PUT with non-CUSTOM_ prefix") + + // Test DELETE /resource_classes/{name} with bad prefix → 400. + log.Info("Testing DELETE /resource_classes/{name} with non-CUSTOM_ prefix") + req, err = http.NewRequestWithContext(ctx, + http.MethodDelete, sc.Endpoint+"/resource_classes/VCPU_CORTEX_E2E_BAD", http.NoBody) + if err != nil { + return fmt.Errorf("failed to create bad-prefix DELETE request: %w", err) + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.7") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send bad-prefix DELETE request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + return fmt.Errorf("DELETE /resource_classes/VCPU_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) + } + log.Info("Correctly received 400 for DELETE with non-CUSTOM_ prefix") + } else { + log.Info("Skipping bad-prefix validation tests (only enforced in crd mode)") + } + return nil } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_classes", run: e2eTestResourceClasses}) + e2eTests = append(e2eTests, e2eTest{name: "resource_classes", run: e2eWrapWithModes(e2eTestResourceClasses)}) } diff --git a/internal/shim/placement/handle_resource_classes_test.go b/internal/shim/placement/handle_resource_classes_test.go index 330cae9be..d3f1c4ee1 100644 --- a/internal/shim/placement/handle_resource_classes_test.go +++ b/internal/shim/placement/handle_resource_classes_test.go @@ -4,11 +4,53 @@ package placement import ( + "bytes" + "context" + "encoding/json" "net/http" + "net/http/httptest" "testing" + + "github.com/cobaltcore-dev/cortex/pkg/resourcelock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestHandleListResourceClasses(t *testing.T) { +func newTestResourceClassConfigMap(namespace, name string, classes []string) *corev1.ConfigMap { + b, err := json.Marshal(classes) + if err != nil { + panic("marshal resource classes: " + err.Error()) + } + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string]string{configMapKeyResourceClasses: string(b)}, + } +} + +func newResourceClassShim(t *testing.T, classes []string) *Shim { + t.Helper() + t.Setenv("POD_NAMESPACE", "default") + objs := []client.Object{newTestResourceClassConfigMap("default", "test-rc-cm", classes)} + cl := newFakeClientWithScheme(t, objs...) + down, up := newTestTimers() + return &Shim{ + Client: cl, + config: config{ + PlacementURL: "http://should-not-be-called:1234", + Features: featuresConfig{ResourceClasses: FeatureModeCRD}, + ResourceClasses: &resourceClassesConfig{ConfigMapName: "test-rc-cm"}, + }, + maxBodyLogSize: 4096, + downstreamRequestTimer: down, + upstreamRequestTimer: up, + resourceLocker: resourcelock.NewResourceLocker(cl, "default"), + } +} + +// --- Passthrough mode tests --- + +func TestHandleListResourceClassesPassthrough(t *testing.T) { var gotPath string s := newTestShim(t, http.StatusOK, `{"resource_classes":[]}`, &gotPath) w := serveHandler(t, "GET", "/resource_classes", s.HandleListResourceClasses, "/resource_classes") @@ -20,7 +62,7 @@ func TestHandleListResourceClasses(t *testing.T) { } } -func TestHandleCreateResourceClass(t *testing.T) { +func TestHandleCreateResourceClassPassthrough(t *testing.T) { s := newTestShim(t, http.StatusCreated, "{}", nil) w := serveHandler(t, "POST", "/resource_classes", s.HandleCreateResourceClass, "/resource_classes") if w.Code != http.StatusCreated { @@ -28,7 +70,7 @@ func TestHandleCreateResourceClass(t *testing.T) { } } -func TestHandleShowResourceClass(t *testing.T) { +func TestHandleShowResourceClassPassthrough(t *testing.T) { var gotPath string s := newTestShim(t, http.StatusOK, "{}", &gotPath) w := serveHandler(t, "GET", "/resource_classes/{name}", s.HandleShowResourceClass, "/resource_classes/VCPU") @@ -40,7 +82,7 @@ func TestHandleShowResourceClass(t *testing.T) { } } -func TestHandleUpdateResourceClass(t *testing.T) { +func TestHandleUpdateResourceClassPassthrough(t *testing.T) { s := newTestShim(t, http.StatusNoContent, "", nil) w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_FOO") if w.Code != http.StatusNoContent { @@ -48,7 +90,7 @@ func TestHandleUpdateResourceClass(t *testing.T) { } } -func TestHandleDeleteResourceClass(t *testing.T) { +func TestHandleDeleteResourceClassPassthrough(t *testing.T) { s := newTestShim(t, http.StatusNoContent, "", nil) w := serveHandler(t, "DELETE", "/resource_classes/{name}", s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_BAR") if w.Code != http.StatusNoContent { @@ -56,98 +98,226 @@ func TestHandleDeleteResourceClass(t *testing.T) { } } -func TestHandleResourceClasses_HybridMode(t *testing.T) { - down, up := newTestTimers() - s := &Shim{ - config: config{ - PlacementURL: "http://should-not-be-called:1234", - Features: featuresConfig{ResourceClasses: FeatureModeHybrid}, - }, - maxBodyLogSize: 4096, - downstreamRequestTimer: down, - upstreamRequestTimer: up, +// --- CRD mode handler tests --- + +func TestHandleListResourceClassesLocal(t *testing.T) { + s := newResourceClassShim(t, []string{"CUSTOM_FOO", "MEMORY_MB", "VCPU"}) + + w := serveHandler(t, "GET", "/resource_classes", s.HandleListResourceClasses, "/resource_classes") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } - t.Run("GET list returns 501", func(t *testing.T) { - w := serveHandler(t, "GET", "/resource_classes", - s.HandleListResourceClasses, "/resource_classes") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("POST returns 501", func(t *testing.T) { - w := serveHandler(t, "POST", "/resource_classes", - s.HandleCreateResourceClass, "/resource_classes") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("GET show returns 501", func(t *testing.T) { - w := serveHandler(t, "GET", "/resource_classes/{name}", - s.HandleShowResourceClass, "/resource_classes/VCPU") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("PUT returns 501", func(t *testing.T) { - w := serveHandler(t, "PUT", "/resource_classes/{name}", - s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_FOO") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + var resp resourceClassesListResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if len(resp.ResourceClasses) != 3 { + t.Fatalf("got %d classes, want 3: %v", len(resp.ResourceClasses), resp.ResourceClasses) + } + want := []string{"CUSTOM_FOO", "MEMORY_MB", "VCPU"} + for i, rc := range resp.ResourceClasses { + if rc.Name != want[i] { + t.Errorf("class[%d] = %q, want %q", i, rc.Name, want[i]) } - }) - t.Run("DELETE returns 501", func(t *testing.T) { - w := serveHandler(t, "DELETE", "/resource_classes/{name}", - s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_BAR") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + if len(rc.Links) != 1 || rc.Links[0].Rel != "self" || rc.Links[0].Href != "/resource_classes/"+rc.Name { + t.Errorf("class[%d] links = %v, want self link", i, rc.Links) } - }) + } +} + +func TestHandleShowResourceClassLocalFound(t *testing.T) { + s := newResourceClassShim(t, []string{"VCPU", "MEMORY_MB"}) + w := serveHandler(t, "GET", "/resource_classes/{name}", s.HandleShowResourceClass, "/resource_classes/VCPU") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } +} + +func TestHandleShowResourceClassLocalNotFound(t *testing.T) { + s := newResourceClassShim(t, []string{"VCPU"}) + w := serveHandler(t, "GET", "/resource_classes/{name}", s.HandleShowResourceClass, "/resource_classes/NONEXISTENT") + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) + } +} + +func TestHandleUpdateResourceClassLocalCreated(t *testing.T) { + s := newResourceClassShim(t, nil) + w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_NEW") + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want %d", w.Code, http.StatusCreated) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_NEW") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if !found { + t.Error("expected resource class to be in store") + } +} + +func TestHandleUpdateResourceClassLocalAlreadyExists(t *testing.T) { + s := newResourceClassShim(t, []string{"CUSTOM_EXISTING"}) + w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_EXISTING") + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) + } +} + +func TestHandleUpdateResourceClassLocalBadPrefix(t *testing.T) { + s := newResourceClassShim(t, nil) + w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/VCPU") + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) + } +} + +func TestHandleCreateResourceClassLocalCreated(t *testing.T) { + s := newResourceClassShim(t, nil) + body := bytes.NewBufferString(`{"name":"CUSTOM_NEW"}`) + w := serveHandlerWithBody(t, "POST", "/resource_classes", s.HandleCreateResourceClass, "/resource_classes", body) + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want %d", w.Code, http.StatusCreated) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_NEW") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if !found { + t.Error("expected resource class to be in store") + } } -func TestHandleResourceClasses_CRDMode(t *testing.T) { +func TestHandleCreateResourceClassLocalConflict(t *testing.T) { + s := newResourceClassShim(t, []string{"CUSTOM_EXISTING"}) + body := bytes.NewBufferString(`{"name":"CUSTOM_EXISTING"}`) + w := serveHandlerWithBody(t, "POST", "/resource_classes", s.HandleCreateResourceClass, "/resource_classes", body) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", w.Code, http.StatusConflict) + } +} + +func TestHandleCreateResourceClassLocalBadPrefix(t *testing.T) { + s := newResourceClassShim(t, nil) + body := bytes.NewBufferString(`{"name":"VCPU"}`) + w := serveHandlerWithBody(t, "POST", "/resource_classes", s.HandleCreateResourceClass, "/resource_classes", body) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) + } +} + +func TestHandleDeleteResourceClassLocal(t *testing.T) { + s := newResourceClassShim(t, []string{"CUSTOM_DEL"}) + w := serveHandler(t, "DELETE", "/resource_classes/{name}", s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_DEL") + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_DEL") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if found { + t.Error("expected resource class to be deleted") + } +} + +func TestHandleDeleteResourceClassLocalNotFound(t *testing.T) { + s := newResourceClassShim(t, nil) + w := serveHandler(t, "DELETE", "/resource_classes/{name}", s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_GONE") + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) + } +} + +func TestHandleDeleteResourceClassLocalBadPrefix(t *testing.T) { + s := newResourceClassShim(t, []string{"VCPU"}) + w := serveHandler(t, "DELETE", "/resource_classes/{name}", s.HandleDeleteResourceClass, "/resource_classes/VCPU") + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) + } +} + +// --- Hybrid mode tests --- + +func newHybridResourceClassShim(t *testing.T, upstreamStatus int, upstreamBody string, classes []string) *Shim { + t.Helper() + t.Setenv("POD_NAMESPACE", "default") + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(upstreamStatus) + if upstreamBody != "" { + if _, err := w.Write([]byte(upstreamBody)); err != nil { + t.Errorf("failed to write upstream body: %v", err) + } + } + })) + t.Cleanup(upstream.Close) + objs := []client.Object{newTestResourceClassConfigMap("default", "test-rc-cm", classes)} + cl := newFakeClientWithScheme(t, objs...) down, up := newTestTimers() - s := &Shim{ + return &Shim{ + Client: cl, config: config{ - PlacementURL: "http://should-not-be-called:1234", - Features: featuresConfig{ResourceClasses: FeatureModeCRD}, + PlacementURL: upstream.URL, + Features: featuresConfig{ResourceClasses: FeatureModeHybrid}, + ResourceClasses: &resourceClassesConfig{ConfigMapName: "test-rc-cm"}, }, + httpClient: upstream.Client(), maxBodyLogSize: 4096, downstreamRequestTimer: down, upstreamRequestTimer: up, + resourceLocker: resourcelock.NewResourceLocker(cl, "default"), + } +} + +func TestHandleListResourceClassesHybridForwards(t *testing.T) { + s := newHybridResourceClassShim(t, http.StatusOK, `{"resource_classes":[{"name":"VCPU"}]}`, nil) + w := serveHandler(t, "GET", "/resource_classes", s.HandleListResourceClasses, "/resource_classes") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } +} + +func TestHandleUpdateResourceClassHybridUpdatesLocal(t *testing.T) { + s := newHybridResourceClassShim(t, http.StatusCreated, "", nil) + w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_HYB") + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want %d", w.Code, http.StatusCreated) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_HYB") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if !found { + t.Error("expected resource class to be added to local configmap in hybrid mode") + } +} + +func TestHandleDeleteResourceClassHybridUpdatesLocal(t *testing.T) { + s := newHybridResourceClassShim(t, http.StatusNoContent, "", []string{"CUSTOM_DEL"}) + w := serveHandler(t, "DELETE", "/resource_classes/{name}", s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_DEL") + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_DEL") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if found { + t.Error("expected resource class to be removed from local configmap in hybrid mode") + } +} + +func TestHandleUpdateResourceClassHybridUpstreamFailure(t *testing.T) { + s := newHybridResourceClassShim(t, http.StatusInternalServerError, "upstream error", nil) + w := serveHandler(t, "PUT", "/resource_classes/{name}", s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_FAIL") + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", w.Code, http.StatusInternalServerError) + } + found, err := s.hasResourceClass(context.Background(), "CUSTOM_FAIL") + if err != nil { + t.Fatalf("hasResourceClass: %v", err) + } + if found { + t.Error("expected resource class NOT to be added when upstream fails") } - t.Run("GET list returns 501", func(t *testing.T) { - w := serveHandler(t, "GET", "/resource_classes", - s.HandleListResourceClasses, "/resource_classes") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("POST returns 501", func(t *testing.T) { - w := serveHandler(t, "POST", "/resource_classes", - s.HandleCreateResourceClass, "/resource_classes") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("GET show returns 501", func(t *testing.T) { - w := serveHandler(t, "GET", "/resource_classes/{name}", - s.HandleShowResourceClass, "/resource_classes/VCPU") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("PUT returns 501", func(t *testing.T) { - w := serveHandler(t, "PUT", "/resource_classes/{name}", - s.HandleUpdateResourceClass, "/resource_classes/CUSTOM_FOO") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) - t.Run("DELETE returns 501", func(t *testing.T) { - w := serveHandler(t, "DELETE", "/resource_classes/{name}", - s.HandleDeleteResourceClass, "/resource_classes/CUSTOM_BAR") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) - } - }) } diff --git a/internal/shim/placement/handle_resource_provider_aggregates.go b/internal/shim/placement/handle_resource_provider_aggregates.go index 3d7205193..3fbea5b97 100644 --- a/internal/shim/placement/handle_resource_provider_aggregates.go +++ b/internal/shim/placement/handle_resource_provider_aggregates.go @@ -4,42 +4,271 @@ package placement import ( + "encoding/json" "net/http" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) +// resourceProviderAggregatesResponse is the JSON body returned by +// GET /resource_providers/{uuid}/aggregates and +// PUT /resource_providers/{uuid}/aggregates (microversion 1.19+). +// +// https://docs.openstack.org/api-ref/placement/#resource-provider-aggregates +type resourceProviderAggregatesResponse struct { + Aggregates []string `json:"aggregates"` + ResourceProviderGeneration int64 `json:"resource_provider_generation"` +} + +// resourceProviderAggregatesRequest is the JSON body expected by +// PUT /resource_providers/{uuid}/aggregates (microversion 1.19+). +type resourceProviderAggregatesRequest struct { + Aggregates []string `json:"aggregates"` + ResourceProviderGeneration int64 `json:"resource_provider_generation"` +} + // HandleListResourceProviderAggregates handles // GET /resource_providers/{uuid}/aggregates requests. // // Returns the list of aggregate UUIDs associated with the resource provider. // Aggregates model relationships among providers such as shared storage, // affinity/anti-affinity groups, and availability zones. Returns an empty -// list if the provider has no aggregate associations. Available since -// microversion 1.1. +// list if the provider has no aggregate associations. +// +// Routing: the uuid is used to determine if the resource provider is a KVM +// hypervisor or vmware/ironic hypervisor. Passthrough mode forwards all +// requests to upstream placement. Hybrid mode uses the hypervisor CRD for +// KVM hypervisors and forwards for anything else. CRD-only mode rejects +// any non-KVM calls with 404. // -// The response format changed at microversion 1.19: earlier versions return -// only a flat array of UUIDs, while 1.19+ returns an object that also -// includes the resource_provider_generation for concurrency tracking. Returns -// 404 if the provider does not exist. +// https://docs.openstack.org/api-ref/placement/#list-resource-provider-aggregates func (s *Shim) HandleListResourceProviderAggregates(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredUUIDPathParam(w, r, "uuid"); !ok { + uuid, ok := requiredUUIDPathParam(w, r, "uuid") + if !ok { return } - s.dispatchPassthroughOnly(w, r, s.config.Features.Aggregates) + switch s.featureModeFromConfOrHeader(r, s.config.Features.Aggregates, true) { + case FeatureModePassthrough: + s.forward(w, r) + case FeatureModeHybrid: + s.listResourceProviderAggregatesHybrid(w, r, uuid) + case FeatureModeCRD: + s.listResourceProviderAggregatesCRD(w, r, uuid) + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + } +} + +// listResourceProviderAggregatesHybrid serves from the CRD if the provider is +// a KVM hypervisor, otherwise forwards to upstream placement. +func (s *Shim) listResourceProviderAggregatesHybrid(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if err != nil || len(hvs.Items) != 1 { + log.Info("resource provider not resolved from kubernetes, forwarding to upstream placement", "uuid", uuid) + s.forward(w, r) + return + } + log.Info("resolved resource provider from CRD, serving aggregates", "uuid", uuid, "hypervisor", hvs.Items[0].Name) + s.writeAggregatesFromCRD(w, &hvs.Items[0]) +} + +// listResourceProviderAggregatesCRD serves exclusively from the CRD, returning +// 404 if the provider is not a known KVM hypervisor. +func (s *Shim) listResourceProviderAggregatesCRD(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { + log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) + http.Error(w, "resource provider not found", http.StatusNotFound) + return + } + if err != nil { + log.Error(err, "failed to list hypervisors with OpenStack ID index") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if len(hvs.Items) > 1 { + log.Error(nil, "multiple hypervisors found with the same OpenStack ID", "uuid", uuid) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + log.Info("serving aggregates from CRD", "uuid", uuid, "hypervisor", hvs.Items[0].Name) + s.writeAggregatesFromCRD(w, &hvs.Items[0]) +} + +func (s *Shim) writeAggregatesFromCRD(w http.ResponseWriter, hv *hv1.Hypervisor) { + aggGroups := hv1.GetAggregates(hv.Spec.Groups) + aggregates := make([]string, 0, len(aggGroups)) + for _, ag := range aggGroups { + aggregates = append(aggregates, ag.UUID) + } + s.writeJSON(w, http.StatusOK, resourceProviderAggregatesResponse{ + Aggregates: aggregates, + ResourceProviderGeneration: hv.Generation, + }) } // HandleUpdateResourceProviderAggregates handles // PUT /resource_providers/{uuid}/aggregates requests. // // Replaces the complete set of aggregate associations for a resource provider. -// Any aggregate UUIDs that do not yet exist are created automatically. The -// request format changed at microversion 1.19: earlier versions accept a -// plain array of UUIDs, while 1.19+ expects an object containing an -// aggregates array and a resource_provider_generation for optimistic -// concurrency control. Returns 409 Conflict if the generation does not match -// (1.19+). Returns 200 with the updated aggregate list on success. +// The request body must include an aggregates array and a +// resource_provider_generation for optimistic concurrency control. Returns +// 409 Conflict if the generation does not match. Returns 200 with the +// updated aggregate list on success. +// +// Routing: same selective per-provider dispatch as GET. +// +// https://docs.openstack.org/api-ref/placement/#update-resource-provider-aggregates func (s *Shim) HandleUpdateResourceProviderAggregates(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredUUIDPathParam(w, r, "uuid"); !ok { + uuid, ok := requiredUUIDPathParam(w, r, "uuid") + if !ok { + return + } + switch s.featureModeFromConfOrHeader(r, s.config.Features.Aggregates, true) { + case FeatureModePassthrough: + s.forward(w, r) + case FeatureModeHybrid: + s.updateResourceProviderAggregatesHybrid(w, r, uuid) + case FeatureModeCRD: + s.updateResourceProviderAggregatesCRD(w, r, uuid) + default: + http.Error(w, "unknown feature mode", http.StatusInternalServerError) + } +} + +// updateResourceProviderAggregatesHybrid updates aggregates via the CRD if the +// provider is a KVM hypervisor, otherwise forwards to upstream placement. +func (s *Shim) updateResourceProviderAggregatesHybrid(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if err != nil || len(hvs.Items) != 1 { + log.Info("resource provider not resolved from kubernetes, forwarding to upstream placement", "uuid", uuid) + s.forward(w, r) return } - s.dispatchPassthroughOnly(w, r, s.config.Features.Aggregates) + hv := &hvs.Items[0] + log.Info("resolved resource provider from CRD, updating aggregates", "uuid", uuid, "hypervisor", hv.Name) + + var req resourceProviderAggregatesRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "malformed request body", http.StatusBadRequest) + return + } + if req.ResourceProviderGeneration != hv.Generation { + log.Info("generation mismatch on aggregate update", + "expected", req.ResourceProviderGeneration, "actual", hv.Generation) + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Aggregate == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + for _, aggUUID := range req.Aggregates { + newGroups = append(newGroups, hv1.Group{ + Aggregate: &hv1.AggregateGroup{Name: aggUUID, UUID: aggUUID}, + }) + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to update hypervisor aggregates") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully updated aggregates via CRD", "uuid", uuid, "aggregateCount", len(req.Aggregates)) + s.writeJSON(w, http.StatusOK, resourceProviderAggregatesResponse{ + Aggregates: req.Aggregates, + ResourceProviderGeneration: hv.Generation, + }) +} + +// updateResourceProviderAggregatesCRD updates aggregates exclusively via the +// CRD, returning 404 if the provider is not a known KVM hypervisor. +func (s *Shim) updateResourceProviderAggregatesCRD(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { + log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) + http.Error(w, "resource provider not found", http.StatusNotFound) + return + } + if err != nil { + log.Error(err, "failed to list hypervisors with OpenStack ID index") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if len(hvs.Items) > 1 { + log.Error(nil, "multiple hypervisors found with the same OpenStack ID", "uuid", uuid) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + hv := &hvs.Items[0] + log.Info("updating aggregates via CRD", "uuid", uuid, "hypervisor", hv.Name) + + var req resourceProviderAggregatesRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "malformed request body", http.StatusBadRequest) + return + } + if req.ResourceProviderGeneration != hv.Generation { + log.Info("generation mismatch on aggregate update", + "expected", req.ResourceProviderGeneration, "actual", hv.Generation) + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Aggregate == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + for _, aggUUID := range req.Aggregates { + newGroups = append(newGroups, hv1.Group{ + Aggregate: &hv1.AggregateGroup{Name: aggUUID, UUID: aggUUID}, + }) + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to update hypervisor aggregates") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully updated aggregates via CRD", "uuid", uuid, "aggregateCount", len(req.Aggregates)) + s.writeJSON(w, http.StatusOK, resourceProviderAggregatesResponse{ + Aggregates: req.Aggregates, + ResourceProviderGeneration: hv.Generation, + }) } diff --git a/internal/shim/placement/handle_resource_provider_aggregates_e2e.go b/internal/shim/placement/handle_resource_provider_aggregates_e2e.go index b673c75f6..3f7f55424 100644 --- a/internal/shim/placement/handle_resource_provider_aggregates_e2e.go +++ b/internal/shim/placement/handle_resource_provider_aggregates_e2e.go @@ -10,8 +10,12 @@ import ( "fmt" "net/http" "slices" + "time" "github.com/cobaltcore-dev/cortex/pkg/conf" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/gophercloud/gophercloud/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -19,15 +23,11 @@ import ( // e2eTestResourceProviderAggregates tests the // /resource_providers/{uuid}/aggregates endpoints. // -// 1. Pre-cleanup: DELETE any leftover test RP (ignore 404). -// 2. POST /resource_providers — create a test RP. -// 3. GET /{uuid}/aggregates — verify aggregates are empty, store generation. -// 4. PUT /{uuid}/aggregates — associate two aggregate UUIDs with the RP. -// 5. GET /{uuid}/aggregates — verify both aggregate UUIDs are present. -// 6. PUT /{uuid}/aggregates — clear aggregates by sending an empty list. -// 7. GET /{uuid}/aggregates — verify aggregates are empty after clear. -// 8. Cleanup: DELETE the test RP (also runs via deferred cleanup on failure). -func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) error { +// In passthrough mode: exercises the upstream placement path with a +// dynamically created resource provider. +// In hybrid/crd mode: exercises the spec.groups-backed CRD path using a +// real KVM hypervisor discovered from the cluster. +func e2eTestResourceProviderAggregates(ctx context.Context, cl client.Client) error { log := logf.FromContext(ctx) log.Info("Running resource provider aggregates endpoint e2e test") config, err := conf.GetConfig[e2eRootConfig]() @@ -43,12 +43,26 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err } log.Info("Successfully created openstack client for resource provider aggregates e2e test") + mode := e2eCurrentMode(ctx) + switch mode { + case FeatureModePassthrough: + return e2ePassthroughResourceProviderAggregates(ctx, sc) + case FeatureModeHybrid, FeatureModeCRD: + return e2eCRDResourceProviderAggregates(ctx, sc, cl) + default: + return fmt.Errorf("unexpected mode %q", mode) + } +} + +func e2ePassthroughResourceProviderAggregates(ctx context.Context, sc *gophercloud.ServiceClient) error { + log := logf.FromContext(ctx) + const testRPUUID = "e2e10000-0000-0000-0000-000000000004" const testRPName = "cortex-e2e-test-rp-agg" const testAggUUID1 = "e2e30000-0000-0000-0000-000000000001" const testAggUUID2 = "e2e30000-0000-0000-0000-000000000002" - // Pre-cleanup: delete any leftover test resource provider from a prior run. + // Pre-cleanup: delete leftover test RP. log.Info("Pre-cleanup: deleting leftover test resource provider", "uuid", testRPUUID) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID, http.NoBody) @@ -107,8 +121,7 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err log.Info("Successfully created test resource provider for aggregates test", "uuid", testRPUUID) - // Deferred cleanup: always delete the test RP on exit so a failed - // assertion doesn't leave the fixed UUID behind. + // Deferred cleanup. defer func() { log.Info("Deferred cleanup: deleting test resource provider", "uuid", testRPUUID) dReq, dErr := http.NewRequestWithContext(ctx, @@ -128,13 +141,11 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err log.Info("Deferred cleanup completed", "status", dResp.StatusCode) }() - // Test GET /resource_providers/{uuid}/aggregates (empty). - log.Info("Testing GET /resource_providers/{uuid}/aggregates (empty)", - "uuid", testRPUUID) + // Test GET (empty). + log.Info("Testing GET /resource_providers/{uuid}/aggregates (empty)", "uuid", testRPUUID) req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/aggregates", http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP aggregates", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -142,44 +153,36 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP aggregates", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP aggregates returned an error", "uuid", testRPUUID) - return err + return fmt.Errorf("GET RP aggregates: unexpected status %d", resp.StatusCode) } var aggResp struct { Aggregates []string `json:"aggregates"` ResourceProviderGeneration int `json:"resource_provider_generation"` } - err = json.NewDecoder(resp.Body).Decode(&aggResp) - if err != nil { - log.Error(err, "failed to decode RP aggregates response", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&aggResp); err != nil { return err } - log.Info("Successfully retrieved empty aggregates for test resource provider", - "uuid", testRPUUID, "aggregates", len(aggResp.Aggregates), - "generation", aggResp.ResourceProviderGeneration) + if len(aggResp.Aggregates) != 0 { + return fmt.Errorf("expected 0 initial aggregates, got %d", len(aggResp.Aggregates)) + } + log.Info("Verified empty aggregates", "generation", aggResp.ResourceProviderGeneration) - // Test PUT /resource_providers/{uuid}/aggregates (set two aggregates). - log.Info("Testing PUT /resource_providers/{uuid}/aggregates to set aggregates", - "uuid", testRPUUID, "agg1", testAggUUID1, "agg2", testAggUUID2) + // Test PUT (associate aggregates). putBody, err := json.Marshal(map[string]any{ "resource_provider_generation": aggResp.ResourceProviderGeneration, "aggregates": []string{testAggUUID1, testAggUUID2}, }) if err != nil { - log.Error(err, "failed to marshal request body") return err } req, err = http.NewRequestWithContext(ctx, http.MethodPut, sc.Endpoint+"/resource_providers/"+testRPUUID+"/aggregates", bytes.NewReader(putBody)) if err != nil { - log.Error(err, "failed to create PUT request for RP aggregates", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -188,35 +191,18 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send PUT request for RP aggregates", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "PUT RP aggregates returned an error", "uuid", testRPUUID) - return err - } - var putAggResp struct { - Aggregates []string `json:"aggregates"` - ResourceProviderGeneration int `json:"resource_provider_generation"` - } - err = json.NewDecoder(resp.Body).Decode(&putAggResp) - if err != nil { - log.Error(err, "failed to decode PUT RP aggregates response", "uuid", testRPUUID) - return err + return fmt.Errorf("PUT RP aggregates: unexpected status %d", resp.StatusCode) } - log.Info("Successfully set aggregates on test resource provider", - "uuid", testRPUUID, "aggregates", len(putAggResp.Aggregates), - "generation", putAggResp.ResourceProviderGeneration) + log.Info("Successfully associated aggregates") - // Test GET /resource_providers/{uuid}/aggregates (after PUT). - log.Info("Testing GET /resource_providers/{uuid}/aggregates (after PUT)", - "uuid", testRPUUID) + // Test GET (after PUT). req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/aggregates", http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP aggregates", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -224,47 +210,29 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP aggregates", "uuid", testRPUUID) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP aggregates returned an error", "uuid", testRPUUID) - return err - } - err = json.NewDecoder(resp.Body).Decode(&aggResp) - if err != nil { - log.Error(err, "failed to decode RP aggregates response", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&aggResp); err != nil { return err } - if len(aggResp.Aggregates) != 2 || - !slices.Contains(aggResp.Aggregates, testAggUUID1) || - !slices.Contains(aggResp.Aggregates, testAggUUID2) { - err := fmt.Errorf("expected aggregates %v, got %v", - []string{testAggUUID1, testAggUUID2}, aggResp.Aggregates) - log.Error(err, "aggregate mismatch", "uuid", testRPUUID) - return err + if !slices.Contains(aggResp.Aggregates, testAggUUID1) || !slices.Contains(aggResp.Aggregates, testAggUUID2) { + return fmt.Errorf("expected aggregates %v and %v, got %v", testAggUUID1, testAggUUID2, aggResp.Aggregates) } - log.Info("Successfully verified aggregates on test resource provider", - "uuid", testRPUUID, "aggregates", aggResp.Aggregates) + log.Info("Verified aggregates present after PUT") // Clear aggregates by PUT with empty list. - log.Info("Testing PUT /resource_providers/{uuid}/aggregates to clear aggregates", - "uuid", testRPUUID) putBody, err = json.Marshal(map[string]any{ "resource_provider_generation": aggResp.ResourceProviderGeneration, "aggregates": []string{}, }) if err != nil { - log.Error(err, "failed to marshal request body") return err } req, err = http.NewRequestWithContext(ctx, http.MethodPut, sc.Endpoint+"/resource_providers/"+testRPUUID+"/aggregates", bytes.NewReader(putBody)) if err != nil { - log.Error(err, "failed to create PUT request to clear RP aggregates", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -273,24 +241,18 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send PUT request to clear RP aggregates", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "PUT to clear RP aggregates returned an error", "uuid", testRPUUID) - return err + return fmt.Errorf("PUT RP aggregates (clear): unexpected status %d", resp.StatusCode) } - log.Info("Successfully cleared aggregates on test resource provider", - "uuid", testRPUUID) + log.Info("Successfully cleared aggregates") - // Verify aggregates are empty after clear. - log.Info("Verifying aggregates are empty after clear", "uuid", testRPUUID) + // Verify empty after clear. req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/aggregates", http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP aggregates", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -298,53 +260,225 @@ func e2eTestResourceProviderAggregates(ctx context.Context, _ client.Client) err req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP aggregates", "uuid", testRPUUID) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP aggregates returned an error", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&aggResp); err != nil { return err } - err = json.NewDecoder(resp.Body).Decode(&aggResp) + if len(aggResp.Aggregates) != 0 { + return fmt.Errorf("expected 0 aggregates after clear, got %d", len(aggResp.Aggregates)) + } + log.Info("Verified aggregates empty after clear") + + // Cleanup. + req, err = http.NewRequestWithContext(ctx, + http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID, http.NoBody) if err != nil { - log.Error(err, "failed to decode RP aggregates response", "uuid", testRPUUID) return err } - if len(aggResp.Aggregates) != 0 { - err := fmt.Errorf("expected 0 aggregates after clear, got %d", len(aggResp.Aggregates)) - log.Error(err, "aggregates not empty after clear", "uuid", testRPUUID) + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.19") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return err + } + resp.Body.Close() + + return nil +} + +// e2eCRDResourceProviderAggregates tests the CRD/hybrid path by discovering a +// real KVM hypervisor in the cluster, seeding spec.groups, and exercising +// GET/PUT through the shim. +func e2eCRDResourceProviderAggregates(ctx context.Context, sc *gophercloud.ServiceClient, cl client.Client) error { + log := logf.FromContext(ctx) + + // Discover a KVM hypervisor with a non-empty OpenStack ID. + var hvs hv1.HypervisorList + if err := cl.List(ctx, &hvs); err != nil { + log.Error(err, "failed to list hypervisors for CRD aggregates path") + return err + } + var kvmHV *hv1.Hypervisor + for i := range hvs.Items { + if hvs.Items[i].Status.HypervisorID != "" { + kvmHV = &hvs.Items[i] + break + } + } + if kvmHV == nil { + log.Info("No KVM hypervisors with OpenStack ID found, skipping CRD aggregates tests") + return nil + } + kvmUUID := kvmHV.Status.HypervisorID + log.Info("Using KVM hypervisor for CRD aggregates e2e tests", "uuid", kvmUUID, "name", kvmHV.Name) + + // Save original groups for restoration. + originalGroups := kvmHV.Spec.Groups + + // Seed spec.groups with test aggregates (preserve non-aggregate groups). + const testAgg1UUID = "e2e40000-0000-0000-0000-000000000001" + const testAgg2UUID = "e2e40000-0000-0000-0000-000000000002" + var nonAggGroups []hv1.Group + for i := range kvmHV.Spec.Groups { + if kvmHV.Spec.Groups[i].Aggregate == nil { + nonAggGroups = append(nonAggGroups, kvmHV.Spec.Groups[i]) + } + } + nonAggGroups = append(nonAggGroups, + hv1.Group{Aggregate: &hv1.AggregateGroup{Name: testAgg1UUID, UUID: testAgg1UUID}}, + hv1.Group{Aggregate: &hv1.AggregateGroup{Name: testAgg2UUID, UUID: testAgg2UUID}}, + ) + kvmHV.Spec.Groups = nonAggGroups + if err := cl.Update(ctx, kvmHV); err != nil { + return fmt.Errorf("failed to seed spec.groups with test aggregates: %w", err) + } + log.Info("Seeded spec.groups with test aggregates", "uuid", kvmUUID) + + // Always restore original groups on exit (retry on conflict). + defer func() { + log.Info("Restoring original spec.groups", "uuid", kvmUUID) + for range 5 { + if err := cl.Get(ctx, client.ObjectKeyFromObject(kvmHV), kvmHV); err != nil { + log.Error(err, "failed to refetch hypervisor for restoration") + return + } + kvmHV.Spec.Groups = originalGroups + if err := cl.Update(ctx, kvmHV); err != nil { + if apierrors.IsConflict(err) { + continue + } + log.Error(err, "failed to restore original spec.groups") + return + } + return + } + log.Error(nil, "exhausted retries restoring original spec.groups") + }() + + // Refetch to get updated generation. + if err := cl.Get(ctx, client.ObjectKeyFromObject(kvmHV), kvmHV); err != nil { + return fmt.Errorf("failed to refetch hypervisor after seed: %w", err) + } + + // Test GET — should return the seeded aggregates. + // Poll because the shim's informer cache may take a moment to observe the update. + log.Info("Testing GET /resource_providers/{uuid}/aggregates (CRD)", "uuid", kvmUUID) + var aggResp struct { + Aggregates []string `json:"aggregates"` + ResourceProviderGeneration int64 `json:"resource_provider_generation"` + } + if err := e2ePollUntil(ctx, 10*time.Second, func() (bool, error) { + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, sc.Endpoint+"/resource_providers/"+kvmUUID+"/aggregates", http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.19") + req.Header.Set("Accept", "application/json") + resp, err := sc.HTTPClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("GET CRD aggregates: expected 200, got %d", resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(&aggResp); err != nil { + return false, fmt.Errorf("failed to decode CRD aggregates response: %w", err) + } + return slices.Contains(aggResp.Aggregates, testAgg1UUID) && + slices.Contains(aggResp.Aggregates, testAgg2UUID), nil + }); err != nil { + return fmt.Errorf("waiting for seeded aggregates: %w (got %v)", err, aggResp.Aggregates) + } + log.Info("Verified GET returns seeded aggregates from CRD", + "aggregates", aggResp.Aggregates, "generation", aggResp.ResourceProviderGeneration) + + // Test PUT — replace aggregates. + const replacementAggUUID = "e2e40000-0000-0000-0000-000000000099" + putBody, err := json.Marshal(map[string]any{ + "resource_provider_generation": aggResp.ResourceProviderGeneration, + "aggregates": []string{replacementAggUUID}, + }) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPut, sc.Endpoint+"/resource_providers/"+kvmUUID+"/aggregates", + bytes.NewReader(putBody)) + if err != nil { return err } - log.Info("Verified aggregates are empty after clear", "uuid", testRPUUID) + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.19") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := sc.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("PUT CRD aggregates: expected 200, got %d", resp.StatusCode) + } + log.Info("Successfully replaced aggregates via PUT (CRD)") - // Cleanup: delete the test resource provider. - log.Info("Cleaning up test resource provider", "uuid", testRPUUID) + // Test PUT with stale generation — should return 409. + putBody, err = json.Marshal(map[string]any{ + "resource_provider_generation": aggResp.ResourceProviderGeneration, + "aggregates": []string{"stale-uuid"}, + }) + if err != nil { + return err + } req, err = http.NewRequestWithContext(ctx, - http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID, http.NoBody) + http.MethodPut, sc.Endpoint+"/resource_providers/"+kvmUUID+"/aggregates", + bytes.NewReader(putBody)) if err != nil { - log.Error(err, "failed to create DELETE request for resource provider", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.19") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send DELETE request for resource provider", "uuid", testRPUUID) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "DELETE /resource_providers/{uuid} returned an error", "uuid", testRPUUID) + if resp.StatusCode != http.StatusConflict { + return fmt.Errorf("PUT CRD aggregates (stale gen): expected 409, got %d", resp.StatusCode) + } + log.Info("Verified generation conflict returns 409") + + // Test GET — verify replacement persisted. + req, err = http.NewRequestWithContext(ctx, + http.MethodGet, sc.Endpoint+"/resource_providers/"+kvmUUID+"/aggregates", http.NoBody) + if err != nil { return err } - log.Info("Successfully deleted test resource provider", "uuid", testRPUUID) + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.19") + req.Header.Set("Accept", "application/json") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(&aggResp); err != nil { + return err + } + if len(aggResp.Aggregates) != 1 || aggResp.Aggregates[0] != replacementAggUUID { + return fmt.Errorf("expected [%s], got %v", replacementAggUUID, aggResp.Aggregates) + } + log.Info("Verified replacement aggregate persisted") return nil } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_provider_aggregates", run: e2eTestResourceProviderAggregates}) + e2eTests = append(e2eTests, e2eTest{name: "resource_provider_aggregates", run: e2eWrapWithModes(e2eTestResourceProviderAggregates)}) } diff --git a/internal/shim/placement/handle_resource_provider_aggregates_test.go b/internal/shim/placement/handle_resource_provider_aggregates_test.go index eb35d665c..5dfac22c6 100644 --- a/internal/shim/placement/handle_resource_provider_aggregates_test.go +++ b/internal/shim/placement/handle_resource_provider_aggregates_test.go @@ -4,8 +4,13 @@ package placement import ( + "encoding/json" "net/http" + "strings" "testing" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func TestHandleListResourceProviderAggregates(t *testing.T) { @@ -51,59 +56,207 @@ func TestHandleUpdateResourceProviderAggregates(t *testing.T) { } func TestHandleResourceProviderAggregates_HybridMode(t *testing.T) { - down, up := newTestTimers() - s := &Shim{ - config: config{ - PlacementURL: "http://should-not-be-called:1234", - Features: featuresConfig{Aggregates: FeatureModeHybrid}, - }, - maxBodyLogSize: 4096, - downstreamRequestTimer: down, - upstreamRequestTimer: up, - } - t.Run("GET returns 501", func(t *testing.T) { + s := newTestShimWithHypervisors(t, http.StatusOK, `{"aggregates":["uuid-1"],"resource_provider_generation":1}`) + s.config.Features.Aggregates = FeatureModeHybrid + t.Run("GET forwards to upstream when provider not in CRD", func(t *testing.T) { w := serveHandler(t, "GET", "/resource_providers/{uuid}/aggregates", s.HandleListResourceProviderAggregates, "/resource_providers/"+validUUID+"/aggregates") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } }) - t.Run("PUT returns 501", func(t *testing.T) { + t.Run("PUT forwards to upstream when provider not in CRD", func(t *testing.T) { w := serveHandler(t, "PUT", "/resource_providers/{uuid}/aggregates", s.HandleUpdateResourceProviderAggregates, "/resource_providers/"+validUUID+"/aggregates") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + }) + + t.Run("GET serves from CRD when provider is KVM", func(t *testing.T) { + hv := testHypervisorWithGroups("kvm-hybrid-agg", validUUID, []hv1.Group{ + {Aggregate: &hv1.AggregateGroup{Name: "az-west", UUID: "agg-uuid-1"}}, + }) + sKVM := newTestShimWithHypervisors(t, http.StatusOK, "{}", hv) + sKVM.config.Features.Aggregates = FeatureModeHybrid + w := serveHandler(t, "GET", "/resource_providers/{uuid}/aggregates", + sKVM.HandleListResourceProviderAggregates, + "/resource_providers/"+validUUID+"/aggregates") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var resp resourceProviderAggregatesResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Aggregates) != 1 || resp.Aggregates[0] != "agg-uuid-1" { + t.Fatalf("expected [agg-uuid-1], got %v", resp.Aggregates) } }) } func TestHandleResourceProviderAggregates_CRDMode(t *testing.T) { - down, up := newTestTimers() - s := &Shim{ - config: config{ - PlacementURL: "http://should-not-be-called:1234", - Features: featuresConfig{Aggregates: FeatureModeCRD}, - }, - maxBodyLogSize: 4096, - downstreamRequestTimer: down, - upstreamRequestTimer: up, + groups := []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "HW_CPU_X86_AVX2"}}, + {Aggregate: &hv1.AggregateGroup{Name: "fast-storage", UUID: "agg-uuid-1"}}, + {Aggregate: &hv1.AggregateGroup{Name: "az-west", UUID: "agg-uuid-2"}}, } - t.Run("GET returns 501", func(t *testing.T) { + hv := testHypervisorWithGroups("kvm-host-1", validUUID, groups) + s := newTestShimWithHypervisors(t, http.StatusOK, "{}", hv) + s.config.Features.Aggregates = FeatureModeCRD + + t.Run("GET returns aggregate UUIDs from spec.groups", func(t *testing.T) { w := serveHandler(t, "GET", "/resource_providers/{uuid}/aggregates", s.HandleListResourceProviderAggregates, "/resource_providers/"+validUUID+"/aggregates") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var resp resourceProviderAggregatesResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Aggregates) != 2 { + t.Fatalf("aggregates count = %d, want 2", len(resp.Aggregates)) + } + if resp.Aggregates[0] != "agg-uuid-1" { + t.Errorf("aggregates[0] = %q, want agg-uuid-1", resp.Aggregates[0]) + } + if resp.Aggregates[1] != "agg-uuid-2" { + t.Errorf("aggregates[1] = %q, want agg-uuid-2", resp.Aggregates[1]) } }) - t.Run("PUT returns 501", func(t *testing.T) { - w := serveHandler(t, "PUT", "/resource_providers/{uuid}/aggregates", + + t.Run("GET returns empty aggregates when spec.groups has no aggregates", func(t *testing.T) { + hvNoAggs := testHypervisorWithGroups("kvm-no-aggs", "b1b2b3b4-c5c6-d7d8-e9e0-f1f2f3f4f5f6", []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "CUSTOM_T"}}, + }) + s2 := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvNoAggs) + s2.config.Features.Aggregates = FeatureModeCRD + w := serveHandler(t, "GET", "/resource_providers/{uuid}/aggregates", + s2.HandleListResourceProviderAggregates, + "/resource_providers/b1b2b3b4-c5c6-d7d8-e9e0-f1f2f3f4f5f6/aggregates") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var resp resourceProviderAggregatesResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Aggregates) != 0 { + t.Fatalf("aggregates count = %d, want 0", len(resp.Aggregates)) + } + }) + + t.Run("GET returns 404 for non-existent provider", func(t *testing.T) { + nonExistUUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + w := serveHandler(t, "GET", "/resource_providers/{uuid}/aggregates", + s.HandleListResourceProviderAggregates, + "/resource_providers/"+nonExistUUID+"/aggregates") + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) + } + }) + + t.Run("PUT replaces aggregates in spec.groups preserving traits", func(t *testing.T) { + hvPut := testHypervisorWithGroups("kvm-put-aggs", "c1c2c3c4-d5d6-e7e8-f9f0-a1a2a3a4a5a6", []hv1.Group{ + {Aggregate: &hv1.AggregateGroup{Name: "old-agg", UUID: "old-uuid"}}, + {Trait: &hv1.TraitGroup{Name: "KEEP_TRAIT"}}, + }) + sPut := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvPut) + sPut.config.Features.Aggregates = FeatureModeCRD + + body := `{"aggregates":["new-uuid-1","new-uuid-2"],"resource_provider_generation":0}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/aggregates", + sPut.HandleUpdateResourceProviderAggregates, + "/resource_providers/c1c2c3c4-d5d6-e7e8-f9f0-a1a2a3a4a5a6/aggregates", strings.NewReader(body)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp resourceProviderAggregatesResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Aggregates) != 2 { + t.Fatalf("aggregates count = %d, want 2", len(resp.Aggregates)) + } + + // Verify traits were preserved. + var updated hv1.Hypervisor + if err := sPut.Get(t.Context(), client.ObjectKeyFromObject(hvPut), &updated); err != nil { + t.Fatalf("failed to get updated hypervisor: %v", err) + } + traits := hv1.GetTraits(updated.Spec.Groups) + if len(traits) != 1 || traits[0].Name != "KEEP_TRAIT" { + t.Fatalf("traits were not preserved: got %+v", traits) + } + }) + + t.Run("PUT returns 409 on generation mismatch", func(t *testing.T) { + hvConflict := testHypervisorWithGroups("kvm-agg-conflict", "d1d2d3d4-e5e6-f7f8-a9a0-b1b2b3b4b5b6", nil) + sConflict := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvConflict) + sConflict.config.Features.Aggregates = FeatureModeCRD + + body := `{"aggregates":["u1"],"resource_provider_generation":999}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/aggregates", + sConflict.HandleUpdateResourceProviderAggregates, + "/resource_providers/d1d2d3d4-e5e6-f7f8-a9a0-b1b2b3b4b5b6/aggregates", strings.NewReader(body)) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", w.Code, http.StatusConflict) + } + }) + + t.Run("PUT returns 404 for non-existent provider", func(t *testing.T) { + body := `{"aggregates":["u1"],"resource_provider_generation":0}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/aggregates", s.HandleUpdateResourceProviderAggregates, - "/resource_providers/"+validUUID+"/aggregates") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + "/resource_providers/e1e2e3e4-f5f6-a7a8-b9b0-c1c2c3c4c5c6/aggregates", strings.NewReader(body)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) + } + }) + + t.Run("PUT with empty list removes all aggregates", func(t *testing.T) { + hvClear := testHypervisorWithGroups("kvm-clear-aggs", "e1e2e3e4-f5f6-a7a8-b9b0-c1c2c3c4c5c6", []hv1.Group{ + {Aggregate: &hv1.AggregateGroup{Name: "remove-me", UUID: "remove-uuid"}}, + {Trait: &hv1.TraitGroup{Name: "KEEP_TRAIT"}}, + }) + sClear := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvClear) + sClear.config.Features.Aggregates = FeatureModeCRD + + body := `{"aggregates":[],"resource_provider_generation":0}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/aggregates", + sClear.HandleUpdateResourceProviderAggregates, + "/resource_providers/e1e2e3e4-f5f6-a7a8-b9b0-c1c2c3c4c5c6/aggregates", strings.NewReader(body)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp resourceProviderAggregatesResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Aggregates) != 0 { + t.Fatalf("expected 0 aggregates, got %d", len(resp.Aggregates)) + } + + var updated hv1.Hypervisor + if err := sClear.Get(t.Context(), client.ObjectKeyFromObject(hvClear), &updated); err != nil { + t.Fatalf("failed to get updated hypervisor: %v", err) + } + traits := hv1.GetTraits(updated.Spec.Groups) + if len(traits) != 1 || traits[0].Name != "KEEP_TRAIT" { + t.Fatalf("traits were not preserved: got %+v", traits) + } + }) + + t.Run("PUT returns 400 for malformed body", func(t *testing.T) { + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/aggregates", + s.HandleUpdateResourceProviderAggregates, + "/resource_providers/"+validUUID+"/aggregates", strings.NewReader("not json")) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) } }) } diff --git a/internal/shim/placement/handle_resource_provider_allocations_e2e.go b/internal/shim/placement/handle_resource_provider_allocations_e2e.go index a63c8cb4f..aea84ec06 100644 --- a/internal/shim/placement/handle_resource_provider_allocations_e2e.go +++ b/internal/shim/placement/handle_resource_provider_allocations_e2e.go @@ -44,6 +44,15 @@ func e2eTestResourceProviderAllocations(ctx context.Context, _ client.Client) er const testRPUUID = "e2e10000-0000-0000-0000-000000000006" const testRPName = "cortex-e2e-test-rp-alloc-view" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/resource_providers/"+testRPUUID+"/allocations") + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete any leftover test resource provider from a prior run. log.Info("Pre-cleanup: deleting leftover test resource provider", "uuid", testRPUUID) req, err := http.NewRequestWithContext(ctx, @@ -227,5 +236,5 @@ func e2eTestResourceProviderAllocations(ctx context.Context, _ client.Client) er } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_provider_allocations", run: e2eTestResourceProviderAllocations}) + e2eTests = append(e2eTests, e2eTest{name: "resource_provider_allocations", run: e2eWrapWithModes(e2eTestResourceProviderAllocations)}) } diff --git a/internal/shim/placement/handle_resource_provider_inventories_e2e.go b/internal/shim/placement/handle_resource_provider_inventories_e2e.go index 354460e81..1462ca87b 100644 --- a/internal/shim/placement/handle_resource_provider_inventories_e2e.go +++ b/internal/shim/placement/handle_resource_provider_inventories_e2e.go @@ -53,6 +53,15 @@ func e2eTestResourceProviderInventories(ctx context.Context, _ client.Client) er const testRC = "CUSTOM_CORTEX_E2E_INV_RC" const apiVersion = "placement 1.26" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/resource_providers/"+testRPUUID+"/inventories") + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete the resource provider (cascades inventories), then // the custom resource class. Ignore 404/409. log.Info("Pre-cleanup: deleting leftover test resources") @@ -488,5 +497,5 @@ func e2eTestResourceProviderInventories(ctx context.Context, _ client.Client) er } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_provider_inventories", run: e2eTestResourceProviderInventories}) + e2eTests = append(e2eTests, e2eTest{name: "resource_provider_inventories", run: e2eWrapWithModes(e2eTestResourceProviderInventories)}) } diff --git a/internal/shim/placement/handle_resource_provider_traits.go b/internal/shim/placement/handle_resource_provider_traits.go index 16978a593..8cc61b421 100644 --- a/internal/shim/placement/handle_resource_provider_traits.go +++ b/internal/shim/placement/handle_resource_provider_traits.go @@ -4,7 +4,7 @@ package placement import ( - "fmt" + "encoding/json" "net/http" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" @@ -16,11 +16,20 @@ import ( // resourceProviderTraitsResponse is the JSON body returned by // GET /resource_providers/{uuid}/traits and // PUT /resource_providers/{uuid}/traits. +// +// https://docs.openstack.org/api-ref/placement/#resource-provider-traits type resourceProviderTraitsResponse struct { Traits []string `json:"traits"` ResourceProviderGeneration int64 `json:"resource_provider_generation"` } +// resourceProviderTraitsRequest is the JSON body expected by +// PUT /resource_providers/{uuid}/traits. +type resourceProviderTraitsRequest struct { + Traits []string `json:"traits"` + ResourceProviderGeneration int64 `json:"resource_provider_generation"` +} + // HandleListResourceProviderTraits handles // GET /resource_providers/{uuid}/traits requests. // @@ -28,16 +37,18 @@ type resourceProviderTraitsResponse struct { // by {uuid}. The response includes an array of trait name strings and the // resource_provider_generation for concurrency tracking. Returns 404 if the // provider does not exist. +// +// https://docs.openstack.org/api-ref/placement/#list-resource-provider-traits func (s *Shim) HandleListResourceProviderTraits(w http.ResponseWriter, r *http.Request) { uuid, ok := requiredUUIDPathParam(w, r, "uuid") if !ok { return } - switch s.config.Features.ResourceProviderTraits.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviderTraits, true) { case FeatureModePassthrough: s.forward(w, r) case FeatureModeHybrid: - s.forward(w, r) + s.listResourceProviderTraitsHybrid(w, r, uuid) case FeatureModeCRD: s.listResourceProviderTraitsCRD(w, r, uuid) default: @@ -45,6 +56,25 @@ func (s *Shim) HandleListResourceProviderTraits(w http.ResponseWriter, r *http.R } } +// listResourceProviderTraitsHybrid serves from the CRD if the provider is a +// KVM hypervisor, otherwise forwards to upstream placement. +func (s *Shim) listResourceProviderTraitsHybrid(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if err != nil || len(hvs.Items) != 1 { + log.Info("resource provider not resolved from kubernetes, forwarding to upstream placement", "uuid", uuid) + s.forward(w, r) + return + } + log.Info("resolved resource provider from CRD, serving traits", "uuid", uuid, "hypervisor", hvs.Items[0].Name) + s.writeTraitsFromCRD(w, &hvs.Items[0]) +} + +// listResourceProviderTraitsCRD serves exclusively from the CRD, returning 404 +// if the provider is not a known KVM hypervisor. func (s *Shim) listResourceProviderTraitsCRD(w http.ResponseWriter, r *http.Request, uuid string) { ctx := r.Context() log := logf.FromContext(ctx) @@ -66,11 +96,15 @@ func (s *Shim) listResourceProviderTraitsCRD(w http.ResponseWriter, r *http.Requ http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } + log.Info("serving traits from CRD", "uuid", uuid, "hypervisor", hvs.Items[0].Name) + s.writeTraitsFromCRD(w, &hvs.Items[0]) +} - hv := hvs.Items[0] - traits := hv.Status.Traits - if traits == nil { - traits = []string{} +func (s *Shim) writeTraitsFromCRD(w http.ResponseWriter, hv *hv1.Hypervisor) { + traitGroups := hv1.GetTraits(hv.Spec.Groups) + traits := make([]string, 0, len(traitGroups)) + for _, tg := range traitGroups { + traits = append(traits, tg.Name) } s.writeJSON(w, http.StatusOK, resourceProviderTraitsResponse{ Traits: traits, @@ -85,25 +119,152 @@ func (s *Shim) listResourceProviderTraitsCRD(w http.ResponseWriter, r *http.Requ // The request body must include a traits array and the // resource_provider_generation for optimistic concurrency control. All // previously associated traits are removed and replaced by the specified set. -// Returns 400 Bad Request if any of the specified traits are invalid (i.e. -// not returned by GET /traits). Returns 409 Conflict if the generation does -// not match. +// Returns 409 Conflict if the generation does not match. +// +// https://docs.openstack.org/api-ref/placement/#update-resource-provider-traits func (s *Shim) HandleUpdateResourceProviderTraits(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredUUIDPathParam(w, r, "uuid"); !ok { + uuid, ok := requiredUUIDPathParam(w, r, "uuid") + if !ok { return } - switch s.config.Features.ResourceProviderTraits.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviderTraits, true) { case FeatureModePassthrough: s.forward(w, r) case FeatureModeHybrid: - s.forward(w, r) + s.updateResourceProviderTraitsHybrid(w, r, uuid) case FeatureModeCRD: - http.Error(w, fmt.Sprintf("%s mode is not yet implemented for resource provider trait writes", s.config.Features.ResourceProviderTraits), http.StatusNotImplemented) + s.updateResourceProviderTraitsCRD(w, r, uuid) default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) } } +// updateResourceProviderTraitsHybrid updates traits via the CRD if the +// provider is a KVM hypervisor, otherwise forwards to upstream placement. +func (s *Shim) updateResourceProviderTraitsHybrid(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if err != nil || len(hvs.Items) != 1 { + log.Info("resource provider not resolved from kubernetes, forwarding to upstream placement", "uuid", uuid) + s.forward(w, r) + return + } + hv := &hvs.Items[0] + log.Info("resolved resource provider from CRD, updating traits", "uuid", uuid, "hypervisor", hv.Name) + + var req resourceProviderTraitsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "malformed request body", http.StatusBadRequest) + return + } + if req.ResourceProviderGeneration != hv.Generation { + log.Info("generation mismatch on trait update", + "expected", req.ResourceProviderGeneration, "actual", hv.Generation) + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Trait == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + for _, name := range req.Traits { + newGroups = append(newGroups, hv1.Group{ + Trait: &hv1.TraitGroup{Name: name}, + }) + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to update hypervisor traits") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully updated traits via CRD", "uuid", uuid, "traitCount", len(req.Traits)) + s.writeJSON(w, http.StatusOK, resourceProviderTraitsResponse{ + Traits: req.Traits, + ResourceProviderGeneration: hv.Generation, + }) +} + +// updateResourceProviderTraitsCRD updates traits exclusively via the CRD, +// returning 404 if the provider is not a known KVM hypervisor. +func (s *Shim) updateResourceProviderTraitsCRD(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { + log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) + http.Error(w, "resource provider not found", http.StatusNotFound) + return + } + if err != nil { + log.Error(err, "failed to list hypervisors with OpenStack ID index") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if len(hvs.Items) > 1 { + log.Error(nil, "multiple hypervisors found with the same OpenStack ID", "uuid", uuid) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + hv := &hvs.Items[0] + log.Info("updating traits via CRD", "uuid", uuid, "hypervisor", hv.Name) + + var req resourceProviderTraitsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "malformed request body", http.StatusBadRequest) + return + } + if req.ResourceProviderGeneration != hv.Generation { + log.Info("generation mismatch on trait update", + "expected", req.ResourceProviderGeneration, "actual", hv.Generation) + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Trait == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + for _, name := range req.Traits { + newGroups = append(newGroups, hv1.Group{ + Trait: &hv1.TraitGroup{Name: name}, + }) + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to update hypervisor traits") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully updated traits via CRD", "uuid", uuid, "traitCount", len(req.Traits)) + s.writeJSON(w, http.StatusOK, resourceProviderTraitsResponse{ + Traits: req.Traits, + ResourceProviderGeneration: hv.Generation, + }) +} + // HandleDeleteResourceProviderTraits handles // DELETE /resource_providers/{uuid}/traits requests. // @@ -113,18 +274,107 @@ func (s *Shim) HandleUpdateResourceProviderTraits(w http.ResponseWriter, r *http // for the same provider, prefer PUT with an empty traits list instead. // Returns 404 if the provider does not exist. Returns 409 Conflict on // concurrent modification. Returns 204 No Content on success. +// +// https://docs.openstack.org/api-ref/placement/#delete-resource-provider-traits func (s *Shim) HandleDeleteResourceProviderTraits(w http.ResponseWriter, r *http.Request) { - if _, ok := requiredUUIDPathParam(w, r, "uuid"); !ok { + uuid, ok := requiredUUIDPathParam(w, r, "uuid") + if !ok { return } - switch s.config.Features.ResourceProviderTraits.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviderTraits, true) { case FeatureModePassthrough: s.forward(w, r) case FeatureModeHybrid: - s.forward(w, r) + s.deleteResourceProviderTraitsHybrid(w, r, uuid) case FeatureModeCRD: - http.Error(w, fmt.Sprintf("%s mode is not yet implemented for resource provider trait writes", s.config.Features.ResourceProviderTraits), http.StatusNotImplemented) + s.deleteResourceProviderTraitsCRD(w, r, uuid) default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) } } + +// deleteResourceProviderTraitsHybrid removes all traits via the CRD if the +// provider is a KVM hypervisor, otherwise forwards to upstream placement. +func (s *Shim) deleteResourceProviderTraitsHybrid(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if err != nil || len(hvs.Items) != 1 { + log.Info("resource provider not resolved from kubernetes, forwarding to upstream placement", "uuid", uuid) + s.forward(w, r) + return + } + hv := &hvs.Items[0] + log.Info("resolved resource provider from CRD, deleting traits", "uuid", uuid, "hypervisor", hv.Name) + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Trait == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to delete hypervisor traits") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully deleted all traits via CRD", "uuid", uuid) + w.WriteHeader(http.StatusNoContent) +} + +// deleteResourceProviderTraitsCRD removes all traits exclusively via the CRD, +// returning 404 if the provider is not a known KVM hypervisor. +func (s *Shim) deleteResourceProviderTraitsCRD(w http.ResponseWriter, r *http.Request, uuid string) { + ctx := r.Context() + log := logf.FromContext(ctx) + + var hvs hv1.HypervisorList + err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) + if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { + log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) + http.Error(w, "resource provider not found", http.StatusNotFound) + return + } + if err != nil { + log.Error(err, "failed to list hypervisors with OpenStack ID index") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if len(hvs.Items) > 1 { + log.Error(nil, "multiple hypervisors found with the same OpenStack ID", "uuid", uuid) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + hv := &hvs.Items[0] + log.Info("deleting all traits via CRD", "uuid", uuid, "hypervisor", hv.Name) + + var newGroups []hv1.Group + for i := range hv.Spec.Groups { + if hv.Spec.Groups[i].Trait == nil { + newGroups = append(newGroups, hv.Spec.Groups[i]) + } + } + hv.Spec.Groups = newGroups + + if err := s.Update(ctx, hv); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "resource provider generation conflict", http.StatusConflict) + return + } + log.Error(err, "failed to delete hypervisor traits") + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + log.Info("successfully deleted all traits via CRD", "uuid", uuid) + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/shim/placement/handle_resource_provider_traits_e2e.go b/internal/shim/placement/handle_resource_provider_traits_e2e.go index c697ca7ff..7dc50b016 100644 --- a/internal/shim/placement/handle_resource_provider_traits_e2e.go +++ b/internal/shim/placement/handle_resource_provider_traits_e2e.go @@ -9,8 +9,13 @@ import ( "encoding/json" "fmt" "net/http" + "slices" + "time" "github.com/cobaltcore-dev/cortex/pkg/conf" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/gophercloud/gophercloud/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -18,15 +23,11 @@ import ( // e2eTestResourceProviderTraits tests the // /resource_providers/{uuid}/traits endpoints. // -// 1. Pre-cleanup: DELETE leftover RP traits, RP, and custom trait (ignore 404). -// 2. Create fixtures: PUT a custom trait, POST a test RP. -// 3. GET /{uuid}/traits — verify the trait list is empty, store generation. -// 4. PUT /{uuid}/traits — associate the custom trait with the RP. -// 5. GET /{uuid}/traits — verify the custom trait is now present. -// 6. DELETE /{uuid}/traits — disassociate all traits from the RP. -// 7. GET /{uuid}/traits — verify the trait list is empty again. -// 8. Cleanup: DELETE the test RP and custom trait. -func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { +// In passthrough mode: exercises the upstream placement path with a +// dynamically created resource provider. +// In hybrid/crd mode: exercises the spec.groups-backed CRD path using a +// real KVM hypervisor discovered from the cluster. +func e2eTestResourceProviderTraits(ctx context.Context, cl client.Client) error { log := logf.FromContext(ctx) log.Info("Running resource provider traits endpoint e2e test") config, err := conf.GetConfig[e2eRootConfig]() @@ -42,6 +43,20 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { } log.Info("Successfully created openstack client for resource provider traits e2e test") + mode := e2eCurrentMode(ctx) + switch mode { + case FeatureModePassthrough: + return e2ePassthroughResourceProviderTraits(ctx, sc) + case FeatureModeHybrid, FeatureModeCRD: + return e2eCRDResourceProviderTraits(ctx, sc, cl) + default: + return fmt.Errorf("unexpected mode %q", mode) + } +} + +func e2ePassthroughResourceProviderTraits(ctx context.Context, sc *gophercloud.ServiceClient) error { + log := logf.FromContext(ctx) + const testRPUUID = "e2e10000-0000-0000-0000-000000000003" const testRPName = "cortex-e2e-test-rp-traits" const testTrait = "CUSTOM_CORTEX_E2E_RP_TRAIT" @@ -136,8 +151,7 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { log.Info("Successfully created test resource provider for RP traits test", "uuid", testRPUUID) - // Deferred cleanup: always delete test fixtures on exit so a failed - // assertion doesn't leave the fixed UUID/trait behind. + // Deferred cleanup. defer func() { log.Info("Deferred cleanup: deleting test resources") for _, c := range []struct { @@ -165,12 +179,11 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { } }() - // Test GET /resource_providers/{uuid}/traits (empty). + // Test GET (empty). log.Info("Testing GET /resource_providers/{uuid}/traits (empty)", "uuid", testRPUUID) req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/traits", http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP traits", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -178,49 +191,36 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP traits", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP traits returned an error", "uuid", testRPUUID) - return err + return fmt.Errorf("GET RP traits: unexpected status %d", resp.StatusCode) } var traitsResp struct { Traits []string `json:"traits"` ResourceProviderGeneration int `json:"resource_provider_generation"` } - err = json.NewDecoder(resp.Body).Decode(&traitsResp) - if err != nil { - log.Error(err, "failed to decode RP traits response", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&traitsResp); err != nil { return err } if len(traitsResp.Traits) != 0 { - err := fmt.Errorf("expected 0 initial traits, got %d", len(traitsResp.Traits)) - log.Error(err, "initial traits not empty", "uuid", testRPUUID) - return err + return fmt.Errorf("expected 0 initial traits, got %d", len(traitsResp.Traits)) } - log.Info("Successfully retrieved empty traits for test resource provider", - "uuid", testRPUUID, "traits", len(traitsResp.Traits), - "generation", traitsResp.ResourceProviderGeneration) + log.Info("Verified empty traits", "generation", traitsResp.ResourceProviderGeneration) - // Test PUT /resource_providers/{uuid}/traits (associate trait). - log.Info("Testing PUT /resource_providers/{uuid}/traits to associate trait", - "uuid", testRPUUID, "trait", testTrait) + // Test PUT (associate trait). putBody, err := json.Marshal(map[string]any{ "resource_provider_generation": traitsResp.ResourceProviderGeneration, "traits": []string{testTrait}, }) if err != nil { - log.Error(err, "failed to marshal request body") return err } req, err = http.NewRequestWithContext(ctx, http.MethodPut, sc.Endpoint+"/resource_providers/"+testRPUUID+"/traits", bytes.NewReader(putBody)) if err != nil { - log.Error(err, "failed to create PUT request for RP traits", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -229,25 +229,18 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send PUT request for RP traits", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "PUT RP traits returned an error", "uuid", testRPUUID) - return err + return fmt.Errorf("PUT RP traits: unexpected status %d", resp.StatusCode) } - log.Info("Successfully associated trait with test resource provider", - "uuid", testRPUUID, "trait", testTrait) + log.Info("Successfully associated trait") - // Test GET /resource_providers/{uuid}/traits (after PUT). - log.Info("Testing GET /resource_providers/{uuid}/traits (after PUT)", - "uuid", testRPUUID) + // Test GET (after PUT). req, err = http.NewRequestWithContext(ctx, http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/traits", http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP traits", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) @@ -255,132 +248,274 @@ func e2eTestResourceProviderTraits(ctx context.Context, _ client.Client) error { req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP traits", "uuid", testRPUUID) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP traits returned an error", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&traitsResp); err != nil { return err } - err = json.NewDecoder(resp.Body).Decode(&traitsResp) - if err != nil { - log.Error(err, "failed to decode RP traits response", "uuid", testRPUUID) - return err - } - if len(traitsResp.Traits) != 1 || traitsResp.Traits[0] != testTrait { - err := fmt.Errorf("expected trait %s, got %v", testTrait, traitsResp.Traits) - log.Error(err, "trait mismatch", "uuid", testRPUUID) - return err + if !slices.Contains(traitsResp.Traits, testTrait) { + return fmt.Errorf("expected trait %s, got %v", testTrait, traitsResp.Traits) } - log.Info("Successfully verified trait on test resource provider", - "uuid", testRPUUID, "traits", traitsResp.Traits) + log.Info("Verified trait present after PUT") - // Test DELETE /resource_providers/{uuid}/traits. - log.Info("Testing DELETE /resource_providers/{uuid}/traits", "uuid", testRPUUID) + // Test DELETE. req, err = http.NewRequestWithContext(ctx, http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID+"/traits", http.NoBody) if err != nil { - log.Error(err, "failed to create DELETE request for RP traits", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.6") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send DELETE request for RP traits", "uuid", testRPUUID) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "DELETE RP traits returned an error", "uuid", testRPUUID) + return fmt.Errorf("DELETE RP traits: unexpected status %d", resp.StatusCode) + } + log.Info("Successfully deleted traits") + + // Cleanup. + req, err = http.NewRequestWithContext(ctx, + http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID, http.NoBody) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + resp, err = sc.HTTPClient.Do(req) + if err != nil { return err } - log.Info("Successfully deleted traits from test resource provider", "uuid", testRPUUID) + resp.Body.Close() - // Verify traits cleared. - log.Info("Verifying traits cleared on test resource provider", "uuid", testRPUUID) req, err = http.NewRequestWithContext(ctx, - http.MethodGet, sc.Endpoint+"/resource_providers/"+testRPUUID+"/traits", http.NoBody) + http.MethodDelete, sc.Endpoint+"/traits/"+testTrait, http.NoBody) if err != nil { - log.Error(err, "failed to create GET request for RP traits", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.6") - req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send GET request for RP traits", "uuid", testRPUUID) + return err + } + resp.Body.Close() + + return nil +} + +// e2eCRDResourceProviderTraits tests the CRD/hybrid path by discovering a +// real KVM hypervisor in the cluster, seeding spec.groups, and exercising +// GET/PUT/DELETE through the shim. +func e2eCRDResourceProviderTraits(ctx context.Context, sc *gophercloud.ServiceClient, cl client.Client) error { + log := logf.FromContext(ctx) + + // Discover a KVM hypervisor with a non-empty OpenStack ID. + var hvs hv1.HypervisorList + if err := cl.List(ctx, &hvs); err != nil { + log.Error(err, "failed to list hypervisors for CRD traits path") + return err + } + var kvmHV *hv1.Hypervisor + for i := range hvs.Items { + if hvs.Items[i].Status.HypervisorID != "" { + kvmHV = &hvs.Items[i] + break + } + } + if kvmHV == nil { + log.Info("No KVM hypervisors with OpenStack ID found, skipping CRD traits tests") + return nil + } + kvmUUID := kvmHV.Status.HypervisorID + log.Info("Using KVM hypervisor for CRD traits e2e tests", "uuid", kvmUUID, "name", kvmHV.Name) + + // Save original groups for restoration. + originalGroups := kvmHV.Spec.Groups + + // Seed spec.groups with test traits (preserve non-trait groups). + const testTrait1 = "CUSTOM_E2E_CRD_TRAIT_1" + const testTrait2 = "CUSTOM_E2E_CRD_TRAIT_2" + var nonTraitGroups []hv1.Group + for i := range kvmHV.Spec.Groups { + if kvmHV.Spec.Groups[i].Trait == nil { + nonTraitGroups = append(nonTraitGroups, kvmHV.Spec.Groups[i]) + } + } + nonTraitGroups = append(nonTraitGroups, + hv1.Group{Trait: &hv1.TraitGroup{Name: testTrait1}}, + hv1.Group{Trait: &hv1.TraitGroup{Name: testTrait2}}, + ) + kvmHV.Spec.Groups = nonTraitGroups + if err := cl.Update(ctx, kvmHV); err != nil { + return fmt.Errorf("failed to seed spec.groups with test traits: %w", err) + } + log.Info("Seeded spec.groups with test traits", "uuid", kvmUUID) + + // Always restore original groups on exit (retry on conflict). + defer func() { + log.Info("Restoring original spec.groups", "uuid", kvmUUID) + for range 5 { + if err := cl.Get(ctx, client.ObjectKeyFromObject(kvmHV), kvmHV); err != nil { + log.Error(err, "failed to refetch hypervisor for restoration") + return + } + kvmHV.Spec.Groups = originalGroups + if err := cl.Update(ctx, kvmHV); err != nil { + if apierrors.IsConflict(err) { + continue + } + log.Error(err, "failed to restore original spec.groups") + return + } + return + } + log.Error(nil, "exhausted retries restoring original spec.groups") + }() + + // Refetch to get updated generation. + if err := cl.Get(ctx, client.ObjectKeyFromObject(kvmHV), kvmHV); err != nil { + return fmt.Errorf("failed to refetch hypervisor after seed: %w", err) + } + + // Test GET — should return the seeded traits. + // Poll because the shim's informer cache may take a moment to observe the update. + log.Info("Testing GET /resource_providers/{uuid}/traits (CRD)", "uuid", kvmUUID) + var traitsResp struct { + Traits []string `json:"traits"` + ResourceProviderGeneration int64 `json:"resource_provider_generation"` + } + if err := e2ePollUntil(ctx, 10*time.Second, func() (bool, error) { + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, sc.Endpoint+"/resource_providers/"+kvmUUID+"/traits", http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + req.Header.Set("Accept", "application/json") + resp, err := sc.HTTPClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("GET CRD traits: expected 200, got %d", resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(&traitsResp); err != nil { + return false, fmt.Errorf("failed to decode CRD traits response: %w", err) + } + return slices.Contains(traitsResp.Traits, testTrait1) && + slices.Contains(traitsResp.Traits, testTrait2), nil + }); err != nil { + return fmt.Errorf("waiting for seeded traits: %w (got %v)", err, traitsResp.Traits) + } + log.Info("Verified GET returns seeded traits from CRD", + "traits", traitsResp.Traits, "generation", traitsResp.ResourceProviderGeneration) + + // Test PUT — replace traits. + const replacementTrait = "CUSTOM_E2E_CRD_REPLACED" + putBody, err := json.Marshal(map[string]any{ + "resource_provider_generation": traitsResp.ResourceProviderGeneration, + "traits": []string{replacementTrait}, + }) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPut, sc.Endpoint+"/resource_providers/"+kvmUUID+"/traits", + bytes.NewReader(putBody)) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := sc.HTTPClient.Do(req) + if err != nil { return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "GET RP traits returned an error", "uuid", testRPUUID) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("PUT CRD traits: expected 200, got %d", resp.StatusCode) + } + log.Info("Successfully replaced traits via PUT (CRD)") + + // Test PUT with stale generation — should return 409. + putBody, err = json.Marshal(map[string]any{ + "resource_provider_generation": traitsResp.ResourceProviderGeneration, + "traits": []string{"STALE"}, + }) + if err != nil { return err } - err = json.NewDecoder(resp.Body).Decode(&traitsResp) + req, err = http.NewRequestWithContext(ctx, + http.MethodPut, sc.Endpoint+"/resource_providers/"+kvmUUID+"/traits", + bytes.NewReader(putBody)) if err != nil { - log.Error(err, "failed to decode RP traits response", "uuid", testRPUUID) return err } - if len(traitsResp.Traits) != 0 { - err := fmt.Errorf("expected 0 traits, got %d", len(traitsResp.Traits)) - log.Error(err, "traits not cleared", "uuid", testRPUUID) + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err = sc.HTTPClient.Do(req) + if err != nil { return err } - log.Info("Verified traits cleared on test resource provider", "uuid", testRPUUID) + defer resp.Body.Close() + if resp.StatusCode != http.StatusConflict { + return fmt.Errorf("PUT CRD traits (stale gen): expected 409, got %d", resp.StatusCode) + } + log.Info("Verified generation conflict returns 409") - // Cleanup: delete the test resource provider and custom trait. - log.Info("Cleaning up test resources") + // Test GET — verify replacement persisted. req, err = http.NewRequestWithContext(ctx, - http.MethodDelete, sc.Endpoint+"/resource_providers/"+testRPUUID, http.NoBody) + http.MethodGet, sc.Endpoint+"/resource_providers/"+kvmUUID+"/traits", http.NoBody) if err != nil { - log.Error(err, "failed to create DELETE request for resource provider", "uuid", testRPUUID) return err } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.6") + req.Header.Set("Accept", "application/json") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send DELETE request for resource provider", "uuid", testRPUUID) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "DELETE resource provider returned an error", "uuid", testRPUUID) + if err := json.NewDecoder(resp.Body).Decode(&traitsResp); err != nil { return err } - log.Info("Successfully deleted test resource provider", "uuid", testRPUUID) + if len(traitsResp.Traits) != 1 || traitsResp.Traits[0] != replacementTrait { + return fmt.Errorf("expected [%s], got %v", replacementTrait, traitsResp.Traits) + } + log.Info("Verified replacement trait persisted") + // Test DELETE — remove all traits. req, err = http.NewRequestWithContext(ctx, - http.MethodDelete, sc.Endpoint+"/traits/"+testTrait, http.NoBody) + http.MethodDelete, sc.Endpoint+"/resource_providers/"+kvmUUID+"/traits", http.NoBody) if err != nil { - log.Error(err, "failed to create DELETE request for trait", "trait", testTrait) return err } req.Header.Set("X-Auth-Token", sc.TokenID) req.Header.Set("OpenStack-API-Version", "placement 1.6") resp, err = sc.HTTPClient.Do(req) if err != nil { - log.Error(err, "failed to send DELETE request for trait", "trait", testTrait) return err } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) - log.Error(err, "DELETE trait returned an error", "trait", testTrait) - return err + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE CRD traits: expected 204, got %d", resp.StatusCode) } - log.Info("Successfully deleted custom trait", "trait", testTrait) + log.Info("Verified DELETE returns 204") return nil } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_provider_traits", run: e2eTestResourceProviderTraits}) + e2eTests = append(e2eTests, e2eTest{name: "resource_provider_traits", run: e2eWrapWithModes(e2eTestResourceProviderTraits)}) } diff --git a/internal/shim/placement/handle_resource_provider_traits_test.go b/internal/shim/placement/handle_resource_provider_traits_test.go index 69ac8cd8c..9483044d4 100644 --- a/internal/shim/placement/handle_resource_provider_traits_test.go +++ b/internal/shim/placement/handle_resource_provider_traits_test.go @@ -6,9 +6,22 @@ package placement import ( "encoding/json" "net/http" + "strings" "testing" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) +func testHypervisorWithGroups(name, openstackID string, groups []hv1.Group) *hv1.Hypervisor { + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: hv1.HypervisorSpec{Groups: groups}, + Status: hv1.HypervisorStatus{HypervisorID: openstackID}, + } +} + func TestHandleListResourceProviderTraits(t *testing.T) { t.Run("valid uuid", func(t *testing.T) { s := newTestShim(t, http.StatusOK, "{}", nil) @@ -73,9 +86,9 @@ func TestHandleDeleteResourceProviderTraits(t *testing.T) { } func TestHandleResourceProviderTraits_HybridMode(t *testing.T) { - s := newTestShim(t, http.StatusOK, `{"traits":["CUSTOM_HW_FPGA"],"resource_provider_generation":1}`, nil) + s := newTestShimWithHypervisors(t, http.StatusOK, `{"traits":["CUSTOM_HW_FPGA"],"resource_provider_generation":1}`) s.config.Features.ResourceProviderTraits = FeatureModeHybrid - t.Run("GET forwards to upstream", func(t *testing.T) { + t.Run("GET forwards to upstream when provider not in CRD", func(t *testing.T) { w := serveHandler(t, "GET", "/resource_providers/{uuid}/traits", s.HandleListResourceProviderTraits, "/resource_providers/"+validUUID+"/traits") @@ -83,7 +96,7 @@ func TestHandleResourceProviderTraits_HybridMode(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } }) - t.Run("PUT forwards to upstream", func(t *testing.T) { + t.Run("PUT forwards to upstream when provider not in CRD", func(t *testing.T) { w := serveHandler(t, "PUT", "/resource_providers/{uuid}/traits", s.HandleUpdateResourceProviderTraits, "/resource_providers/"+validUUID+"/traits") @@ -92,9 +105,9 @@ func TestHandleResourceProviderTraits_HybridMode(t *testing.T) { } }) - sDel := newTestShim(t, http.StatusNoContent, "", nil) + sDel := newTestShimWithHypervisors(t, http.StatusNoContent, "") sDel.config.Features.ResourceProviderTraits = FeatureModeHybrid - t.Run("DELETE forwards to upstream", func(t *testing.T) { + t.Run("DELETE forwards to upstream when provider not in CRD", func(t *testing.T) { w := serveHandler(t, "DELETE", "/resource_providers/{uuid}/traits", sDel.HandleDeleteResourceProviderTraits, "/resource_providers/"+validUUID+"/traits") @@ -102,15 +115,41 @@ func TestHandleResourceProviderTraits_HybridMode(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) } }) + + t.Run("GET serves from CRD when provider is KVM", func(t *testing.T) { + hv := testHypervisorWithGroups("kvm-hybrid", validUUID, []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "CUSTOM_KVM_TRAIT"}}, + }) + sKVM := newTestShimWithHypervisors(t, http.StatusOK, "{}", hv) + sKVM.config.Features.ResourceProviderTraits = FeatureModeHybrid + w := serveHandler(t, "GET", "/resource_providers/{uuid}/traits", + sKVM.HandleListResourceProviderTraits, + "/resource_providers/"+validUUID+"/traits") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var resp resourceProviderTraitsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Traits) != 1 || resp.Traits[0] != "CUSTOM_KVM_TRAIT" { + t.Fatalf("expected [CUSTOM_KVM_TRAIT], got %v", resp.Traits) + } + }) } func TestHandleResourceProviderTraits_CRDMode(t *testing.T) { - hv := testHypervisorFull("kvm-host-1", validUUID, nil, []string{"CUSTOM_HW_FPGA", "HW_CPU_X86_SSE42"}, nil) - s := newTestShimWithHypervisors(t, http.StatusOK, "{}", &hv) + groups := []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "CUSTOM_HW_FPGA"}}, + {Trait: &hv1.TraitGroup{Name: "HW_CPU_X86_SSE42"}}, + {Aggregate: &hv1.AggregateGroup{Name: "az1", UUID: "agg-uuid-1"}}, + } + hv := testHypervisorWithGroups("kvm-host-1", validUUID, groups) + s := newTestShimWithHypervisors(t, http.StatusOK, "{}", hv) s.config.Features.ResourceProviderTraits = FeatureModeCRD s.config.Features.ResourceProviders = FeatureModeCRD - t.Run("GET returns traits from CRD for KVM provider", func(t *testing.T) { + t.Run("GET returns traits from spec.groups", func(t *testing.T) { w := serveHandler(t, "GET", "/resource_providers/{uuid}/traits", s.HandleListResourceProviderTraits, "/resource_providers/"+validUUID+"/traits") @@ -124,8 +163,36 @@ func TestHandleResourceProviderTraits_CRDMode(t *testing.T) { if len(resp.Traits) != 2 { t.Fatalf("traits count = %d, want 2", len(resp.Traits)) } + if resp.Traits[0] != "CUSTOM_HW_FPGA" { + t.Errorf("traits[0] = %q, want CUSTOM_HW_FPGA", resp.Traits[0]) + } + if resp.Traits[1] != "HW_CPU_X86_SSE42" { + t.Errorf("traits[1] = %q, want HW_CPU_X86_SSE42", resp.Traits[1]) + } }) - t.Run("GET returns 404 for non-KVM provider", func(t *testing.T) { + + t.Run("GET returns empty traits when spec.groups has no traits", func(t *testing.T) { + hvNoTraits := testHypervisorWithGroups("kvm-no-traits", "b1b2b3b4-c5c6-d7d8-e9e0-f1f2f3f4f5f6", []hv1.Group{ + {Aggregate: &hv1.AggregateGroup{Name: "az1", UUID: "agg-1"}}, + }) + s2 := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvNoTraits) + s2.config.Features.ResourceProviderTraits = FeatureModeCRD + w := serveHandler(t, "GET", "/resource_providers/{uuid}/traits", + s2.HandleListResourceProviderTraits, + "/resource_providers/b1b2b3b4-c5c6-d7d8-e9e0-f1f2f3f4f5f6/traits") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var resp resourceProviderTraitsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Traits) != 0 { + t.Fatalf("traits count = %d, want 0", len(resp.Traits)) + } + }) + + t.Run("GET returns 404 for non-existent provider", func(t *testing.T) { nonKVMUUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" w := serveHandler(t, "GET", "/resource_providers/{uuid}/traits", s.HandleListResourceProviderTraits, @@ -134,20 +201,100 @@ func TestHandleResourceProviderTraits_CRDMode(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) } }) - t.Run("PUT returns 501", func(t *testing.T) { - w := serveHandler(t, "PUT", "/resource_providers/{uuid}/traits", + + t.Run("PUT replaces traits in spec.groups preserving aggregates", func(t *testing.T) { + hvPut := testHypervisorWithGroups("kvm-put-traits", "c1c2c3c4-d5d6-e7e8-f9f0-a1a2a3a4a5a6", []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "OLD_TRAIT"}}, + {Aggregate: &hv1.AggregateGroup{Name: "keep-me", UUID: "keep-uuid"}}, + }) + sPut := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvPut) + sPut.config.Features.ResourceProviderTraits = FeatureModeCRD + + body := `{"traits":["NEW_TRAIT_1","NEW_TRAIT_2"],"resource_provider_generation":0}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/traits", + sPut.HandleUpdateResourceProviderTraits, + "/resource_providers/c1c2c3c4-d5d6-e7e8-f9f0-a1a2a3a4a5a6/traits", strings.NewReader(body)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp resourceProviderTraitsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if len(resp.Traits) != 2 { + t.Fatalf("traits count = %d, want 2", len(resp.Traits)) + } + + // Verify aggregates were preserved by fetching the updated object. + var updated hv1.Hypervisor + if err := sPut.Get(t.Context(), client.ObjectKeyFromObject(hvPut), &updated); err != nil { + t.Fatalf("failed to get updated hypervisor: %v", err) + } + aggs := hv1.GetAggregates(updated.Spec.Groups) + if len(aggs) != 1 || aggs[0].UUID != "keep-uuid" { + t.Fatalf("aggregates were not preserved: got %+v", aggs) + } + }) + + t.Run("PUT returns 409 on generation mismatch", func(t *testing.T) { + hvConflict := testHypervisorWithGroups("kvm-conflict", "d1d2d3d4-e5e6-f7f8-a9a0-b1b2b3b4b5b6", nil) + sConflict := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvConflict) + sConflict.config.Features.ResourceProviderTraits = FeatureModeCRD + + body := `{"traits":["T1"],"resource_provider_generation":999}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/traits", + sConflict.HandleUpdateResourceProviderTraits, + "/resource_providers/d1d2d3d4-e5e6-f7f8-a9a0-b1b2b3b4b5b6/traits", strings.NewReader(body)) + if w.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", w.Code, http.StatusConflict) + } + }) + + t.Run("PUT returns 404 for non-existent provider", func(t *testing.T) { + body := `{"traits":["T1"],"resource_provider_generation":0}` + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/traits", s.HandleUpdateResourceProviderTraits, - "/resource_providers/"+validUUID+"/traits") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + "/resource_providers/e1e2e3e4-f5f6-a7a8-b9b0-c1c2c3c4c5c6/traits", strings.NewReader(body)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound) } }) - t.Run("DELETE returns 501", func(t *testing.T) { + + t.Run("PUT returns 400 for malformed body", func(t *testing.T) { + w := serveHandlerWithBody(t, "PUT", "/resource_providers/{uuid}/traits", + s.HandleUpdateResourceProviderTraits, + "/resource_providers/"+validUUID+"/traits", strings.NewReader("not json")) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) + } + }) + + t.Run("DELETE removes all traits preserving aggregates", func(t *testing.T) { + hvDel := testHypervisorWithGroups("kvm-del-traits", "f1f2f3f4-a5a6-b7b8-c9c0-d1d2d3d4d5d6", []hv1.Group{ + {Trait: &hv1.TraitGroup{Name: "REMOVE_ME"}}, + {Aggregate: &hv1.AggregateGroup{Name: "stay", UUID: "stay-uuid"}}, + }) + sDel := newTestShimWithHypervisors(t, http.StatusOK, "{}", hvDel) + sDel.config.Features.ResourceProviderTraits = FeatureModeCRD + w := serveHandler(t, "DELETE", "/resource_providers/{uuid}/traits", - s.HandleDeleteResourceProviderTraits, - "/resource_providers/"+validUUID+"/traits") - if w.Code != http.StatusNotImplemented { - t.Fatalf("status = %d, want %d", w.Code, http.StatusNotImplemented) + sDel.HandleDeleteResourceProviderTraits, + "/resource_providers/f1f2f3f4-a5a6-b7b8-c9c0-d1d2d3d4d5d6/traits") + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) + } + + var updated hv1.Hypervisor + if err := sDel.Get(t.Context(), client.ObjectKeyFromObject(hvDel), &updated); err != nil { + t.Fatalf("failed to get updated hypervisor: %v", err) + } + traits := hv1.GetTraits(updated.Spec.Groups) + if len(traits) != 0 { + t.Fatalf("expected no traits, got %+v", traits) + } + aggs := hv1.GetAggregates(updated.Spec.Groups) + if len(aggs) != 1 || aggs[0].UUID != "stay-uuid" { + t.Fatalf("aggregates were not preserved: got %+v", aggs) } }) } diff --git a/internal/shim/placement/handle_resource_provider_usages_e2e.go b/internal/shim/placement/handle_resource_provider_usages_e2e.go index c548162ba..05965b23c 100644 --- a/internal/shim/placement/handle_resource_provider_usages_e2e.go +++ b/internal/shim/placement/handle_resource_provider_usages_e2e.go @@ -44,6 +44,15 @@ func e2eTestResourceProviderUsages(ctx context.Context, _ client.Client) error { const testRPUUID = "e2e10000-0000-0000-0000-000000000005" const testRPName = "cortex-e2e-test-rp-usages" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/resource_providers/"+testRPUUID+"/usages") + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Pre-cleanup: delete any leftover test resource provider from a prior run. log.Info("Pre-cleanup: deleting leftover test resource provider", "uuid", testRPUUID) req, err := http.NewRequestWithContext(ctx, @@ -227,5 +236,5 @@ func e2eTestResourceProviderUsages(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_provider_usages", run: e2eTestResourceProviderUsages}) + e2eTests = append(e2eTests, e2eTest{name: "resource_provider_usages", run: e2eWrapWithModes(e2eTestResourceProviderUsages)}) } diff --git a/internal/shim/placement/handle_resource_providers.go b/internal/shim/placement/handle_resource_providers.go index 7d4dd1fff..6f73e27d6 100644 --- a/internal/shim/placement/handle_resource_providers.go +++ b/internal/shim/placement/handle_resource_providers.go @@ -115,7 +115,8 @@ func (s *Shim) HandleCreateResourceProvider(w http.ResponseWriter, r *http.Reque ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.ResourceProviders.orDefault() { + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviders, true) + switch mode { case FeatureModePassthrough: s.forward(w, r) return @@ -184,7 +185,7 @@ func (s *Shim) HandleCreateResourceProvider(w http.ResponseWriter, r *http.Reque } // No conflict — forward to upstream placement (hybrid) or reject (crd). - if s.config.Features.ResourceProviders.orDefault() == FeatureModeCRD { + if mode == FeatureModeCRD { log.Info("crd mode: non-kvm resource provider create not supported", "name", req.Name) http.Error(w, "resource provider not found", http.StatusNotFound) return @@ -209,7 +210,8 @@ func (s *Shim) HandleShowResourceProvider(w http.ResponseWriter, r *http.Request ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.ResourceProviders.orDefault() { + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviders, true) + switch mode { case FeatureModePassthrough: s.forward(w, r) return @@ -229,7 +231,7 @@ func (s *Shim) HandleShowResourceProvider(w http.ResponseWriter, r *http.Request var hvs hv1.HypervisorList err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { - if s.config.Features.ResourceProviders.orDefault() == FeatureModeCRD { + if mode == FeatureModeCRD { log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) http.Error(w, "resource provider not found", http.StatusNotFound) return @@ -278,7 +280,8 @@ func (s *Shim) HandleUpdateResourceProvider(w http.ResponseWriter, r *http.Reque ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.ResourceProviders.orDefault() { + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviders, true) + switch mode { case FeatureModePassthrough: s.forward(w, r) return @@ -315,7 +318,7 @@ func (s *Shim) HandleUpdateResourceProvider(w http.ResponseWriter, r *http.Reque var hvs hv1.HypervisorList err = s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { - if s.config.Features.ResourceProviders.orDefault() == FeatureModeCRD { + if mode == FeatureModeCRD { log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) http.Error(w, "resource provider not found", http.StatusNotFound) return @@ -373,7 +376,8 @@ func (s *Shim) HandleDeleteResourceProvider(w http.ResponseWriter, r *http.Reque ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.ResourceProviders.orDefault() { + mode := s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviders, true) + switch mode { case FeatureModePassthrough: s.forward(w, r) return @@ -393,7 +397,7 @@ func (s *Shim) HandleDeleteResourceProvider(w http.ResponseWriter, r *http.Reque var hvs hv1.HypervisorList err := s.List(ctx, &hvs, client.MatchingFields{idxHypervisorOpenStackId: uuid}) if apierrors.IsNotFound(err) || len(hvs.Items) == 0 { - if s.config.Features.ResourceProviders.orDefault() == FeatureModeCRD { + if mode == FeatureModeCRD { log.Info("resource provider not found in kubernetes (crd mode)", "uuid", uuid) http.Error(w, "resource provider not found", http.StatusNotFound) return @@ -448,7 +452,7 @@ type listResourceProvidersResponse struct { // // See: https://docs.openstack.org/api-ref/placement/#list-resource-providers func (s *Shim) HandleListResourceProviders(w http.ResponseWriter, r *http.Request) { - switch s.config.Features.ResourceProviders.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.ResourceProviders, true) { case FeatureModePassthrough: s.forward(w, r) case FeatureModeHybrid: diff --git a/internal/shim/placement/handle_resource_providers_e2e.go b/internal/shim/placement/handle_resource_providers_e2e.go index 90b850369..963f44533 100644 --- a/internal/shim/placement/handle_resource_providers_e2e.go +++ b/internal/shim/placement/handle_resource_providers_e2e.go @@ -58,14 +58,21 @@ func e2eTestResourceProviders(ctx context.Context, cl client.Client) error { // ==================== Phase 1: VMware path ==================== - log.Info("=== VMware path: passthrough resource provider tests ===") - if err := e2eVMwareResourceProviders(ctx, sc); err != nil { - return fmt.Errorf("VMware path: %w", err) + // The VMware path creates synthetic test RPs against upstream placement. + // In crd mode there is no upstream, so skip it. + mode := e2eCurrentMode(ctx) + if mode != FeatureModeCRD { + log.Info("=== VMware path: passthrough resource provider tests ===") + if err := e2eVMwareResourceProviders(ctx, sc); err != nil { + return fmt.Errorf("VMware path: %w", err) + } + } else { + log.Info("Skipping VMware path because mode is crd (no upstream placement)") } // ==================== Phase 2: KVM path ==================== - if config.Features.ResourceProviders.orDefault() == FeatureModePassthrough { + if mode == FeatureModePassthrough { log.Info("Skipping KVM resource provider e2e tests because resourceProviders mode is passthrough") } else { log.Info("=== KVM path: hypervisor-backed resource provider tests ===") @@ -506,5 +513,5 @@ func e2eKVMResourceProviders(ctx context.Context, sc *gophercloud.ServiceClient, } func init() { - e2eTests = append(e2eTests, e2eTest{name: "resource_providers", run: e2eTestResourceProviders}) + e2eTests = append(e2eTests, e2eTest{name: "resource_providers", run: e2eWrapWithModes(e2eTestResourceProviders)}) } diff --git a/internal/shim/placement/handle_root.go b/internal/shim/placement/handle_root.go index acad69dcb..45ada64fe 100644 --- a/internal/shim/placement/handle_root.go +++ b/internal/shim/placement/handle_root.go @@ -50,7 +50,7 @@ func (s *Shim) HandleGetRoot(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.Root.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.Root, s.config.Versioning != nil) { case FeatureModePassthrough: log.Info("forwarding GET / to upstream placement") s.forward(w, r) diff --git a/internal/shim/placement/handle_root_e2e.go b/internal/shim/placement/handle_root_e2e.go index 2e558705a..e4a785606 100644 --- a/internal/shim/placement/handle_root_e2e.go +++ b/internal/shim/placement/handle_root_e2e.go @@ -51,5 +51,5 @@ func e2eTestGetRoot(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "root", run: e2eTestGetRoot}) + e2eTests = append(e2eTests, e2eTest{name: "root", run: e2eWrapWithModes(e2eTestGetRoot)}) } diff --git a/internal/shim/placement/handle_traits.go b/internal/shim/placement/handle_traits.go index b509b2599..c64d6887e 100644 --- a/internal/shim/placement/handle_traits.go +++ b/internal/shim/placement/handle_traits.go @@ -7,16 +7,13 @@ import ( "context" "encoding/json" "fmt" - "math/rand" + "io" "net/http" - "net/url" "os" "sort" "strings" "time" - "github.com/go-logr/logr" - "github.com/gophercloud/gophercloud/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,24 +24,6 @@ import ( const configMapKeyTraits = "traits" -func (s *Shim) staticTraitsConfigMapKey() client.ObjectKey { - return client.ObjectKey{ - Namespace: os.Getenv("POD_NAMESPACE"), - Name: s.config.Traits.ConfigMapName, - } -} - -func (s *Shim) customTraitsConfigMapKey() client.ObjectKey { - return client.ObjectKey{ - Namespace: os.Getenv("POD_NAMESPACE"), - Name: s.config.Traits.ConfigMapName + "-custom", - } -} - -func (s *Shim) traitsLockName() string { - return s.config.Traits.ConfigMapName + "-custom-lock" -} - // traitsListResponse matches the OpenStack Placement GET /traits response. type traitsListResponse struct { Traits []string `json:"traits"` @@ -52,30 +31,30 @@ type traitsListResponse struct { // HandleListTraits handles GET /traits requests. // -// Returns a sorted list of trait strings merged from the static (Helm-managed) -// and dynamic (CUSTOM_*) ConfigMaps. Supports optional query parameter "name" -// for filtering: "in:TRAIT_A,TRAIT_B" returns only named traits, -// "startswith:CUSTOM_" returns prefix matches. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream placement. +// - crd: serves the trait list from the local ConfigMap. // // See: https://docs.openstack.org/api-ref/placement/#list-traits func (s *Shim) HandleListTraits(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.Traits.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.Traits, s.config.Traits != nil) { case FeatureModePassthrough, FeatureModeHybrid: s.forward(w, r) return case FeatureModeCRD: - // Serve from local ConfigMaps. + // Serve from local ConfigMap. default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) return } - traitSet, err := s.getAllTraits(ctx) + traitSet, err := s.getTraits(ctx) if err != nil { - log.Error(err, "failed to list traits from configmaps") + log.Error(err, "failed to list traits from configmap") http.Error(w, "failed to list traits", http.StatusInternalServerError) return } @@ -124,20 +103,22 @@ func (s *Shim) HandleListTraits(w http.ResponseWriter, r *http.Request) { // HandleShowTrait handles GET /traits/{name} requests. // -// Checks whether a trait with the given name exists in either the static -// or dynamic ConfigMap. Returns 204 No Content if found, 404 Not Found otherwise. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream placement. +// - crd: checks the local ConfigMap for the trait. // // See: https://docs.openstack.org/api-ref/placement/#show-traits func (s *Shim) HandleShowTrait(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.Traits.orDefault() { + switch s.featureModeFromConfOrHeader(r, s.config.Features.Traits, s.config.Traits != nil) { case FeatureModePassthrough, FeatureModeHybrid: s.forward(w, r) return case FeatureModeCRD: - // Serve from local ConfigMaps. + // Serve from local ConfigMap. default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) return @@ -164,22 +145,26 @@ func (s *Shim) HandleShowTrait(w http.ResponseWriter, r *http.Request) { // HandleUpdateTrait handles PUT /traits/{name} requests. // -// Creates a new custom trait in the dynamic ConfigMap. Only traits prefixed -// with CUSTOM_ may be created. Returns 201 Created if the trait is newly -// inserted, or 204 No Content if it already exists (in either ConfigMap). -// Returns 400 Bad Request if the name does not carry the CUSTOM_ prefix. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream; on success, adds the trait to the local ConfigMap. +// - crd: writes the trait to the local ConfigMap (CUSTOM_ prefix required). // // See: https://docs.openstack.org/api-ref/placement/#update-trait func (s *Shim) HandleUpdateTrait(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.Traits.orDefault() { - case FeatureModePassthrough, FeatureModeHybrid: + mode := s.featureModeFromConfOrHeader(r, s.config.Features.Traits, s.config.Traits != nil) + switch mode { + case FeatureModePassthrough: s.forward(w, r) return + case FeatureModeHybrid: + s.handleUpdateTraitHybrid(w, r) + return case FeatureModeCRD: - // Serve from local ConfigMaps. + // Handle locally. default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) return @@ -195,115 +180,73 @@ func (s *Shim) HandleUpdateTrait(w http.ResponseWriter, r *http.Request) { return } - // Fast path: trait already exists in either ConfigMap (no lock needed). - allTraits, err := s.getAllTraits(ctx) + created, err := s.addTraitToConfigMap(ctx, name) if err != nil { - log.Error(err, "failed to read traits for existence check", "trait", name) + log.Error(err, "failed to create trait", "trait", name) http.Error(w, "failed to create trait", http.StatusInternalServerError) return } - if _, exists := allTraits[name]; exists { - log.Info("trait already exists, nothing to do", "trait", name) + if created { + w.WriteHeader(http.StatusCreated) + } else { w.WriteHeader(http.StatusNoContent) - return } +} - // Slow path: acquire lock, read/create dynamic ConfigMap, add trait. - host, err := os.Hostname() - if err != nil { - host = "unknown" - } - lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) - if err := s.resourceLocker.AcquireLock(ctx, s.traitsLockName(), lockerID); err != nil { - log.Error(err, "failed to acquire traits lock", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) +// handleUpdateTraitHybrid forwards PUT /traits/{name} to upstream, then +// updates the local ConfigMap on success. +func (s *Shim) handleUpdateTraitHybrid(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logf.FromContext(ctx) + + name, ok := requiredPathParam(w, r, "name") + if !ok { return } - defer func() { - releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.resourceLocker.ReleaseLock(releaseCtx, s.traitsLockName(), lockerID); err != nil { - log.Error(err, "failed to release traits lock") - } - }() - cm := &corev1.ConfigMap{} - err = s.Get(ctx, s.customTraitsConfigMapKey(), cm) - if apierrors.IsNotFound(err) { - // Dynamic ConfigMap does not exist yet — create it with the new trait. - cm = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: s.customTraitsConfigMapKey().Name, - Namespace: s.customTraitsConfigMapKey().Namespace, - }, - Data: map[string]string{configMapKeyTraits: "[]"}, - } - current := map[string]struct{}{name: {}} - if err := s.writeTraits(cm, current); err != nil { - log.Error(err, "failed to serialize traits", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return + s.forwardWithHook(w, r, func(w http.ResponseWriter, resp *http.Response) { + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } } - if err := s.Create(ctx, cm); err != nil { - log.Error(err, "failed to create custom traits configmap", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return + w.WriteHeader(resp.StatusCode) + if resp.Body != nil { + if _, err := io.Copy(w, resp.Body); err != nil { + log.Error(err, "hybrid: failed to copy upstream response body") + } } - log.Info("created custom traits configmap with new trait", "trait", name) - s.syncTraitToUpstream(ctx, name, r.Header) - w.WriteHeader(http.StatusCreated) - return - } - if err != nil { - log.Error(err, "failed to get custom traits configmap", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return - } - current, err := parseTraits(cm) - if err != nil { - log.Error(err, "failed to parse custom traits configmap", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return - } - if _, exists := current[name]; exists { - log.Info("trait already exists in custom configmap after lock acquisition", "trait", name) - w.WriteHeader(http.StatusNoContent) - return - } - current[name] = struct{}{} - if err := s.writeTraits(cm, current); err != nil { - log.Error(err, "failed to serialize traits", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return - } - if err := s.Update(ctx, cm); err != nil { - log.Error(err, "failed to update custom traits configmap", "trait", name) - http.Error(w, "failed to create trait", http.StatusInternalServerError) - return - } - log.Info("added custom trait to configmap", "trait", name) - s.syncTraitToUpstream(ctx, name, r.Header) - w.WriteHeader(http.StatusCreated) + if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent { + if _, err := s.addTraitToConfigMap(ctx, name); err != nil { + log.Error(err, "hybrid: failed to add trait to local configmap", "trait", name) + } + } + }) } // HandleDeleteTrait handles DELETE /traits/{name} requests. // -// Deletes a custom trait from the dynamic ConfigMap. Standard traits (those -// without the CUSTOM_ prefix) cannot be deleted and return 400 Bad Request. -// Returns 404 if the trait does not exist. Returns 204 No Content on success. +// Feature modes: +// - passthrough: forwards to upstream placement. +// - hybrid: forwards to upstream; on success, removes the trait from the local ConfigMap. +// - crd: removes the trait from the local ConfigMap (CUSTOM_ prefix required). // // See: https://docs.openstack.org/api-ref/placement/#delete-traits func (s *Shim) HandleDeleteTrait(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := logf.FromContext(ctx) - switch s.config.Features.Traits.orDefault() { - case FeatureModePassthrough, FeatureModeHybrid: + mode := s.featureModeFromConfOrHeader(r, s.config.Features.Traits, s.config.Traits != nil) + switch mode { + case FeatureModePassthrough: s.forward(w, r) return + case FeatureModeHybrid: + s.handleDeleteTraitHybrid(w, r) + return case FeatureModeCRD: - // Serve from local ConfigMaps. + // Handle locally. default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) return @@ -319,101 +262,62 @@ func (s *Shim) HandleDeleteTrait(w http.ResponseWriter, r *http.Request) { return } - host, err := os.Hostname() - if err != nil { - host = "unknown" - } - lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) - if err := s.resourceLocker.AcquireLock(ctx, s.traitsLockName(), lockerID); err != nil { - log.Error(err, "failed to acquire traits lock", "trait", name) - http.Error(w, "failed to delete trait", http.StatusInternalServerError) - return - } - defer func() { - releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.resourceLocker.ReleaseLock(releaseCtx, s.traitsLockName(), lockerID); err != nil { - log.Error(err, "failed to release traits lock") - } - }() - - cm := &corev1.ConfigMap{} - err = s.Get(ctx, s.customTraitsConfigMapKey(), cm) - if apierrors.IsNotFound(err) { - log.Info("custom traits configmap not found, trait does not exist", "trait", name) - http.Error(w, "trait not found", http.StatusNotFound) - return - } + removed, err := s.removeTraitFromConfigMap(ctx, name) if err != nil { - log.Error(err, "failed to get custom traits configmap", "trait", name) + log.Error(err, "failed to delete trait", "trait", name) http.Error(w, "failed to delete trait", http.StatusInternalServerError) return } - current, err := parseTraits(cm) - if err != nil { - log.Error(err, "failed to parse custom traits configmap", "trait", name) - http.Error(w, "failed to delete trait", http.StatusInternalServerError) - return - } - if _, exists := current[name]; !exists { - log.Info("trait not found in custom configmap", "trait", name) + if !removed { + log.Info("trait not found in configmap", "trait", name) http.Error(w, "trait not found", http.StatusNotFound) return } - delete(current, name) - if err := s.writeTraits(cm, current); err != nil { - log.Error(err, "failed to serialize traits", "trait", name) - http.Error(w, "failed to delete trait", http.StatusInternalServerError) - return - } - if err := s.Update(ctx, cm); err != nil { - log.Error(err, "failed to update custom traits configmap", "trait", name) - http.Error(w, "failed to delete trait", http.StatusInternalServerError) - return - } - log.Info("deleted custom trait from configmap", "trait", name) + log.Info("deleted trait from configmap", "trait", name) w.WriteHeader(http.StatusNoContent) } -// getStaticTraits reads traits from the Helm-managed static ConfigMap. -func (s *Shim) getStaticTraits(ctx context.Context) (map[string]struct{}, error) { - cm := &corev1.ConfigMap{} - if err := s.Get(ctx, s.staticTraitsConfigMapKey(), cm); err != nil { - return nil, fmt.Errorf("get static configmap %s: %w", s.config.Traits.ConfigMapName, err) +// handleDeleteTraitHybrid forwards DELETE /traits/{name} to upstream, then +// updates the local ConfigMap on success. +func (s *Shim) handleDeleteTraitHybrid(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logf.FromContext(ctx) + + name, ok := requiredPathParam(w, r, "name") + if !ok { + return } - return parseTraits(cm) + + s.forwardWithHook(w, r, func(w http.ResponseWriter, resp *http.Response) { + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + if resp.Body != nil { + if _, err := io.Copy(w, resp.Body); err != nil { + log.Error(err, "hybrid: failed to copy upstream response body") + } + } + + if resp.StatusCode == http.StatusNoContent { + if _, err := s.removeTraitFromConfigMap(ctx, name); err != nil { + log.Error(err, "hybrid: failed to remove trait from local configmap", "trait", name) + } + } + }) } -// getCustomTraits reads traits from the dynamic ConfigMap created by the shim. -// Returns an empty set if the ConfigMap does not exist yet. -func (s *Shim) getCustomTraits(ctx context.Context) (map[string]struct{}, error) { +// getTraits reads traits from the single ConfigMap. +func (s *Shim) getTraits(ctx context.Context) (map[string]struct{}, error) { cm := &corev1.ConfigMap{} - err := s.Get(ctx, s.customTraitsConfigMapKey(), cm) - if apierrors.IsNotFound(err) { - return make(map[string]struct{}), nil - } - if err != nil { - return nil, fmt.Errorf("get custom configmap %s-custom: %w", s.config.Traits.ConfigMapName, err) + if err := s.Get(ctx, client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.Traits.ConfigMapName}, cm); err != nil { + return nil, fmt.Errorf("get traits configmap %s: %w", s.config.Traits.ConfigMapName, err) } return parseTraits(cm) } -// getAllTraits merges static and custom traits into a single set. -func (s *Shim) getAllTraits(ctx context.Context) (map[string]struct{}, error) { - static, err := s.getStaticTraits(ctx) - if err != nil { - return nil, err - } - custom, err := s.getCustomTraits(ctx) - if err != nil { - return nil, err - } - for t := range custom { - static[t] = struct{}{} - } - return static, nil -} - // parseTraits extracts the trait set from a ConfigMap. func parseTraits(cm *corev1.ConfigMap) (map[string]struct{}, error) { raw, ok := cm.Data[configMapKeyTraits] @@ -432,7 +336,7 @@ func parseTraits(cm *corev1.ConfigMap) (map[string]struct{}, error) { } func (s *Shim) hasTrait(ctx context.Context, name string) (bool, error) { - traits, err := s.getAllTraits(ctx) + traits, err := s.getTraits(ctx) if err != nil { return false, err } @@ -440,8 +344,8 @@ func (s *Shim) hasTrait(ctx context.Context, name string) (bool, error) { return ok, nil } -// writeTraits serializes the trait set into the ConfigMap's data field. -func (s *Shim) writeTraits(cm *corev1.ConfigMap, traitSet map[string]struct{}) error { +// writeTraitsToConfigMap serializes the trait set into the ConfigMap's data field. +func writeTraitsToConfigMap(cm *corev1.ConfigMap, traitSet map[string]struct{}) error { traits := make([]string, 0, len(traitSet)) for t := range traitSet { traits = append(traits, t) @@ -459,121 +363,114 @@ func (s *Shim) writeTraits(cm *corev1.ConfigMap, traitSet map[string]struct{}) e return nil } -// syncTraitToUpstream best-effort creates the trait in upstream placement so -// that endpoints forwarded to upstream (e.g. PUT /resource_providers/{uuid}/traits) -// can reference locally-created custom traits. Errors are logged but never -// propagated — upstream may be unreachable and that is acceptable. -func (s *Shim) syncTraitToUpstream(ctx context.Context, name string, incomingHeader http.Header) { - log := logf.FromContext(ctx) - if s.httpClient == nil { - log.V(1).Info("skipping upstream trait sync, no http client configured", "trait", name) - return - } - u, err := url.Parse(s.config.PlacementURL) +// addTraitToConfigMap adds a trait to the ConfigMap under the resource lock. +// Returns true if the trait was newly created, false if it already existed. +func (s *Shim) addTraitToConfigMap(ctx context.Context, name string) (bool, error) { + // Fast path: trait already exists (no lock needed). + traits, err := s.getTraits(ctx) if err != nil { - log.Error(err, "failed to parse placement URL for trait sync", "trait", name) - return + return false, err } - u.Path, err = url.JoinPath(u.Path, "/traits/"+name) - if err != nil { - log.Error(err, "failed to build upstream trait URL", "trait", name) - return + if _, exists := traits[name]; exists { + return false, nil } - req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), http.NoBody) + + // Slow path: acquire lock, re-read, add trait. + host, err := os.Hostname() if err != nil { - log.Error(err, "failed to create upstream trait request", "trait", name) - return + return false, fmt.Errorf("get hostname: %w", err) } - // Forward authentication headers so upstream placement accepts the request. - req.Header = incomingHeader.Clone() - resp, err := s.httpClient.Do(req) - if err != nil { - log.Info("best-effort upstream trait sync failed, upstream may be down", "trait", name, "error", err.Error()) - return + lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) + if err := s.resourceLocker.AcquireLock(ctx, s.config.Traits.ConfigMapName+"-lock", lockerID); err != nil { + return false, fmt.Errorf("acquire traits lock: %w", err) } - defer resp.Body.Close() - log.Info("synced custom trait to upstream placement", "trait", name, "status", resp.StatusCode) -} + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.resourceLocker.ReleaseLock(releaseCtx, s.config.Traits.ConfigMapName+"-lock", lockerID); err != nil { + ctrl.Log.WithName("placement-shim").Error(err, "failed to release traits lock") + } + }() -// startTraitSyncLoop runs a periodic goroutine that fetches traits from -// upstream placement and writes them into the static ConfigMap. Only active -// when features.traits is hybrid. The loop exits when ctx is cancelled. -func (s *Shim) startTraitSyncLoop(ctx context.Context) { - if s.config.Features.Traits.orDefault() != FeatureModeHybrid { - return + cm := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.Traits.ConfigMapName} + if err := s.Get(ctx, key, cm); err != nil { + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Data: map[string]string{configMapKeyTraits: "[]"}, + } + current := map[string]struct{}{name: {}} + if err := writeTraitsToConfigMap(cm, current); err != nil { + return false, err + } + if err := s.Create(ctx, cm); err != nil { + return false, fmt.Errorf("create traits configmap: %w", err) + } + return true, nil + } + return false, fmt.Errorf("get traits configmap: %w", err) } - log := ctrl.Log.WithName("placement-shim").WithName("trait-sync") - jitter := time.Duration(rand.Int63n(int64(30 * time.Second))) //nolint:gosec - log.Info("starting trait sync loop", "jitter", jitter) - select { - case <-ctx.Done(): - return - case <-time.After(jitter): + current, err := parseTraits(cm) + if err != nil { + return false, err } - - s.syncTraitsFromUpstream(ctx, log) - - ticker := time.NewTicker(60 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.syncTraitsFromUpstream(ctx, log) - } + if _, exists := current[name]; exists { + return false, nil } -} - -// syncTraitsFromUpstream fetches GET /traits from upstream placement and -// writes the result into the static ConfigMap so that the shim's local -// view stays in sync with upstream. Uses the gophercloud ServiceClient -// for automatic token management (including reauth on 401). -func (s *Shim) syncTraitsFromUpstream(ctx context.Context, log logr.Logger) { - if s.placementServiceClient == nil { - log.V(1).Info("skipping upstream trait sync, no placement service client configured") - return + current[name] = struct{}{} + if err := writeTraitsToConfigMap(cm, current); err != nil { + return false, err } - u, err := url.JoinPath(s.placementServiceClient.Endpoint, "/traits") - if err != nil { - log.Error(err, "failed to build upstream traits URL") - return + if err := s.Update(ctx, cm); err != nil { + return false, fmt.Errorf("update traits configmap: %w", err) } - resp, err := s.placementServiceClient.Request(ctx, http.MethodGet, u, &gophercloud.RequestOpts{ - OkCodes: []int{http.StatusOK}, - MoreHeaders: map[string]string{ - "OpenStack-API-Version": "placement 1.6", - }, - KeepResponseBody: true, - }) + return true, nil +} + +// removeTraitFromConfigMap removes a trait from the ConfigMap under the +// resource lock. Returns true if the trait was found and removed. +func (s *Shim) removeTraitFromConfigMap(ctx context.Context, name string) (bool, error) { + host, err := os.Hostname() if err != nil { - log.Info("upstream trait sync failed", "error", err.Error()) - return + return false, fmt.Errorf("get hostname: %w", err) } - defer resp.Body.Close() - var body traitsListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - log.Error(err, "failed to decode upstream trait list") - return + lockerID := fmt.Sprintf("shim-%s-%d", host, time.Now().UnixNano()) + if err := s.resourceLocker.AcquireLock(ctx, s.config.Traits.ConfigMapName+"-lock", lockerID); err != nil { + return false, fmt.Errorf("acquire traits lock: %w", err) } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.resourceLocker.ReleaseLock(releaseCtx, s.config.Traits.ConfigMapName+"-lock", lockerID); err != nil { + ctrl.Log.WithName("placement-shim").Error(err, "failed to release traits lock") + } + }() cm := &corev1.ConfigMap{} - if err := s.Get(ctx, s.staticTraitsConfigMapKey(), cm); err != nil { - log.Error(err, "failed to get static traits configmap for sync") - return + if err := s.Get(ctx, client.ObjectKey{Namespace: os.Getenv("POD_NAMESPACE"), Name: s.config.Traits.ConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("get traits configmap: %w", err) } - traitSet := make(map[string]struct{}, len(body.Traits)) - for _, t := range body.Traits { - traitSet[t] = struct{}{} + current, err := parseTraits(cm) + if err != nil { + return false, err } - if err := s.writeTraits(cm, traitSet); err != nil { - log.Error(err, "failed to serialize synced traits") - return + if _, exists := current[name]; !exists { + return false, nil + } + delete(current, name) + if err := writeTraitsToConfigMap(cm, current); err != nil { + return false, err } if err := s.Update(ctx, cm); err != nil { - log.Error(err, "failed to update static traits configmap with upstream data") - return + return false, fmt.Errorf("update traits configmap: %w", err) } - log.Info("synced traits from upstream placement", "count", len(body.Traits)) + return true, nil } diff --git a/internal/shim/placement/handle_traits_e2e.go b/internal/shim/placement/handle_traits_e2e.go index 8a904b935..9c204e0f8 100644 --- a/internal/shim/placement/handle_traits_e2e.go +++ b/internal/shim/placement/handle_traits_e2e.go @@ -80,10 +80,11 @@ func e2eTestTraits(ctx context.Context, _ client.Client) error { if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { return fmt.Errorf("failed to decode GET /traits response: %w", err) } - // When traits are served locally (hybrid or crd mode) the static list may - // be empty. Only require at least one trait when forwarding to upstream + // When traits are served locally (hybrid or crd mode) the list may be + // empty. Only require at least one trait when forwarding to upstream // placement, which always has standard traits. - if config.Features.Traits.orDefault() == FeatureModePassthrough && len(listResp.Traits) == 0 { + traitsMode := e2eCurrentMode(ctx) + if traitsMode == FeatureModePassthrough && len(listResp.Traits) == 0 { return errors.New("GET /traits: expected at least one trait, got 0") } log.Info("Successfully retrieved traits", "count", len(listResp.Traits)) @@ -131,14 +132,9 @@ func e2eTestTraits(ctx context.Context, _ client.Client) error { } log.Info("Correctly received 404 for nonexistent trait") - // ==================== Phase 2: CRUD tests (feature-gated) ==================== + // ==================== Phase 2: CRUD tests ==================== - if config.Features.Traits.orDefault() == FeatureModePassthrough { - log.Info("Skipping trait CRUD e2e tests because traits mode is passthrough") - return nil - } - - log.Info("=== Phase 2: CRUD trait tests (traits mode non-passthrough) ===") + log.Info("=== Phase 2: CRUD trait tests ===") const testTrait = "CUSTOM_CORTEX_E2E_TRAIT" @@ -287,47 +283,53 @@ func e2eTestTraits(ctx context.Context, _ client.Client) error { } log.Info("Verified test trait was deleted", "trait", testTrait) - // Test PUT /traits/{name} with bad prefix → 400. - log.Info("Testing PUT /traits/{name} with non-CUSTOM_ prefix") - req, err = http.NewRequestWithContext(ctx, - http.MethodPut, sc.Endpoint+"/traits/HW_CORTEX_E2E_BAD", http.NoBody) - if err != nil { - return fmt.Errorf("failed to create bad-prefix PUT request: %w", err) - } - req.Header.Set("X-Auth-Token", sc.TokenID) - req.Header.Set("OpenStack-API-Version", "placement 1.6") - resp, err = sc.HTTPClient.Do(req) - if err != nil { - return fmt.Errorf("failed to send bad-prefix PUT request: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - return fmt.Errorf("PUT /traits/HW_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) - } - log.Info("Correctly received 400 for PUT with non-CUSTOM_ prefix") + // Bad-prefix validation is only enforced by the shim in crd mode. + // In hybrid mode, writes forward to upstream which has different behavior. + if traitsMode == FeatureModeCRD { + // Test PUT /traits/{name} with bad prefix → 400. + log.Info("Testing PUT /traits/{name} with non-CUSTOM_ prefix") + req, err = http.NewRequestWithContext(ctx, + http.MethodPut, sc.Endpoint+"/traits/HW_CORTEX_E2E_BAD", http.NoBody) + if err != nil { + return fmt.Errorf("failed to create bad-prefix PUT request: %w", err) + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send bad-prefix PUT request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + return fmt.Errorf("PUT /traits/HW_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) + } + log.Info("Correctly received 400 for PUT with non-CUSTOM_ prefix") - // Test DELETE /traits/{name} with bad prefix → 400. - log.Info("Testing DELETE /traits/{name} with non-CUSTOM_ prefix") - req, err = http.NewRequestWithContext(ctx, - http.MethodDelete, sc.Endpoint+"/traits/HW_CORTEX_E2E_BAD", http.NoBody) - if err != nil { - return fmt.Errorf("failed to create bad-prefix DELETE request: %w", err) - } - req.Header.Set("X-Auth-Token", sc.TokenID) - req.Header.Set("OpenStack-API-Version", "placement 1.6") - resp, err = sc.HTTPClient.Do(req) - if err != nil { - return fmt.Errorf("failed to send bad-prefix DELETE request: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - return fmt.Errorf("DELETE /traits/HW_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) + // Test DELETE /traits/{name} with bad prefix → 400. + log.Info("Testing DELETE /traits/{name} with non-CUSTOM_ prefix") + req, err = http.NewRequestWithContext(ctx, + http.MethodDelete, sc.Endpoint+"/traits/HW_CORTEX_E2E_BAD", http.NoBody) + if err != nil { + return fmt.Errorf("failed to create bad-prefix DELETE request: %w", err) + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + resp, err = sc.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send bad-prefix DELETE request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + return fmt.Errorf("DELETE /traits/HW_CORTEX_E2E_BAD: expected 400, got %d", resp.StatusCode) + } + log.Info("Correctly received 400 for DELETE with non-CUSTOM_ prefix") + } else { + log.Info("Skipping bad-prefix validation tests (only enforced in crd mode)") } - log.Info("Correctly received 400 for DELETE with non-CUSTOM_ prefix") return nil } func init() { - e2eTests = append(e2eTests, e2eTest{name: "traits", run: e2eTestTraits}) + e2eTests = append(e2eTests, e2eTest{name: "traits", run: e2eWrapWithModes(e2eTestTraits)}) } diff --git a/internal/shim/placement/handle_traits_test.go b/internal/shim/placement/handle_traits_test.go index bf692fd41..c7531796a 100644 --- a/internal/shim/placement/handle_traits_test.go +++ b/internal/shim/placement/handle_traits_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "net/http" - "net/http/httptest" "testing" "github.com/cobaltcore-dev/cortex/pkg/resourcelock" @@ -42,13 +41,12 @@ func newTestConfigMap(namespace, name string, traits []string) *corev1.ConfigMap } } -func newTraitShim(t *testing.T, staticTraits []string, customTraits ...string) *Shim { +func newTraitShim(t *testing.T, traits []string, extraTraits ...string) *Shim { t.Helper() t.Setenv("POD_NAMESPACE", "default") - objs := []client.Object{newTestConfigMap("default", "test-cm", staticTraits)} - if len(customTraits) > 0 { - objs = append(objs, newTestConfigMap("default", "test-cm-custom", customTraits)) - } + all := append([]string{}, traits...) + all = append(all, extraTraits...) + objs := []client.Object{newTestConfigMap("default", "test-cm", all)} cl := newFakeClientWithScheme(t, objs...) down, up := newTestTimers() return &Shim{ @@ -264,39 +262,11 @@ func TestHandleUpdateTraitLocalBadPrefix(t *testing.T) { } } -func TestHandleUpdateTraitLocalSyncsToUpstream(t *testing.T) { - var gotMethod, gotPath string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotMethod = r.Method - gotPath = r.URL.Path - w.WriteHeader(http.StatusNoContent) - })) - t.Cleanup(upstream.Close) +func TestHandleUpdateTraitLocalNoUpstreamContact(t *testing.T) { s := newTraitShim(t, nil) - s.config.PlacementURL = upstream.URL - s.httpClient = upstream.Client() - - w := serveHandler(t, "PUT", "/traits/{name}", s.HandleUpdateTrait, "/traits/CUSTOM_NEW") - if w.Code != http.StatusCreated { - t.Fatalf("status = %d, want %d", w.Code, http.StatusCreated) - } - if gotMethod != "PUT" || gotPath != "/traits/CUSTOM_NEW" { - t.Fatalf("upstream got %s %s, want PUT /traits/CUSTOM_NEW", gotMethod, gotPath) - } -} - -func TestHandleUpdateTraitLocalUpstreamDown(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - t.Cleanup(upstream.Close) - s := newTraitShim(t, nil) - s.config.PlacementURL = upstream.URL - s.httpClient = upstream.Client() - w := serveHandler(t, "PUT", "/traits/{name}", s.HandleUpdateTrait, "/traits/CUSTOM_NEW") if w.Code != http.StatusCreated { - t.Fatalf("status = %d, want %d; upstream failure should not block local creation", w.Code, http.StatusCreated) + t.Fatalf("status = %d, want %d; CRD mode should not contact upstream", w.Code, http.StatusCreated) } } diff --git a/internal/shim/placement/handle_usages_e2e.go b/internal/shim/placement/handle_usages_e2e.go index c7ac8c965..66f5c40a9 100644 --- a/internal/shim/placement/handle_usages_e2e.go +++ b/internal/shim/placement/handle_usages_e2e.go @@ -39,6 +39,15 @@ func e2eTestUsages(ctx context.Context, _ client.Client) error { const apiVersion = "placement 1.9" + // Probe: for non-passthrough modes, verify endpoint returns 501. + unimplemented, err := e2eProbeUnimplemented(ctx, sc, sc.Endpoint+"/usages?project_id=test") + if err != nil { + return fmt.Errorf("probe: %w", err) + } + if unimplemented { + return nil + } + // Get the list of projects from the identity service, so that we can test // the /usages endpoint with a valid project id. log.Info("Getting list of projects from identity service for usages e2e test") @@ -113,5 +122,5 @@ func e2eTestUsages(ctx context.Context, _ client.Client) error { } func init() { - e2eTests = append(e2eTests, e2eTest{name: "usages", run: e2eTestUsages}) + e2eTests = append(e2eTests, e2eTest{name: "usages", run: e2eWrapWithModes(e2eTestUsages)}) } diff --git a/internal/shim/placement/shim.go b/internal/shim/placement/shim.go index 273273983..77ff72a66 100644 --- a/internal/shim/placement/shim.go +++ b/internal/shim/placement/shim.go @@ -52,6 +52,18 @@ type requestIDContextKey struct{} // header value through the request lifecycle for tracing. var requestIDKey = requestIDContextKey{} +// featureModeOverrideContextKey is a separate type for the per-request feature +// mode override injected via the X-Cortex-Feature-Mode header. +type featureModeOverrideContextKey struct{} + +// featureModeOverrideKey is the context key used to propagate the feature mode +// override from the middleware to handlers. +var featureModeOverrideKey = featureModeOverrideContextKey{} + +// headerFeatureModeOverride is the HTTP header that allows e2e tests to +// override the configured feature mode on a per-request basis. +const headerFeatureModeOverride = "X-Cortex-Feature-Mode" + // FeatureMode controls how an endpoint group interacts with upstream // placement and the hypervisor CRD. type FeatureMode string @@ -88,18 +100,42 @@ func (m FeatureMode) valid() bool { } // dispatchPassthroughOnly forwards in passthrough mode, returns 501 for -// hybrid/crd, and 500 for unknown modes. +// hybrid/crd, and 500 for unknown modes. These endpoints have no backing +// config requirement so we pass true — the 501 response already guards +// against actual nil dereferences. func (s *Shim) dispatchPassthroughOnly(w http.ResponseWriter, r *http.Request, mode FeatureMode) { - switch mode.orDefault() { + resolved := s.featureModeFromConfOrHeader(r, mode, true) + switch resolved { case FeatureModePassthrough: s.forward(w, r) case FeatureModeHybrid, FeatureModeCRD: - http.Error(w, fmt.Sprintf("%s mode is not yet implemented for this endpoint", mode), http.StatusNotImplemented) + http.Error(w, fmt.Sprintf("%s mode is not yet implemented for this endpoint", resolved), http.StatusNotImplemented) default: http.Error(w, "unknown feature mode", http.StatusInternalServerError) } } +// featureModeFromConfOrHeader returns the effective feature mode for the +// current request. If a valid override is present in the request context +// (injected by wrapHandler from the X-Cortex-Feature-Mode header), the +// override takes precedence — unless it would escalate to hybrid/crd without +// the endpoint's backing config being available. Callers pass hasBackingConfig +// to indicate whether the infrastructure required by hybrid/crd mode for their +// specific endpoint was validated at startup. +func (s *Shim) featureModeFromConfOrHeader(r *http.Request, configured FeatureMode, hasBackingConfig bool) FeatureMode { + override, ok := r.Context().Value(featureModeOverrideKey).(FeatureMode) + if !ok { + return configured.orDefault() + } + resolved := override.orDefault() + if resolved == FeatureModeHybrid || resolved == FeatureModeCRD { + if !hasBackingConfig { + return configured.orDefault() + } + } + return resolved +} + // featuresConfig controls the feature mode for each endpoint group. // Every field defaults to passthrough (zero value) when omitted. type featuresConfig struct { @@ -133,6 +169,14 @@ type traitsConfig struct { ConfigMapName string `json:"configMapName"` } +// resourceClassesConfig configures the local resource class store used when +// features.resourceClasses is hybrid or crd. +type resourceClassesConfig struct { + // ConfigMapName is the name of the ConfigMap used to persist resource classes. + // Must exist in the same namespace as the shim pod. + ConfigMapName string `json:"configMapName"` +} + // config holds configuration for the placement shim. type config struct { // SSO is an optional configuration for the certificates the http client @@ -178,6 +222,9 @@ type config struct { // Traits configures the local trait store used when // features.traits is hybrid or crd. Traits *traitsConfig `json:"traits,omitempty"` + // ResourceClasses configures the local resource class store used when + // features.resourceClasses is hybrid or crd. + ResourceClasses *resourceClassesConfig `json:"resourceClasses,omitempty"` } // validate checks the config for required fields and returns an error if the @@ -213,15 +260,27 @@ func (c *config) validate() error { } } traitsMode := c.Features.Traits.orDefault() - if traitsMode == FeatureModeHybrid || traitsMode == FeatureModeCRD { - if c.Traits == nil { - return fmt.Errorf("traits config is required when features.traits is %s", traitsMode) - } + if traitsMode != FeatureModePassthrough && c.Traits == nil { + return fmt.Errorf("traits config is required when features.traits is %s", traitsMode) + } + if c.Traits != nil { if c.Traits.ConfigMapName == "" { - return fmt.Errorf("traits.configMapName is required when features.traits is %s", traitsMode) + return errors.New("traits.configMapName is required when traits config is present") + } + if os.Getenv("POD_NAMESPACE") == "" { + return errors.New("pod namespace (POD_NAMESPACE) is required when traits config is present") + } + } + rcMode := c.Features.ResourceClasses.orDefault() + if rcMode != FeatureModePassthrough && c.ResourceClasses == nil { + return fmt.Errorf("resourceClasses config is required when features.resourceClasses is %s", rcMode) + } + if c.ResourceClasses != nil { + if c.ResourceClasses.ConfigMapName == "" { + return errors.New("resourceClasses.configMapName is required when resourceClasses config is present") } - if traitsMode == FeatureModeCRD && os.Getenv("POD_NAMESPACE") == "" { - return errors.New("pod namespace (POD_NAMESPACE) is required when features.traits is crd") + if os.Getenv("POD_NAMESPACE") == "" { + return errors.New("pod namespace (POD_NAMESPACE) is required when resourceClasses config is present") } } if c.Auth != nil && c.KeystoneURL == "" { @@ -269,8 +328,8 @@ type Shim struct { tokenCache *tokenCache // tokenIntrospector validates tokens against Keystone. tokenIntrospector tokenIntrospector - // resourceLocker serializes writes to the custom traits ConfigMap - // across replicas using a Kubernetes Lease. + // resourceLocker serializes writes to ConfigMaps across replicas + // using a Kubernetes Lease. resourceLocker *resourcelock.ResourceLocker // placementServiceClient is an authenticated gophercloud service client // used by background tasks (trait sync) to make requests to upstream @@ -399,7 +458,36 @@ func (s *Shim) Start(ctx context.Context) error { if err := s.initPlacementServiceClient(ctx); err != nil { return err } - go s.startTraitSyncLoop(ctx) + if s.config.Traits != nil { + ts := NewTraitSyncer( + s.Client, + s.config.Traits.ConfigMapName, + os.Getenv("POD_NAMESPACE"), + s.placementServiceClient, + s.resourceLocker, + ) + if err := ts.Init(ctx); err != nil { + return err + } + if s.config.Features.Traits.orDefault() != FeatureModeCRD { + go ts.Run(ctx) + } + } + if s.config.ResourceClasses != nil { + rs := NewResourceClassSyncer( + s.Client, + s.config.ResourceClasses.ConfigMapName, + os.Getenv("POD_NAMESPACE"), + s.placementServiceClient, + s.resourceLocker, + ) + if err := rs.Init(ctx); err != nil { + return err + } + if s.config.Features.ResourceClasses.orDefault() != FeatureModeCRD { + go rs.Run(ctx) + } + } return nil } @@ -472,13 +560,10 @@ func (s *Shim) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (err erro Buckets: prometheus.DefBuckets, }, []string{"method", "pattern", "responsecode"}) - traitsMode := s.config.Features.Traits.orDefault() - if traitsMode == FeatureModeHybrid || traitsMode == FeatureModeCRD { - s.resourceLocker = resourcelock.NewResourceLocker( - s.Client, - os.Getenv("POD_NAMESPACE"), - ) - } + s.resourceLocker = resourcelock.NewResourceLocker( + s.Client, + os.Getenv("POD_NAMESPACE"), + ) // Check that the provided client is a multicluster client, since we need // that to watch for hypervisors across clusters. diff --git a/internal/shim/placement/shim_e2e.go b/internal/shim/placement/shim_e2e.go index e7f9e30f3..b678fa6e5 100644 --- a/internal/shim/placement/shim_e2e.go +++ b/internal/shim/placement/shim_e2e.go @@ -66,7 +66,7 @@ func makeE2EServiceClient(ctx context.Context, rc e2eRootConfig) (*gophercloud.S log.Info("No SSO config provided, using plain transport for placement API") transport = &http.Transport{} } - provider.HTTPClient.Transport = transport + provider.HTTPClient.Transport = &e2eModeTransport{base: transport} if err := openstack.Authenticate(ctx, provider, authOpts); err != nil { log.Error(err, "Failed to authenticate with keystone") return nil, fmt.Errorf("failed to authenticate with keystone: %w", err) @@ -88,6 +88,105 @@ type e2eTest struct { // e2eTests is populated by init() functions in the handle_*_e2e.go files. var e2eTests []e2eTest +// e2eAllModes is the list of feature modes exercised by e2e tests when +// AllowModeOverride is enabled. +var e2eAllModes = []FeatureMode{ + FeatureModePassthrough, + FeatureModeHybrid, + FeatureModeCRD, +} + +// setFeatureModeHeader sets the X-Cortex-Feature-Mode override header on the +// request so the shim dispatches to the specified mode regardless of its +// configured mode. +func setFeatureModeHeader(req *http.Request, mode FeatureMode) { + if mode != "" { + req.Header.Set(headerFeatureModeOverride, string(mode)) + } +} + +// e2eModeContextKey is used to pass the current test mode through context. +type e2eModeContextKey struct{} + +// e2eCurrentMode retrieves the feature mode from context (set by +// e2eWrapWithModes). Returns empty string if not set. +func e2eCurrentMode(ctx context.Context) FeatureMode { + if m, ok := ctx.Value(e2eModeContextKey{}).(FeatureMode); ok { + return m + } + return "" +} + +// e2eWrapWithModes returns a test function that iterates over all feature +// modes. For each mode it injects the mode into context (retrievable via +// e2eCurrentMode) so that the e2eModeTransport sets the override header on +// every outgoing request. +func e2eWrapWithModes(fn func(ctx context.Context, cl client.Client) error) func(ctx context.Context, cl client.Client) error { + return func(ctx context.Context, cl client.Client) error { + log := logf.FromContext(ctx) + for _, mode := range e2eAllModes { + modeLog := log.WithName(string(mode)) + modeCtx := context.WithValue(ctx, e2eModeContextKey{}, mode) + modeCtx = logf.IntoContext(modeCtx, modeLog) + modeLog.Info("Starting mode") + if err := fn(modeCtx, cl); err != nil { + return fmt.Errorf("mode %s: %w", mode, err) + } + modeLog.Info("Mode passed") + } + return nil + } +} + +// e2eProbeUnimplemented sends a single GET request with the mode override +// header to verify the endpoint returns 501 Not Implemented. Returns true if +// the endpoint is unimplemented for this mode (test should skip). Returns +// false if the endpoint returned a success status (test should continue). +// Returns an error for unexpected status codes (4xx/5xx other than 501). +func e2eProbeUnimplemented(ctx context.Context, sc *gophercloud.ServiceClient, probeURL string) (bool, error) { + log := logf.FromContext(ctx) + mode := e2eCurrentMode(ctx) + if mode == "" || mode == FeatureModePassthrough { + return false, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("X-Auth-Token", sc.TokenID) + req.Header.Set("OpenStack-API-Version", "placement 1.6") + setFeatureModeHeader(req, mode) + resp, err := sc.HTTPClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotImplemented { + log.Info("Endpoint correctly returns 501 for unimplemented mode", "mode", mode) + return true, nil + } + if resp.StatusCode >= http.StatusBadRequest { + return false, fmt.Errorf("probe %s in mode %s returned unexpected status %d", probeURL, mode, resp.StatusCode) + } + return false, nil +} + +// e2eModeTransport wraps an http.RoundTripper to automatically inject the +// X-Cortex-Feature-Mode header based on the mode stored in the request's +// context (via e2eModeContextKey). This avoids manually calling +// setFeatureModeHeader on every request in every e2e test. +type e2eModeTransport struct { + base http.RoundTripper +} + +func (t *e2eModeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if mode := e2eCurrentMode(req.Context()); mode != "" { + req = req.Clone(req.Context()) + req.Header.Set(headerFeatureModeOverride, string(mode)) + } + return t.base.RoundTrip(req) +} + // RunE2E executes end-to-end tests for all placement shim handlers. // It stops on the first failure and returns the error. func RunE2E(ctx context.Context, cl client.Client) error { @@ -113,3 +212,27 @@ func RunE2E(ctx context.Context, cl client.Client) error { "took_ms", time.Since(totalStart).Milliseconds()) return nil } + +// e2ePollUntil retries check at short intervals until it returns true or the +// timeout expires. Used to wait for the informer cache to pick up a CRD +// update before asserting via the HTTP API. +func e2ePollUntil(ctx context.Context, timeout time.Duration, check func() (bool, error)) error { + deadline := time.Now().Add(timeout) + for { + ok, err := check() + if err != nil { + return err + } + if ok { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for condition", timeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} diff --git a/internal/shim/placement/shim_io.go b/internal/shim/placement/shim_io.go index 98d5ba0bc..792f8edc3 100644 --- a/internal/shim/placement/shim_io.go +++ b/internal/shim/placement/shim_io.go @@ -113,6 +113,14 @@ func (s *Shim) wrapHandler(pattern string, next http.HandlerFunc) http.HandlerFu log = log.WithValues("requestID", reqID) ctx = context.WithValue(ctx, requestIDKey, reqID) } + + // Read the feature mode override header and store in context. + if raw := r.Header.Get(headerFeatureModeOverride); raw != "" { + if fm := FeatureMode(raw); fm.valid() && fm != "" { + ctx = context.WithValue(ctx, featureModeOverrideKey, fm) + } + } + ctx = logf.IntoContext(ctx, log) r = r.WithContext(ctx) diff --git a/internal/shim/placement/shim_test.go b/internal/shim/placement/shim_test.go index 503b94c72..786831553 100644 --- a/internal/shim/placement/shim_test.go +++ b/internal/shim/placement/shim_test.go @@ -83,6 +83,18 @@ func serveHandler(t *testing.T, method, pattern string, handler http.HandlerFunc return w } +// serveHandlerWithBody is like serveHandler but allows providing a request body. +func serveHandlerWithBody(t *testing.T, method, pattern string, handler http.HandlerFunc, reqPath string, body io.Reader) *httptest.ResponseRecorder { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc(method+" "+pattern, handler) + req := httptest.NewRequest(method, reqPath, body) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + return w +} + func TestForward(t *testing.T) { tests := []struct { name string @@ -482,6 +494,30 @@ func TestConfigValidateTraitsCRDRequiresConfig(t *testing.T) { } } +func TestConfigValidateResourceClassesCRDRequiresConfig(t *testing.T) { + t.Setenv("POD_NAMESPACE", "") + + c := config{ + PlacementURL: "http://placement:8778", + Features: featuresConfig{ResourceClasses: FeatureModeCRD}, + } + if err := c.validate(); err == nil { + t.Fatal("expected error when resourceClasses mode is crd without config") + } + c.ResourceClasses = &resourceClassesConfig{} + if err := c.validate(); err == nil { + t.Fatal("expected error when resourceClasses.configMapName is empty") + } + c.ResourceClasses.ConfigMapName = "cortex-placement-shim-resource-classes" + if err := c.validate(); err == nil { + t.Fatal("expected error when POD_NAMESPACE is not set") + } + t.Setenv("POD_NAMESPACE", "default") + if err := c.validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + func TestWrapHandlerWithAuth(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) @@ -561,3 +597,137 @@ func TestWrapHandlerWithAuth(t *testing.T) { } }) } + +func TestFeatureModeFromConfOrHeader(t *testing.T) { + s := &Shim{config: config{ + Traits: &traitsConfig{ConfigMapName: "test"}, + }} + + t.Run("returns configured mode when no override", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + got := s.featureModeFromConfOrHeader(req, FeatureModeHybrid, true) + if got != FeatureModeHybrid { + t.Fatalf("got %q, want %q", got, FeatureModeHybrid) + } + }) + + t.Run("defaults empty configured mode to passthrough", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + got := s.featureModeFromConfOrHeader(req, "", true) + if got != FeatureModePassthrough { + t.Fatalf("got %q, want %q", got, FeatureModePassthrough) + } + }) + + t.Run("returns override when present in context and backing config exists", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + ctx := context.WithValue(req.Context(), featureModeOverrideKey, FeatureModeCRD) + req = req.WithContext(ctx) + got := s.featureModeFromConfOrHeader(req, FeatureModePassthrough, true) + if got != FeatureModeCRD { + t.Fatalf("got %q, want %q", got, FeatureModeCRD) + } + }) + + t.Run("override to hybrid/crd ignored when no backing config", func(t *testing.T) { + bare := &Shim{} + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + ctx := context.WithValue(req.Context(), featureModeOverrideKey, FeatureModeCRD) + req = req.WithContext(ctx) + got := bare.featureModeFromConfOrHeader(req, FeatureModePassthrough, false) + if got != FeatureModePassthrough { + t.Fatalf("got %q, want %q (override should be rejected without backing config)", got, FeatureModePassthrough) + } + }) + + t.Run("override to passthrough always allowed", func(t *testing.T) { + bare := &Shim{} + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + ctx := context.WithValue(req.Context(), featureModeOverrideKey, FeatureModePassthrough) + req = req.WithContext(ctx) + got := bare.featureModeFromConfOrHeader(req, FeatureModeHybrid, false) + if got != FeatureModePassthrough { + t.Fatalf("got %q, want %q", got, FeatureModePassthrough) + } + }) + + t.Run("override defaults empty to passthrough", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", http.NoBody) + ctx := context.WithValue(req.Context(), featureModeOverrideKey, FeatureMode("")) + req = req.WithContext(ctx) + got := s.featureModeFromConfOrHeader(req, FeatureModeHybrid, true) + if got != FeatureModePassthrough { + t.Fatalf("got %q, want %q", got, FeatureModePassthrough) + } + }) +} + +func TestWrapHandlerFeatureModeOverride(t *testing.T) { + t.Run("valid header injects override into context", func(t *testing.T) { + var gotMode FeatureMode + down, up := newTestTimers() + s := &Shim{ + config: config{PlacementURL: "http://unused"}, + maxBodyLogSize: 4096, + downstreamRequestTimer: down, + upstreamRequestTimer: up, + } + wrapped := s.wrapHandler("/test", func(w http.ResponseWriter, r *http.Request) { + if override, ok := r.Context().Value(featureModeOverrideKey).(FeatureMode); ok { + gotMode = override + } + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/test", http.NoBody) + req.Header.Set(headerFeatureModeOverride, string(FeatureModeCRD)) + w := httptest.NewRecorder() + wrapped(w, req) + if gotMode != FeatureModeCRD { + t.Fatalf("context override = %q, want %q", gotMode, FeatureModeCRD) + } + }) + + t.Run("invalid header value is ignored", func(t *testing.T) { + var gotOverride bool + down, up := newTestTimers() + s := &Shim{ + config: config{PlacementURL: "http://unused"}, + maxBodyLogSize: 4096, + downstreamRequestTimer: down, + upstreamRequestTimer: up, + } + wrapped := s.wrapHandler("/test", func(w http.ResponseWriter, r *http.Request) { + _, gotOverride = r.Context().Value(featureModeOverrideKey).(FeatureMode) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/test", http.NoBody) + req.Header.Set(headerFeatureModeOverride, "bogus") + w := httptest.NewRecorder() + wrapped(w, req) + if gotOverride { + t.Fatal("override should not be set for invalid mode value") + } + }) + + t.Run("empty header value is ignored", func(t *testing.T) { + var gotOverride bool + down, up := newTestTimers() + s := &Shim{ + config: config{PlacementURL: "http://unused"}, + maxBodyLogSize: 4096, + downstreamRequestTimer: down, + upstreamRequestTimer: up, + } + wrapped := s.wrapHandler("/test", func(w http.ResponseWriter, r *http.Request) { + _, gotOverride = r.Context().Value(featureModeOverrideKey).(FeatureMode) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/test", http.NoBody) + req.Header.Set(headerFeatureModeOverride, "") + w := httptest.NewRecorder() + wrapped(w, req) + if gotOverride { + t.Fatal("override should not be set for empty header") + } + }) +} diff --git a/internal/shim/placement/syncer.go b/internal/shim/placement/syncer.go new file mode 100644 index 000000000..f00487fe9 --- /dev/null +++ b/internal/shim/placement/syncer.go @@ -0,0 +1,19 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import "context" + +// Syncer manages the lifecycle of a ConfigMap-backed local store: +// creating the ConfigMap on startup, and running a periodic background +// sync from upstream placement. +type Syncer interface { + // Init creates the ConfigMap if it does not exist. Called once during + // Shim.Start before any requests are served. + Init(ctx context.Context) error + + // Run starts the periodic background sync from upstream. Blocks until + // ctx is cancelled. Called as a goroutine from Shim.Start. + Run(ctx context.Context) +} diff --git a/internal/shim/placement/syncer_resource_classes.go b/internal/shim/placement/syncer_resource_classes.go new file mode 100644 index 000000000..a6aa854b5 --- /dev/null +++ b/internal/shim/placement/syncer_resource_classes.go @@ -0,0 +1,196 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "os" + "time" + + "github.com/cobaltcore-dev/cortex/pkg/resourcelock" + "github.com/gophercloud/gophercloud/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const configMapKeyResourceClasses = "resource_classes" + +// resourceClassesListResponse matches the OpenStack Placement GET /resource_classes response. +type resourceClassesListResponse struct { + ResourceClasses []resourceClassEntry `json:"resource_classes"` +} + +type resourceClassEntry struct { + Name string `json:"name"` + Links []resourceClassLink `json:"links,omitempty"` +} + +type resourceClassLink struct { + Rel string `json:"rel"` + Href string `json:"href"` +} + +// ResourceClassSyncer manages the lifecycle of the resource classes ConfigMap. +// It creates the ConfigMap on startup and periodically syncs from upstream. +type ResourceClassSyncer struct { + client client.Client + configMapName string + namespace string + placementClient *gophercloud.ServiceClient + resourceLocker *resourcelock.ResourceLocker +} + +func NewResourceClassSyncer( + cl client.Client, + configMapName string, + namespace string, + placementClient *gophercloud.ServiceClient, + resourceLocker *resourcelock.ResourceLocker, +) *ResourceClassSyncer { + + return &ResourceClassSyncer{ + client: cl, + configMapName: configMapName, + namespace: namespace, + placementClient: placementClient, + resourceLocker: resourceLocker, + } +} + +// Init creates the resource classes ConfigMap if it does not already exist. +func (rs *ResourceClassSyncer) Init(ctx context.Context) error { + log := ctrl.Log.WithName("placement-shim").WithName("resource-class-syncer") + cm := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: rs.namespace, Name: rs.configMapName} + err := rs.client.Get(ctx, key, cm) + if err == nil { + log.Info("Resource classes ConfigMap already exists", "name", rs.configMapName) + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("checking resource classes configmap: %w", err) + } + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: rs.configMapName, + Namespace: rs.namespace, + }, + Data: map[string]string{configMapKeyResourceClasses: "[]"}, + } + if err := rs.client.Create(ctx, cm); err != nil { + if apierrors.IsAlreadyExists(err) { + log.Info("Resource classes ConfigMap was created concurrently", "name", rs.configMapName) + return nil + } + return fmt.Errorf("creating resource classes configmap: %w", err) + } + log.Info("Created resource classes ConfigMap", "name", rs.configMapName) + return nil +} + +// Run starts the periodic background sync from upstream placement. +// Blocks until ctx is cancelled. +func (rs *ResourceClassSyncer) Run(ctx context.Context) { + log := ctrl.Log.WithName("placement-shim").WithName("resource-class-syncer") + if rs.placementClient == nil { + log.Info("No placement service client configured, resource class sync loop will not run") + return + } + + jitter := time.Duration(rand.Int63n(int64(30 * time.Second))) //nolint:gosec + log.Info("Starting resource class sync loop", "jitter", jitter) + + select { + case <-ctx.Done(): + return + case <-time.After(jitter): + } + + rs.sync(ctx) + + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + rs.sync(ctx) + } + } +} + +// sync fetches GET /resource_classes from upstream placement and writes the +// result into the ConfigMap under the resource lock. +func (rs *ResourceClassSyncer) sync(ctx context.Context) { + log := ctrl.Log.WithName("placement-shim").WithName("resource-class-syncer") + u, err := url.JoinPath(rs.placementClient.Endpoint, "/resource_classes") + if err != nil { + log.Error(err, "Failed to build upstream resource classes URL") + return + } + resp, err := rs.placementClient.Request(ctx, http.MethodGet, u, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusOK}, + MoreHeaders: map[string]string{ + "OpenStack-API-Version": "placement 1.7", + }, + KeepResponseBody: true, + }) + if err != nil { + log.Info("Upstream resource class sync failed", "error", err.Error()) + return + } + defer resp.Body.Close() + var body resourceClassesListResponse + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + log.Error(err, "Failed to decode upstream resource class list") + return + } + + host, err := os.Hostname() + if err != nil { + log.Error(err, "Failed to get hostname for resource class sync lock") + return + } + lockerID := fmt.Sprintf("syncer-%s-%d", host, time.Now().UnixNano()) + lockName := rs.configMapName + "-lock" + if err := rs.resourceLocker.AcquireLock(ctx, lockName, lockerID); err != nil { + log.Error(err, "Failed to acquire lock for resource class sync") + return + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := rs.resourceLocker.ReleaseLock(releaseCtx, lockName, lockerID); err != nil { + log.Error(err, "Failed to release lock after resource class sync") + } + }() + + cm := &corev1.ConfigMap{} + if err := rs.client.Get(ctx, client.ObjectKey{Namespace: rs.namespace, Name: rs.configMapName}, cm); err != nil { + log.Error(err, "Failed to get resource classes ConfigMap for sync") + return + } + rcSet := make(map[string]struct{}, len(body.ResourceClasses)) + for _, rc := range body.ResourceClasses { + rcSet[rc.Name] = struct{}{} + } + if err := writeResourceClassesToConfigMap(cm, rcSet); err != nil { + log.Error(err, "Failed to serialize synced resource classes") + return + } + if err := rs.client.Update(ctx, cm); err != nil { + log.Error(err, "Failed to update resource classes ConfigMap with upstream data") + return + } + log.Info("Synced resource classes from upstream placement", "count", len(body.ResourceClasses)) +} diff --git a/internal/shim/placement/syncer_resource_classes_test.go b/internal/shim/placement/syncer_resource_classes_test.go new file mode 100644 index 000000000..83d4d3ad0 --- /dev/null +++ b/internal/shim/placement/syncer_resource_classes_test.go @@ -0,0 +1,149 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/cobaltcore-dev/cortex/pkg/resourcelock" + "github.com/gophercloud/gophercloud/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestResourceClassSyncerInitCreatesConfigMap(t *testing.T) { + cl := newFakeClientWithScheme(t) + rs := NewResourceClassSyncer(cl, "test-rc", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + if err := rs.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-rc"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyResourceClasses] != "[]" { + t.Fatalf("expected empty resource classes array, got %q", cm.Data[configMapKeyResourceClasses]) + } +} + +func TestResourceClassSyncerInitIdempotent(t *testing.T) { + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rc", Namespace: "default"}, + Data: map[string]string{configMapKeyResourceClasses: `["CUSTOM_EXISTING"]`}, + } + cl := newFakeClientWithScheme(t, existing) + rs := NewResourceClassSyncer(cl, "test-rc", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + if err := rs.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-rc"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyResourceClasses] != `["CUSTOM_EXISTING"]` { + t.Fatalf("Init overwrote existing data: got %q", cm.Data[configMapKeyResourceClasses]) + } +} + +func TestResourceClassSyncerRunNoClient(t *testing.T) { + cl := newFakeClientWithScheme(t) + rs := NewResourceClassSyncer(cl, "test-rc", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + rs.Run(ctx) +} + +func TestResourceClassSyncerSyncWritesUpstreamClasses(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/resource_classes" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resourceClassesListResponse{ + ResourceClasses: []resourceClassEntry{ + {Name: "VCPU"}, + {Name: "MEMORY_MB"}, + {Name: "CUSTOM_SYNCED"}, + }, + }); err != nil { + t.Errorf("encode response: %v", err) + } + })) + t.Cleanup(upstream.Close) + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rc", Namespace: "default"}, + Data: map[string]string{configMapKeyResourceClasses: "[]"}, + } + cl := newFakeClientWithScheme(t, existing) + + sc := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{}, + Endpoint: upstream.URL, + } + sc.HTTPClient = *upstream.Client() + + rs := NewResourceClassSyncer(cl, "test-rc", "default", sc, resourcelock.NewResourceLocker(cl, "default")) + rs.sync(context.Background()) + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-rc"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + + var classes []string + if err := json.Unmarshal([]byte(cm.Data[configMapKeyResourceClasses]), &classes); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(classes) != 3 { + t.Fatalf("expected 3 classes, got %d: %v", len(classes), classes) + } + want := map[string]bool{"VCPU": true, "MEMORY_MB": true, "CUSTOM_SYNCED": true} + for _, c := range classes { + if !want[c] { + t.Errorf("unexpected class: %s", c) + } + } +} + +func TestResourceClassSyncerSyncUpstreamError(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rc", Namespace: "default"}, + Data: map[string]string{configMapKeyResourceClasses: `["CUSTOM_ORIGINAL"]`}, + } + cl := newFakeClientWithScheme(t, existing) + + sc := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{}, + Endpoint: upstream.URL, + } + sc.HTTPClient = *upstream.Client() + + rs := NewResourceClassSyncer(cl, "test-rc", "default", sc, resourcelock.NewResourceLocker(cl, "default")) + rs.sync(context.Background()) + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-rc"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyResourceClasses] != `["CUSTOM_ORIGINAL"]` { + t.Fatalf("sync should not have modified ConfigMap on error, got %q", cm.Data[configMapKeyResourceClasses]) + } +} diff --git a/internal/shim/placement/syncer_traits.go b/internal/shim/placement/syncer_traits.go new file mode 100644 index 000000000..f9748d66d --- /dev/null +++ b/internal/shim/placement/syncer_traits.go @@ -0,0 +1,179 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "os" + "time" + + "github.com/cobaltcore-dev/cortex/pkg/resourcelock" + "github.com/gophercloud/gophercloud/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TraitSyncer manages the lifecycle of the single traits ConfigMap. +// It creates the ConfigMap on startup and periodically syncs from upstream. +type TraitSyncer struct { + client client.Client + configMapName string + namespace string + placementClient *gophercloud.ServiceClient + resourceLocker *resourcelock.ResourceLocker +} + +func NewTraitSyncer( + cl client.Client, + configMapName string, + namespace string, + placementClient *gophercloud.ServiceClient, + resourceLocker *resourcelock.ResourceLocker, +) *TraitSyncer { + + return &TraitSyncer{ + client: cl, + configMapName: configMapName, + namespace: namespace, + placementClient: placementClient, + resourceLocker: resourceLocker, + } +} + +// Init creates the traits ConfigMap if it does not already exist. +func (ts *TraitSyncer) Init(ctx context.Context) error { + log := ctrl.Log.WithName("placement-shim").WithName("trait-syncer") + cm := &corev1.ConfigMap{} + key := client.ObjectKey{Namespace: ts.namespace, Name: ts.configMapName} + err := ts.client.Get(ctx, key, cm) + if err == nil { + log.Info("Traits ConfigMap already exists", "name", ts.configMapName) + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("checking traits configmap: %w", err) + } + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ts.configMapName, + Namespace: ts.namespace, + }, + Data: map[string]string{configMapKeyTraits: "[]"}, + } + if err := ts.client.Create(ctx, cm); err != nil { + if apierrors.IsAlreadyExists(err) { + log.Info("Traits ConfigMap was created concurrently", "name", ts.configMapName) + return nil + } + return fmt.Errorf("creating traits configmap: %w", err) + } + log.Info("Created traits ConfigMap", "name", ts.configMapName) + return nil +} + +// Run starts the periodic background sync from upstream placement. +// Blocks until ctx is cancelled. +func (ts *TraitSyncer) Run(ctx context.Context) { + log := ctrl.Log.WithName("placement-shim").WithName("trait-syncer") + if ts.placementClient == nil { + log.Info("No placement service client configured, trait sync loop will not run") + return + } + + jitter := time.Duration(rand.Int63n(int64(30 * time.Second))) //nolint:gosec + log.Info("Starting trait sync loop", "jitter", jitter) + + select { + case <-ctx.Done(): + return + case <-time.After(jitter): + } + + ts.sync(ctx) + + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ts.sync(ctx) + } + } +} + +// sync fetches GET /traits from upstream placement and writes the result +// into the ConfigMap under the resource lock. +func (ts *TraitSyncer) sync(ctx context.Context) { + log := ctrl.Log.WithName("placement-shim").WithName("trait-syncer") + u, err := url.JoinPath(ts.placementClient.Endpoint, "/traits") + if err != nil { + log.Error(err, "Failed to build upstream traits URL") + return + } + resp, err := ts.placementClient.Request(ctx, http.MethodGet, u, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusOK}, + MoreHeaders: map[string]string{ + "OpenStack-API-Version": "placement 1.6", + }, + KeepResponseBody: true, + }) + if err != nil { + log.Info("Upstream trait sync failed", "error", err.Error()) + return + } + defer resp.Body.Close() + var body traitsListResponse + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + log.Error(err, "Failed to decode upstream trait list") + return + } + + host, err := os.Hostname() + if err != nil { + log.Error(err, "Failed to get hostname for trait sync lock") + return + } + lockerID := fmt.Sprintf("syncer-%s-%d", host, time.Now().UnixNano()) + lockName := ts.configMapName + "-lock" + if err := ts.resourceLocker.AcquireLock(ctx, lockName, lockerID); err != nil { + log.Error(err, "Failed to acquire lock for trait sync") + return + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := ts.resourceLocker.ReleaseLock(releaseCtx, lockName, lockerID); err != nil { + log.Error(err, "Failed to release lock after trait sync") + } + }() + + cm := &corev1.ConfigMap{} + if err := ts.client.Get(ctx, client.ObjectKey{Namespace: ts.namespace, Name: ts.configMapName}, cm); err != nil { + log.Error(err, "Failed to get traits ConfigMap for sync") + return + } + traitSet := make(map[string]struct{}, len(body.Traits)) + for _, t := range body.Traits { + traitSet[t] = struct{}{} + } + if err := writeTraitsToConfigMap(cm, traitSet); err != nil { + log.Error(err, "Failed to serialize synced traits") + return + } + if err := ts.client.Update(ctx, cm); err != nil { + log.Error(err, "Failed to update traits ConfigMap with upstream data") + return + } + log.Info("Synced traits from upstream placement", "count", len(body.Traits)) +} diff --git a/internal/shim/placement/syncer_traits_test.go b/internal/shim/placement/syncer_traits_test.go new file mode 100644 index 000000000..23cada155 --- /dev/null +++ b/internal/shim/placement/syncer_traits_test.go @@ -0,0 +1,145 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/cobaltcore-dev/cortex/pkg/resourcelock" + "github.com/gophercloud/gophercloud/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestTraitSyncerInitCreatesConfigMap(t *testing.T) { + cl := newFakeClientWithScheme(t) + ts := NewTraitSyncer(cl, "test-traits", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + if err := ts.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-traits"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyTraits] != "[]" { + t.Fatalf("expected empty traits array, got %q", cm.Data[configMapKeyTraits]) + } +} + +func TestTraitSyncerInitIdempotent(t *testing.T) { + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-traits", Namespace: "default"}, + Data: map[string]string{configMapKeyTraits: `["CUSTOM_EXISTING"]`}, + } + cl := newFakeClientWithScheme(t, existing) + ts := NewTraitSyncer(cl, "test-traits", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + if err := ts.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-traits"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyTraits] != `["CUSTOM_EXISTING"]` { + t.Fatalf("Init overwrote existing data: got %q", cm.Data[configMapKeyTraits]) + } +} + +func TestTraitSyncerRunNoClient(t *testing.T) { + cl := newFakeClientWithScheme(t) + ts := NewTraitSyncer(cl, "test-traits", "default", nil, resourcelock.NewResourceLocker(cl, "default")) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + ts.Run(ctx) +} + +func TestTraitSyncerSyncWritesUpstreamTraits(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/traits" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(traitsListResponse{ + Traits: []string{"HW_CPU_X86_AVX2", "CUSTOM_SYNCED"}, + }); err != nil { + t.Errorf("encode response: %v", err) + } + })) + t.Cleanup(upstream.Close) + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-traits", Namespace: "default"}, + Data: map[string]string{configMapKeyTraits: "[]"}, + } + cl := newFakeClientWithScheme(t, existing) + + sc := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{}, + Endpoint: upstream.URL, + } + sc.HTTPClient = *upstream.Client() + + ts := NewTraitSyncer(cl, "test-traits", "default", sc, resourcelock.NewResourceLocker(cl, "default")) + ts.sync(context.Background()) + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-traits"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + + var traits []string + if err := json.Unmarshal([]byte(cm.Data[configMapKeyTraits]), &traits); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(traits) != 2 { + t.Fatalf("expected 2 traits, got %d: %v", len(traits), traits) + } + want := map[string]bool{"CUSTOM_SYNCED": true, "HW_CPU_X86_AVX2": true} + for _, tr := range traits { + if !want[tr] { + t.Errorf("unexpected trait: %s", tr) + } + } +} + +func TestTraitSyncerSyncUpstreamError(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-traits", Namespace: "default"}, + Data: map[string]string{configMapKeyTraits: `["CUSTOM_ORIGINAL"]`}, + } + cl := newFakeClientWithScheme(t, existing) + + sc := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{}, + Endpoint: upstream.URL, + } + sc.HTTPClient = *upstream.Client() + + ts := NewTraitSyncer(cl, "test-traits", "default", sc, resourcelock.NewResourceLocker(cl, "default")) + ts.sync(context.Background()) + + cm := &corev1.ConfigMap{} + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-traits"}, cm); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if cm.Data[configMapKeyTraits] != `["CUSTOM_ORIGINAL"]` { + t.Fatalf("sync should not have modified ConfigMap on error, got %q", cm.Data[configMapKeyTraits]) + } +} diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 73ecfebd3..552a067da 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:trixie-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a +FROM debian:trixie-slim@sha256:cedb1ef40439206b673ee8b33a46a03a0c9fa90bf3732f54704f99cb061d2c5a # explicitly set user/group IDs RUN set -eux; \ @@ -64,10 +64,10 @@ RUN set -ex; \ gpgconf --kill all; \ rm -rf "$GNUPGHOME" -ENV PG_MAJOR 17 +ENV PG_MAJOR 18 ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV PG_VERSION 17.9-1.pgdg13+1 +ENV PG_VERSION 18.3-1.pgdg13+1 RUN set -ex; \ \ @@ -75,7 +75,7 @@ RUN set -ex; \ export PYTHONDONTWRITEBYTECODE=1; \ \ dpkgArch="$(dpkg --print-architecture)"; \ - aptRepo="[ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] http://apt.postgresql.org/pub/repos/apt/ trixie-pgdg main $PG_MAJOR"; \ + aptRepo="[ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] http://apt.postgresql.org/pub/repos/apt trixie-pgdg main $PG_MAJOR"; \ case "$dpkgArch" in \ amd64 | arm64 | ppc64el) \ # arches officially built by upstream @@ -135,6 +135,10 @@ RUN set -ex; \ apt-get install -y --no-install-recommends \ "postgresql-$PG_MAJOR=$PG_VERSION" \ ; \ +# https://github.com/docker-library/postgres/pull/1344#issuecomment-2936578203 (JIT is a separate package in 18+, but only supported for a subset of architectures) + if apt-get install -s "postgresql-$PG_MAJOR-jit" > /dev/null 2>&1; then \ + apt-get install -y --no-install-recommends "postgresql-$PG_MAJOR-jit=$PG_VERSION"; \ + fi; \ \ rm -rf /var/lib/apt/lists/*; \ \ @@ -159,10 +163,8 @@ RUN set -eux; \ RUN install --verbose --directory --owner postgres --group postgres --mode 3777 /var/run/postgresql -ENV PGDATA /var/lib/postgresql/data -# this 1777 will be replaced by 0700 at runtime (allows semi-arbitrary "--user" values) -RUN install --verbose --directory --owner postgres --group postgres --mode 1777 "$PGDATA" -VOLUME /var/lib/postgresql/data +ENV PGDATA /var/lib/postgresql/18/docker +VOLUME /var/lib/postgresql COPY docker-entrypoint.sh docker-ensure-initdb.sh /usr/local/bin/ RUN ln -sT docker-ensure-initdb.sh /usr/local/bin/docker-enforce-initdb.sh diff --git a/postgres/docker-ensure-initdb.sh b/postgres/docker-ensure-initdb.sh index 07c08899f..e9b15ef77 100755 --- a/postgres/docker-ensure-initdb.sh +++ b/postgres/docker-ensure-initdb.sh @@ -69,4 +69,4 @@ else exit 99 ;; esac -fi \ No newline at end of file +fi diff --git a/postgres/docker-entrypoint.sh b/postgres/docker-entrypoint.sh index 6057a20ba..d4442d8a5 100755 --- a/postgres/docker-entrypoint.sh +++ b/postgres/docker-entrypoint.sh @@ -103,24 +103,6 @@ docker_init_database_dir() { # print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' # assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] docker_verify_minimum_env() { - case "${PG_MAJOR:-}" in - 13) # https://github.com/postgres/postgres/commit/67a472d71c98c3d2fa322a1b4013080b20720b98 - # check password first so we can output the warning before postgres - # messes it up - if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then - cat >&2 <<-'EOWARN' - - WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. - - This will not work if used via PGPASSWORD with "psql". - - https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) - https://github.com/docker-library/postgres/issues/507 - - EOWARN - fi - ;; - esac if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then # The - option suppresses leading tabs but *not* spaces. :) cat >&2 <<-'EOE' @@ -168,8 +150,14 @@ docker_error_old_databases() { Counter to that, there appears to be PostgreSQL data in: ${OLD_DATABASES[*]} - This is usually the result of upgrading the Docker image without upgrading - the underlying database using "pg_upgrade" (which requires both versions). + This is usually the result of upgrading the Docker image without + upgrading the underlying database using "pg_upgrade" (which requires both + versions). + + The suggested container configuration for 18+ is to place a single mount + at /var/lib/postgresql which will then place PostgreSQL data in a + subdirectory, allowing usage of "pg_upgrade --link" without mount point + boundary issues. See https://github.com/docker-library/postgres/issues/37 for a (long) discussion around this process, and suggestions for how to do so. @@ -264,6 +252,14 @@ docker_setup_env() { OLD_DATABASES+=( "$d" ) fi done + if [ "${#OLD_DATABASES[@]}" -eq 0 ] && [ "$PG_MAJOR" -ge 18 ] && { + # in BusyBox, "mountpoint" only checks dev vs ino (https://github.com/tianon/mirror-busybox/blob/be7d1b7b1701d225379bc1665487ed0871b592a5/util-linux/mountpoint.c#L78) which will notably miss bind mounts entirely (which almost all Docker volume mounts are) + # coreutils checks /proc/self/mountinfo, so we have a fallback to mimic that and directly check "/proc/self/mountinfo" to catch that case + mountpoint -q /var/lib/postgresql/data \ + || awk '$5 == "/var/lib/postgresql/data" { found = 1 } END { exit !found }' /proc/self/mountinfo + }; then + OLD_DATABASES+=( '/var/lib/postgresql/data (unused mount/volume)' ) + fi fi } @@ -388,4 +384,4 @@ _main() { if ! _is_sourced; then _main "$@" -fi \ No newline at end of file +fi diff --git a/tools/visualize-committed-resources/main.go b/tools/visualize-committed-resources/main.go new file mode 100644 index 000000000..afa16a372 --- /dev/null +++ b/tools/visualize-committed-resources/main.go @@ -0,0 +1,596 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Tool to visualize CommittedResource CRDs and their child Reservation slots. +// +// Usage: +// +// go run tools/visualize-committed-resources/main.go [flags] +// +// Flags: +// +// --context=ctx Kubernetes context (default: current context) +// --filter-project=id Show only CRs for this project ID (substring match) +// --filter-az=az Show only CRs in this availability zone (substring match) +// --filter-group=name Show only CRs for this flavor group (substring match) +// --filter-state=state Show only CRs in this state (e.g. confirmed, reserving) +// --active Shorthand: show only confirmed/guaranteed CRs +// --views=v1,v2,... Views to show (default: all). Available: summary, commitments, reservations, allocations +// --hide=v1,v2,... Views to hide (applied after --views) +// --watch=interval Refresh interval (e.g. 2s, 5s). Clears screen between refreshes. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(v1alpha1.AddToScheme(scheme)) +} + +// ── ANSI colours ────────────────────────────────────────────────────────────── + +const ( + colReset = "\033[0m" + colBold = "\033[1m" + colGreen = "\033[32m" + colYellow = "\033[33m" + colRed = "\033[31m" + colCyan = "\033[36m" + colGray = "\033[90m" +) + +func green(s string) string { return colGreen + s + colReset } +func yellow(s string) string { return colYellow + s + colReset } +func red(s string) string { return colRed + s + colReset } +func cyan(s string) string { return colCyan + s + colReset } +func gray(s string) string { return colGray + s + colReset } +func bold(s string) string { return colBold + s + colReset } + +// ── Views ───────────────────────────────────────────────────────────────────── + +const ( + viewSummary = "summary" + viewCommitments = "commitments" + viewReservations = "reservations" + viewAllocations = "allocations" +) + +var allViews = []string{viewSummary, viewCommitments, viewReservations, viewAllocations} + +type viewSet map[string]bool + +func parseViews(s string) viewSet { + vs := make(viewSet) + if s == "all" || s == "" { + for _, v := range allViews { + vs[v] = true + } + return vs + } + for _, v := range strings.Split(s, ",") { + vs[strings.TrimSpace(v)] = true + } + return vs +} + +func (vs viewSet) hide(s string) { + if s == "" { + return + } + for _, v := range strings.Split(s, ",") { + delete(vs, strings.TrimSpace(v)) + } +} + +func (vs viewSet) has(v string) bool { return vs[v] } + +// ── k8s client ──────────────────────────────────────────────────────────────── + +func newClient(contextName string) (client.Client, error) { + if contextName == "" { + c, err := config.GetConfig() + if err != nil { + return nil, fmt.Errorf("getting kubeconfig: %w", err) + } + return client.New(c, client.Options{Scheme: scheme}) + } + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + loadingRules, + &clientcmd.ConfigOverrides{CurrentContext: contextName}, + ) + c, err := kubeConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("getting kubeconfig for context %q: %w", contextName, err) + } + return client.New(c, client.Options{Scheme: scheme}) +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func printHeader(title string) { + line := strings.Repeat("─", 80) + fmt.Println() + fmt.Println(bold(line)) + fmt.Printf("%s %s\n", bold("▶"), bold(title)) + fmt.Println(bold(line)) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-1] + "…" +} + +func age(t *metav1.Time) string { + if t == nil { + return gray("—") + } + d := time.Since(t.Time).Round(time.Second) + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", int(d.Seconds())) + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} + +func crReadyStatus(cr v1alpha1.CommittedResource) string { + cond := apimeta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + return gray("unknown") + } + switch cond.Reason { + case v1alpha1.CommittedResourceReasonAccepted: + return green("Accepted") + case v1alpha1.CommittedResourceReasonRejected: + return red("Rejected") + case v1alpha1.CommittedResourceReasonReserving: + return yellow("Reserving") + case v1alpha1.CommittedResourceReasonPlanned: + return gray("Planned") + default: + return yellow(cond.Reason) + } +} + +func resReadyStatus(res v1alpha1.Reservation) string { + cond := apimeta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil { + return gray("pending") + } + if cond.Status == metav1.ConditionTrue { + return green("Ready") + } + return red("NotReady: " + truncate(cond.Message, 40)) +} + +func stateColour(state v1alpha1.CommitmentStatus) string { + switch state { + case v1alpha1.CommitmentStatusConfirmed, v1alpha1.CommitmentStatusGuaranteed: + return green(string(state)) + case v1alpha1.CommitmentStatusPlanned, v1alpha1.CommitmentStatusPending: + return yellow(string(state)) + case v1alpha1.CommitmentStatusExpired, v1alpha1.CommitmentStatusSuperseded: + return gray(string(state)) + default: + return string(state) + } +} + +// ── filters ─────────────────────────────────────────────────────────────────── + +type filters struct { + project string + az string + group string + state string + active bool +} + +func (f filters) match(cr v1alpha1.CommittedResource) bool { + if f.project != "" && !strings.Contains(cr.Spec.ProjectID, f.project) { + return false + } + if f.az != "" && !strings.Contains(cr.Spec.AvailabilityZone, f.az) { + return false + } + if f.group != "" && !strings.Contains(cr.Spec.FlavorGroupName, f.group) { + return false + } + if f.state != "" && !strings.EqualFold(string(cr.Spec.State), f.state) { + return false + } + if f.active { + s := cr.Spec.State + if s != v1alpha1.CommitmentStatusConfirmed && s != v1alpha1.CommitmentStatusGuaranteed { + return false + } + } + return true +} + +// ── views ───────────────────────────────────────────────────────────────────── + +func printSummary(crs []v1alpha1.CommittedResource, reservations []v1alpha1.Reservation) { + printHeader("Summary") + + byState := make(map[v1alpha1.CommitmentStatus]int) + byReady := map[string]int{"Accepted": 0, "Reserving": 0, "Rejected": 0, "Planned": 0, "Unknown": 0} + for _, cr := range crs { + byState[cr.Spec.State]++ + cond := apimeta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + byReady["Unknown"]++ + } else { + byReady[cond.Reason]++ + } + } + + resReady, resNotReady, resPending := 0, 0, 0 + for _, res := range reservations { + cond := apimeta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + switch { + case cond == nil: + resPending++ + case cond.Status == metav1.ConditionTrue: + resReady++ + default: + resNotReady++ + } + } + + fmt.Printf(" CommittedResources : %s\n", bold(fmt.Sprintf("%d total", len(crs)))) + for _, s := range []v1alpha1.CommitmentStatus{ + v1alpha1.CommitmentStatusConfirmed, + v1alpha1.CommitmentStatusGuaranteed, + v1alpha1.CommitmentStatusPending, + v1alpha1.CommitmentStatusPlanned, + v1alpha1.CommitmentStatusExpired, + v1alpha1.CommitmentStatusSuperseded, + } { + if n := byState[s]; n > 0 { + fmt.Printf(" %-14s %d\n", string(s)+":", n) + } + } + fmt.Println() + fmt.Printf(" Ready conditions : %s accepted, %s reserving, %s rejected\n", + green(strconv.Itoa(byReady["Accepted"])), + yellow(strconv.Itoa(byReady["Reserving"])), + red(strconv.Itoa(byReady["Rejected"])), + ) + fmt.Println() + fmt.Printf(" Reservation slots : %s total — %s ready, %s not-ready, %s pending\n", + bold(strconv.Itoa(len(reservations))), + green(strconv.Itoa(resReady)), + red(strconv.Itoa(resNotReady)), + yellow(strconv.Itoa(resPending)), + ) +} + +func printCommitments(crs []v1alpha1.CommittedResource) { + printHeader(fmt.Sprintf("CommittedResources (%d)", len(crs))) + + if len(crs) == 0 { + fmt.Println(gray(" (none)")) + return + } + + for _, cr := range crs { + fmt.Printf("\n %s %s\n", + bold(cyan(cr.Spec.CommitmentUUID)), + crReadyStatus(cr), + ) + fmt.Printf(" project=%-36s group=%-20s az=%s\n", + cr.Spec.ProjectID, cr.Spec.FlavorGroupName, cr.Spec.AvailabilityZone) + fmt.Printf(" state=%-14s amount=%-10s accepted=%s\n", + stateColour(cr.Spec.State), + cr.Spec.Amount.String(), + func() string { + if cr.Status.AcceptedAmount == nil { + return gray("—") + } + return cr.Status.AcceptedAmount.String() + }(), + ) + + if cr.Status.UsedAmount != nil { + fmt.Printf(" used=%-12s\n", cr.Status.UsedAmount.String()) + } + + endStr := gray("no expiry") + if cr.Spec.EndTime != nil { + remaining := time.Until(cr.Spec.EndTime.Time).Round(time.Minute) + if remaining < 0 { + endStr = red(fmt.Sprintf("expired %s ago", age(cr.Spec.EndTime))) + } else { + endStr = fmt.Sprintf("expires in %s (at %s)", remaining, cr.Spec.EndTime.Format(time.RFC3339)) + } + } + fmt.Printf(" age=%-8s %s\n", age(&cr.CreationTimestamp), endStr) + } +} + +func printReservations(crs []v1alpha1.CommittedResource, reservations []v1alpha1.Reservation, showAllocations bool) { + // Index reservations by CommitmentUUID for display under each CR. + byUUID := make(map[string][]v1alpha1.Reservation) + for _, res := range reservations { + if res.Spec.CommittedResourceReservation == nil { + continue + } + uuid := res.Spec.CommittedResourceReservation.CommitmentUUID + byUUID[uuid] = append(byUUID[uuid], res) + } + + printHeader("Reservation Slots") + + if len(reservations) == 0 { + fmt.Println(gray(" (none)")) + return + } + + for _, cr := range crs { + slots := byUUID[cr.Spec.CommitmentUUID] + if len(slots) == 0 { + continue + } + fmt.Printf("\n %s %s %s\n", + bold(cyan(cr.Spec.CommitmentUUID)), + gray(cr.Spec.FlavorGroupName), + gray(fmt.Sprintf("%d slot(s)", len(slots))), + ) + + sort.Slice(slots, func(i, j int) bool { + return slots[i].Name < slots[j].Name + }) + + for _, res := range slots { + targetHost := res.Spec.TargetHost + statusHost := res.Status.Host + var hostStr string + switch { + case statusHost == "": + hostStr = yellow(targetHost) + gray(" (not yet placed)") + case statusHost != targetHost: + hostStr = red(fmt.Sprintf("target=%s status=%s (migrating?)", targetHost, statusHost)) + default: + hostStr = green(targetHost) + } + + genOK := "" + if s := res.Status.CommittedResourceReservation; s != nil { + spec := res.Spec.CommittedResourceReservation + if spec != nil && s.ObservedParentGeneration != spec.ParentGeneration { + genOK = yellow(fmt.Sprintf(" [gen: spec=%d observed=%d]", + spec.ParentGeneration, s.ObservedParentGeneration)) + } + } + + resources := "" + var resourcesSb391 strings.Builder + for rname, qty := range res.Spec.Resources { + fmt.Fprintf(&resourcesSb391, "%s=%s ", rname, qty.String()) + } + resources += resourcesSb391.String() + + fmt.Printf(" %s host=%s %s %s%s\n", + truncate(res.Name, 40), + hostStr, + resReadyStatus(res), + gray(strings.TrimSpace(resources)), + genOK, + ) + + if showAllocations { + specAllocs := 0 + statusAllocs := 0 + if res.Spec.CommittedResourceReservation != nil { + specAllocs = len(res.Spec.CommittedResourceReservation.Allocations) + } + if res.Status.CommittedResourceReservation != nil { + statusAllocs = len(res.Status.CommittedResourceReservation.Allocations) + } + + if specAllocs > 0 || statusAllocs > 0 { + fmt.Printf(" allocations: spec=%d confirmed=%d\n", specAllocs, statusAllocs) + if res.Spec.CommittedResourceReservation != nil { + statusAlloc := map[string]string{} + if res.Status.CommittedResourceReservation != nil { + statusAlloc = res.Status.CommittedResourceReservation.Allocations + } + for vmUUID, alloc := range res.Spec.CommittedResourceReservation.Allocations { + resources := "" + var resourcesSb422 strings.Builder + for rname, qty := range alloc.Resources { + fmt.Fprintf(&resourcesSb422, "%s=%s ", rname, qty.String()) + } + resources += resourcesSb422.String() + confirmedHost, confirmed := statusAlloc[vmUUID] + state := "" + if confirmed { + state = green("confirmed on " + confirmedHost) + } else { + state = yellow(fmt.Sprintf("spec-only (grace since %s)", age(&alloc.CreationTimestamp))) + } + fmt.Printf(" vm=%s %s %s\n", + truncate(vmUUID, 36), + gray(strings.TrimSpace(resources)), + state, + ) + } + } + } + } + } + } +} + +// ── main ────────────────────────────────────────────────────────────────────── + +func main() { + k8sContext := flag.String("context", "", "Kubernetes context (default: current context)") + filterProject := flag.String("filter-project", "", "Show only CRs for this project ID (substring match)") + filterAZ := flag.String("filter-az", "", "Show only CRs in this availability zone (substring match)") + filterGroup := flag.String("filter-group", "", "Show only CRs for this flavor group (substring match)") + filterState := flag.String("filter-state", "", "Show only CRs in this state") + activeOnly := flag.Bool("active", false, "Show only confirmed/guaranteed CRs") + viewsFlag := flag.String("views", "all", "Views: all, summary, commitments, reservations, allocations") + hideFlag := flag.String("hide", "", "Views to hide (applied after --views)") + watchInterval := flag.Duration("watch", 0, "Refresh interval (e.g. 2s, 5s). 0 = run once.") + limitFlag := flag.Int("limit", 200, "Max CRs to fetch (0 = unlimited)") + flag.Parse() + + views := parseViews(*viewsFlag) + views.hide(*hideFlag) + + f := filters{ + project: *filterProject, + az: *filterAZ, + group: *filterGroup, + state: *filterState, + active: *activeOnly, + } + + cl, err := newClient(*k8sContext) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + ctx := context.Background() + var prevDigest string + first := true + for { + crs, reservations := fetchSnapshot(ctx, cl, f, *limitFlag) + if d := snapshotDigest(crs, reservations); first || d != prevDigest { + if !first { + fmt.Printf("\n%s %s %s\n", + bold("━━━ changed at"), + bold(time.Now().Format(time.RFC3339)), + bold("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"), + ) + } + printSnapshot(crs, reservations, f, views) + prevDigest = d + first = false + } + if *watchInterval == 0 { + break + } + time.Sleep(*watchInterval) + } +} + +// snapshotDigest returns a string that changes whenever any CRD is added, removed, or updated. +func snapshotDigest(crs []v1alpha1.CommittedResource, reservations []v1alpha1.Reservation) string { + var b strings.Builder + for _, cr := range crs { + fmt.Fprintf(&b, "%s:%s ", cr.Name, cr.ResourceVersion) + } + for _, res := range reservations { + fmt.Fprintf(&b, "%s:%s ", res.Name, res.ResourceVersion) + } + return b.String() +} + +func fetchSnapshot(ctx context.Context, cl client.Client, f filters, limit int) ([]v1alpha1.CommittedResource, []v1alpha1.Reservation) { + var listOpts []client.ListOption + if limit > 0 { + listOpts = append(listOpts, client.Limit(int64(limit))) + } + + var crList v1alpha1.CommittedResourceList + if err := cl.List(ctx, &crList, listOpts...); err != nil { + fmt.Fprintf(os.Stderr, "error listing CommittedResources: %v\n", err) + os.Exit(1) + } + + var resList v1alpha1.ReservationList + if err := cl.List(ctx, &resList, append(listOpts, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + })...); err != nil { + fmt.Fprintf(os.Stderr, "error listing Reservations: %v\n", err) + os.Exit(1) + } + + if crList.Continue != "" { + fmt.Fprintf(os.Stderr, yellow("warning: CR list truncated at %d — use --limit=0 or a higher value to see all\n"), limit) + } + if resList.Continue != "" { + fmt.Fprintf(os.Stderr, yellow("warning: Reservation list truncated at %d — use --limit=0 or a higher value to see all\n"), limit) + } + var crs []v1alpha1.CommittedResource + for _, cr := range crList.Items { + if f.match(cr) { + crs = append(crs, cr) + } + } + sort.Slice(crs, func(i, j int) bool { + if crs[i].Spec.FlavorGroupName != crs[j].Spec.FlavorGroupName { + return crs[i].Spec.FlavorGroupName < crs[j].Spec.FlavorGroupName + } + return crs[i].Spec.CommitmentUUID < crs[j].Spec.CommitmentUUID + }) + + matchedUUIDs := make(map[string]bool, len(crs)) + for _, cr := range crs { + matchedUUIDs[cr.Spec.CommitmentUUID] = true + } + var reservations []v1alpha1.Reservation + for _, res := range resList.Items { + if res.Spec.CommittedResourceReservation == nil { + continue + } + if matchedUUIDs[res.Spec.CommittedResourceReservation.CommitmentUUID] { + reservations = append(reservations, res) + } + } + return crs, reservations +} + +func printSnapshot(crs []v1alpha1.CommittedResource, reservations []v1alpha1.Reservation, f filters, views viewSet) { + fmt.Printf("\n%s — %s\n", + bold("visualize-committed-resources"), + gray(time.Now().Format(time.RFC3339)), + ) + if f.project != "" || f.az != "" || f.group != "" || f.state != "" || f.active { + fmt.Printf("%s project=%q az=%q group=%q state=%q active=%v\n", + gray("filters:"), f.project, f.az, f.group, f.state, f.active) + } + + if views.has(viewSummary) { + printSummary(crs, reservations) + } + if views.has(viewCommitments) { + printCommitments(crs) + } + if views.has(viewReservations) { + printReservations(crs, reservations, views.has(viewAllocations)) + } + + fmt.Println() +}