From 66ba1500cfc08127c2442cdd320ed35295f78da4 Mon Sep 17 00:00:00 2001
From: Evan Nemerson <evan.nemerson@cloudzero.com>
Date: Wed, 4 Mar 2026 09:43:01 -0500
Subject: [PATCH] CP-37371: Use distroless Prometheus image by default

Prometheus 3.10.0 introduced an official distroless image variant that
eliminates shells, package managers, and other utilities from the container.
This significantly reduces the attack surface of the Prometheus container,
addressing security concerns around the nodes/proxy RBAC permission
(see docs/wiki/The-nodes-proxy-Problem.md).

Functional Change:

Before: The Helm chart defaulted to the standard Prometheus image
(busybox-based, includes shell and utilities) at version v3.7.3. The
Prometheus image tag was resolved by the `cloudzero-agent.prometheusImageTag`
helper with an inline fallback chain duplicated across templates.

After: The Helm chart defaults to the distroless Prometheus image at
version v3.10.0. The image tag is resolved by a new
`cloudzero-agent.Values.components.prometheus.image.tag` helper that
appends "-distroless" to Chart.AppVersion when no explicit tag is set.
The deprecated server.image.tag compat override is handled by
generateImage's compat layer, not the helper itself.

Solution:

1. Bumped Chart.AppVersion from v3.7.3 to v3.10.0 in helm/Chart.yaml

2. Added `cloudzero-agent.Values.components.prometheus.image.tag` helper
   in helm/templates/_helpers.tpl that resolves
   components.prometheus.image.tag with a fallback to
   `{Chart.AppVersion}-distroless`

3. Removed the old `cloudzero-agent.prometheusImageTag` helper and updated
   all call sites (agent-deploy.yaml, agent-daemonset.yaml,
   prometheusAgentFlag) to use the new helper

4. Removed outdated inline comments about version-specific distroless
   logic from agent-deploy.yaml and agent-daemonset.yaml

5. Regenerated Helm template test snapshots (alloy.yaml, cert-manager.yaml,
   federated.yaml, istio.yaml, manifest.yaml) to reflect the new default
   image tag

Validation:

- Helm template snapshots regenerated and verified
---
 helm/Chart.yaml                       |  2 +-
 helm/templates/_helpers.tpl           | 24 ++++++--
 helm/templates/agent-daemonset.yaml   |  6 +-
 helm/templates/agent-deploy.yaml      | 13 ++++-
 tests/helm/template/alloy.yaml        | 70 +++++++++++-----------
 tests/helm/template/cert-manager.yaml | 74 +++++++++++------------
 tests/helm/template/federated.yaml    | 84 +++++++++++++--------------
 tests/helm/template/istio.yaml        | 78 ++++++++++++-------------
 tests/helm/template/manifest.yaml     | 74 +++++++++++------------
 9 files changed, 226 insertions(+), 199 deletions(-)

diff --git a/helm/Chart.yaml b/helm/Chart.yaml
index 740ac190a..d1e8260ff 100644
--- a/helm/Chart.yaml
+++ b/helm/Chart.yaml
@@ -8,7 +8,7 @@ kubeVersion: ">= 1.21.0-0"
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
-appVersion: "v3.7.3"
+appVersion: "v3.10.0"
 dependencies:
   - name: kube-state-metrics
     version: "5.36.*"
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index 6563e4677..29b62a259 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -1389,6 +1389,23 @@ prometheus.yml
 {{- end -}}
 {{- end -}}
 
+{{/*
+Resolve the Prometheus image tag.
+
+Resolves the tag from components.prometheus.image.tag, falling back to
+Chart.AppVersion with "-distroless" appended to use the official distroless
+image variant (no shell, minimal attack surface).
+
+Note: the deprecated server.image.tag compat override is handled by
+generateImage's compat layer at the call site, not here.
+
+Usage: {{ include "cloudzero-agent.Values.components.prometheus.image.tag" . }}
+Returns: string (e.g., "v3.10.0-distroless", "v3.7.3")
+*/}}
+{{- define "cloudzero-agent.Values.components.prometheus.image.tag" -}}
+  {{- .Values.components.prometheus.image.tag | default (printf "%s-distroless" .Chart.AppVersion) -}}
+{{- end -}}
+
 {{/*
 Get the appropriate Prometheus agent mode flag based on version and mode
 
@@ -1401,8 +1418,8 @@ The cloudzero-agent.Values.components.agent.mode helper already handles all the
 complex mode derivation logic, so we just check if it returns "agent" or "federated"
 and then determine the appropriate version-specific flag.
 
-Uses the same tag fallback chain as image generation:
-server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion
+Uses the same tag fallback chain as image generation via
+cloudzero-agent.Values.components.prometheus.image.tag
 
 Usage: {{ include "cloudzero-agent.prometheusAgentFlag" . }}
 Returns: string (either "--agent", "--enable-feature=agent", or empty string)
@@ -1410,8 +1427,7 @@ Returns: string (either "--agent", "--enable-feature=agent", or empty string)
 {{- define "cloudzero-agent.prometheusAgentFlag" -}}
   {{- $mode := include "cloudzero-agent.Values.components.agent.mode" . -}}
   {{- if or (eq $mode "agent") (eq $mode "federated") -}}
-    {{- /* Use same fallback chain as image generation: server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion */ -}}
-    {{- $tag := .Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion -}}
+    {{- $tag := include "cloudzero-agent.Values.components.prometheus.image.tag" . -}}
     {{- if hasPrefix "v2." $tag -}}
       --enable-feature=agent
     {{- else -}}
diff --git a/helm/templates/agent-daemonset.yaml b/helm/templates/agent-daemonset.yaml
index d9d6f8cb9..f47561072 100644
--- a/helm/templates/agent-daemonset.yaml
+++ b/helm/templates/agent-daemonset.yaml
@@ -127,8 +127,7 @@ spec:
               readOnly: true
         {{- end }}
         - name: {{ template "cloudzero-agent.name" . }}-server
-          {{/* This is a little special because we want to fall back on the .Chart.AppVersion */}}
-          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.defaults.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (.Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
+          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.defaults.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (include "cloudzero-agent.Values.components.prometheus.image.tag" .) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
           env:
             - name: NODE_NAME
               valueFrom:
@@ -145,6 +144,9 @@ spec:
             {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
             {{- if $agentFlag }}
             - {{ $agentFlag }}
+            - --storage.agent.path=/data
+            {{- else }}
+            - --storage.tsdb.path=/data
             {{- end }}
           ports:
             - containerPort: 9090
diff --git a/helm/templates/agent-deploy.yaml b/helm/templates/agent-deploy.yaml
index 4660d1cf8..188c704ed 100644
--- a/helm/templates/agent-deploy.yaml
+++ b/helm/templates/agent-deploy.yaml
@@ -153,8 +153,14 @@ spec:
         {{- if ne (include "cloudzero-agent.Values.components.agent.mode" .) "clustered" }}
         # Prometheus server container
         - name: {{ template "cloudzero-agent.name" . }}-server
-          {{/* This is a little special because we want to fall back on the .Chart.AppVersion */}}
-          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.defaults.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (.Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
+          {{- include "cloudzero-agent.generateImage" (dict
+            "defaults" .Values.defaults.image
+            "image" .Values.components.prometheus.image
+            "compat" (dict
+              "repository" .Values.server.image.repository
+              "tag" (include "cloudzero-agent.Values.components.prometheus.image.tag" .)
+              "digest" .Values.server.image.digest
+              "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
           {{- if .Values.server.env }}
           env:
 {{ toYaml .Values.server.env | indent 12}}
@@ -182,6 +188,9 @@ spec:
             {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
             {{- if $agentFlag }}
             - {{ $agentFlag }}
+            - --storage.agent.path={{ .Values.server.persistentVolume.mountPath }}
+            {{- else }}
+            - --storage.tsdb.path={{ .Values.server.persistentVolume.mountPath }}
             {{- end }}
             - --log.level={{ .Values.server.logging.level | default "info" }}
           ports:
diff --git a/tests/helm/template/alloy.yaml b/tests/helm/template/alloy.yaml
index 22092d5f4..47094764e 100644
--- a/tests/helm/template/alloy.yaml
+++ b/tests/helm/template/alloy.yaml
@@ -30,7 +30,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -51,7 +51,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -72,7 +72,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -107,7 +107,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -122,7 +122,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -137,7 +137,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -154,7 +154,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -169,7 +169,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -546,7 +546,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -900,7 +900,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1737,7 +1737,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1844,7 +1844,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -2073,7 +2073,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -2164,7 +2164,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2231,7 +2231,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2253,7 +2253,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2304,7 +2304,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2327,7 +2327,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2443,7 +2443,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2461,7 +2461,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2736,7 +2736,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2753,7 +2753,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2892,7 +2892,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2910,7 +2910,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: webhook-server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -3011,7 +3011,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: onetime
     job-type: backfill
@@ -3025,7 +3025,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: backfill
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
         job-category: onetime
         job-type: backfill
@@ -3093,7 +3093,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3105,7 +3105,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: validator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3223,7 +3223,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3235,7 +3235,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: helmless
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3291,7 +3291,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3302,7 +3302,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: init-cert
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3356,7 +3356,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: cronjob
     job-type: backfill
@@ -3376,7 +3376,7 @@ spec:
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: backfill
             app.kubernetes.io/part-of: cloudzero-agent
-            app.kubernetes.io/version: v3.7.3
+            app.kubernetes.io/version: v3.10.0
             helm.sh/chart: cloudzero-agent-1.1.0-dev
             job-category: cronjob
             job-type: backfill
@@ -3442,7 +3442,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/cert-manager.yaml b/tests/helm/template/cert-manager.yaml
index 9ff067bb0..a49875d4d 100644
--- a/tests/helm/template/cert-manager.yaml
+++ b/tests/helm/template/cert-manager.yaml
@@ -30,7 +30,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -51,7 +51,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -72,7 +72,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -107,7 +107,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -122,7 +122,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -137,7 +137,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -154,7 +154,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -463,7 +463,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -817,7 +817,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1653,7 +1653,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1760,7 +1760,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -1989,7 +1989,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -2080,7 +2080,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2147,7 +2147,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2169,7 +2169,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2220,7 +2220,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2243,7 +2243,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2359,7 +2359,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2377,7 +2377,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2519,8 +2519,7 @@ spec:
               readOnly: true
         # Prometheus server container
         - name: cloudzero-agent-server
-          
-          image: "quay.io/prometheus/prometheus:v3.7.3"
+          image: "quay.io/prometheus/prometheus:v3.10.0-distroless"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2546,6 +2545,7 @@ spec:
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --agent
+            - --storage.agent.path=/data
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2637,7 +2637,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2654,7 +2654,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2793,7 +2793,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2811,7 +2811,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: webhook-server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2912,7 +2912,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: onetime
     job-type: backfill
@@ -2926,7 +2926,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: backfill
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
         job-category: onetime
         job-type: backfill
@@ -2994,7 +2994,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3006,7 +3006,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: validator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3124,7 +3124,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3136,7 +3136,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: helmless
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3192,7 +3192,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: cronjob
     job-type: backfill
@@ -3212,7 +3212,7 @@ spec:
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: backfill
             app.kubernetes.io/part-of: cloudzero-agent
-            app.kubernetes.io/version: v3.7.3
+            app.kubernetes.io/version: v3.10.0
             helm.sh/chart: cloudzero-agent-1.1.0-dev
             job-category: cronjob
             job-type: backfill
@@ -3278,7 +3278,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -3289,7 +3289,7 @@ spec:
       app.kubernetes.io/managed-by: Helm
       app.kubernetes.io/name: webhook-server
       app.kubernetes.io/part-of: cloudzero-agent
-      app.kubernetes.io/version: v3.7.3
+      app.kubernetes.io/version: v3.10.0
       helm.sh/chart: cloudzero-agent-1.1.0-dev
   privateKey:
     algorithm: RSA
@@ -3316,7 +3316,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -3333,7 +3333,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/federated.yaml b/tests/helm/template/federated.yaml
index d6635ce24..78d52bb8c 100644
--- a/tests/helm/template/federated.yaml
+++ b/tests/helm/template/federated.yaml
@@ -30,7 +30,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -51,7 +51,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -72,7 +72,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -107,7 +107,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -122,7 +122,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -137,7 +137,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -154,7 +154,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -169,7 +169,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -390,7 +390,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-daemonset-cm
   namespace: cz-agent
@@ -551,7 +551,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -905,7 +905,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1741,7 +1741,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1848,7 +1848,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -2077,7 +2077,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -2168,7 +2168,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2235,7 +2235,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2257,7 +2257,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2308,7 +2308,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2331,7 +2331,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2356,7 +2356,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-daemonset
@@ -2373,7 +2373,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2458,8 +2458,7 @@ spec:
               mountPath: /etc/config
               readOnly: true
         - name: cloudzero-agent-server
-          
-          image: "quay.io/prometheus/prometheus:v3.7.3"
+          image: "quay.io/prometheus/prometheus:v3.10.0-distroless"
           imagePullPolicy: "IfNotPresent"
           env:
             - name: NODE_NAME
@@ -2472,6 +2471,7 @@ spec:
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --agent
+            - --storage.agent.path=/data
           ports:
             - containerPort: 9090
           readinessProbe:
@@ -2642,7 +2642,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2660,7 +2660,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2802,8 +2802,7 @@ spec:
               readOnly: true
         # Prometheus server container
         - name: cloudzero-agent-server
-          
-          image: "quay.io/prometheus/prometheus:v3.7.3"
+          image: "quay.io/prometheus/prometheus:v3.10.0-distroless"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2829,6 +2828,7 @@ spec:
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --agent
+            - --storage.agent.path=/data
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2920,7 +2920,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2937,7 +2937,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -3076,7 +3076,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -3094,7 +3094,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: webhook-server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -3195,7 +3195,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: onetime
     job-type: backfill
@@ -3209,7 +3209,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: backfill
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
         job-category: onetime
         job-type: backfill
@@ -3277,7 +3277,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3289,7 +3289,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: validator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3407,7 +3407,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3419,7 +3419,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: helmless
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3475,7 +3475,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3486,7 +3486,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: init-cert
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3540,7 +3540,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: cronjob
     job-type: backfill
@@ -3560,7 +3560,7 @@ spec:
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: backfill
             app.kubernetes.io/part-of: cloudzero-agent
-            app.kubernetes.io/version: v3.7.3
+            app.kubernetes.io/version: v3.10.0
             helm.sh/chart: cloudzero-agent-1.1.0-dev
             job-category: cronjob
             job-type: backfill
@@ -3626,7 +3626,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/istio.yaml b/tests/helm/template/istio.yaml
index 84b4ab518..5ce24d5ec 100644
--- a/tests/helm/template/istio.yaml
+++ b/tests/helm/template/istio.yaml
@@ -30,7 +30,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -51,7 +51,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -72,7 +72,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -107,7 +107,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -122,7 +122,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -137,7 +137,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -154,7 +154,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -169,7 +169,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -478,7 +478,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -832,7 +832,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1668,7 +1668,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1775,7 +1775,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -2004,7 +2004,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -2095,7 +2095,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2162,7 +2162,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2184,7 +2184,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2235,7 +2235,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2258,7 +2258,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2374,7 +2374,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2392,7 +2392,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2534,8 +2534,7 @@ spec:
               readOnly: true
         # Prometheus server container
         - name: cloudzero-agent-server
-          
-          image: "quay.io/prometheus/prometheus:v3.7.3"
+          image: "quay.io/prometheus/prometheus:v3.10.0-distroless"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2561,6 +2560,7 @@ spec:
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --agent
+            - --storage.agent.path=/data
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2652,7 +2652,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2669,7 +2669,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2808,7 +2808,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2826,7 +2826,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: webhook-server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2928,7 +2928,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: onetime
     job-type: backfill
@@ -2942,7 +2942,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: backfill
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
         job-category: onetime
         job-type: backfill
@@ -3011,7 +3011,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3023,7 +3023,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: validator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3141,7 +3141,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3153,7 +3153,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: helmless
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3209,7 +3209,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3220,7 +3220,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: init-cert
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3274,7 +3274,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: cronjob
     job-type: backfill
@@ -3294,7 +3294,7 @@ spec:
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: backfill
             app.kubernetes.io/part-of: cloudzero-agent
-            app.kubernetes.io/version: v3.7.3
+            app.kubernetes.io/version: v3.10.0
             helm.sh/chart: cloudzero-agent-1.1.0-dev
             job-category: cronjob
             job-type: backfill
@@ -3362,7 +3362,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -3387,7 +3387,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
@@ -3448,7 +3448,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
diff --git a/tests/helm/template/manifest.yaml b/tests/helm/template/manifest.yaml
index 0e2de51ad..342089b76 100644
--- a/tests/helm/template/manifest.yaml
+++ b/tests/helm/template/manifest.yaml
@@ -30,7 +30,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -51,7 +51,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -72,7 +72,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -107,7 +107,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -122,7 +122,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -137,7 +137,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -154,7 +154,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -169,7 +169,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -478,7 +478,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -832,7 +832,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1668,7 +1668,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1775,7 +1775,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -2004,7 +2004,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -2095,7 +2095,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2162,7 +2162,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2184,7 +2184,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2235,7 +2235,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2258,7 +2258,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2374,7 +2374,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2392,7 +2392,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2534,8 +2534,7 @@ spec:
               readOnly: true
         # Prometheus server container
         - name: cloudzero-agent-server
-          
-          image: "quay.io/prometheus/prometheus:v3.7.3"
+          image: "quay.io/prometheus/prometheus:v3.10.0-distroless"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2561,6 +2560,7 @@ spec:
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
             - --agent
+            - --storage.agent.path=/data
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2652,7 +2652,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2669,7 +2669,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2808,7 +2808,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2826,7 +2826,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: webhook-server
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2927,7 +2927,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: onetime
     job-type: backfill
@@ -2941,7 +2941,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: backfill
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
         job-category: onetime
         job-type: backfill
@@ -3009,7 +3009,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: validator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3021,7 +3021,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: validator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3139,7 +3139,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: helmless
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3151,7 +3151,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: helmless
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3207,7 +3207,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: init-cert
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3218,7 +3218,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: init-cert
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v3.7.3
+        app.kubernetes.io/version: v3.10.0
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3272,7 +3272,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: backfill
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-category: cronjob
     job-type: backfill
@@ -3292,7 +3292,7 @@ spec:
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: backfill
             app.kubernetes.io/part-of: cloudzero-agent
-            app.kubernetes.io/version: v3.7.3
+            app.kubernetes.io/version: v3.10.0
             helm.sh/chart: cloudzero-agent-1.1.0-dev
             job-category: cronjob
             job-type: backfill
@@ -3358,7 +3358,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: webhook-server
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v3.7.3
+    app.kubernetes.io/version: v3.10.0
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation