From 24d87f5b5f6fce47a6f9531107e07fa1afe3f3e6 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Wed, 11 Mar 2026 17:08:47 +0100 Subject: [PATCH 01/17] Add nrr-controller Helm chart --- .github/ci/ct.yaml | 5 + .github/workflows/helm.yaml | 70 ++++ charts/node-readiness-controller/.helmignore | 22 ++ charts/node-readiness-controller/README.md | 101 ++++++ charts/nrr-controller/.helmignore | 22 ++ charts/nrr-controller/Chart.yaml | 15 + charts/nrr-controller/README.md | 101 ++++++ ...eadinessrules.readiness.node.x-k8s.io.yaml | 334 ++++++++++++++++++ charts/nrr-controller/templates/_helpers.tpl | 76 ++++ .../templates/certificates.yaml | 37 ++ .../nrr-controller/templates/deployment.yaml | 107 ++++++ charts/nrr-controller/templates/issuer.yaml | 11 + .../templates/nodereadinessrules.yaml | 17 + charts/nrr-controller/templates/rbac.yaml | 167 +++++++++ .../templates/serviceaccount.yaml | 13 + .../templates/servicemetrics.yaml | 15 + .../templates/servicewebhook.yaml | 17 + .../validatingwebhookconfiguration.yaml | 28 ++ charts/nrr-controller/tests/.gitignore | 1 + .../tests/annotations_test.yaml | 42 +++ .../nrr-controller/tests/deployment_test.yaml | 47 +++ .../tests/webhook_and_metrics_test.yaml | 27 ++ charts/nrr-controller/values.yaml | 148 ++++++++ 23 files changed, 1423 insertions(+) create mode 100644 .github/ci/ct.yaml create mode 100644 .github/workflows/helm.yaml create mode 100644 charts/node-readiness-controller/.helmignore create mode 100644 charts/node-readiness-controller/README.md create mode 100644 charts/nrr-controller/.helmignore create mode 100644 charts/nrr-controller/Chart.yaml create mode 100644 charts/nrr-controller/README.md create mode 100644 charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml create mode 100644 charts/nrr-controller/templates/_helpers.tpl create mode 100644 charts/nrr-controller/templates/certificates.yaml create mode 100644 charts/nrr-controller/templates/deployment.yaml create mode 100644 charts/nrr-controller/templates/issuer.yaml create mode 100644 charts/nrr-controller/templates/nodereadinessrules.yaml create mode 100644 charts/nrr-controller/templates/rbac.yaml create mode 100644 charts/nrr-controller/templates/serviceaccount.yaml create mode 100644 charts/nrr-controller/templates/servicemetrics.yaml create mode 100644 charts/nrr-controller/templates/servicewebhook.yaml create mode 100644 charts/nrr-controller/templates/validatingwebhookconfiguration.yaml create mode 100644 charts/nrr-controller/tests/.gitignore create mode 100644 charts/nrr-controller/tests/annotations_test.yaml create mode 100644 charts/nrr-controller/tests/deployment_test.yaml create mode 100644 charts/nrr-controller/tests/webhook_and_metrics_test.yaml create mode 100644 charts/nrr-controller/values.yaml diff --git a/.github/ci/ct.yaml b/.github/ci/ct.yaml new file mode 100644 index 0000000..e71665a --- /dev/null +++ b/.github/ci/ct.yaml @@ -0,0 +1,5 @@ +chart-dirs: + - charts +helm-extra-args: "--timeout=5m" +check-version-increment: false +target-branch: master diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 0000000..a1a24ea --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,70 @@ +name: Helm + +on: + push: + branches: + - master + - release-* + paths: + - 'charts/**' + - '.github/workflows/helm.yaml' + - '.github/ci/ct.yaml' + pull_request: + paths: + - 'charts/**' + - '.github/workflows/helm.yaml' + - '.github/ci/ct.yaml' + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v4.2.0 + with: + version: v3.15.1 + + - uses: actions/setup-python@v5.1.1 + with: + python-version: 3.12 + + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.8.0 + with: + version: v3.11.0 + + - name: Install Helm Unit Test Plugin + run: | + helm plugin install --version 1.0.3 https://github.com/helm-unittest/helm-unittest + + - name: Run Helm Unit Tests + run: | + helm unittest charts/nrr-controller --strict -d + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config=.github/ci/ct.yaml) + if [[ -n "$changed" ]]; then + echo "changed=true" >> $GITHUB_OUTPUT + fi + + - name: Run chart-testing (lint) + run: ct lint --config=.github/ci/ct.yaml --validate-maintainers=false + + # Need a multi node cluster so controller can run with leadership + - name: Create multi node Kind cluster + run: make kind-multi-node + + # helm-extra-set-args only available after ct 3.6.0 + - name: Run chart-testing (install) + run: ct install --config=.github/ci/ct.yaml --helm-extra-set-args='--set=kind=Deployment' diff --git a/charts/node-readiness-controller/.helmignore b/charts/node-readiness-controller/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/node-readiness-controller/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/node-readiness-controller/README.md b/charts/node-readiness-controller/README.md new file mode 100644 index 0000000..3e0871d --- /dev/null +++ b/charts/node-readiness-controller/README.md @@ -0,0 +1,101 @@ +# Node Readiness Controller for Kubernetes + +[Node Readiness Controller](https://github.com/kubernetes-sigs/node-readiness-controller) for Kubernetes is a controller that manages node readiness rules and conditions. It allows you to define custom readiness conditions for nodes based on various criteria, enabling more sophisticated node lifecycle management and workload scheduling decisions. + +## TL;DR: + +```shell +helm repo add node-readiness-controller https://kubernetes-sigs.github.io/node-readiness/ +helm install my-release --namespace kube-system node-readiness-controller/nrr-controller +``` + +## Introduction + +This chart bootstraps a [node-readiness-controller](https://github.com/kubernetes-sigs/node-readiness) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.25+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```shell +helm install --namespace kube-system my-release node-readiness-controller/nrr-controller +``` + +The command deploys the _node-readiness-controller_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```shell +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the _node-readiness-controller_ chart and their default values. + +| Parameter | Description | Default | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| `image.repository` | Docker repository to use | `registry.k8s.io/node-readiness-controller/node-readiness-controller` | +| `image.tag` | Docker tag to use | `v[chart appVersion]` | +| `image.pullPolicy` | Docker image pull policy | `IfNotPresent` | +| `imagePullSecrets` | Docker repository secrets | `[]` | +| `nameOverride` | String to partially override `nrr-controller.fullname` template (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override `nrr-controller.fullname` template | `""` | +| `namespaceOverride` | Override the deployment namespace; defaults to .Release.Namespace | `""` | +| `replicaCount` | The replica count for Deployment | `1` | +| `leaderElection.enabled` | Enable leader election to support multiple replicas | `false` | +| `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | +| `rbac.create` | If `true`, create & use RBAC resources | `true` | +| `resources` | Node Readiness Controller container CPU and memory requests/limits | _see values.yaml_ | +| `serviceAccount.create` | If `true`, create a service account | `true` | +| `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | +| `serviceAccount.annotations` | Specifies custom annotations for the serviceAccount | `{}` | +| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{}` | +| `podLabels` | Labels to add to the node-readiness-controller Pods | `{}` | +| `commonLabels` | Labels to apply to all resources | `{}` | +| `podSecurityContext` | Security context for pod | _see values.yaml_ | +| `securityContext` | Security context for container | _see values.yaml_ | +| `terminationGracePeriodSeconds` | Time to wait before forcefully terminating the pod | `10` | +| `healthProbeBindAddress` | The bind address for health probes | `:8081` | +| `livenessProbe` | Liveness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `readinessProbe` | Readiness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `metrics.secure` | Enable secure metrics endpoint | `false` | +| `metrics.bindAddress` | The bind address for metrics server | `:8443` | +| `metrics.service.port` | The port exposed by the metrics service | `8443` | +| `metrics.service.targetPort` | The target port for the metrics service | `8443` | +| `metrics.certDir` | Directory for metrics server certificates | `/tmp/k8s-metrics-server/metrics-certs` | +| `metrics.certSecretName` | Name of the secret containing metrics server certificates | `metrics-server-cert` | +| `webhook.enabled` | Enable the webhook server | `false` | +| `webhook.port` | The port for the webhook server | `9443` | +| `webhook.service.port` | The port exposed by the webhook service | `8443` | +| `webhook.service.targetPort` | The target port for the webhook service | `9443` | +| `webhook.certDir` | Directory for webhook server certificates | `/tmp/k8s-webhook-server/serving-certs` | +| `webhook.certSecretName` | Name of the secret containing webhook server certificates | `webhook-server-certs` | +| `certManager.enabled` | Enable cert-manager integration for automatic TLS certificate generation | `false` | +| `certManager.issuer.create` | Create a cert-manager issuer | `true` | +| `certManager.issuer.name` | Name of the cert-manager issuer | `selfsigned-issuer` | +| `certManager.metricsCertificate.create` | Create a cert-manager certificate for metrics server | `true` | +| `certManager.metricsCertificate.name` | Name of the metrics certificate | `metrics-certs` | +| `certManager.webhookCertificate.create` | Create a cert-manager certificate for webhook server | `true` | +| `certManager.webhookCertificate.name` | Name of the webhook certificate | `serving-cert` | +| `validatingWebhook.enabled` | Enable the validating webhook | `false` | +| `validatingWebhook.name` | Name of the ValidatingWebhookConfiguration resource | `validating-webhook-configuration` | +| `validatingWebhook.webhookName` | Name of the webhook | `vnodereadinessrule.kb.io` | +| `validatingWebhook.failurePolicy` | Failure policy for the webhook | `Fail` | +| `validatingWebhook.sideEffects` | Side effects for the webhook | `None` | +| `validatingWebhook.path` | The path for the webhook | `/validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule` | +| `validatingWebhook.admissionReviewVersions` | Admission review versions supported by the webhook | `["v1"]` | +| `nodeSelector` | Node selectors to run the controller on specific nodes | `nil` | +| `tolerations` | Tolerations to run the controller on specific nodes | `nil` | +| `affinity` | Node affinity to run the controller on specific nodes | `nil` | +| `nodeReadinessRules` | Custom readiness rules to create as NodeReadinessRule resources. CRD must be preinstalled or installation will fail. | `[]` | diff --git a/charts/nrr-controller/.helmignore b/charts/nrr-controller/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/nrr-controller/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/nrr-controller/Chart.yaml b/charts/nrr-controller/Chart.yaml new file mode 100644 index 0000000..f79fa2d --- /dev/null +++ b/charts/nrr-controller/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: nrr-controller +description: A Helm chart for the Node Readiness Controller +type: application +version: 0.1.0 +appVersion: "v0.2.0" +kubeVersion: ">=1.25.0-0" +home: https://github.com/kubernetes-sigs/node-readiness-controller +sources: + - https://github.com/kubernetes-sigs/node-readiness-controller +keywords: + - kubernetes + - controller + - readiness + - node diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md new file mode 100644 index 0000000..8121e43 --- /dev/null +++ b/charts/nrr-controller/README.md @@ -0,0 +1,101 @@ +# Node Readiness Controller for Kubernetes + +[Node Readiness Controller](https://github.com/kubernetes-sigs/node-readiness-controller) for Kubernetes a Kubernetes controller that provides fine-grained, declarative readiness for nodes. It ensures nodes only accept workloads when all required components eg: network agents, GPU drivers, storage drivers or custom health-checks, are fully ready on the node. + +## TL;DR: + +```shell +helm repo add nrr-controller https://kubernetes-sigs.github.io/nrr-controller/ +helm install my-release --namespace kube-system nrr-controller/nrr-controller +``` + +## Introduction + +This chart bootstraps a [node-readiness-controller](https://github.com/kubernetes-sigs/node-readiness-controller) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.25+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```shell +helm install --namespace kube-system my-release nrr-controller/nrr-controller +``` + +The command deploys the _node-readiness-controller_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```shell +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the _node-readiness-controller_ chart and their default values. + +| Parameter | Description | Default | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| `image.repository` | Docker repository to use | `registry.k8s.io/node-readiness-controller/node-readiness-controller` | +| `image.tag` | Docker tag to use | `v[chart appVersion]` | +| `image.pullPolicy` | Docker image pull policy | `IfNotPresent` | +| `imagePullSecrets` | Docker repository secrets | `[]` | +| `nameOverride` | String to partially override `nrr-controller.fullname` template (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override `nrr-controller.fullname` template | `""` | +| `namespaceOverride` | Override the deployment namespace; defaults to .Release.Namespace | `""` | +| `replicaCount` | The replica count for Deployment | `1` | +| `leaderElection.enabled` | Enable leader election to support multiple replicas | `false` | +| `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | +| `rbac.create` | If `true`, create & use RBAC resources | `true` | +| `resources` | Node Readiness Controller container CPU and memory requests/limits | _see values.yaml_ | +| `serviceAccount.create` | If `true`, create a service account | `true` | +| `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | +| `serviceAccount.annotations` | Specifies custom annotations for the serviceAccount | `{}` | +| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{}` | +| `podLabels` | Labels to add to the node-readiness-controller Pods | `{}` | +| `commonLabels` | Labels to apply to all resources | `{}` | +| `podSecurityContext` | Security context for pod | _see values.yaml_ | +| `securityContext` | Security context for container | _see values.yaml_ | +| `terminationGracePeriodSeconds` | Time to wait before forcefully terminating the pod | `10` | +| `healthProbeBindAddress` | The bind address for health probes | `:8081` | +| `livenessProbe` | Liveness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `readinessProbe` | Readiness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `metrics.secure` | Enable secure metrics endpoint | `false` | +| `metrics.bindAddress` | The bind address for metrics server | `:8443` | +| `metrics.service.port` | The port exposed by the metrics service | `8443` | +| `metrics.service.targetPort` | The target port for the metrics service | `8443` | +| `metrics.certDir` | Directory for metrics server certificates | `/tmp/k8s-metrics-server/metrics-certs` | +| `metrics.certSecretName` | Name of the secret containing metrics server certificates | `metrics-server-cert` | +| `webhook.enabled` | Enable the webhook server | `false` | +| `webhook.port` | The port for the webhook server | `9443` | +| `webhook.service.port` | The port exposed by the webhook service | `8443` | +| `webhook.service.targetPort` | The target port for the webhook service | `9443` | +| `webhook.certDir` | Directory for webhook server certificates | `/tmp/k8s-webhook-server/serving-certs` | +| `webhook.certSecretName` | Name of the secret containing webhook server certificates | `webhook-server-certs` | +| `certManager.enabled` | Enable cert-manager integration for automatic TLS certificate generation | `false` | +| `certManager.issuer.create` | Create a cert-manager issuer | `true` | +| `certManager.issuer.name` | Name of the cert-manager issuer | `selfsigned-issuer` | +| `certManager.metricsCertificate.create` | Create a cert-manager certificate for metrics server | `true` | +| `certManager.metricsCertificate.name` | Name of the metrics certificate | `metrics-certs` | +| `certManager.webhookCertificate.create` | Create a cert-manager certificate for webhook server | `true` | +| `certManager.webhookCertificate.name` | Name of the webhook certificate | `serving-cert` | +| `validatingWebhook.enabled` | Enable the validating webhook | `false` | +| `validatingWebhook.name` | Name of the ValidatingWebhookConfiguration resource | `validating-webhook-configuration` | +| `validatingWebhook.webhookName` | Name of the webhook | `vnodereadinessrule.kb.io` | +| `validatingWebhook.failurePolicy` | Failure policy for the webhook | `Fail` | +| `validatingWebhook.sideEffects` | Side effects for the webhook | `None` | +| `validatingWebhook.path` | The path for the webhook | `/validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule` | +| `validatingWebhook.admissionReviewVersions` | Admission review versions supported by the webhook | `["v1"]` | +| `nodeSelector` | Node selectors to run the controller on specific nodes | `nil` | +| `tolerations` | Tolerations to run the controller on specific nodes | `nil` | +| `affinity` | Node affinity to run the controller on specific nodes | `nil` | +| `nodeReadinessRules` | Custom NodeReadinessRule resources to create. CRD must be preinstalled or installation will fail. | `[]` | diff --git a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml new file mode 100644 index 0000000..1ceb152 --- /dev/null +++ b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml @@ -0,0 +1,334 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: nodereadinessrules.readiness.node.x-k8s.io +spec: + group: readiness.node.x-k8s.io + names: + kind: NodeReadinessRule + listKind: NodeReadinessRuleList + plural: nodereadinessrules + shortNames: + - nrr + singular: nodereadinessrule + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - description: Continuous, Bootstrap or Dryrun - shows if the rule is in enforcement or dryrun mode. + jsonPath: .spec.enforcementMode + name: Mode + type: string + - description: The readiness taint applied by this rule. + jsonPath: .spec.taint.key + name: Taint + type: string + - description: The age of this resource + jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + description: NodeReadinessRule is the Schema for the NodeReadinessRules API. + type: object + required: + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + type: string + metadata: + type: object + spec: + description: spec defines the desired state of NodeReadinessRule + type: object + required: + - conditions + - enforcementMode + - nodeSelector + - taint + properties: + conditions: + description: |- + conditions contains a list of the Node conditions that defines the specific + criteria that must be met for taints to be managed on the target Node. + The presence or status of these conditions directly triggers the application or removal of Node taints. + type: array + minItems: 1 + maxItems: 32 + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + items: + type: object + description: |- + ConditionRequirement defines a specific Node condition and the status value + required to trigger the controller's action. + required: + - requiredStatus + - type + properties: + requiredStatus: + description: requiredStatus is status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: |- + type of Node condition + + Following kubebuilder validation is referred from https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition + type: string + minLength: 1 + maxLength: 316 + dryRun: + description: |- + dryRun when set to true, The controller will evaluate Node conditions and log intended taint modifications + without persisting changes to the cluster. Proposed actions are reflected in the resource status. + type: boolean + enforcementMode: + description: |- + enforcementMode specifies how the controller maintains the desired state. + enforcementMode is one of bootstrap-only, continuous. + "bootstrap-only" applies the configuration once during initial setup. + "continuous" ensures the state is monitored and corrected throughout the resource lifecycle. + type: string + enum: + - bootstrap-only + - continuous + nodeSelector: + description: nodeSelector limits the scope of this rule to a specific subset of Nodes. + type: object + x-kubernetes-map-type: atomic + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + x-kubernetes-list-type: atomic + items: + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + type: array + x-kubernetes-list-type: atomic + items: + type: string + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions. + type: object + additionalProperties: + type: string + taint: + description: |- + taint defines the specific Taint (Key, Value, and Effect) to be managed + on Nodes that meet the defined condition criteria. + type: object + required: + - effect + - key + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. + type: string + format: date-time + value: + description: The taint value corresponding to the taint key. + type: string + status: + description: status defines the observed state of NodeReadinessRule + type: object + minProperties: 1 + properties: + appliedNodes: + description: |- + appliedNodes lists the names of Nodes where the taint has been successfully managed. + type: array + maxItems: 5000 + x-kubernetes-list-type: set + items: + type: string + maxLength: 253 + dryRunResults: + description: |- + dryRunResults captures the outcome of the rule evaluation when DryRun is enabled. + type: object + minProperties: 1 + required: + - summary + properties: + affectedNodes: + description: affectedNodes is the total count of Nodes that match the rule's criteria. + type: integer + format: int32 + minimum: 0 + riskyOperations: + description: |- + riskyOperations represents the count of Nodes where required conditions + are missing entirely, potentially indicating an ambiguous node state. + type: integer + format: int32 + minimum: 0 + summary: + description: |- + summary provides a human-readable overview of the dry run evaluation, + highlighting key findings or warnings. + type: string + minLength: 1 + maxLength: 4096 + taintsToAdd: + description: taintsToAdd is the number of Nodes that currently lack the specified taint and would have it applied. + type: integer + format: int32 + minimum: 0 + taintsToRemove: + description: |- + taintsToRemove is the number of Nodes that currently possess the taint but no longer meet the criteria. + type: integer + format: int32 + minimum: 0 + failedNodes: + description: |- + failedNodes lists the Nodes where the rule evaluation encountered an error. + type: array + maxItems: 5000 + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + items: + type: object + required: + - lastEvaluationTime + - nodeName + properties: + lastEvaluationTime: + description: lastEvaluationTime is the timestamp of the last rule check failed for this Node. + type: string + format: date-time + message: + description: message is a human-readable message indicating details about the evaluation. + type: string + minLength: 1 + maxLength: 10240 + nodeName: + description: |- + nodeName is the name of the failed Node. + type: string + minLength: 1 + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + reason: + description: reason provides a brief explanation of the evaluation result. + type: string + minLength: 1 + maxLength: 256 + nodeEvaluations: + description: |- + nodeEvaluations provides detailed insight into the rule's assessment for individual Nodes. + type: array + maxItems: 5000 + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + items: + type: object + required: + - conditionResults + - lastEvaluationTime + - nodeName + - taintStatus + properties: + conditionResults: + description: |- + conditionResults provides a detailed breakdown of each condition evaluation for this Node. + type: array + maxItems: 5000 + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + items: + type: object + required: + - currentStatus + - requiredStatus + - type + properties: + currentStatus: + type: string + enum: + - "True" + - "False" + - Unknown + requiredStatus: + type: string + enum: + - "True" + - "False" + - Unknown + type: + type: string + minLength: 1 + maxLength: 316 + lastEvaluationTime: + description: lastEvaluationTime is the timestamp when the controller last assessed this Node. + type: string + format: date-time + nodeName: + description: nodeName is the name of the evaluated Node. + type: string + minLength: 1 + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + taintStatus: + description: taintStatus represents the taint status on the Node, one of Present, Absent. + type: string + enum: + - Present + - Absent + observedGeneration: + description: observedGeneration reflects the generation of the most recently observed NodeReadinessRule by the controller. + type: integer + format: int64 + minimum: 1 + subresources: + status: {} diff --git a/charts/nrr-controller/templates/_helpers.tpl b/charts/nrr-controller/templates/_helpers.tpl new file mode 100644 index 0000000..7dffee1 --- /dev/null +++ b/charts/nrr-controller/templates/_helpers.tpl @@ -0,0 +1,76 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "nrr-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nrr-controller.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := include "nrr-controller.name" . -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Expand the namespace of the release. +Allows overriding it for multi-namespace deployments in combined charts. +*/}} +{{- define "nrr-controller.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nrr-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "nrr-controller.labels" -}} +app.kubernetes.io/name: {{ include "nrr-controller.name" . }} +helm.sh/chart: {{ include "nrr-controller.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "nrr-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nrr-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +control-plane: controller-manager +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nrr-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "nrr-controller.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/nrr-controller/templates/certificates.yaml b/charts/nrr-controller/templates/certificates.yaml new file mode 100644 index 0000000..174877f --- /dev/null +++ b/charts/nrr-controller/templates/certificates.yaml @@ -0,0 +1,37 @@ +{{- if .Values.certManager.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.metricsCertificate.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + secretName: {{ .Values.metrics.certSecretName }} + dnsNames: + - {{ include "nrr-controller.fullname" . }}-metrics-service.{{ include "nrr-controller.namespace" . }}.svc + - {{ include "nrr-controller.fullname" . }}-metrics-service.{{ include "nrr-controller.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} + +{{- if and .Values.webhook.enabled .Values.certManager.webhookCertificate.create }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.webhookCertificate.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + secretName: {{ .Values.webhook.certSecretName }} + dnsNames: + - {{ include "nrr-controller.fullname" . }}-webhook-service.{{ include "nrr-controller.namespace" . }}.svc + - {{ include "nrr-controller.fullname" . }}-webhook-service.{{ include "nrr-controller.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} +{{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/deployment.yaml b/charts/nrr-controller/templates/deployment.yaml new file mode 100644 index 0000000..8c66a85 --- /dev/null +++ b/charts/nrr-controller/templates/deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "nrr-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.podAnnotations }} + {{- .Values.podAnnotations | toYaml | nindent 10 }} + {{- end }} + labels: + {{- include "nrr-controller.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "nrr-controller.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: manager + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /manager + args: + {{- if .Values.leaderElection.enabled }} + - --leader-elect + {{- end }} + - --health-probe-bind-address={{ .Values.healthProbeBindAddress }} + - --enable-webhook={{ ternary "true" "false" .Values.webhook.enabled }} + - --metrics-bind-address={{ .Values.metrics.bindAddress }} + {{- if .Values.metrics.secure }} + - --metrics-secure + - --metrics-cert-dir={{ .Values.metrics.certDir }} + {{- end }} + {{- if .Values.webhook.enabled }} + ports: + - name: webhook-server + containerPort: {{ .Values.webhook.port }} + protocol: TCP + {{- end }} + livenessProbe: + httpGet: + path: {{ .Values.livenessProbe.path }} + port: 8081 + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.path }} + port: 8081 + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + {{- if .Values.webhook.enabled }} + - name: cert + mountPath: {{ .Values.webhook.certDir }} + readOnly: true + {{- end }} + {{- if .Values.metrics.secure }} + - name: metrics-certs + mountPath: {{ .Values.metrics.certDir }} + readOnly: true + {{- end }} + volumes: + {{- if .Values.webhook.enabled }} + - name: cert + secret: + secretName: {{ .Values.webhook.certSecretName }} + defaultMode: 420 + {{- end }} + {{- if .Values.metrics.secure }} + - name: metrics-certs + secret: + secretName: {{ .Values.metrics.certSecretName }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/nrr-controller/templates/issuer.yaml b/charts/nrr-controller/templates/issuer.yaml new file mode 100644 index 0000000..eed5c29 --- /dev/null +++ b/charts/nrr-controller/templates/issuer.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.certManager.enabled .Values.certManager.issuer.create }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} diff --git a/charts/nrr-controller/templates/nodereadinessrules.yaml b/charts/nrr-controller/templates/nodereadinessrules.yaml new file mode 100644 index 0000000..82dd51f --- /dev/null +++ b/charts/nrr-controller/templates/nodereadinessrules.yaml @@ -0,0 +1,17 @@ +{{- if .Values.nodeReadinessRules }} +{{- range $rule := .Values.nodeReadinessRules }} +--- +apiVersion: readiness.node.x-k8s.io/v1alpha1 +kind: NodeReadinessRule +metadata: + name: {{ $rule.name | quote }} +spec: + conditions: +{{- toYaml $rule.conditions | nindent 4 }} + taint: +{{- toYaml $rule.taint | nindent 4 }} + enforcementMode: {{ $rule.enforcementMode | quote }} + nodeSelector: +{{- toYaml $rule.nodeSelector | nindent 4 }} +{{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/rbac.yaml b/charts/nrr-controller/templates/rbac.yaml new file mode 100644 index 0000000..a7308f0 --- /dev/null +++ b/charts/nrr-controller/templates/rbac.yaml @@ -0,0 +1,167 @@ +{{- if .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "nrr-controller.fullname" . }}-leader-election-role + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-leader-election-rolebinding + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nrr-controller.fullname" . }}-leader-election-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: [""] + resources: ["nodes/finalizers"] + verbs: ["update"] + - apiGroups: [""] + resources: ["nodes/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/finalizers"] + verbs: ["update"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get", "patch", "update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager-rolebinding + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "nrr-controller.fullname" . }}-manager-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-rolebinding + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-reader + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - nonResourceURLs: ["/metrics"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-admin-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["*"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-editor-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-viewer-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["get", "list", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] +{{- end }} diff --git a/charts/nrr-controller/templates/serviceaccount.yaml b/charts/nrr-controller/templates/serviceaccount.yaml new file mode 100644 index 0000000..61f2d55 --- /dev/null +++ b/charts/nrr-controller/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/servicemetrics.yaml b/charts/nrr-controller/templates/servicemetrics.yaml new file mode 100644 index 0000000..8ca74ce --- /dev/null +++ b/charts/nrr-controller/templates/servicemetrics.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-service + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + ports: + - name: https + port: {{ .Values.metrics.service.port }} + protocol: TCP + targetPort: {{ .Values.metrics.service.targetPort }} + selector: + {{- include "nrr-controller.selectorLabels" . | nindent 4 }} diff --git a/charts/nrr-controller/templates/servicewebhook.yaml b/charts/nrr-controller/templates/servicewebhook.yaml new file mode 100644 index 0000000..4a5c7ee --- /dev/null +++ b/charts/nrr-controller/templates/servicewebhook.yaml @@ -0,0 +1,17 @@ +{{- if .Values.webhook.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nrr-controller.fullname" . }}-webhook-service + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + ports: + - name: https + port: {{ .Values.webhook.service.port }} + protocol: TCP + targetPort: {{ .Values.webhook.service.targetPort }} + selector: + {{- include "nrr-controller.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml new file mode 100644 index 0000000..2a4407c --- /dev/null +++ b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.webhook.enabled .Values.validatingWebhook.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.validatingWebhook.name }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} + {{- if .Values.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "nrr-controller.namespace" . }}/{{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.webhookCertificate.name }} + {{- end }} +webhooks: + - name: {{ .Values.validatingWebhook.webhookName }} + admissionReviewVersions: + {{- toYaml .Values.validatingWebhook.admissionReviewVersions | nindent 6 }} + clientConfig: + service: + name: {{ include "nrr-controller.fullname" . }}-webhook-service + namespace: {{ include "nrr-controller.namespace" . }} + path: {{ .Values.validatingWebhook.path }} + failurePolicy: {{ .Values.validatingWebhook.failurePolicy }} + sideEffects: {{ .Values.validatingWebhook.sideEffects }} + rules: + - apiGroups: ["readiness.node.x-k8s.io"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["nodereadinessrules"] +{{- end }} diff --git a/charts/nrr-controller/tests/.gitignore b/charts/nrr-controller/tests/.gitignore new file mode 100644 index 0000000..75292be --- /dev/null +++ b/charts/nrr-controller/tests/.gitignore @@ -0,0 +1 @@ +__snapshot__ \ No newline at end of file diff --git a/charts/nrr-controller/tests/annotations_test.yaml b/charts/nrr-controller/tests/annotations_test.yaml new file mode 100644 index 0000000..adce08a --- /dev/null +++ b/charts/nrr-controller/tests/annotations_test.yaml @@ -0,0 +1,42 @@ +suite: Test Node Readiness Controller Pod Annotations and Labels + +templates: + - "*.yaml" + +release: + name: nrr-controller + +tests: + - it: adds pod annotations and labels when set + template: templates/deployment.yaml + set: + podAnnotations: + monitoring.company.com/scrape: "true" + description: "test pod" + podLabels: + environment: "test" + team: "platform" + asserts: + - equal: + path: spec.template.metadata.annotations["monitoring.company.com/scrape"] + value: "true" + - equal: + path: spec.template.metadata.annotations.description + value: "test pod" + - equal: + path: spec.template.metadata.labels.environment + value: "test" + - equal: + path: spec.template.metadata.labels.team + value: "platform" + + - it: does not add metadata when not set + template: templates/deployment.yaml + asserts: + - isNull: + path: spec.template.metadata.annotations + - isNotNull: + path: spec.template.metadata.labels + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: nrr-controller diff --git a/charts/nrr-controller/tests/deployment_test.yaml b/charts/nrr-controller/tests/deployment_test.yaml new file mode 100644 index 0000000..d79439a --- /dev/null +++ b/charts/nrr-controller/tests/deployment_test.yaml @@ -0,0 +1,47 @@ +suite: Test Node Readiness Controller Deployment + +templates: + - "*.yaml" + +release: + name: nrr-controller + +set: + replicaCount: 1 + +tests: + - it: creates Deployment + template: templates/deployment.yaml + asserts: + - isKind: + of: Deployment + + - it: enables leader-election + set: + leaderElection: + enabled: true + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --leader-elect + + - it: adds metrics secure args when enabled + set: + metrics: + secure: true + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-secure + + - it: exposes webhook port when webhook enabled + set: + webhook: + enabled: true + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].ports[0].name + content: webhook-server diff --git a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml new file mode 100644 index 0000000..9fec19b --- /dev/null +++ b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml @@ -0,0 +1,27 @@ +suite: Test Node Readiness Controller Webhook and Metrics Config + +templates: + - "*.yaml" + +release: + name: nrr-controller + +set: + webhook: + enabled: true + metrics: + secure: true + +tests: + - it: mounts webhook cert volume when webhook enabled + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.volumes[0].name + content: cert + - it: mounts metrics cert volume when metrics.secure enabled + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.volumes[?(@.name=="metrics-certs")].name + content: metrics-certs diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml new file mode 100644 index 0000000..f2af1f3 --- /dev/null +++ b/charts/nrr-controller/values.yaml @@ -0,0 +1,148 @@ +# Default values for node-readiness-controller. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: registry.k8s.io/node-readiness-controller/node-readiness-controller + # Overrides the image tag whose default is the chart version + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: container-registry-secret + +resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi + +nameOverride: "" + +fullnameOverride: "" + +# -- Override the deployment namespace; defaults to .Release.Namespace +namespaceOverride: "" + +commonLabels: {} + +# Specifies the replica count for Deployment +# Enable leaderElection if you want to use more than 1 replica +# Set affinity.podAntiAffinity rule if you want to schedule onto a node +replicaCount: 1 + +priorityClassName: system-cluster-critical + +podAnnotations: + kubectl.kubernetes.io/default-container: manager + +podLabels: {} + +# podSecurityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +terminationGracePeriodSeconds: 10 + +rbac: + create: true + +serviceAccount: + create: true + name: "" + annotations: {} + +# Enable leader election to support multiple replicas +leaderElection: + enabled: false + +# Enable the webhook server +healthProbeBindAddress: ":8081" + +# Liveness probe configuration +livenessProbe: + path: /healthz + initialDelaySeconds: 15 + periodSeconds: 20 + +# Readiness probe configuration +readinessProbe: + path: /readyz + initialDelaySeconds: 5 + periodSeconds: 10 + +# Metrics server configuration +metrics: + secure: false + bindAddress: ":8443" + service: + port: 8443 + targetPort: 8443 + certDir: /tmp/k8s-metrics-server/metrics-certs + certSecretName: metrics-server-cert + +# Webhook server configuration +webhook: + enabled: false + port: 9443 + service: + port: 8443 + targetPort: 9443 + certDir: /tmp/k8s-webhook-server/serving-certs + certSecretName: webhook-server-certs + +# cert-manager configuration for generating TLS certificates for the webhook and metrics server +certManager: + enabled: false + issuer: + create: true + name: selfsigned-issuer + metricsCertificate: + create: true + name: metrics-certs + webhookCertificate: + create: true + name: serving-cert + +validatingWebhook: + enabled: false + name: validating-webhook-configuration + webhookName: vnodereadinessrule.kb.io + failurePolicy: Fail + sideEffects: None + path: /validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule + admissionReviewVersions: + - v1 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# NOTE: Before enabling these rules. The CRD `nodereadinessrules.readiness.node.x-k8s.io` must already be installed in the cluster +# If the CRD is not installed, Helm will fail during installation. +nodeReadinessRules: [] + # - name: kube-proxy-unhealthy-noschedule + # conditions: + # - type: "KubeProxyUnhealthy" + # requiredStatus: "False" + # taint: + # key: "readiness.k8s.io/KubeProxyUnhealthy" + # value: "true" + # effect: "NoSchedule" + # enforcementMode: "continuous" + # nodeSelector: + # matchLabels: + # kubernetes.io/os: linux From a18bbfb5347ca2c372db66a5941d86866edaf2d8 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Wed, 11 Mar 2026 17:16:13 +0100 Subject: [PATCH 02/17] Update README.md & remove temp files --- charts/node-readiness-controller/.helmignore | 22 ---- charts/node-readiness-controller/README.md | 101 ------------------- charts/nrr-controller/README.md | 6 +- 3 files changed, 3 insertions(+), 126 deletions(-) delete mode 100644 charts/node-readiness-controller/.helmignore delete mode 100644 charts/node-readiness-controller/README.md diff --git a/charts/node-readiness-controller/.helmignore b/charts/node-readiness-controller/.helmignore deleted file mode 100644 index 50af031..0000000 --- a/charts/node-readiness-controller/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/node-readiness-controller/README.md b/charts/node-readiness-controller/README.md deleted file mode 100644 index 3e0871d..0000000 --- a/charts/node-readiness-controller/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Node Readiness Controller for Kubernetes - -[Node Readiness Controller](https://github.com/kubernetes-sigs/node-readiness-controller) for Kubernetes is a controller that manages node readiness rules and conditions. It allows you to define custom readiness conditions for nodes based on various criteria, enabling more sophisticated node lifecycle management and workload scheduling decisions. - -## TL;DR: - -```shell -helm repo add node-readiness-controller https://kubernetes-sigs.github.io/node-readiness/ -helm install my-release --namespace kube-system node-readiness-controller/nrr-controller -``` - -## Introduction - -This chart bootstraps a [node-readiness-controller](https://github.com/kubernetes-sigs/node-readiness) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. - -## Prerequisites - -- Kubernetes 1.25+ - -## Installing the Chart - -To install the chart with the release name `my-release`: - -```shell -helm install --namespace kube-system my-release node-readiness-controller/nrr-controller -``` - -The command deploys the _node-readiness-controller_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. - -> **Tip**: List all releases using `helm list` - -## Uninstalling the Chart - -To uninstall/delete the `my-release` deployment: - -```shell -helm delete my-release -``` - -The command removes all the Kubernetes components associated with the chart and deletes the release. - -## Configuration - -The following table lists the configurable parameters of the _node-readiness-controller_ chart and their default values. - -| Parameter | Description | Default | -| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| `image.repository` | Docker repository to use | `registry.k8s.io/node-readiness-controller/node-readiness-controller` | -| `image.tag` | Docker tag to use | `v[chart appVersion]` | -| `image.pullPolicy` | Docker image pull policy | `IfNotPresent` | -| `imagePullSecrets` | Docker repository secrets | `[]` | -| `nameOverride` | String to partially override `nrr-controller.fullname` template (will prepend the release name) | `""` | -| `fullnameOverride` | String to fully override `nrr-controller.fullname` template | `""` | -| `namespaceOverride` | Override the deployment namespace; defaults to .Release.Namespace | `""` | -| `replicaCount` | The replica count for Deployment | `1` | -| `leaderElection.enabled` | Enable leader election to support multiple replicas | `false` | -| `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | -| `rbac.create` | If `true`, create & use RBAC resources | `true` | -| `resources` | Node Readiness Controller container CPU and memory requests/limits | _see values.yaml_ | -| `serviceAccount.create` | If `true`, create a service account | `true` | -| `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | -| `serviceAccount.annotations` | Specifies custom annotations for the serviceAccount | `{}` | -| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{}` | -| `podLabels` | Labels to add to the node-readiness-controller Pods | `{}` | -| `commonLabels` | Labels to apply to all resources | `{}` | -| `podSecurityContext` | Security context for pod | _see values.yaml_ | -| `securityContext` | Security context for container | _see values.yaml_ | -| `terminationGracePeriodSeconds` | Time to wait before forcefully terminating the pod | `10` | -| `healthProbeBindAddress` | The bind address for health probes | `:8081` | -| `livenessProbe` | Liveness probe configuration for the node-readiness-controller container | _see values.yaml_ | -| `readinessProbe` | Readiness probe configuration for the node-readiness-controller container | _see values.yaml_ | -| `metrics.secure` | Enable secure metrics endpoint | `false` | -| `metrics.bindAddress` | The bind address for metrics server | `:8443` | -| `metrics.service.port` | The port exposed by the metrics service | `8443` | -| `metrics.service.targetPort` | The target port for the metrics service | `8443` | -| `metrics.certDir` | Directory for metrics server certificates | `/tmp/k8s-metrics-server/metrics-certs` | -| `metrics.certSecretName` | Name of the secret containing metrics server certificates | `metrics-server-cert` | -| `webhook.enabled` | Enable the webhook server | `false` | -| `webhook.port` | The port for the webhook server | `9443` | -| `webhook.service.port` | The port exposed by the webhook service | `8443` | -| `webhook.service.targetPort` | The target port for the webhook service | `9443` | -| `webhook.certDir` | Directory for webhook server certificates | `/tmp/k8s-webhook-server/serving-certs` | -| `webhook.certSecretName` | Name of the secret containing webhook server certificates | `webhook-server-certs` | -| `certManager.enabled` | Enable cert-manager integration for automatic TLS certificate generation | `false` | -| `certManager.issuer.create` | Create a cert-manager issuer | `true` | -| `certManager.issuer.name` | Name of the cert-manager issuer | `selfsigned-issuer` | -| `certManager.metricsCertificate.create` | Create a cert-manager certificate for metrics server | `true` | -| `certManager.metricsCertificate.name` | Name of the metrics certificate | `metrics-certs` | -| `certManager.webhookCertificate.create` | Create a cert-manager certificate for webhook server | `true` | -| `certManager.webhookCertificate.name` | Name of the webhook certificate | `serving-cert` | -| `validatingWebhook.enabled` | Enable the validating webhook | `false` | -| `validatingWebhook.name` | Name of the ValidatingWebhookConfiguration resource | `validating-webhook-configuration` | -| `validatingWebhook.webhookName` | Name of the webhook | `vnodereadinessrule.kb.io` | -| `validatingWebhook.failurePolicy` | Failure policy for the webhook | `Fail` | -| `validatingWebhook.sideEffects` | Side effects for the webhook | `None` | -| `validatingWebhook.path` | The path for the webhook | `/validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule` | -| `validatingWebhook.admissionReviewVersions` | Admission review versions supported by the webhook | `["v1"]` | -| `nodeSelector` | Node selectors to run the controller on specific nodes | `nil` | -| `tolerations` | Tolerations to run the controller on specific nodes | `nil` | -| `affinity` | Node affinity to run the controller on specific nodes | `nil` | -| `nodeReadinessRules` | Custom readiness rules to create as NodeReadinessRule resources. CRD must be preinstalled or installation will fail. | `[]` | diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md index 8121e43..e7c4e89 100644 --- a/charts/nrr-controller/README.md +++ b/charts/nrr-controller/README.md @@ -5,8 +5,8 @@ ## TL;DR: ```shell -helm repo add nrr-controller https://kubernetes-sigs.github.io/nrr-controller/ -helm install my-release --namespace kube-system nrr-controller/nrr-controller +helm repo add node-readiness-controller https://kubernetes-sigs.github.io/node-readiness-controller/ +helm install my-release --namespace kube-system node-readiness-controller/nrr-controller ``` ## Introduction @@ -22,7 +22,7 @@ This chart bootstraps a [node-readiness-controller](https://github.com/kubernete To install the chart with the release name `my-release`: ```shell -helm install --namespace kube-system my-release nrr-controller/nrr-controller +helm install --namespace kube-system my-release node-readiness-controller/nrr-controller ``` The command deploys the _node-readiness-controller_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. From 78d2eece1a83c116e972996be24905bf4be5125d Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Thu, 12 Mar 2026 10:39:11 +0100 Subject: [PATCH 03/17] Add test config --- .github/ci/ct.yaml | 2 +- Makefile | 21 ++++++++++++++++++++- hack/kind_config.yaml | 18 ++++++++++++++++++ hack/verify-chart.sh | 1 + 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 hack/kind_config.yaml create mode 100755 hack/verify-chart.sh diff --git a/.github/ci/ct.yaml b/.github/ci/ct.yaml index e71665a..983c22a 100644 --- a/.github/ci/ct.yaml +++ b/.github/ci/ct.yaml @@ -2,4 +2,4 @@ chart-dirs: - charts helm-extra-args: "--timeout=5m" check-version-increment: false -target-branch: master +target-branch: main diff --git a/Makefile b/Makefile index 0204bc0..a2fcab1 100644 --- a/Makefile +++ b/Makefile @@ -489,4 +489,23 @@ crd-ref-docs: --source-path=${PWD}/api/v1alpha1/ \ --config=crd-ref-docs.yaml \ --renderer=markdown \ - --output-path=${PWD}/docs/book/src/reference/api-spec.md \ No newline at end of file + --output-path=${PWD}/docs/book/src/reference/api-spec.md + +# helm + +ensure-helm-install: +ifndef HAS_HELM + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && chmod 700 ./get_helm.sh && ./get_helm.sh +endif + +lint-chart: ensure-helm-install + helm lint ./charts/nrr-controller + +build-helm: + helm package ./charts/nrr-controller --dependency-update --destination ./bin/chart + +kind-multi-node: + kind create cluster --name kind --config ./hack/kind_config.yaml --wait 2m + +ct-helm: + ./hack/verify-chart.sh \ No newline at end of file diff --git a/hack/kind_config.yaml b/hack/kind_config.yaml new file mode 100644 index 0000000..c405702 --- /dev/null +++ b/hack/kind_config.yaml @@ -0,0 +1,18 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "topology.kubernetes.io/zone=local-a" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "topology.kubernetes.io/zone=local-b" diff --git a/hack/verify-chart.sh b/hack/verify-chart.sh new file mode 100755 index 0000000..df829fc --- /dev/null +++ b/hack/verify-chart.sh @@ -0,0 +1 @@ +${CONTAINER_ENGINE:-docker} run -it --rm --network host --workdir=/data --volume ~/.kube/config:/root/.kube/config:ro --volume $(pwd):/data quay.io/helmpack/chart-testing:v3.7.0 /bin/bash -c "git config --global --add safe.directory /data; ct install --config=.github/ci/ct.yaml --helm-extra-set-args=\"--set=kind=Deployment\"" From eaead3b0a67969a616437fbcf07e390c17aef1e3 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Thu, 12 Mar 2026 10:47:36 +0100 Subject: [PATCH 04/17] Add missing eof --- Makefile | 2 +- charts/nrr-controller/tests/.gitignore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a2fcab1..e41fbbb 100644 --- a/Makefile +++ b/Makefile @@ -508,4 +508,4 @@ kind-multi-node: kind create cluster --name kind --config ./hack/kind_config.yaml --wait 2m ct-helm: - ./hack/verify-chart.sh \ No newline at end of file + ./hack/verify-chart.sh diff --git a/charts/nrr-controller/tests/.gitignore b/charts/nrr-controller/tests/.gitignore index 75292be..bae41db 100644 --- a/charts/nrr-controller/tests/.gitignore +++ b/charts/nrr-controller/tests/.gitignore @@ -1 +1 @@ -__snapshot__ \ No newline at end of file +__snapshot__ From 7a909c7a0e7717fe670674524ea15908133c33f3 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Thu, 12 Mar 2026 11:40:44 +0100 Subject: [PATCH 05/17] Fix wrong helm branch - github workflows --- .github/workflows/helm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index a1a24ea..73f809a 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -3,7 +3,7 @@ name: Helm on: push: branches: - - master + - main - release-* paths: - 'charts/**' From 91cd05724de8a5d8d4b15d2a4ae2b268f2dd0383 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Fri, 20 Mar 2026 13:39:25 +0100 Subject: [PATCH 06/17] Update crds & rbac & tolerations for image v0.3.0 --- charts/nrr-controller/Chart.yaml | 2 +- ...eadinessrules.readiness.node.x-k8s.io.yaml | 697 ++++++++++-------- charts/nrr-controller/templates/rbac.yaml | 18 +- charts/nrr-controller/values.yaml | 6 +- 4 files changed, 406 insertions(+), 317 deletions(-) diff --git a/charts/nrr-controller/Chart.yaml b/charts/nrr-controller/Chart.yaml index f79fa2d..1b721f2 100644 --- a/charts/nrr-controller/Chart.yaml +++ b/charts/nrr-controller/Chart.yaml @@ -3,7 +3,7 @@ name: nrr-controller description: A Helm chart for the Node Readiness Controller type: application version: 0.1.0 -appVersion: "v0.2.0" +appVersion: "v0.3.0" kubeVersion: ">=1.25.0-0" home: https://github.com/kubernetes-sigs/node-readiness-controller sources: diff --git a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml index 1ceb152..cf914d3 100644 --- a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml +++ b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml @@ -11,324 +11,415 @@ spec: listKind: NodeReadinessRuleList plural: nodereadinessrules shortNames: - - nrr + - nrr singular: nodereadinessrule scope: Cluster versions: - - name: v1alpha1 - served: true - storage: true - additionalPrinterColumns: - - description: Continuous, Bootstrap or Dryrun - shows if the rule is in enforcement or dryrun mode. - jsonPath: .spec.enforcementMode - name: Mode - type: string - - description: The readiness taint applied by this rule. - jsonPath: .spec.taint.key - name: Taint - type: string - - description: The age of this resource - jsonPath: .metadata.creationTimestamp - name: Age - type: date - schema: - openAPIV3Schema: - description: NodeReadinessRule is the Schema for the NodeReadinessRules API. - type: object - required: - - spec - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - type: string - metadata: - type: object - spec: - description: spec defines the desired state of NodeReadinessRule - type: object - required: - - conditions - - enforcementMode - - nodeSelector - - taint - properties: - conditions: + - additionalPrinterColumns: + - description: Continuous, Bootstrap or Dryrun - shows if the rule is in enforcement + or dryrun mode. + jsonPath: .spec.enforcementMode + name: Mode + type: string + - description: The readiness taint applied by this rule. + jsonPath: .spec.taint.key + name: Taint + type: string + - description: The age of this resource + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeReadinessRule is the Schema for the NodeReadinessRules API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of NodeReadinessRule + properties: + conditions: + description: |- + conditions contains a list of the Node conditions that defines the specific + criteria that must be met for taints to be managed on the target Node. + The presence or status of these conditions directly triggers the application or removal of Node taints. + items: description: |- - conditions contains a list of the Node conditions that defines the specific - criteria that must be met for taints to be managed on the target Node. - The presence or status of these conditions directly triggers the application or removal of Node taints. - type: array - minItems: 1 - maxItems: 32 - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - items: - type: object - description: |- - ConditionRequirement defines a specific Node condition and the status value - required to trigger the controller's action. - required: - - requiredStatus - - type - properties: - requiredStatus: - description: requiredStatus is status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: |- - type of Node condition + ConditionRequirement defines a specific Node condition and the status value + required to trigger the controller's action. + properties: + requiredStatus: + description: requiredStatus is status of the condition, one + of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of Node condition - Following kubebuilder validation is referred from https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition - type: string - minLength: 1 - maxLength: 316 - dryRun: - description: |- - dryRun when set to true, The controller will evaluate Node conditions and log intended taint modifications - without persisting changes to the cluster. Proposed actions are reflected in the resource status. - type: boolean - enforcementMode: - description: |- - enforcementMode specifies how the controller maintains the desired state. - enforcementMode is one of bootstrap-only, continuous. - "bootstrap-only" applies the configuration once during initial setup. - "continuous" ensures the state is monitored and corrected throughout the resource lifecycle. - type: string - enum: - - bootstrap-only - - continuous - nodeSelector: - description: nodeSelector limits the scope of this rule to a specific subset of Nodes. + Following kubebuilder validation is referred from https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition + maxLength: 316 + minLength: 1 + type: string + required: + - requiredStatus + - type type: object - x-kubernetes-map-type: atomic - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - type: array - x-kubernetes-list-type: atomic - items: - type: object - required: - - key - - operator - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. - type: array - x-kubernetes-list-type: atomic - items: - type: string - matchLabels: + maxItems: 32 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: conditions is immutable + rule: self == oldSelf + dryRun: + description: |- + dryRun when set to true, The controller will evaluate Node conditions and log intended taint modifications + without persisting changes to the cluster. Proposed actions are reflected in the resource status. + type: boolean + enforcementMode: + description: |- + enforcementMode specifies how the controller maintains the desired state. + enforcementMode is one of bootstrap-only, continuous. + "bootstrap-only" applies the configuration once during initial setup. + "continuous" ensures the state is monitored and corrected throughout the resource lifecycle. + enum: + - bootstrap-only + - continuous + type: string + x-kubernetes-validations: + - message: enforcementMode is immutable + rule: self == oldSelf + nodeSelector: + description: nodeSelector limits the scope of this rule to a specific + subset of Nodes. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions. + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - additionalProperties: - type: string - taint: - description: |- - taint defines the specific Taint (Key, Value, and Effect) to be managed - on Nodes that meet the defined condition criteria. - type: object - required: - - effect - - key + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: nodeSelector is immutable + rule: self == oldSelf + taint: + description: |- + taint defines the specific Taint (Key, Value, and Effect) to be managed + on Nodes that meet the defined condition criteria. + + The taint key must follow Kubernetes qualified name format: prefix/name + where prefix is 'readiness.k8s.io' (DNS subdomain) and name is a qualified + name (max 63 chars, alphanumeric, '-', '_', '.', must start and end with alphanumeric). + ref: git.k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/api/validate/content/kube.go#L24-L72 + + Supported effects: NoSchedule, PreferNoSchedule, NoExecute. + Caution: NoExecute evicts existing pods and can cause significant disruption + when combined with continuous enforcement mode. Prefer NoSchedule for most use cases. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + x-kubernetes-validations: + - message: taint key must start with 'readiness.k8s.io/' + rule: self.key.startsWith('readiness.k8s.io/') + - message: taint key length must be at most 253 characters + rule: self.key.size() <= 253 + - message: taint key must have exactly one '/' separator (prefix/name + format) + rule: size(self.key.split('/')) == 2 + - message: taint key name part must be 1-63 characters + rule: size(self.key.split('/')[1]) > 0 && size(self.key.split('/')[1]) + <= 63 + - message: taint key name part must consist of alphanumeric characters, + '-', '_' or '.', and must start and end with an alphanumeric character + rule: self.key.split('/')[1].matches('^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$') + - message: taint value length must be at most 63 characters + rule: '!has(self.value) || self.value.size() <= 63' + - message: taint effect must be one of 'NoSchedule', 'PreferNoSchedule', + 'NoExecute' + rule: self.effect in ['NoSchedule', 'PreferNoSchedule', 'NoExecute'] + - message: taint key is immutable + rule: '!has(oldSelf.key) || self.key == oldSelf.key' + - message: taint effect is immutable + rule: '!has(oldSelf.effect) || self.effect == oldSelf.effect' + - message: taint value is immutable + rule: '!has(oldSelf.value) || self.value == oldSelf.value' + required: + - conditions + - enforcementMode + - nodeSelector + - taint + type: object + status: + description: status defines the observed state of NodeReadinessRule + minProperties: 1 + properties: + appliedNodes: + description: |- + appliedNodes lists the names of Nodes where the taint has been successfully managed. + This provides a quick reference to the scope of impact for this rule. + items: + maxLength: 253 + type: string + maxItems: 5000 + type: array + x-kubernetes-list-type: set + dryRunResults: + description: |- + dryRunResults captures the outcome of the rule evaluation when DryRun is enabled. + This field provides visibility into the actions the controller would have taken, + allowing users to preview taint changes before they are committed. + minProperties: 1 + properties: + affectedNodes: + description: affectedNodes is the total count of Nodes that match + the rule's criteria. + format: int32 + minimum: 0 + type: integer + riskyOperations: + description: |- + riskyOperations represents the count of Nodes where required conditions + are missing entirely, potentially indicating an ambiguous node state. + format: int32 + minimum: 0 + type: integer + summary: + description: |- + summary provides a human-readable overview of the dry run evaluation, + highlighting key findings or warnings. + maxLength: 4096 + minLength: 1 + type: string + taintsToAdd: + description: taintsToAdd is the number of Nodes that currently + lack the specified taint and would have it applied. + format: int32 + minimum: 0 + type: integer + taintsToRemove: + description: |- + taintsToRemove is the number of Nodes that currently possess the + taint but no longer meet the criteria, leading to its removal. + format: int32 + minimum: 0 + type: integer + required: + - summary + type: object + failedNodes: + description: |- + failedNodes lists the Nodes where the rule evaluation encountered an error. + This is used for troubleshooting configuration issues, such as invalid selectors during node lookup. + items: + description: NodeFailure provides diagnostic details for Nodes that + could not be successfully evaluated by the rule. properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + lastEvaluationTime: + description: lastEvaluationTime is the timestamp of the last + rule check failed for this Node. + format: date-time type: string - key: - description: Required. The taint key to be applied to a node. + message: + description: message is a human-readable message indicating + details about the evaluation. + maxLength: 10240 + minLength: 1 type: string - timeAdded: - description: TimeAdded represents the time at which the taint was added. + nodeName: + description: |- + nodeName is the name of the failed Node. + + Following kubebuilder validation is referred from + https://github.com/kubernetes/apimachinery/blob/84d740c9e27f3ccc94c8bc4d13f1b17f60f7080b/pkg/util/validation/validation.go#L198 + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - format: date-time - value: - description: The taint value corresponding to the taint key. + reason: + description: reason provides a brief explanation of the evaluation + result. + maxLength: 256 + minLength: 1 type: string - status: - description: status defines the observed state of NodeReadinessRule - type: object - minProperties: 1 - properties: - appliedNodes: - description: |- - appliedNodes lists the names of Nodes where the taint has been successfully managed. - type: array - maxItems: 5000 - x-kubernetes-list-type: set - items: - type: string - maxLength: 253 - dryRunResults: - description: |- - dryRunResults captures the outcome of the rule evaluation when DryRun is enabled. - type: object - minProperties: 1 required: - - summary + - lastEvaluationTime + - nodeName + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + nodeEvaluations: + description: |- + nodeEvaluations provides detailed insight into the rule's assessment for individual Nodes. + This is primarily used for auditing and debugging why specific Nodes were or + were not targeted by the rule. + items: + description: NodeEvaluation provides a detailed audit of a single + Node's compliance with the rule. properties: - affectedNodes: - description: affectedNodes is the total count of Nodes that match the rule's criteria. - type: integer - format: int32 - minimum: 0 - riskyOperations: + conditionResults: description: |- - riskyOperations represents the count of Nodes where required conditions - are missing entirely, potentially indicating an ambiguous node state. - type: integer - format: int32 - minimum: 0 - summary: - description: |- - summary provides a human-readable overview of the dry run evaluation, - highlighting key findings or warnings. + conditionResults provides a detailed breakdown of each condition evaluation + for this Node. This allows for granular auditing of which specific + criteria passed or failed during the rule assessment. + items: + description: |- + ConditionEvaluationResult provides a detailed report of the comparison between + the Node's observed condition and the rule's requirement. + properties: + currentStatus: + description: currentStatus is the actual status value + observed on the Node, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + requiredStatus: + description: requiredStatus is the status value defined + in the rule that must be matched, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type corresponds to the Node condition type + being evaluated. + maxLength: 316 + minLength: 1 + type: string + required: + - currentStatus + - requiredStatus + - type + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastEvaluationTime: + description: lastEvaluationTime is the timestamp when the controller + last assessed this Node. + format: date-time type: string + nodeName: + description: nodeName is the name of the evaluated Node. + maxLength: 253 minLength: 1 - maxLength: 4096 - taintsToAdd: - description: taintsToAdd is the number of Nodes that currently lack the specified taint and would have it applied. - type: integer - format: int32 - minimum: 0 - taintsToRemove: - description: |- - taintsToRemove is the number of Nodes that currently possess the taint but no longer meet the criteria. - type: integer - format: int32 - minimum: 0 - failedNodes: - description: |- - failedNodes lists the Nodes where the rule evaluation encountered an error. - type: array - maxItems: 5000 - x-kubernetes-list-map-keys: - - nodeName - x-kubernetes-list-type: map - items: - type: object - required: - - lastEvaluationTime - - nodeName - properties: - lastEvaluationTime: - description: lastEvaluationTime is the timestamp of the last rule check failed for this Node. - type: string - format: date-time - message: - description: message is a human-readable message indicating details about the evaluation. - type: string - minLength: 1 - maxLength: 10240 - nodeName: - description: |- - nodeName is the name of the failed Node. - type: string - minLength: 1 - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - reason: - description: reason provides a brief explanation of the evaluation result. - type: string - minLength: 1 - maxLength: 256 - nodeEvaluations: - description: |- - nodeEvaluations provides detailed insight into the rule's assessment for individual Nodes. - type: array - maxItems: 5000 - x-kubernetes-list-map-keys: - - nodeName - x-kubernetes-list-type: map - items: - type: object - required: - - conditionResults - - lastEvaluationTime - - nodeName - - taintStatus - properties: - conditionResults: - description: |- - conditionResults provides a detailed breakdown of each condition evaluation for this Node. - type: array - maxItems: 5000 - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - items: - type: object - required: - - currentStatus - - requiredStatus - - type - properties: - currentStatus: - type: string - enum: - - "True" - - "False" - - Unknown - requiredStatus: - type: string - enum: - - "True" - - "False" - - Unknown - type: - type: string - minLength: 1 - maxLength: 316 - lastEvaluationTime: - description: lastEvaluationTime is the timestamp when the controller last assessed this Node. - type: string - format: date-time - nodeName: - description: nodeName is the name of the evaluated Node. - type: string - minLength: 1 - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - taintStatus: - description: taintStatus represents the taint status on the Node, one of Present, Absent. - type: string - enum: - - Present - - Absent - observedGeneration: - description: observedGeneration reflects the generation of the most recently observed NodeReadinessRule by the controller. - type: integer - format: int64 - minimum: 1 - subresources: - status: {} + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + taintStatus: + description: taintStatus represents the taint status on the + Node, one of Present, Absent. + enum: + - Present + - Absent + type: string + required: + - conditionResults + - lastEvaluationTime + - nodeName + - taintStatus + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + observedGeneration: + description: observedGeneration reflects the generation of the most + recently observed NodeReadinessRule by the controller. + format: int64 + minimum: 1 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nrr-controller/templates/rbac.yaml b/charts/nrr-controller/templates/rbac.yaml index a7308f0..485210d 100644 --- a/charts/nrr-controller/templates/rbac.yaml +++ b/charts/nrr-controller/templates/rbac.yaml @@ -8,15 +8,9 @@ metadata: labels: {{- include "nrr-controller.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -44,17 +38,17 @@ metadata: {{- include "nrr-controller.labels" . | nindent 4 }} rules: - apiGroups: [""] - resources: ["nodes"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + resources: ["events"] + verbs: ["create", "patch"] - apiGroups: [""] - resources: ["nodes/finalizers"] - verbs: ["update"] + resources: ["nodes"] + verbs: ["get", "list", "patch", "update", "watch"] - apiGroups: [""] resources: ["nodes/status"] - verbs: ["get", "patch", "update"] + verbs: ["get"] - apiGroups: ["readiness.node.x-k8s.io"] resources: ["nodereadinessrules"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + verbs: ["get", "list", "patch", "update", "watch"] - apiGroups: ["readiness.node.x-k8s.io"] resources: ["nodereadinessrules/finalizers"] verbs: ["update"] diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml index f2af1f3..517d92f 100644 --- a/charts/nrr-controller/values.yaml +++ b/charts/nrr-controller/values.yaml @@ -127,7 +127,11 @@ validatingWebhook: nodeSelector: {} -tolerations: [] +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists affinity: {} From 03252a6cc4d4d559645e072b16539aa895806bfa Mon Sep 17 00:00:00 2001 From: Sahitya Chandra Date: Fri, 8 May 2026 15:31:53 +0530 Subject: [PATCH 07/17] Fix Helm chart unit test assertions --- charts/nrr-controller/tests/annotations_test.yaml | 7 ++++--- charts/nrr-controller/tests/deployment_test.yaml | 4 ++-- charts/nrr-controller/tests/webhook_and_metrics_test.yaml | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/charts/nrr-controller/tests/annotations_test.yaml b/charts/nrr-controller/tests/annotations_test.yaml index adce08a..b549136 100644 --- a/charts/nrr-controller/tests/annotations_test.yaml +++ b/charts/nrr-controller/tests/annotations_test.yaml @@ -30,11 +30,12 @@ tests: path: spec.template.metadata.labels.team value: "platform" - - it: does not add metadata when not set + - it: adds default annotations and labels when not set template: templates/deployment.yaml asserts: - - isNull: - path: spec.template.metadata.annotations + - equal: + path: spec.template.metadata.annotations["kubectl.kubernetes.io/default-container"] + value: manager - isNotNull: path: spec.template.metadata.labels - equal: diff --git a/charts/nrr-controller/tests/deployment_test.yaml b/charts/nrr-controller/tests/deployment_test.yaml index d79439a..7a2d6b8 100644 --- a/charts/nrr-controller/tests/deployment_test.yaml +++ b/charts/nrr-controller/tests/deployment_test.yaml @@ -42,6 +42,6 @@ tests: enabled: true template: templates/deployment.yaml asserts: - - contains: + - equal: path: spec.template.spec.containers[0].ports[0].name - content: webhook-server + value: webhook-server diff --git a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml index 9fec19b..180945e 100644 --- a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml +++ b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml @@ -16,12 +16,12 @@ tests: - it: mounts webhook cert volume when webhook enabled template: templates/deployment.yaml asserts: - - contains: + - equal: path: spec.template.spec.volumes[0].name - content: cert + value: cert - it: mounts metrics cert volume when metrics.secure enabled template: templates/deployment.yaml asserts: - - contains: + - equal: path: spec.template.spec.volumes[?(@.name=="metrics-certs")].name - content: metrics-certs + value: metrics-certs From 5b86372a1ccb21c3312817091edd79007fe93d80 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Fri, 8 May 2026 12:24:09 +0200 Subject: [PATCH 08/17] Align pod annotation assertions with defaults --- charts/nrr-controller/README.md | 2 +- charts/nrr-controller/tests/annotations_test.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md index e7c4e89..ecc54b0 100644 --- a/charts/nrr-controller/README.md +++ b/charts/nrr-controller/README.md @@ -60,7 +60,7 @@ The following table lists the configurable parameters of the _node-readiness-con | `serviceAccount.create` | If `true`, create a service account | `true` | | `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | | `serviceAccount.annotations` | Specifies custom annotations for the serviceAccount | `{}` | -| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{}` | +| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{"kubectl.kubernetes.io/default-container":"manager"}` | | `podLabels` | Labels to add to the node-readiness-controller Pods | `{}` | | `commonLabels` | Labels to apply to all resources | `{}` | | `podSecurityContext` | Security context for pod | _see values.yaml_ | diff --git a/charts/nrr-controller/tests/annotations_test.yaml b/charts/nrr-controller/tests/annotations_test.yaml index b549136..786dcc9 100644 --- a/charts/nrr-controller/tests/annotations_test.yaml +++ b/charts/nrr-controller/tests/annotations_test.yaml @@ -23,6 +23,9 @@ tests: - equal: path: spec.template.metadata.annotations.description value: "test pod" + - equal: + path: spec.template.metadata.annotations["kubectl.kubernetes.io/default-container"] + value: manager - equal: path: spec.template.metadata.labels.environment value: "test" From 11699bf32816420e351230dd07b9df0703bf75e3 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Fri, 8 May 2026 12:27:37 +0200 Subject: [PATCH 09/17] Add boilerplate header to verify-chart.sh --- hack/verify-chart.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hack/verify-chart.sh b/hack/verify-chart.sh index df829fc..90603be 100755 --- a/hack/verify-chart.sh +++ b/hack/verify-chart.sh @@ -1 +1,17 @@ +#!/usr/bin/env bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ${CONTAINER_ENGINE:-docker} run -it --rm --network host --workdir=/data --volume ~/.kube/config:/root/.kube/config:ro --volume $(pwd):/data quay.io/helmpack/chart-testing:v3.7.0 /bin/bash -c "git config --global --add safe.directory /data; ct install --config=.github/ci/ct.yaml --helm-extra-set-args=\"--set=kind=Deployment\"" From 2f0d613401f8e1393b240e4d9f5b76f9a6b980e8 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 15:59:11 +0200 Subject: [PATCH 10/17] Fix wire validating webhook service port --- charts/nrr-controller/README.md | 2 +- .../templates/servicewebhook.yaml | 2 +- .../validatingwebhookconfiguration.yaml | 1 + .../tests/webhook_and_metrics_test.yaml | 41 +++++++++++++++++++ charts/nrr-controller/values.yaml | 2 +- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md index ecc54b0..c441b1a 100644 --- a/charts/nrr-controller/README.md +++ b/charts/nrr-controller/README.md @@ -77,7 +77,7 @@ The following table lists the configurable parameters of the _node-readiness-con | `metrics.certSecretName` | Name of the secret containing metrics server certificates | `metrics-server-cert` | | `webhook.enabled` | Enable the webhook server | `false` | | `webhook.port` | The port for the webhook server | `9443` | -| `webhook.service.port` | The port exposed by the webhook service | `8443` | +| `webhook.service.port` | The port exposed by the webhook service | `443` | | `webhook.service.targetPort` | The target port for the webhook service | `9443` | | `webhook.certDir` | Directory for webhook server certificates | `/tmp/k8s-webhook-server/serving-certs` | | `webhook.certSecretName` | Name of the secret containing webhook server certificates | `webhook-server-certs` | diff --git a/charts/nrr-controller/templates/servicewebhook.yaml b/charts/nrr-controller/templates/servicewebhook.yaml index 4a5c7ee..10dfc53 100644 --- a/charts/nrr-controller/templates/servicewebhook.yaml +++ b/charts/nrr-controller/templates/servicewebhook.yaml @@ -8,7 +8,7 @@ metadata: {{- include "nrr-controller.labels" . | nindent 4 }} spec: ports: - - name: https + - name: webhook port: {{ .Values.webhook.service.port }} protocol: TCP targetPort: {{ .Values.webhook.service.targetPort }} diff --git a/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml index 2a4407c..cb22b1d 100644 --- a/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml +++ b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml @@ -17,6 +17,7 @@ webhooks: service: name: {{ include "nrr-controller.fullname" . }}-webhook-service namespace: {{ include "nrr-controller.namespace" . }} + port: {{ .Values.webhook.service.port }} path: {{ .Values.validatingWebhook.path }} failurePolicy: {{ .Values.validatingWebhook.failurePolicy }} sideEffects: {{ .Values.validatingWebhook.sideEffects }} diff --git a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml index 180945e..ca554b1 100644 --- a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml +++ b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml @@ -9,10 +9,51 @@ release: set: webhook: enabled: true + validatingWebhook: + enabled: true metrics: secure: true tests: + - it: exposes webhook service on upstream service port + template: templates/servicewebhook.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.ports[0].name + value: webhook + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: wires validating webhook to webhook service port + template: templates/validatingwebhookconfiguration.yaml + asserts: + - equal: + path: webhooks[0].clientConfig.service.name + value: nrr-controller-webhook-service + - equal: + path: webhooks[0].clientConfig.service.port + value: 443 + - equal: + path: webhooks[0].clientConfig.service.path + value: /validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule + + - it: supports overriding webhook service port + template: templates/validatingwebhookconfiguration.yaml + set: + webhook: + service: + port: 8443 + asserts: + - equal: + path: webhooks[0].clientConfig.service.port + value: 8443 + - it: mounts webhook cert volume when webhook enabled template: templates/deployment.yaml asserts: diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml index 517d92f..7c16b11 100644 --- a/charts/nrr-controller/values.yaml +++ b/charts/nrr-controller/values.yaml @@ -97,7 +97,7 @@ webhook: enabled: false port: 9443 service: - port: 8443 + port: 443 targetPort: 9443 certDir: /tmp/k8s-webhook-server/serving-certs certSecretName: webhook-server-certs From 90557d2b269418ad4483e1a244113a2e2579a060 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 16:01:45 +0200 Subject: [PATCH 11/17] Render full NodeReadinessRule specs --- charts/nrr-controller/README.md | 6 +- .../templates/nodereadinessrules.yaml | 12 ++-- .../tests/nodereadinessrules_test.yaml | 57 +++++++++++++++++++ charts/nrr-controller/values.yaml | 3 +- 4 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 charts/nrr-controller/tests/nodereadinessrules_test.yaml diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md index c441b1a..c5eb015 100644 --- a/charts/nrr-controller/README.md +++ b/charts/nrr-controller/README.md @@ -29,6 +29,10 @@ The command deploys the _node-readiness-controller_ on the Kubernetes cluster in > **Tip**: List all releases using `helm list` +## CRD Upgrades + +Helm installs CRDs from the chart `crds/` directory during initial install, but Helm does not upgrade or delete CRDs from that directory during `helm upgrade` or `helm uninstall`. Before upgrading to a chart version that changes the `NodeReadinessRule` schema, apply the updated CRD from the release artifacts or from `charts/nrr-controller/crds`. + ## Uninstalling the Chart To uninstall/delete the `my-release` deployment: @@ -98,4 +102,4 @@ The following table lists the configurable parameters of the _node-readiness-con | `nodeSelector` | Node selectors to run the controller on specific nodes | `nil` | | `tolerations` | Tolerations to run the controller on specific nodes | `nil` | | `affinity` | Node affinity to run the controller on specific nodes | `nil` | -| `nodeReadinessRules` | Custom NodeReadinessRule resources to create. CRD must be preinstalled or installation will fail. | `[]` | +| `nodeReadinessRules` | Custom NodeReadinessRule resources to create. When validating webhooks are enabled, apply rules after the webhook is ready. | `[]` | diff --git a/charts/nrr-controller/templates/nodereadinessrules.yaml b/charts/nrr-controller/templates/nodereadinessrules.yaml index 82dd51f..b840021 100644 --- a/charts/nrr-controller/templates/nodereadinessrules.yaml +++ b/charts/nrr-controller/templates/nodereadinessrules.yaml @@ -1,17 +1,15 @@ {{- if .Values.nodeReadinessRules }} {{- range $rule := .Values.nodeReadinessRules }} +{{- $spec := omit $rule "name" }} +{{- if not (hasKey $spec "nodeSelector") }} +{{- $_ := set $spec "nodeSelector" (dict) }} +{{- end }} --- apiVersion: readiness.node.x-k8s.io/v1alpha1 kind: NodeReadinessRule metadata: name: {{ $rule.name | quote }} spec: - conditions: -{{- toYaml $rule.conditions | nindent 4 }} - taint: -{{- toYaml $rule.taint | nindent 4 }} - enforcementMode: {{ $rule.enforcementMode | quote }} - nodeSelector: -{{- toYaml $rule.nodeSelector | nindent 4 }} +{{- toYaml $spec | nindent 2 }} {{- end }} {{- end }} diff --git a/charts/nrr-controller/tests/nodereadinessrules_test.yaml b/charts/nrr-controller/tests/nodereadinessrules_test.yaml new file mode 100644 index 0000000..2e1b3b6 --- /dev/null +++ b/charts/nrr-controller/tests/nodereadinessrules_test.yaml @@ -0,0 +1,57 @@ +suite: Test Node Readiness Controller NodeReadinessRules + +templates: + - templates/nodereadinessrules.yaml + +release: + name: nrr-controller + +tests: + - it: renders full rule spec and defaults nodeSelector when omitted + set: + nodeReadinessRules: + - name: kube-proxy-unhealthy-noschedule + dryRun: true + conditions: + - type: KubeProxyUnhealthy + requiredStatus: "False" + taint: + key: readiness.k8s.io/KubeProxyUnhealthy + value: "true" + effect: NoSchedule + enforcementMode: continuous + asserts: + - equal: + path: metadata.name + value: kube-proxy-unhealthy-noschedule + - equal: + path: spec.dryRun + value: true + - equal: + path: spec.nodeSelector + value: {} + - equal: + path: spec.conditions[0].type + value: KubeProxyUnhealthy + - equal: + path: spec.taint.key + value: readiness.k8s.io/KubeProxyUnhealthy + + - it: preserves provided nodeSelector + set: + nodeReadinessRules: + - name: linux-nodes + conditions: + - type: KubeProxyUnhealthy + requiredStatus: "False" + taint: + key: readiness.k8s.io/KubeProxyUnhealthy + effect: NoSchedule + enforcementMode: continuous + nodeSelector: + matchLabels: + kubernetes.io/os: linux + asserts: + - equal: + path: spec.nodeSelector.matchLabels["kubernetes.io/os"] + value: linux diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml index 7c16b11..3efc458 100644 --- a/charts/nrr-controller/values.yaml +++ b/charts/nrr-controller/values.yaml @@ -135,8 +135,7 @@ tolerations: affinity: {} -# NOTE: Before enabling these rules. The CRD `nodereadinessrules.readiness.node.x-k8s.io` must already be installed in the cluster -# If the CRD is not installed, Helm will fail during installation. +# NOTE: When using validating webhooks, apply NodeReadinessRule resources after the controller webhook is ready, for example in a separate helm upgrade. nodeReadinessRules: [] # - name: kube-proxy-unhealthy-noschedule # conditions: From f27dd4d5941ae4695681b7d384ef7a3ef6946727 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 16:06:23 +0200 Subject: [PATCH 12/17] Add CRD drift verification --- .github/workflows/helm.yaml | 12 ++++++++ ...eadinessrules.readiness.node.x-k8s.io.yaml | 1 + hack/verify-all.sh | 1 + hack/verify-chart-drift.sh | 30 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100755 hack/verify-chart-drift.sh diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index 73f809a..b70c531 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -6,14 +6,22 @@ on: - main - release-* paths: + - 'api/**' - 'charts/**' + - 'config/crd/**' - '.github/workflows/helm.yaml' - '.github/ci/ct.yaml' + - 'hack/verify-chart-drift.sh' + - 'Makefile' pull_request: paths: + - 'api/**' - 'charts/**' + - 'config/crd/**' - '.github/workflows/helm.yaml' - '.github/ci/ct.yaml' + - 'hack/verify-chart-drift.sh' + - 'Makefile' jobs: lint-and-test: @@ -37,6 +45,10 @@ jobs: with: go-version-file: 'go.mod' + - name: Run chart drift check + run: | + hack/verify-chart-drift.sh + - name: Set up chart-testing uses: helm/chart-testing-action@v2.8.0 with: diff --git a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml index cf914d3..ac68b93 100644 --- a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml +++ b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/hack/verify-all.sh b/hack/verify-all.sh index 52b9bc9..e466537 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -20,5 +20,6 @@ set -o pipefail # Run all verification scripts hack/verify-boilerplate.sh +hack/verify-chart-drift.sh hack/verify-links.sh hack/verify-govulncheck.sh diff --git a/hack/verify-chart-drift.sh b/hack/verify-chart-drift.sh new file mode 100755 index 0000000..da4f7c2 --- /dev/null +++ b/hack/verify-chart-drift.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks that the Helm chart CRDs match controller-gen output. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT="$(dirname "${BASH_SOURCE[0]}")/.." +cd "${KUBE_ROOT}" + +make manifests + +diff -u \ + config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml \ + charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml From b9e97e698290112047ce7305e3ffb212f182e476 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 16:08:30 +0200 Subject: [PATCH 13/17] Reuse existing kind multi-node config --- Makefile | 2 +- hack/kind_config.yaml | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 hack/kind_config.yaml diff --git a/Makefile b/Makefile index e41fbbb..ebea31f 100644 --- a/Makefile +++ b/Makefile @@ -505,7 +505,7 @@ build-helm: helm package ./charts/nrr-controller --dependency-update --destination ./bin/chart kind-multi-node: - kind create cluster --name kind --config ./hack/kind_config.yaml --wait 2m + kind create cluster --name $(KIND_CLUSTER) --config ./config/testing/kind/kind-3node-config.yaml --wait 2m ct-helm: ./hack/verify-chart.sh diff --git a/hack/kind_config.yaml b/hack/kind_config.yaml deleted file mode 100644 index c405702..0000000 --- a/hack/kind_config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane -- role: worker - kubeadmConfigPatches: - - | - kind: JoinConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "topology.kubernetes.io/zone=local-a" -- role: worker - kubeadmConfigPatches: - - | - kind: JoinConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "topology.kubernetes.io/zone=local-b" From 78cbb1416a18e0565880a536864c64cb30bbe2b8 Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 16:40:21 +0200 Subject: [PATCH 14/17] Add dryRun to rule example --- charts/nrr-controller/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml index 3efc458..1e65357 100644 --- a/charts/nrr-controller/values.yaml +++ b/charts/nrr-controller/values.yaml @@ -138,6 +138,7 @@ affinity: {} # NOTE: When using validating webhooks, apply NodeReadinessRule resources after the controller webhook is ready, for example in a separate helm upgrade. nodeReadinessRules: [] # - name: kube-proxy-unhealthy-noschedule + # dryRun: false # conditions: # - type: "KubeProxyUnhealthy" # requiredStatus: "False" From 9667275c9decc9d35f9a7f7510ef1f949307954e Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Mon, 11 May 2026 17:00:06 +0200 Subject: [PATCH 15/17] Sync chart CRD with generated output --- .../nodereadinessrules.readiness.node.x-k8s.io.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml index ac68b93..1ae7cb8 100644 --- a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml +++ b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml @@ -17,8 +17,7 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - description: Continuous, Bootstrap or Dryrun - shows if the rule is in enforcement - or dryrun mode. + - description: 'The enforcement mode of the rule: bootstrap-only or continuous.' jsonPath: .spec.enforcementMode name: Mode type: string @@ -26,6 +25,14 @@ spec: jsonPath: .spec.taint.key name: Taint type: string + - description: 'The taint effect: NoSchedule, PreferNoSchedule or NoExecute.' + jsonPath: .spec.taint.effect + name: Effect + type: string + - description: Whether the rule is in dry-run mode and only previews taint changes. + jsonPath: .spec.dryRun + name: DryRun + type: boolean - description: The age of this resource jsonPath: .metadata.creationTimestamp name: Age From 296ea331ccb228bbf071f71d3ee8c2c50a2b85ff Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Fri, 15 May 2026 11:16:24 +0200 Subject: [PATCH 16/17] Set leaderElection enabled by default --- charts/nrr-controller/README.md | 2 +- charts/nrr-controller/tests/deployment_test.yaml | 5 +---- charts/nrr-controller/values.yaml | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md index c5eb015..1914d36 100644 --- a/charts/nrr-controller/README.md +++ b/charts/nrr-controller/README.md @@ -57,7 +57,7 @@ The following table lists the configurable parameters of the _node-readiness-con | `fullnameOverride` | String to fully override `nrr-controller.fullname` template | `""` | | `namespaceOverride` | Override the deployment namespace; defaults to .Release.Namespace | `""` | | `replicaCount` | The replica count for Deployment | `1` | -| `leaderElection.enabled` | Enable leader election to support multiple replicas | `false` | +| `leaderElection.enabled` | Enable leader election to support multiple replicas | `true` | | `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | | `rbac.create` | If `true`, create & use RBAC resources | `true` | | `resources` | Node Readiness Controller container CPU and memory requests/limits | _see values.yaml_ | diff --git a/charts/nrr-controller/tests/deployment_test.yaml b/charts/nrr-controller/tests/deployment_test.yaml index 7a2d6b8..eb38d9d 100644 --- a/charts/nrr-controller/tests/deployment_test.yaml +++ b/charts/nrr-controller/tests/deployment_test.yaml @@ -16,10 +16,7 @@ tests: - isKind: of: Deployment - - it: enables leader-election - set: - leaderElection: - enabled: true + - it: enables leader-election by default template: templates/deployment.yaml asserts: - contains: diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml index 1e65357..3a1e19f 100644 --- a/charts/nrr-controller/values.yaml +++ b/charts/nrr-controller/values.yaml @@ -29,7 +29,6 @@ namespaceOverride: "" commonLabels: {} # Specifies the replica count for Deployment -# Enable leaderElection if you want to use more than 1 replica # Set affinity.podAntiAffinity rule if you want to schedule onto a node replicaCount: 1 @@ -65,7 +64,7 @@ serviceAccount: # Enable leader election to support multiple replicas leaderElection: - enabled: false + enabled: true # Enable the webhook server healthProbeBindAddress: ":8081" From 38feff0c5fe4f5a3c6a16c74f33332c96aaaa36b Mon Sep 17 00:00:00 2001 From: Hong Hai Nguyen Date: Fri, 15 May 2026 11:17:37 +0200 Subject: [PATCH 17/17] Remove extra arg - helm test --- .github/workflows/helm.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index b70c531..01a6ff4 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -77,6 +77,5 @@ jobs: - name: Create multi node Kind cluster run: make kind-multi-node - # helm-extra-set-args only available after ct 3.6.0 - name: Run chart-testing (install) - run: ct install --config=.github/ci/ct.yaml --helm-extra-set-args='--set=kind=Deployment' + run: ct install --config=.github/ci/ct.yaml