diff --git a/.github/ci/ct.yaml b/.github/ci/ct.yaml new file mode 100644 index 0000000..983c22a --- /dev/null +++ b/.github/ci/ct.yaml @@ -0,0 +1,5 @@ +chart-dirs: + - charts +helm-extra-args: "--timeout=5m" +check-version-increment: false +target-branch: main diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 0000000..01a6ff4 --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,81 @@ +name: Helm + +on: + push: + branches: + - main + - release-* + paths: + - 'api/**' + - 'charts/**' + - 'config/crd/**' + - '.github/workflows/helm.yaml' + - '.github/ci/ct.yaml' + - 'hack/verify-chart-drift.sh' + - 'Makefile' + pull_request: + paths: + - 'api/**' + - 'charts/**' + - 'config/crd/**' + - '.github/workflows/helm.yaml' + - '.github/ci/ct.yaml' + - 'hack/verify-chart-drift.sh' + - 'Makefile' + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v4.2.0 + with: + version: v3.15.1 + + - uses: actions/setup-python@v5.1.1 + with: + python-version: 3.12 + + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Run chart drift check + run: | + hack/verify-chart-drift.sh + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.8.0 + with: + version: v3.11.0 + + - name: Install Helm Unit Test Plugin + run: | + helm plugin install --version 1.0.3 https://github.com/helm-unittest/helm-unittest + + - name: Run Helm Unit Tests + run: | + helm unittest charts/nrr-controller --strict -d + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config=.github/ci/ct.yaml) + if [[ -n "$changed" ]]; then + echo "changed=true" >> $GITHUB_OUTPUT + fi + + - name: Run chart-testing (lint) + run: ct lint --config=.github/ci/ct.yaml --validate-maintainers=false + + # Need a multi node cluster so controller can run with leadership + - name: Create multi node Kind cluster + run: make kind-multi-node + + - name: Run chart-testing (install) + run: ct install --config=.github/ci/ct.yaml diff --git a/Makefile b/Makefile index 0204bc0..ebea31f 100644 --- a/Makefile +++ b/Makefile @@ -489,4 +489,23 @@ crd-ref-docs: --source-path=${PWD}/api/v1alpha1/ \ --config=crd-ref-docs.yaml \ --renderer=markdown \ - --output-path=${PWD}/docs/book/src/reference/api-spec.md \ No newline at end of file + --output-path=${PWD}/docs/book/src/reference/api-spec.md + +# helm + +ensure-helm-install: +ifndef HAS_HELM + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && chmod 700 ./get_helm.sh && ./get_helm.sh +endif + +lint-chart: ensure-helm-install + helm lint ./charts/nrr-controller + +build-helm: + helm package ./charts/nrr-controller --dependency-update --destination ./bin/chart + +kind-multi-node: + kind create cluster --name $(KIND_CLUSTER) --config ./config/testing/kind/kind-3node-config.yaml --wait 2m + +ct-helm: + ./hack/verify-chart.sh diff --git a/charts/nrr-controller/.helmignore b/charts/nrr-controller/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/nrr-controller/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/nrr-controller/Chart.yaml b/charts/nrr-controller/Chart.yaml new file mode 100644 index 0000000..1b721f2 --- /dev/null +++ b/charts/nrr-controller/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: nrr-controller +description: A Helm chart for the Node Readiness Controller +type: application +version: 0.1.0 +appVersion: "v0.3.0" +kubeVersion: ">=1.25.0-0" +home: https://github.com/kubernetes-sigs/node-readiness-controller +sources: + - https://github.com/kubernetes-sigs/node-readiness-controller +keywords: + - kubernetes + - controller + - readiness + - node diff --git a/charts/nrr-controller/README.md b/charts/nrr-controller/README.md new file mode 100644 index 0000000..1914d36 --- /dev/null +++ b/charts/nrr-controller/README.md @@ -0,0 +1,105 @@ +# Node Readiness Controller for Kubernetes + +[Node Readiness Controller](https://github.com/kubernetes-sigs/node-readiness-controller) for Kubernetes a Kubernetes controller that provides fine-grained, declarative readiness for nodes. It ensures nodes only accept workloads when all required components eg: network agents, GPU drivers, storage drivers or custom health-checks, are fully ready on the node. + +## TL;DR: + +```shell +helm repo add node-readiness-controller https://kubernetes-sigs.github.io/node-readiness-controller/ +helm install my-release --namespace kube-system node-readiness-controller/nrr-controller +``` + +## Introduction + +This chart bootstraps a [node-readiness-controller](https://github.com/kubernetes-sigs/node-readiness-controller) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.25+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```shell +helm install --namespace kube-system my-release node-readiness-controller/nrr-controller +``` + +The command deploys the _node-readiness-controller_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## CRD Upgrades + +Helm installs CRDs from the chart `crds/` directory during initial install, but Helm does not upgrade or delete CRDs from that directory during `helm upgrade` or `helm uninstall`. Before upgrading to a chart version that changes the `NodeReadinessRule` schema, apply the updated CRD from the release artifacts or from `charts/nrr-controller/crds`. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```shell +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the _node-readiness-controller_ chart and their default values. + +| Parameter | Description | Default | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| `image.repository` | Docker repository to use | `registry.k8s.io/node-readiness-controller/node-readiness-controller` | +| `image.tag` | Docker tag to use | `v[chart appVersion]` | +| `image.pullPolicy` | Docker image pull policy | `IfNotPresent` | +| `imagePullSecrets` | Docker repository secrets | `[]` | +| `nameOverride` | String to partially override `nrr-controller.fullname` template (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override `nrr-controller.fullname` template | `""` | +| `namespaceOverride` | Override the deployment namespace; defaults to .Release.Namespace | `""` | +| `replicaCount` | The replica count for Deployment | `1` | +| `leaderElection.enabled` | Enable leader election to support multiple replicas | `true` | +| `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | +| `rbac.create` | If `true`, create & use RBAC resources | `true` | +| `resources` | Node Readiness Controller container CPU and memory requests/limits | _see values.yaml_ | +| `serviceAccount.create` | If `true`, create a service account | `true` | +| `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | +| `serviceAccount.annotations` | Specifies custom annotations for the serviceAccount | `{}` | +| `podAnnotations` | Annotations to add to the node-readiness-controller Pods | `{"kubectl.kubernetes.io/default-container":"manager"}` | +| `podLabels` | Labels to add to the node-readiness-controller Pods | `{}` | +| `commonLabels` | Labels to apply to all resources | `{}` | +| `podSecurityContext` | Security context for pod | _see values.yaml_ | +| `securityContext` | Security context for container | _see values.yaml_ | +| `terminationGracePeriodSeconds` | Time to wait before forcefully terminating the pod | `10` | +| `healthProbeBindAddress` | The bind address for health probes | `:8081` | +| `livenessProbe` | Liveness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `readinessProbe` | Readiness probe configuration for the node-readiness-controller container | _see values.yaml_ | +| `metrics.secure` | Enable secure metrics endpoint | `false` | +| `metrics.bindAddress` | The bind address for metrics server | `:8443` | +| `metrics.service.port` | The port exposed by the metrics service | `8443` | +| `metrics.service.targetPort` | The target port for the metrics service | `8443` | +| `metrics.certDir` | Directory for metrics server certificates | `/tmp/k8s-metrics-server/metrics-certs` | +| `metrics.certSecretName` | Name of the secret containing metrics server certificates | `metrics-server-cert` | +| `webhook.enabled` | Enable the webhook server | `false` | +| `webhook.port` | The port for the webhook server | `9443` | +| `webhook.service.port` | The port exposed by the webhook service | `443` | +| `webhook.service.targetPort` | The target port for the webhook service | `9443` | +| `webhook.certDir` | Directory for webhook server certificates | `/tmp/k8s-webhook-server/serving-certs` | +| `webhook.certSecretName` | Name of the secret containing webhook server certificates | `webhook-server-certs` | +| `certManager.enabled` | Enable cert-manager integration for automatic TLS certificate generation | `false` | +| `certManager.issuer.create` | Create a cert-manager issuer | `true` | +| `certManager.issuer.name` | Name of the cert-manager issuer | `selfsigned-issuer` | +| `certManager.metricsCertificate.create` | Create a cert-manager certificate for metrics server | `true` | +| `certManager.metricsCertificate.name` | Name of the metrics certificate | `metrics-certs` | +| `certManager.webhookCertificate.create` | Create a cert-manager certificate for webhook server | `true` | +| `certManager.webhookCertificate.name` | Name of the webhook certificate | `serving-cert` | +| `validatingWebhook.enabled` | Enable the validating webhook | `false` | +| `validatingWebhook.name` | Name of the ValidatingWebhookConfiguration resource | `validating-webhook-configuration` | +| `validatingWebhook.webhookName` | Name of the webhook | `vnodereadinessrule.kb.io` | +| `validatingWebhook.failurePolicy` | Failure policy for the webhook | `Fail` | +| `validatingWebhook.sideEffects` | Side effects for the webhook | `None` | +| `validatingWebhook.path` | The path for the webhook | `/validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule` | +| `validatingWebhook.admissionReviewVersions` | Admission review versions supported by the webhook | `["v1"]` | +| `nodeSelector` | Node selectors to run the controller on specific nodes | `nil` | +| `tolerations` | Tolerations to run the controller on specific nodes | `nil` | +| `affinity` | Node affinity to run the controller on specific nodes | `nil` | +| `nodeReadinessRules` | Custom NodeReadinessRule resources to create. When validating webhooks are enabled, apply rules after the webhook is ready. | `[]` | diff --git a/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml new file mode 100644 index 0000000..1ae7cb8 --- /dev/null +++ b/charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml @@ -0,0 +1,433 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: nodereadinessrules.readiness.node.x-k8s.io +spec: + group: readiness.node.x-k8s.io + names: + kind: NodeReadinessRule + listKind: NodeReadinessRuleList + plural: nodereadinessrules + shortNames: + - nrr + singular: nodereadinessrule + scope: Cluster + versions: + - additionalPrinterColumns: + - description: 'The enforcement mode of the rule: bootstrap-only or continuous.' + jsonPath: .spec.enforcementMode + name: Mode + type: string + - description: The readiness taint applied by this rule. + jsonPath: .spec.taint.key + name: Taint + type: string + - description: 'The taint effect: NoSchedule, PreferNoSchedule or NoExecute.' + jsonPath: .spec.taint.effect + name: Effect + type: string + - description: Whether the rule is in dry-run mode and only previews taint changes. + jsonPath: .spec.dryRun + name: DryRun + type: boolean + - description: The age of this resource + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeReadinessRule is the Schema for the NodeReadinessRules API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of NodeReadinessRule + properties: + conditions: + description: |- + conditions contains a list of the Node conditions that defines the specific + criteria that must be met for taints to be managed on the target Node. + The presence or status of these conditions directly triggers the application or removal of Node taints. + items: + description: |- + ConditionRequirement defines a specific Node condition and the status value + required to trigger the controller's action. + properties: + requiredStatus: + description: requiredStatus is status of the condition, one + of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of Node condition + + Following kubebuilder validation is referred from https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition + maxLength: 316 + minLength: 1 + type: string + required: + - requiredStatus + - type + type: object + maxItems: 32 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: conditions is immutable + rule: self == oldSelf + dryRun: + description: |- + dryRun when set to true, The controller will evaluate Node conditions and log intended taint modifications + without persisting changes to the cluster. Proposed actions are reflected in the resource status. + type: boolean + enforcementMode: + description: |- + enforcementMode specifies how the controller maintains the desired state. + enforcementMode is one of bootstrap-only, continuous. + "bootstrap-only" applies the configuration once during initial setup. + "continuous" ensures the state is monitored and corrected throughout the resource lifecycle. + enum: + - bootstrap-only + - continuous + type: string + x-kubernetes-validations: + - message: enforcementMode is immutable + rule: self == oldSelf + nodeSelector: + description: nodeSelector limits the scope of this rule to a specific + subset of Nodes. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: nodeSelector is immutable + rule: self == oldSelf + taint: + description: |- + taint defines the specific Taint (Key, Value, and Effect) to be managed + on Nodes that meet the defined condition criteria. + + The taint key must follow Kubernetes qualified name format: prefix/name + where prefix is 'readiness.k8s.io' (DNS subdomain) and name is a qualified + name (max 63 chars, alphanumeric, '-', '_', '.', must start and end with alphanumeric). + ref: git.k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/api/validate/content/kube.go#L24-L72 + + Supported effects: NoSchedule, PreferNoSchedule, NoExecute. + Caution: NoExecute evicts existing pods and can cause significant disruption + when combined with continuous enforcement mode. Prefer NoSchedule for most use cases. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + x-kubernetes-validations: + - message: taint key must start with 'readiness.k8s.io/' + rule: self.key.startsWith('readiness.k8s.io/') + - message: taint key length must be at most 253 characters + rule: self.key.size() <= 253 + - message: taint key must have exactly one '/' separator (prefix/name + format) + rule: size(self.key.split('/')) == 2 + - message: taint key name part must be 1-63 characters + rule: size(self.key.split('/')[1]) > 0 && size(self.key.split('/')[1]) + <= 63 + - message: taint key name part must consist of alphanumeric characters, + '-', '_' or '.', and must start and end with an alphanumeric character + rule: self.key.split('/')[1].matches('^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$') + - message: taint value length must be at most 63 characters + rule: '!has(self.value) || self.value.size() <= 63' + - message: taint effect must be one of 'NoSchedule', 'PreferNoSchedule', + 'NoExecute' + rule: self.effect in ['NoSchedule', 'PreferNoSchedule', 'NoExecute'] + - message: taint key is immutable + rule: '!has(oldSelf.key) || self.key == oldSelf.key' + - message: taint effect is immutable + rule: '!has(oldSelf.effect) || self.effect == oldSelf.effect' + - message: taint value is immutable + rule: '!has(oldSelf.value) || self.value == oldSelf.value' + required: + - conditions + - enforcementMode + - nodeSelector + - taint + type: object + status: + description: status defines the observed state of NodeReadinessRule + minProperties: 1 + properties: + appliedNodes: + description: |- + appliedNodes lists the names of Nodes where the taint has been successfully managed. + This provides a quick reference to the scope of impact for this rule. + items: + maxLength: 253 + type: string + maxItems: 5000 + type: array + x-kubernetes-list-type: set + dryRunResults: + description: |- + dryRunResults captures the outcome of the rule evaluation when DryRun is enabled. + This field provides visibility into the actions the controller would have taken, + allowing users to preview taint changes before they are committed. + minProperties: 1 + properties: + affectedNodes: + description: affectedNodes is the total count of Nodes that match + the rule's criteria. + format: int32 + minimum: 0 + type: integer + riskyOperations: + description: |- + riskyOperations represents the count of Nodes where required conditions + are missing entirely, potentially indicating an ambiguous node state. + format: int32 + minimum: 0 + type: integer + summary: + description: |- + summary provides a human-readable overview of the dry run evaluation, + highlighting key findings or warnings. + maxLength: 4096 + minLength: 1 + type: string + taintsToAdd: + description: taintsToAdd is the number of Nodes that currently + lack the specified taint and would have it applied. + format: int32 + minimum: 0 + type: integer + taintsToRemove: + description: |- + taintsToRemove is the number of Nodes that currently possess the + taint but no longer meet the criteria, leading to its removal. + format: int32 + minimum: 0 + type: integer + required: + - summary + type: object + failedNodes: + description: |- + failedNodes lists the Nodes where the rule evaluation encountered an error. + This is used for troubleshooting configuration issues, such as invalid selectors during node lookup. + items: + description: NodeFailure provides diagnostic details for Nodes that + could not be successfully evaluated by the rule. + properties: + lastEvaluationTime: + description: lastEvaluationTime is the timestamp of the last + rule check failed for this Node. + format: date-time + type: string + message: + description: message is a human-readable message indicating + details about the evaluation. + maxLength: 10240 + minLength: 1 + type: string + nodeName: + description: |- + nodeName is the name of the failed Node. + + Following kubebuilder validation is referred from + https://github.com/kubernetes/apimachinery/blob/84d740c9e27f3ccc94c8bc4d13f1b17f60f7080b/pkg/util/validation/validation.go#L198 + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + reason: + description: reason provides a brief explanation of the evaluation + result. + maxLength: 256 + minLength: 1 + type: string + required: + - lastEvaluationTime + - nodeName + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + nodeEvaluations: + description: |- + nodeEvaluations provides detailed insight into the rule's assessment for individual Nodes. + This is primarily used for auditing and debugging why specific Nodes were or + were not targeted by the rule. + items: + description: NodeEvaluation provides a detailed audit of a single + Node's compliance with the rule. + properties: + conditionResults: + description: |- + conditionResults provides a detailed breakdown of each condition evaluation + for this Node. This allows for granular auditing of which specific + criteria passed or failed during the rule assessment. + items: + description: |- + ConditionEvaluationResult provides a detailed report of the comparison between + the Node's observed condition and the rule's requirement. + properties: + currentStatus: + description: currentStatus is the actual status value + observed on the Node, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + requiredStatus: + description: requiredStatus is the status value defined + in the rule that must be matched, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type corresponds to the Node condition type + being evaluated. + maxLength: 316 + minLength: 1 + type: string + required: + - currentStatus + - requiredStatus + - type + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastEvaluationTime: + description: lastEvaluationTime is the timestamp when the controller + last assessed this Node. + format: date-time + type: string + nodeName: + description: nodeName is the name of the evaluated Node. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + taintStatus: + description: taintStatus represents the taint status on the + Node, one of Present, Absent. + enum: + - Present + - Absent + type: string + required: + - conditionResults + - lastEvaluationTime + - nodeName + - taintStatus + type: object + maxItems: 5000 + type: array + x-kubernetes-list-map-keys: + - nodeName + x-kubernetes-list-type: map + observedGeneration: + description: observedGeneration reflects the generation of the most + recently observed NodeReadinessRule by the controller. + format: int64 + minimum: 1 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nrr-controller/templates/_helpers.tpl b/charts/nrr-controller/templates/_helpers.tpl new file mode 100644 index 0000000..7dffee1 --- /dev/null +++ b/charts/nrr-controller/templates/_helpers.tpl @@ -0,0 +1,76 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "nrr-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nrr-controller.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := include "nrr-controller.name" . -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Expand the namespace of the release. +Allows overriding it for multi-namespace deployments in combined charts. +*/}} +{{- define "nrr-controller.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nrr-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "nrr-controller.labels" -}} +app.kubernetes.io/name: {{ include "nrr-controller.name" . }} +helm.sh/chart: {{ include "nrr-controller.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "nrr-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nrr-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +control-plane: controller-manager +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nrr-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "nrr-controller.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/nrr-controller/templates/certificates.yaml b/charts/nrr-controller/templates/certificates.yaml new file mode 100644 index 0000000..174877f --- /dev/null +++ b/charts/nrr-controller/templates/certificates.yaml @@ -0,0 +1,37 @@ +{{- if .Values.certManager.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.metricsCertificate.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + secretName: {{ .Values.metrics.certSecretName }} + dnsNames: + - {{ include "nrr-controller.fullname" . }}-metrics-service.{{ include "nrr-controller.namespace" . }}.svc + - {{ include "nrr-controller.fullname" . }}-metrics-service.{{ include "nrr-controller.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} + +{{- if and .Values.webhook.enabled .Values.certManager.webhookCertificate.create }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.webhookCertificate.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + secretName: {{ .Values.webhook.certSecretName }} + dnsNames: + - {{ include "nrr-controller.fullname" . }}-webhook-service.{{ include "nrr-controller.namespace" . }}.svc + - {{ include "nrr-controller.fullname" . }}-webhook-service.{{ include "nrr-controller.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} +{{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/deployment.yaml b/charts/nrr-controller/templates/deployment.yaml new file mode 100644 index 0000000..8c66a85 --- /dev/null +++ b/charts/nrr-controller/templates/deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "nrr-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.podAnnotations }} + {{- .Values.podAnnotations | toYaml | nindent 10 }} + {{- end }} + labels: + {{- include "nrr-controller.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "nrr-controller.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: manager + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /manager + args: + {{- if .Values.leaderElection.enabled }} + - --leader-elect + {{- end }} + - --health-probe-bind-address={{ .Values.healthProbeBindAddress }} + - --enable-webhook={{ ternary "true" "false" .Values.webhook.enabled }} + - --metrics-bind-address={{ .Values.metrics.bindAddress }} + {{- if .Values.metrics.secure }} + - --metrics-secure + - --metrics-cert-dir={{ .Values.metrics.certDir }} + {{- end }} + {{- if .Values.webhook.enabled }} + ports: + - name: webhook-server + containerPort: {{ .Values.webhook.port }} + protocol: TCP + {{- end }} + livenessProbe: + httpGet: + path: {{ .Values.livenessProbe.path }} + port: 8081 + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.path }} + port: 8081 + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + {{- if .Values.webhook.enabled }} + - name: cert + mountPath: {{ .Values.webhook.certDir }} + readOnly: true + {{- end }} + {{- if .Values.metrics.secure }} + - name: metrics-certs + mountPath: {{ .Values.metrics.certDir }} + readOnly: true + {{- end }} + volumes: + {{- if .Values.webhook.enabled }} + - name: cert + secret: + secretName: {{ .Values.webhook.certSecretName }} + defaultMode: 420 + {{- end }} + {{- if .Values.metrics.secure }} + - name: metrics-certs + secret: + secretName: {{ .Values.metrics.certSecretName }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/nrr-controller/templates/issuer.yaml b/charts/nrr-controller/templates/issuer.yaml new file mode 100644 index 0000000..eed5c29 --- /dev/null +++ b/charts/nrr-controller/templates/issuer.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.certManager.enabled .Values.certManager.issuer.create }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.issuer.name }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} diff --git a/charts/nrr-controller/templates/nodereadinessrules.yaml b/charts/nrr-controller/templates/nodereadinessrules.yaml new file mode 100644 index 0000000..b840021 --- /dev/null +++ b/charts/nrr-controller/templates/nodereadinessrules.yaml @@ -0,0 +1,15 @@ +{{- if .Values.nodeReadinessRules }} +{{- range $rule := .Values.nodeReadinessRules }} +{{- $spec := omit $rule "name" }} +{{- if not (hasKey $spec "nodeSelector") }} +{{- $_ := set $spec "nodeSelector" (dict) }} +{{- end }} +--- +apiVersion: readiness.node.x-k8s.io/v1alpha1 +kind: NodeReadinessRule +metadata: + name: {{ $rule.name | quote }} +spec: +{{- toYaml $spec | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/rbac.yaml b/charts/nrr-controller/templates/rbac.yaml new file mode 100644 index 0000000..485210d --- /dev/null +++ b/charts/nrr-controller/templates/rbac.yaml @@ -0,0 +1,161 @@ +{{- if .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "nrr-controller.fullname" . }}-leader-election-role + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-leader-election-rolebinding + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nrr-controller.fullname" . }}-leader-election-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "update", "watch"] + - apiGroups: [""] + resources: ["nodes/status"] + verbs: ["get"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["get", "list", "patch", "update", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/finalizers"] + verbs: ["update"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get", "patch", "update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-manager-rolebinding + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "nrr-controller.fullname" . }}-manager-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-rolebinding + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "nrr-controller.fullname" . }}-metrics-auth-role +subjects: + - kind: ServiceAccount + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-reader + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - nonResourceURLs: ["/metrics"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-admin-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["*"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-editor-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nrr-controller.fullname" . }}-nodereadinessrule-viewer-role + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +rules: + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules"] + verbs: ["get", "list", "watch"] + - apiGroups: ["readiness.node.x-k8s.io"] + resources: ["nodereadinessrules/status"] + verbs: ["get"] +{{- end }} diff --git a/charts/nrr-controller/templates/serviceaccount.yaml b/charts/nrr-controller/templates/serviceaccount.yaml new file mode 100644 index 0000000..61f2d55 --- /dev/null +++ b/charts/nrr-controller/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nrr-controller.serviceAccountName" . }} + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nrr-controller/templates/servicemetrics.yaml b/charts/nrr-controller/templates/servicemetrics.yaml new file mode 100644 index 0000000..8ca74ce --- /dev/null +++ b/charts/nrr-controller/templates/servicemetrics.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nrr-controller.fullname" . }}-metrics-service + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + ports: + - name: https + port: {{ .Values.metrics.service.port }} + protocol: TCP + targetPort: {{ .Values.metrics.service.targetPort }} + selector: + {{- include "nrr-controller.selectorLabels" . | nindent 4 }} diff --git a/charts/nrr-controller/templates/servicewebhook.yaml b/charts/nrr-controller/templates/servicewebhook.yaml new file mode 100644 index 0000000..10dfc53 --- /dev/null +++ b/charts/nrr-controller/templates/servicewebhook.yaml @@ -0,0 +1,17 @@ +{{- if .Values.webhook.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nrr-controller.fullname" . }}-webhook-service + namespace: {{ include "nrr-controller.namespace" . }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} +spec: + ports: + - name: webhook + port: {{ .Values.webhook.service.port }} + protocol: TCP + targetPort: {{ .Values.webhook.service.targetPort }} + selector: + {{- include "nrr-controller.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml new file mode 100644 index 0000000..cb22b1d --- /dev/null +++ b/charts/nrr-controller/templates/validatingwebhookconfiguration.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.webhook.enabled .Values.validatingWebhook.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "nrr-controller.fullname" . }}-{{ .Values.validatingWebhook.name }} + labels: + {{- include "nrr-controller.labels" . | nindent 4 }} + {{- if .Values.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "nrr-controller.namespace" . }}/{{ include "nrr-controller.fullname" . }}-{{ .Values.certManager.webhookCertificate.name }} + {{- end }} +webhooks: + - name: {{ .Values.validatingWebhook.webhookName }} + admissionReviewVersions: + {{- toYaml .Values.validatingWebhook.admissionReviewVersions | nindent 6 }} + clientConfig: + service: + name: {{ include "nrr-controller.fullname" . }}-webhook-service + namespace: {{ include "nrr-controller.namespace" . }} + port: {{ .Values.webhook.service.port }} + path: {{ .Values.validatingWebhook.path }} + failurePolicy: {{ .Values.validatingWebhook.failurePolicy }} + sideEffects: {{ .Values.validatingWebhook.sideEffects }} + rules: + - apiGroups: ["readiness.node.x-k8s.io"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["nodereadinessrules"] +{{- end }} diff --git a/charts/nrr-controller/tests/.gitignore b/charts/nrr-controller/tests/.gitignore new file mode 100644 index 0000000..bae41db --- /dev/null +++ b/charts/nrr-controller/tests/.gitignore @@ -0,0 +1 @@ +__snapshot__ diff --git a/charts/nrr-controller/tests/annotations_test.yaml b/charts/nrr-controller/tests/annotations_test.yaml new file mode 100644 index 0000000..786dcc9 --- /dev/null +++ b/charts/nrr-controller/tests/annotations_test.yaml @@ -0,0 +1,46 @@ +suite: Test Node Readiness Controller Pod Annotations and Labels + +templates: + - "*.yaml" + +release: + name: nrr-controller + +tests: + - it: adds pod annotations and labels when set + template: templates/deployment.yaml + set: + podAnnotations: + monitoring.company.com/scrape: "true" + description: "test pod" + podLabels: + environment: "test" + team: "platform" + asserts: + - equal: + path: spec.template.metadata.annotations["monitoring.company.com/scrape"] + value: "true" + - equal: + path: spec.template.metadata.annotations.description + value: "test pod" + - equal: + path: spec.template.metadata.annotations["kubectl.kubernetes.io/default-container"] + value: manager + - equal: + path: spec.template.metadata.labels.environment + value: "test" + - equal: + path: spec.template.metadata.labels.team + value: "platform" + + - it: adds default annotations and labels when not set + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.metadata.annotations["kubectl.kubernetes.io/default-container"] + value: manager + - isNotNull: + path: spec.template.metadata.labels + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: nrr-controller diff --git a/charts/nrr-controller/tests/deployment_test.yaml b/charts/nrr-controller/tests/deployment_test.yaml new file mode 100644 index 0000000..eb38d9d --- /dev/null +++ b/charts/nrr-controller/tests/deployment_test.yaml @@ -0,0 +1,44 @@ +suite: Test Node Readiness Controller Deployment + +templates: + - "*.yaml" + +release: + name: nrr-controller + +set: + replicaCount: 1 + +tests: + - it: creates Deployment + template: templates/deployment.yaml + asserts: + - isKind: + of: Deployment + + - it: enables leader-election by default + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --leader-elect + + - it: adds metrics secure args when enabled + set: + metrics: + secure: true + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-secure + + - it: exposes webhook port when webhook enabled + set: + webhook: + enabled: true + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].name + value: webhook-server diff --git a/charts/nrr-controller/tests/nodereadinessrules_test.yaml b/charts/nrr-controller/tests/nodereadinessrules_test.yaml new file mode 100644 index 0000000..2e1b3b6 --- /dev/null +++ b/charts/nrr-controller/tests/nodereadinessrules_test.yaml @@ -0,0 +1,57 @@ +suite: Test Node Readiness Controller NodeReadinessRules + +templates: + - templates/nodereadinessrules.yaml + +release: + name: nrr-controller + +tests: + - it: renders full rule spec and defaults nodeSelector when omitted + set: + nodeReadinessRules: + - name: kube-proxy-unhealthy-noschedule + dryRun: true + conditions: + - type: KubeProxyUnhealthy + requiredStatus: "False" + taint: + key: readiness.k8s.io/KubeProxyUnhealthy + value: "true" + effect: NoSchedule + enforcementMode: continuous + asserts: + - equal: + path: metadata.name + value: kube-proxy-unhealthy-noschedule + - equal: + path: spec.dryRun + value: true + - equal: + path: spec.nodeSelector + value: {} + - equal: + path: spec.conditions[0].type + value: KubeProxyUnhealthy + - equal: + path: spec.taint.key + value: readiness.k8s.io/KubeProxyUnhealthy + + - it: preserves provided nodeSelector + set: + nodeReadinessRules: + - name: linux-nodes + conditions: + - type: KubeProxyUnhealthy + requiredStatus: "False" + taint: + key: readiness.k8s.io/KubeProxyUnhealthy + effect: NoSchedule + enforcementMode: continuous + nodeSelector: + matchLabels: + kubernetes.io/os: linux + asserts: + - equal: + path: spec.nodeSelector.matchLabels["kubernetes.io/os"] + value: linux diff --git a/charts/nrr-controller/tests/webhook_and_metrics_test.yaml b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml new file mode 100644 index 0000000..ca554b1 --- /dev/null +++ b/charts/nrr-controller/tests/webhook_and_metrics_test.yaml @@ -0,0 +1,68 @@ +suite: Test Node Readiness Controller Webhook and Metrics Config + +templates: + - "*.yaml" + +release: + name: nrr-controller + +set: + webhook: + enabled: true + validatingWebhook: + enabled: true + metrics: + secure: true + +tests: + - it: exposes webhook service on upstream service port + template: templates/servicewebhook.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.ports[0].name + value: webhook + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: wires validating webhook to webhook service port + template: templates/validatingwebhookconfiguration.yaml + asserts: + - equal: + path: webhooks[0].clientConfig.service.name + value: nrr-controller-webhook-service + - equal: + path: webhooks[0].clientConfig.service.port + value: 443 + - equal: + path: webhooks[0].clientConfig.service.path + value: /validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule + + - it: supports overriding webhook service port + template: templates/validatingwebhookconfiguration.yaml + set: + webhook: + service: + port: 8443 + asserts: + - equal: + path: webhooks[0].clientConfig.service.port + value: 8443 + + - it: mounts webhook cert volume when webhook enabled + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.volumes[0].name + value: cert + - it: mounts metrics cert volume when metrics.secure enabled + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="metrics-certs")].name + value: metrics-certs diff --git a/charts/nrr-controller/values.yaml b/charts/nrr-controller/values.yaml new file mode 100644 index 0000000..3a1e19f --- /dev/null +++ b/charts/nrr-controller/values.yaml @@ -0,0 +1,151 @@ +# Default values for node-readiness-controller. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: registry.k8s.io/node-readiness-controller/node-readiness-controller + # Overrides the image tag whose default is the chart version + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: container-registry-secret + +resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi + +nameOverride: "" + +fullnameOverride: "" + +# -- Override the deployment namespace; defaults to .Release.Namespace +namespaceOverride: "" + +commonLabels: {} + +# Specifies the replica count for Deployment +# Set affinity.podAntiAffinity rule if you want to schedule onto a node +replicaCount: 1 + +priorityClassName: system-cluster-critical + +podAnnotations: + kubectl.kubernetes.io/default-container: manager + +podLabels: {} + +# podSecurityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +terminationGracePeriodSeconds: 10 + +rbac: + create: true + +serviceAccount: + create: true + name: "" + annotations: {} + +# Enable leader election to support multiple replicas +leaderElection: + enabled: true + +# Enable the webhook server +healthProbeBindAddress: ":8081" + +# Liveness probe configuration +livenessProbe: + path: /healthz + initialDelaySeconds: 15 + periodSeconds: 20 + +# Readiness probe configuration +readinessProbe: + path: /readyz + initialDelaySeconds: 5 + periodSeconds: 10 + +# Metrics server configuration +metrics: + secure: false + bindAddress: ":8443" + service: + port: 8443 + targetPort: 8443 + certDir: /tmp/k8s-metrics-server/metrics-certs + certSecretName: metrics-server-cert + +# Webhook server configuration +webhook: + enabled: false + port: 9443 + service: + port: 443 + targetPort: 9443 + certDir: /tmp/k8s-webhook-server/serving-certs + certSecretName: webhook-server-certs + +# cert-manager configuration for generating TLS certificates for the webhook and metrics server +certManager: + enabled: false + issuer: + create: true + name: selfsigned-issuer + metricsCertificate: + create: true + name: metrics-certs + webhookCertificate: + create: true + name: serving-cert + +validatingWebhook: + enabled: false + name: validating-webhook-configuration + webhookName: vnodereadinessrule.kb.io + failurePolicy: Fail + sideEffects: None + path: /validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule + admissionReviewVersions: + - v1 + +nodeSelector: {} + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +affinity: {} + +# NOTE: When using validating webhooks, apply NodeReadinessRule resources after the controller webhook is ready, for example in a separate helm upgrade. +nodeReadinessRules: [] + # - name: kube-proxy-unhealthy-noschedule + # dryRun: false + # conditions: + # - type: "KubeProxyUnhealthy" + # requiredStatus: "False" + # taint: + # key: "readiness.k8s.io/KubeProxyUnhealthy" + # value: "true" + # effect: "NoSchedule" + # enforcementMode: "continuous" + # nodeSelector: + # matchLabels: + # kubernetes.io/os: linux diff --git a/hack/verify-all.sh b/hack/verify-all.sh index 52b9bc9..e466537 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -20,5 +20,6 @@ set -o pipefail # Run all verification scripts hack/verify-boilerplate.sh +hack/verify-chart-drift.sh hack/verify-links.sh hack/verify-govulncheck.sh diff --git a/hack/verify-chart-drift.sh b/hack/verify-chart-drift.sh new file mode 100755 index 0000000..da4f7c2 --- /dev/null +++ b/hack/verify-chart-drift.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks that the Helm chart CRDs match controller-gen output. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT="$(dirname "${BASH_SOURCE[0]}")/.." +cd "${KUBE_ROOT}" + +make manifests + +diff -u \ + config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml \ + charts/nrr-controller/crds/nodereadinessrules.readiness.node.x-k8s.io.yaml diff --git a/hack/verify-chart.sh b/hack/verify-chart.sh new file mode 100755 index 0000000..90603be --- /dev/null +++ b/hack/verify-chart.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +${CONTAINER_ENGINE:-docker} run -it --rm --network host --workdir=/data --volume ~/.kube/config:/root/.kube/config:ro --volume $(pwd):/data quay.io/helmpack/chart-testing:v3.7.0 /bin/bash -c "git config --global --add safe.directory /data; ct install --config=.github/ci/ct.yaml --helm-extra-set-args=\"--set=kind=Deployment\""