From 397dee6a94aceb558ec170f2fcf4d90f5be75681 Mon Sep 17 00:00:00 2001 From: Cam Hutchison Date: Wed, 6 May 2026 12:22:28 +1000 Subject: [PATCH] Add plugin for putting k8s node into maintenance Add a script that puts a k8s node into maintenance mode, and takes it out again. Maintenance mode ensures there are no pods running on the node as much as possible (something could tolerate the taint that gets added, but it's a custom taint so should not happen). Add a symlink for the completion script to allow shell command line completion to work with it. This is a kubectl plugin which works similarly to a git plugin - a script named kubectl-X can be run as `kubectl X`. kubectl completion runs `kubectl_complete-X` to handle completion for sub-command X. --- kubectl-maintain | 74 +++++++++++++++++++++++++++++++++++++++ kubectl_complete-maintain | 1 + 2 files changed, 75 insertions(+) create mode 100755 kubectl-maintain create mode 120000 kubectl_complete-maintain diff --git a/kubectl-maintain b/kubectl-maintain new file mode 100755 index 0000000..c6ebcf3 --- /dev/null +++ b/kubectl-maintain @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# kubectl-maintain is a kubectl plugin for putting a node into and taking out +# of maintenance. When a node is in maintenance, no pods run on it, including +# DaemonSets. It allows the kubelet to be shutdown so the node can be rebooted +# or other maintenance activities performed. +# +# If this script is installed somewhere other than the git repository it comes +# from, a symlink of kubectl_complete-maintain should be created to link to +# this script, alongside it. This will allow command line completion to work. + +main() { + # taint name is not particularly relevant, just the :NoEexecute when applied. + taint='xdna.net/maintenance' + + case "${0##*/}" in + kubectl-maintain) + maintain "$@" + ;; + kubectl_complete-maintain) + maintain_complete "$@" + ;; + *) + echo "unknown invocation: $0" >&2 + exit 1 + ;; + esac +} + +maintain() { + # set -e makes this not idempotent, but that's intentional. Continuing when + # something does not run as expected is not desirable as you can end up + # being more destructive than planned. + set -euo pipefail + case "${1-}" in + start) + node="${2:?usage: kubectl maintain start }" + kubectl drain --ignore-daemonsets --delete-emptydir-data -- "$node" + kubectl taint node -- "$node" "${taint}:NoExecute" + echo "Node $node marked for maintenance. Pods evicted" + ;; + stop) + node="${2:?usage: kubectl maintain stop }" + kubectl taint node -- "$node" "${taint}:NoExecute-" + kubectl uncordon -- "$node" + echo "Node $node ready for workloads" + ;; + *) + echo "usage: kubectl maintain {start|stop} " >&2 + return 1 + ;; + esac +} + +maintain_complete() { + set -euo pipefail + trap 'echo :4' EXIT # BashCompDirectiveNoFileComp - disable filename completion + local nodes_jq='.items[] | select(.spec.taints // [] | map(.key) | index("%s")%s) | .metadata.name' + case "$#" in + 1) printf '%s\n' start stop ;; + 2) + # shellcheck disable=SC2059 # I have format strings in the var + if [[ "$1" == start ]]; then + printf -v jq "$nodes_jq" "$taint" ' | not' + elif [[ "$1" == stop ]]; then + printf -v jq "$nodes_jq" "$taint" '' + else + return 1 + fi + kubectl get nodes -o json | jq -r "$jq" + ;; + esac +} +main "$@" diff --git a/kubectl_complete-maintain b/kubectl_complete-maintain new file mode 120000 index 0000000..fcbba45 --- /dev/null +++ b/kubectl_complete-maintain @@ -0,0 +1 @@ +kubectl-maintain \ No newline at end of file