Maintenance mode for homelab

maint.sh

#!/bin/bash

ARGO="argocd"
VERYBASICS="home-assistant mosquitto ssher zigbee2mqtt2"
SYSTEM="kube-system longhorn-system metallb-system $ARGO"

COLUMNCOMMAND="column -t" # switch to `cat` if you don't have column from bsdmainutils

CONSIDER_NAMESPACES=$(kubectl get namespace | grep -v NAME | cut -f1 -d' ' | tr '\n' ' ')

while [[ $# -gt 0 ]]; do
  case $1 in
    --everything)
      EVERYTHING=YES
      shift # past argument
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift # past argument
      ;;
  esac
done

if [ "YES" == "$EVERYTHING" ]; then
        NAMESPACES=$(echo $ARGO $CONSIDER_NAMESPACES | tr ' ' '\n' | grep -vF $(for ns in $SYSTEM; do echo "-e $ns"; done))
else
        NAMESPACES=$(echo $ARGO $CONSIDER_NAMESPACES | tr ' ' '\n' | grep -vF $(for ns in $SYSTEM $VERYBASICS; do echo "-e $ns"; done))
fi

echo "Maintenance mode will scale down the statefulsets and deployments, set suspend on all cronjobs"
echo " and set a nodeselector of 'non-existing: true' in the following namespaces : "
echo ""
echo $ARGO $NAMESPACES | ${COLUMNCOMMAND}
echo ""
echo "(It will start with argocd because, well, otherwise, argo will see maintenance mode as "
echo " something to repair)"
echo ""
echo "If you do not wish to continue, press ^c now, otherwise, hit enter and we will go ahead"
read

for ns in $ARGO $NAMESPACES; do
        echo "Shuttering $ns..."
        kubectl scale deploy,statefulset -n $ns --all --replicas=0
        for ds in $(kubectl get daemonset -n $ns -o custom-columns=NAME:.metadata.name| grep -v NAME); do
                kubectl patch daemonset -n $ns $ds -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
        done
        for cj in $(kubectl get cronjob -n $ns -o custom-columns=NAME:.metadata.name | grep -v NAME); do
                kubectl patch cronjob -n $ns $cj -p '{"spec" : {"suspend" : true }}'
        done
done

echo "To manually bring back a namespace, use "
echo "> kubectl scale deploy,statefulset --replicas=1 --all -n <namespace> # and hope you had 1!"
echo "To manually bring back a daemonset use the following command (or scale up argo)"
echo "> kubectl -n <namespace> patch daemonset <name-of-daemon-set> --type json -p='[{\"op\": \"remove\", \"path\": \"/spec/template/spec/nodeSelector/non-existing\"}]'"
echo "To manually enable a cronjob : "
echo "> kubectl patch cronjob <job-name> [-n namespace] -p '{\"spec\" : {\"suspend\" : true }}'"

Used to shut down all the things for cluster maintenance, optionally and by default excluding mosquitto, homeassistant, zigbee2mqtt2 (my second install of) and the fat container that I use as a linux terminal to my systems that runs ssh. --everything shuts down everything except longhorn, metallb and kubernetes internals.