Force cleanup script for stuck node when EC2 instance is confirmed terminated ( Karpenter + AWS EKS AutoMode)

force-cleanup-stuck-node.sh

#!/bin/bash
# Force cleanup script for stuck node when EC2 instance is confirmed terminated
# Usage: ./force-cleanup-stuck-node.sh i-09f7b4edb715392f4

set -euo pipefail

NODE_ID="${1:-}"
if [ -z "$NODE_ID" ]; then
    echo "Usage: $0 <node-instance-id>"
    exit 1
fi

NODE_NAME="$NODE_ID"

echo "=========================================="
echo "🧹 Force Cleanup: $NODE_NAME"
echo "=========================================="
echo ""

# Find NodeClaim
NODECLAIM=$(kubectl get nodeclaim -A -o json 2>/dev/null | \
    jq -r ".items[] | select(.status.nodeName == \"$NODE_NAME\") | \"\(.metadata.namespace // \"\")|\(.metadata.name)\"" | head -1)

if [ -n "$NODECLAIM" ]; then
    NODECLAIM_NS=$(echo "$NODECLAIM" | cut -d'|' -f1)
    NODECLAIM_NAME=$(echo "$NODECLAIM" | cut -d'|' -f2)
    
    echo "Found NodeClaim: $NODECLAIM_NAME (namespace: ${NODECLAIM_NS:-default})"
    
    # Remove NodeClaim finalizer
    echo "Removing NodeClaim finalizer..."
    if [ -n "$NODECLAIM_NS" ] && [ "$NODECLAIM_NS" != "null" ] && [ "$NODECLAIM_NS" != "" ]; then
        kubectl patch nodeclaim "$NODECLAIM_NAME" -n "$NODECLAIM_NS" \
            --type json \
            -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true
    else
        # Try without namespace (cluster-scoped)
        kubectl patch nodeclaim "$NODECLAIM_NAME" \
            --type json \
            -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true
    fi
    echo "✅ NodeClaim finalizer removed"
    echo ""
fi

# Remove node finalizer
echo "Removing node finalizer..."
kubectl patch node "$NODE_NAME" \
    --type json \
    -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || {
    echo "⚠️  Failed to patch node, trying force delete..."
    kubectl delete node "$NODE_NAME" --force --grace-period=0 2>/dev/null || true
}

echo "✅ Node finalizer removed"
echo ""
echo "Node should be deleted now. Verifying..."
sleep 2

if kubectl get node "$NODE_NAME" &>/dev/null; then
    echo "⚠️  Node still exists. Checking status..."
    kubectl get node "$NODE_NAME" -o yaml | grep -A 5 "deletionTimestamp\|finalizers" || true
else
    echo "✅ Node successfully deleted!"
fi