partition-nvme-rook-ceph-k3s-ansible.yaml

---
- name: Repartition NVMe drives for K3s etcd
  become: true
  hosts: masters
  vars:
    nvme_device: "/dev/nvme0n1"
    etcd_partition_size: "20GiB"
    rook_partition_size: "20GiB"
    rook_partition_end: "40GiB"
    ceph_partition_size: "100%FREE"
    etcd_mount_point: "/var/lib/rancher/k3s"
    rook_mount_point: "/var/lib/rook"
    
  tasks:
    - name: Ping hosts
      ping:

    - name: Stop K3s service if running
      systemd:
        name: k3s
        state: stopped
      ignore_errors: yes

    - name: Unmount any existing mount points
      mount:
        path: "{{ item }}"
        state: unmounted
      ignore_errors: yes
      loop:
        - "{{ etcd_mount_point }}"
        - "{{ rook_mount_point }}"

    - name: Clean mount point directories (remove any files if not mounted)
      shell: |
        # Check if directory exists and is not a mount point, then clean it
        if [ -d "{{ item }}" ] && ! mountpoint -q "{{ item }}"; then
          echo "Cleaning directory: {{ item }}"
          rm -rf "{{ item }}"/*
          rm -rf "{{ item }}"/.*
        fi
      ignore_errors: yes
      loop:
        - "{{ etcd_mount_point }}"
        - "{{ rook_mount_point }}"

    - name: Remove any existing Ceph LVM volumes
      shell: |
        # Remove any existing Ceph LVM volumes
        for vg in $(vgs --noheadings -o vg_name | grep ceph || true); do
          echo "Removing VG: $vg"
          vgremove -f "$vg" || true
        done
        
        # Remove any existing physical volumes on NVMe
        for pv in $(pvs --noheadings -o pv_name | grep nvme || true); do
          echo "Removing PV: $pv"
          pvremove -f "$pv" || true
        done
      ignore_errors: yes

    - name: Wipe NVMe device completely
      shell: |
        # Stop any processes using the device
        fuser -k {{ nvme_device }} || true
        
        # Wipe filesystem signatures
        wipefs -af {{ nvme_device }}
        
        # Zero out the beginning of the device
        dd if=/dev/zero of={{ nvme_device }} bs=1M count=100 status=progress || true
        
        # Zero out the end of the device  
        dd if=/dev/zero of={{ nvme_device }} bs=1M seek=$(( $(blockdev --getsz {{ nvme_device }}) / 2048 - 100 )) count=100 status=progress || true
        
        # Inform kernel of partition changes
        partprobe {{ nvme_device }} || true
        
        echo "NVMe device {{ nvme_device }} wiped successfully"

    - name: Create GPT partition table and partitions
      shell: |
        # Create GPT partition table
        parted -s {{ nvme_device }} mklabel gpt
        
        # Create etcd partition ({{etcd_partition_size}})
        parted -s {{ nvme_device }} mkpart etcd ext4 1MiB {{ etcd_partition_size }}
        
        # Calculate rook partition end (etcd_size + rook_size)
        rook_end={{ rook_partition_end }}
        
        # Create rook partition ({{rook_partition_size}})  
        parted -s {{ nvme_device }} mkpart rook ext4 {{ etcd_partition_size }} ${rook_end}
        
        # Create Ceph partition (remaining space)
        parted -s {{ nvme_device }} mkpart notset ext4 ${rook_end} 100%
        
        # Set partition names
        parted -s {{ nvme_device }} name 1 etcd
        parted -s {{ nvme_device }} name 2 rook
        parted -s {{ nvme_device }} name 3 notset
        
        # Inform kernel of changes
        partprobe {{ nvme_device }}
        
        # Wait for device nodes to appear
        sleep 2
        
        echo "Partitions created successfully"

    - name: Format etcd partition with ext4
      filesystem:
        fstype: ext4
        dev: "{{ nvme_device }}p1"
        opts: "-L etcd-data"

    - name: Format rook partition with ext4
      filesystem:
        fstype: ext4
        dev: "{{ nvme_device }}p2"
        opts: "-L rook-data"

    - name: Create mount point directories
      file:
        path: "{{ item }}"
        state: directory
        mode: '0755'
      loop:
        - "{{ etcd_mount_point }}"
        - "{{ rook_mount_point }}"

    - name: Mount etcd partition
      mount:
        path: "{{ etcd_mount_point }}"
        src: "{{ nvme_device }}p1"
        fstype: ext4
        opts: defaults,noatime
        state: mounted

    - name: Mount rook partition
      mount:
        path: "{{ rook_mount_point }}"
        src: "{{ nvme_device }}p2"
        fstype: ext4
        opts: defaults,noatime
        state: mounted

    - name: Add etcd partition to fstab
      mount:
        path: "{{ etcd_mount_point }}"
        src: "{{ nvme_device }}p1"
        fstype: ext4
        opts: defaults,noatime
        state: present

    - name: Add rook partition to fstab
      mount:
        path: "{{ rook_mount_point }}"
        src: "{{ nvme_device }}p2"
        fstype: ext4
        opts: defaults,noatime
        state: present

    - name: Set correct ownership for mount directories
      file:
        path: "{{ item }}"
        owner: root
        group: root
        mode: '0755'
      loop:
        - "{{ etcd_mount_point }}"
        - "{{ rook_mount_point }}"

    - name: Display partition information
      shell: |
        echo "=== Partition table for {{ nvme_device }} ==="
        lsblk {{ nvme_device }}
        echo ""
        echo "=== Filesystem information ==="
        df -h {{ etcd_mount_point }} {{ rook_mount_point }}
        echo ""
        echo "=== Available space for Ceph ==="
        lsblk {{ nvme_device }}p3
      register: partition_info

    - name: Show partition results
      debug:
        msg: "{{ partition_info.stdout_lines }}"

  handlers:
    - name: Reload systemd
      systemd:
        daemon_reload: yes