Script to start the self sufficient bundle on LInux using `virt-install`

crc-ng.sh

#!/bin/bash

set -xeuo pipefail

PASS_DEVELOPER="${PASS_DEVELOPER:-P@ssd3v3loper}"
PASS_KUBEADMIN="${PASS_DEVELOPER:-P@sskub3admin}"
CRC_BUNDLE_PATH="${CRC_BUNDLE_PATH:-$HOME/Downloads/crc_vfkit_4.19.3_arm64.crcbundle}"

SSH="ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -i ${PUB_KEY_PATH%.*}"
SCP="scp -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -i ${PUB_KEY_PATH%.*}"

OS=$(uname)

if [[ "${PULL_SECRET_PATH}" == "" ]]; then
    echo -n "Path to Pull secret file needs to be set using PULL_SECRET_PATH env variable"
    exit 1
fi

if [[ "${PUB_KEY_PATH}" == "" ]]; then
    echo -n "Path to the SSH Public key needs to be set using PUB_KEY_PATH env variable"
    exit 1
fi

if [[ "${CRC_BUNDLE_PATH}" == "" ]]; then
    echo -n "Path to a CRC bundle needs to be set using CRC_BUNDLE_PATH env variable"
    exit 1
fi

PULL_SECRET=$(cat ${PULL_SECRET_PATH})
PUB_KEY=$(cat ${PUB_KEY_PATH})

function gen_cloud_init() {
    echo -n "Generating cloud-init user-data..."
    rm -rf seed.iso
    cat <<EOF > user-data
#cloud-config

runcmd:
  - systemctl enable --now kubelet
write_files:
- path: /home/core/.ssh/authorized_keys
  content: '$PUB_KEY'
  owner: core
  permissions: '0600'
- path: /opt/crc/id_rsa.pub
  content: '$PUB_KEY'
  owner: root:root
  permissions: '0644'
- path: /etc/sysconfig/crc-env
  content: |
    CRC_CLOUD=1
    CRC_NETWORK_MODE_USER=0
  owner: root:root
  permissions: '0644'
- path: /usr/local/bin/crc-check-cloud-env.sh
  content: |
    #!/bin/bash
    exit 0
  owner: root:root
  permissions: '0777'
- path: /opt/crc/pull-secret
  content: |
    $PULL_SECRET
  permissions: '0644'
- path: /opt/crc/pass_kubeadmin
  content: '$PASS_KUBEADMIN'
  permissions: '0644'
- path: /opt/crc/pass_developer
  content: '$PASS_DEVELOPER'
  permissions: '0644'
- path: /opt/crc/ocp-custom-domain.service.done
  permissions: '0644'
EOF

    # create cloud-init ISO
    # touch meta-data
    # mkisofs -output seed.iso -volid cidata -joliet -rock user-data meta-data
    # macos: hdiutil makehybrid -o seed.iso -hfs -joliet -iso -default-volume-name cidata seedconfig/
}

function extract_disk_img() {
    echo -n "Extracting VM image from CRC bundle ..."
    zstd -d --format=zstd -o bundle.tar "${CRC_BUNDLE_PATH}"
    bundle_name=$(basename "${CRC_BUNDLE_PATH}")
    if [ "${OS}" == "Darwin" ]; then
        tar -O -xvf bundle.tar "${bundle_name%.*}"/crc.img > crc.img
    elif [ "${OS}" == "Linux" ]; then
        tar -O -xvf bundle.tar "${bundle_name%.*}"/crc.qcow2 > crc.qcow2
    fi
    rm -rf bundle.tar
}

function create_libvirt_vm() {
    crc_disk_path="$(pwd)/crc.qcow2"
    vm_name=${1}

    # sudo chown qemu:qemu ${crc_disk_path}
    # sudo chown qemu:qemu ${cloud_init_iso}

    echo -n "Creating VM..."

    sudo virt-install \
        --name ${vm_name} \
        --vcpus 4 \
        --memory 14000 \
        --disk path=${crc_disk_path},format=qcow2,bus=virtio \
        --import \
        --os-variant=generic \
        --nographics \
        --cloud-init disable=on,user-data=./user-data \
        --noautoconsole
}

function create_vfkit_vm() {
    crc_disk_path="$(pwd)/crc.qcow2"
    vm_name=${1}

    echo -n "Creating VM..."

    vfkit --cpus 4 \
        --memory 10752 \
        --bootloader efi,variable-store=$HOME/.crc/machines/crc/efistore.nvram,create \
        --device virtio-serial,logFilePath=$HOME/.crc/machines/crc/vfkit.log \
        --device virtio-fs,sharedDir=$HOME,mountTag=dir0 \
        --device virtio-rng \
        --device virtio-blk,path=$(pwd)/crc.img \
        --device virtio-net,nat,mac=52:54:00:70:2b:71 \
        --timesync vsockPort=1234 --cloud-init $(pwd)/user-data --gui --log-level debug
}

function get_kubeconfig() {
    echo -n "Waiting 3mins for VM to start ..."
    sleep 180

    if [ "${OS}" == "Darwin" ]; then
        VM_IP=$(cat /var/db/dhcpd_leases | grep -w5 crc | grep ip_address | cut -d '=' -f 2)
    elif [ ${OS} == "Linux" ]; then
        vm_name=${1}
        VM_IP=$(sudo virsh domifaddr ${vm_name} | tail -2 | head -1 | awk '{print $4}' | cut -d/ -f1)
    fi

    while ! ${SSH} core@${VM_IP} -- exit 0; do
        sleep 5
        echo -n "Waiting for SSH to be available ..."
    done

    echo -n "VM is running ..."

    while ! ${SSH} core@${VM_IP} -- 'sudo oc get node --kubeconfig /opt/crc/kubeconfig --context system:admin'; do
        sleep 30
        echo -n "Waiting for CA to be rotated ..."
    done

    ${SCP} core@${VM_IP}:/opt/kubeconfig .
    oc config set clusters.api-crc-testing:6443.server https://${VM_IP}:6443 --kubeconfig ./kubeconfig
    oc config set clusters.crc.server https://${VM_IP}:6443 --kubeconfig ./kubeconfig
}

gen_cloud_init

extract_disk_img

case "${OS}" in
    Darwin)
        create_vfkit_vm crc-ng &
	;;
    Linux)
        create_libvirt_vm crc-ng &
	;;
    *)
        echo "OS Not supported!!"
	;;
esac

get_kubeconfig crc-ng

I want to avoid making multiple ssh calls if possible so in case of kubelet service up and running instead of ${SSH} core@${VM_IP} -- 'systemctl is-active kubelet' better to just scp the kubeconfig on host and then check if resource is available.

function get_kubeconfig() {
    vm_name=${1}

    VM_IP=$(sudo virsh domifaddr ${vm_name} | tail -2 | head -1 | awk '{print $4}' | cut -d/ -f1)

    while ! ${SSH} core@${VM_IP} -- exit 0; do
        sleep 5
        echo -n "Waiting for SSH to be available ..."
    done

    echo -n "VM is running ..."

    ${SCP} core@${VM_IP}:/opt/kubeconfig .
    oc config set clusters.api-crc-testing:6443.server https://${vm_ip}:6443 --config ./kubeconfig
    oc config set clusters.crc.server https://${vm_ip}:6443 --config ./kubeconfig

    while oc get node --kubeconfig ./kubeconfig --context system:admin'; do
        sleep 30
        echo -n "Waiting for apiserver"
    done
}

If we order kubelet.service After cloud-final.service, it might be possible to always enable the kubelet service and remove runcmd: systemctl enable --now kubelet from the cloud-init file

@cfergeau this means adding a drop-in file for kubelet service and changing the way current bundle work locally. As of now bundles which use with crc are not depend on cloud-init so not sure if having this change cause issue for these scenario.

Yes, with the way we currently start the bundle, it's not possible, but when we fully switch to the self sufficient bundle, then we can consider it.

@anjannath https://gist.github.com/anjannath/51f78600568a3e29334ace77e8a32e3b#file-crc-ng-sh-L130-L131 need to be updated with VM_IP instead vm_ip and https://gist.github.com/anjannath/51f78600568a3e29334ace77e8a32e3b#file-crc-ng-sh-L131 should be changed from --config ./kubeconfig to --kubeconfig ./kubeconfig .

@anjannath https://gist.github.com/anjannath/51f78600568a3e29334ace77e8a32e3b#file-crc-ng-sh-L130-L131 need to be updated with VM_IP instead vm_ip and https://gist.github.com/anjannath/51f78600568a3e29334ace77e8a32e3b#file-crc-ng-sh-L131 should be changed from --config ./kubeconfig to --kubeconfig ./kubeconfig .

updated, thanks!

updated the script to also work with vfkit on macOS