Firecracker parallel VM test

fc-step5-parallel.sh

#!/bin/bash
set -eux

echo "=========================================="
echo "  Firecracker Parallel VM Test (Step 5)"
echo "  3 VMs from same snapshot via netns"
echo "=========================================="

TOTAL_START=$(date +%s%3N)

# ==================================================
# Phase 0: Prerequisites
# ==================================================
echo "=== Phase 0: Prerequisites ==="

# Enable KVM
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm.rules
sudo udevadm control --reload-rules && sudo udevadm trigger

# Install Firecracker
FC_VERSION=v1.12.0
curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/${FC_VERSION}/firecracker-${FC_VERSION}-x86_64.tgz | sudo tar xz -C /tmp
sudo mv /tmp/release-${FC_VERSION}-x86_64/firecracker-${FC_VERSION}-x86_64 /usr/local/bin/firecracker
sudo mv /tmp/release-${FC_VERSION}-x86_64/jailer-${FC_VERSION}-x86_64 /usr/local/bin/jailer
firecracker --version

# Install sshpass
sudo apt-get update -qq && sudo apt-get install -y -qq sshpass

# Extract host kernel
KVER=$(uname -r)
curl -fsSL https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux -o /tmp/extract-vmlinux
chmod +x /tmp/extract-vmlinux
sudo /tmp/extract-vmlinux /boot/vmlinuz-$KVER > /tmp/vmlinux

# ==================================================
# Phase 1: Build Rootfs
# ==================================================
echo "=== Phase 1: Build Rootfs ==="

docker run --name rootfs-builder -d ubuntu:24.04 sleep 3600
docker exec rootfs-builder bash -c "
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -qq
  apt-get install -y -qq curl iptables iproute2 kmod systemd dbus udev openssh-server
  curl -fsSL https://get.docker.com | sh
  systemctl enable docker
  curl -Lo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 && chmod +x /usr/local/bin/kind
  curl -LO https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/
  curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
  mkdir -p /root/.ssh
  echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
  echo 'root:firecracker' | chpasswd
  systemctl enable ssh
  apt-get clean && rm -rf /var/lib/apt/lists/*
"

dd if=/dev/zero of=/tmp/rootfs.ext4 bs=1M count=10240
mkfs.ext4 /tmp/rootfs.ext4
sudo mkdir -p /tmp/rootfs_mnt
sudo mount /tmp/rootfs.ext4 /tmp/rootfs_mnt
docker export rootfs-builder | sudo tar x -C /tmp/rootfs_mnt

# Copy kernel modules
sudo mkdir -p /tmp/rootfs_mnt/lib/modules/
sudo cp -r /lib/modules/$KVER /tmp/rootfs_mnt/lib/modules/

# firecracker-init.service
sudo tee /tmp/rootfs_mnt/etc/systemd/system/firecracker-init.service <<'SVCEOF'
[Unit]
Description=Firecracker Init (load modules + configure network)
Before=docker.service containerd.service
After=systemd-modules-load.service

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/bash -c 'modprobe overlay || true; modprobe br_netfilter || true; modprobe iptable_nat || true; modprobe iptable_filter || true; modprobe veth || true; modprobe nf_conntrack || true; ip addr add 172.16.0.2/24 dev eth0; ip link set eth0 up; ip route add default via 172.16.0.1; echo "nameserver 8.8.8.8" > /etc/resolv.conf'

[Install]
WantedBy=multi-user.target
SVCEOF
sudo chroot /tmp/rootfs_mnt systemctl enable firecracker-init.service

# auto-kind.sh (standalone script to avoid systemd % escaping)
sudo tee /tmp/rootfs_mnt/usr/local/bin/auto-kind.sh <<'SCRIPTEOF'
#!/bin/bash
set -x
echo "=== Waiting for Docker socket ==="
for i in $(seq 1 60); do
  if docker info >/dev/null 2>&1; then
    echo "Docker ready after ${i}s"
    break
  fi
  sleep 1
done
echo "=== Creating kind cluster ==="
KIND_START=$(date +%s%3N)
kind create cluster --name fc-test --wait 120s 2>&1
KIND_RC=$?
KIND_END=$(date +%s%3N)
echo "=== kind create time: $((KIND_END - KIND_START))ms ==="
echo "=== kind exit code: $KIND_RC ==="
kubectl get nodes 2>&1
kind export kubeconfig --name fc-test --kubeconfig /etc/kind-kubeconfig
chmod 644 /etc/kind-kubeconfig
echo "=== STEP3_COMPLETE ==="
SCRIPTEOF
sudo chmod +x /tmp/rootfs_mnt/usr/local/bin/auto-kind.sh

sudo tee /tmp/rootfs_mnt/etc/systemd/system/auto-kind.service <<'KINDEOF'
[Unit]
Description=Auto create kind cluster
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
StandardOutput=journal+console
StandardError=journal+console
ExecStart=/usr/local/bin/auto-kind.sh

[Install]
WantedBy=multi-user.target
KINDEOF
sudo chroot /tmp/rootfs_mnt systemctl enable auto-kind.service

sudo ln -sf /lib/systemd/systemd /tmp/rootfs_mnt/sbin/init 2>/dev/null || true
sudo umount /tmp/rootfs_mnt

# Stop host Docker to free memory
docker rm -f rootfs-builder
sudo systemctl stop docker

echo "=== Rootfs built ==="

# ==================================================
# Phase 2: Boot VM, create kind cluster, install cert-manager, snapshot
# ==================================================
echo "=== Phase 2: Boot VM + provision + snapshot ==="

# Setup TAP networking
sudo ip tuntap add tap0 mode tap
sudo ip addr add 172.16.0.1/24 dev tap0
sudo ip link set tap0 up
sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
sudo iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
sudo iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
sudo iptables -A FORWARD -i tap0 -o eth0 -j ACCEPT

# Start Firecracker
rm -f /tmp/firecracker.sock
firecracker --api-sock /tmp/firecracker.sock --id fc-snap 2>&1 &
FC_PID=$!
sleep 0.5

# Configure VM — 4096 MiB per VM to allow 3 VMs later (4*3=12GB)
curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/boot-source \
  -H "Content-Type: application/json" \
  -d '{"kernel_image_path": "/tmp/vmlinux", "boot_args": "console=ttyS0 reboot=k panic=1 init=/sbin/init systemd.unified_cgroup_hierarchy=1"}'

curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/drives/rootfs \
  -H "Content-Type: application/json" \
  -d '{"drive_id": "rootfs", "path_on_host": "/tmp/rootfs.ext4", "is_root_device": true, "is_read_only": false}'

curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/machine-config \
  -H "Content-Type: application/json" \
  -d '{"vcpu_count": 2, "mem_size_mib": 4096}'

curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/network-interfaces/eth0 \
  -H "Content-Type: application/json" \
  -d '{"iface_id": "eth0", "guest_mac": "AA:FC:00:00:00:01", "host_dev_name": "tap0"}'

curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/actions \
  -H "Content-Type: application/json" \
  -d '{"action_type": "InstanceStart"}'

# Wait for SSH
SSH_BASE="sshpass -p firecracker ssh -n -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@172.16.0.2"
echo "Waiting for SSH..."
for i in $(seq 1 120); do
  if $SSH_BASE "echo SSH_OK" 2>/dev/null | grep -q SSH_OK; then
    echo "SSH ready after ${i}s"
    break
  fi
  sleep 1
done

# Wait for kind cluster
echo "Waiting for kind cluster..."
for i in $(seq 1 180); do
  if $SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig kubectl get nodes 2>/dev/null" | grep -q Ready; then
    echo "Cluster ready after ${i}s"
    break
  fi
  sleep 1
done

$SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig kubectl get nodes"
$SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig kubectl get pods -A"

# Install cert-manager
echo "Installing cert-manager..."
CM_START=$(date +%s%3N)
$SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig helm repo add jetstack https://charts.jetstack.io --force-update 2>&1"
$SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.17.2 --set crds.enabled=true --wait --timeout 180s 2>&1"
CM_END=$(date +%s%3N)
echo "=== cert-manager install time: $((CM_END - CM_START))ms ==="

# Create test resources (to verify they survive snapshot)
$SSH_BASE 'KUBECONFIG=/etc/kind-kubeconfig kubectl apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: selfsigned-issuer
spec:
  selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: pre-snapshot-cert
  namespace: default
spec:
  secretName: pre-snapshot-cert-tls
  issuerRef:
    name: selfsigned-issuer
    kind: ClusterIssuer
  commonName: pre-snapshot.example.com
  dnsNames:
    - pre-snapshot.example.com
EOF'

# Wait for cert to be ready
sleep 5
$SSH_BASE "KUBECONFIG=/etc/kind-kubeconfig kubectl get certificate -A"

# Create snapshot
echo "Creating snapshot..."
SNAP_START=$(date +%s%3N)
curl -s --unix-socket /tmp/firecracker.sock -X PATCH http://localhost/vm \
  -H "Content-Type: application/json" \
  -d '{"state": "Paused"}'

mkdir -p /tmp/fc-snapshot
curl -s --unix-socket /tmp/firecracker.sock -X PUT http://localhost/snapshot/create \
  -H "Content-Type: application/json" \
  -d '{"snapshot_type": "Full", "snapshot_path": "/tmp/fc-snapshot/vmstate", "mem_file_path": "/tmp/fc-snapshot/mem"}'
SNAP_END=$(date +%s%3N)
echo "=== Snapshot create time: $((SNAP_END - SNAP_START))ms ==="

ls -lh /tmp/fc-snapshot/

# Kill original VM
kill $FC_PID 2>/dev/null || true
wait $FC_PID 2>/dev/null || true

# Clean up original TAP
sudo ip link del tap0 2>/dev/null || true

echo "=== Original VM stopped, snapshot ready ==="

# ==================================================
# Phase 3: Start 3 VMs simultaneously from snapshot
# ==================================================
echo "=== Phase 3: Parallel VM startup ==="

NUM_VMS=3
PARALLEL_START=$(date +%s%3N)

# Setup function for each VM
setup_vm() {
  local N=$1
  local VM_START=$(date +%s%3N)

  echo "[VM${N}] Setting up network namespace..."

  # Create network namespace
  sudo ip netns add vm${N}

  # Create TAP inside namespace
  sudo ip netns exec vm${N} ip tuntap add tap0 mode tap
  sudo ip netns exec vm${N} ip addr add 172.16.0.1/24 dev tap0
  sudo ip netns exec vm${N} ip link set tap0 up
  sudo ip netns exec vm${N} ip link set lo up

  # Copy rootfs
  echo "[VM${N}] Copying rootfs..."
  cp /tmp/rootfs.ext4 /tmp/rootfs-vm${N}.ext4

  # Start Firecracker inside namespace
  echo "[VM${N}] Starting Firecracker..."
  rm -f /tmp/fc-${N}.sock
  sudo ip netns exec vm${N} firecracker \
    --api-sock /tmp/fc-${N}.sock \
    --id fc-vm${N} &
  eval "FC_VM${N}_PID=$!"
  sleep 0.5

  # Load snapshot
  local LOAD_START=$(date +%s%3N)
  sudo ip netns exec vm${N} curl -s --unix-socket /tmp/fc-${N}.sock \
    -X PUT http://localhost/snapshot/load \
    -H "Content-Type: application/json" \
    -d "{
      \"snapshot_path\": \"/tmp/fc-snapshot/vmstate\",
      \"mem_backend\": {
        \"backend_type\": \"File\",
        \"backend_path\": \"/tmp/fc-snapshot/mem\"
      },
      \"enable_diff_snapshots\": false,
      \"resume_vm\": true
    }"
  local LOAD_END=$(date +%s%3N)
  echo "[VM${N}] Snapshot load time: $((LOAD_END - LOAD_START))ms"

  local VM_END=$(date +%s%3N)
  echo "[VM${N}] Total setup time: $((VM_END - VM_START))ms"
}

# Start all 3 VMs simultaneously
for N in $(seq 1 $NUM_VMS); do
  setup_vm $N &
done
wait

PARALLEL_END=$(date +%s%3N)
echo "=== All VMs started in $((PARALLEL_END - PARALLEL_START))ms ==="

# ==================================================
# Phase 4: Wait for SSH on all VMs
# ==================================================
echo "=== Phase 4: Wait for SSH ==="

SSH_START=$(date +%s%3N)

wait_for_ssh() {
  local N=$1
  local VM_SSH_START=$(date +%s%3N)
  for i in $(seq 1 60); do
    if sudo ip netns exec vm${N} sshpass -p firecracker ssh -n \
      -o StrictHostKeyChecking=no -o ConnectTimeout=2 root@172.16.0.2 \
      "echo SSH_OK" 2>/dev/null | grep -q SSH_OK; then
      local VM_SSH_END=$(date +%s%3N)
      echo "[VM${N}] SSH ready after $((VM_SSH_END - VM_SSH_START))ms"
      return 0
    fi
    sleep 0.5
  done
  echo "[VM${N}] SSH TIMEOUT"
  return 1
}

for N in $(seq 1 $NUM_VMS); do
  wait_for_ssh $N &
done
wait

SSH_END=$(date +%s%3N)
echo "=== All VMs SSH-ready in $((SSH_END - SSH_START))ms ==="

# ==================================================
# Phase 5: Verify each VM
# ==================================================
echo "=== Phase 5: Verification ==="

verify_vm() {
  local N=$1
  local NSSH="sudo ip netns exec vm${N} sshpass -p firecracker ssh -n -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@172.16.0.2"

  echo ""
  echo "==============================="
  echo "  Verifying VM${N}"
  echo "==============================="

  # Wait for API server to stabilize after restore (TLS handshake may timeout initially)
  echo "[VM${N}] Waiting for API server..."
  local API_START=$(date +%s%3N)
  for attempt in $(seq 1 30); do
    if $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get nodes" 2>&1 | grep -q Ready; then
      local API_END=$(date +%s%3N)
      echo "[VM${N}] API server ready after $((API_END - API_START))ms (attempt ${attempt})"
      break
    fi
    if [ "$attempt" -eq 30 ]; then
      echo "[VM${N}] API server TIMEOUT after 30 attempts"
      $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get nodes" 2>&1 || true
      return 1
    fi
    sleep 2
  done

  # Check node status
  echo "[VM${N}] Node status:"
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get nodes" 2>&1

  # Check all pods
  echo "[VM${N}] Pods:"
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get pods -A --no-headers" 2>&1

  # Check pre-snapshot cert-manager resources
  echo "[VM${N}] Pre-snapshot Certificate:"
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get certificate -A" 2>&1

  # Create unique ConfigMap (isolation test)
  echo "[VM${N}] Creating unique ConfigMap..."
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl create configmap vm${N}-marker --from-literal=vm=vm${N}" 2>&1

  # Create new Certificate post-restore (cert-manager functionality test)
  echo "[VM${N}] Creating post-restore Certificate..."
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: post-restore-cert-vm${N}
  namespace: default
spec:
  secretName: post-restore-cert-vm${N}-tls
  issuerRef:
    name: selfsigned-issuer
    kind: ClusterIssuer
  commonName: vm${N}.example.com
  dnsNames:
    - vm${N}.example.com
EOF" 2>&1

  # Wait for cert
  sleep 5
  echo "[VM${N}] Post-restore Certificate status:"
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get certificate -A" 2>&1

  # Verify isolation — this VM should NOT have other VMs' ConfigMaps
  echo "[VM${N}] ConfigMaps (isolation check):"
  $NSSH "KUBECONFIG=/etc/kind-kubeconfig kubectl get configmap -n default" 2>&1
}

for N in $(seq 1 $NUM_VMS); do
  verify_vm $N
done

# ==================================================
# Summary
# ==================================================
TOTAL_END=$(date +%s%3N)

echo ""
echo "=========================================="
echo "  RESULTS SUMMARY"
echo "=========================================="
echo "Total time: $((TOTAL_END - TOTAL_START))ms"
echo "Parallel VM start time: $((PARALLEL_END - PARALLEL_START))ms"
echo "All VMs SSH-ready time: $((SSH_END - SSH_START))ms"
echo "Number of VMs: $NUM_VMS"
echo "Memory per VM: 4096 MiB"
echo "Network isolation: Linux network namespaces"
echo "Guest IP: 172.16.0.2 (same in all VMs, isolated by netns)"
echo "=========================================="

# Cleanup
echo "=== Cleaning up ==="
for N in $(seq 1 $NUM_VMS); do
  sudo ip netns pids vm${N} 2>/dev/null | xargs -r sudo kill 2>/dev/null || true
  sudo ip netns del vm${N} 2>/dev/null || true
  rm -f /tmp/rootfs-vm${N}.ext4
  rm -f /tmp/fc-${N}.sock
done
rm -f /tmp/fc-snapshot/mem /tmp/fc-snapshot/vmstate
echo "=== Done ==="