bootstrap-oc-cluster-with-custom-certs.sh

#!/bin/bash
set -e

# -----------------------------------------------------------------------------
# CONFIGURATION
# -----------------------------------------------------------------------------
CLUSTER="aaffinit"
RESOURCEGROUP="aro-aaffinit-rg"
CLUSTER_DOMAIN="aaffinit.com"
WORK_DIR="$HOME/dev/aro-bootstrap"
CERT_DIR="${WORK_DIR}/certs"
BUNDLE_FILE="$CERT_DIR/full-ca-bundle.crt"
mkdir -p "$CERT_DIR"

API_DOMAIN="api.${CLUSTER_DOMAIN}"
APPS_DOMAIN="*.apps.${CLUSTER_DOMAIN}"
CONSOLE_DOMAIN="console-openshift-console.apps.${CLUSTER_DOMAIN}"
OAUTH_DOMAIN="oauth-openshift.apps.${CLUSTER_DOMAIN}"

echo "================================================================"
echo " Bootstrapping ARO Cluster: $CLUSTER"
echo " Working Directory: $WORK_DIR"
echo "================================================================"

# -----------------------------------------------------------------------------
# 1. FETCH DETAILS FROM AZURE (IPs & Credentials)
# -----------------------------------------------------------------------------
echo "--> [1/5] Fetching connection details from Azure..."

# FIX: Added quotes around queries to prevent Zsh globbing errors
API_IP=$(az aro show -n $CLUSTER -g $RESOURCEGROUP --query "apiserverProfile.ip" -o tsv)
if [ -z "$API_IP" ]; then
  echo "❌ Error: Could not get API IP"
  exit 1
fi
echo "    API IP: $API_IP"

# FIX: Added quotes and specifically targeted the 'ip' field of the first profile
INGRESS_IP=$(az aro show -n $CLUSTER -g $RESOURCEGROUP --query "ingressProfiles[0].ip" -o tsv)
if [ -z "$INGRESS_IP" ]; then
  echo "❌ Error: Could not get Ingress IP"
  exit 1
fi
echo "    Ingress IP: $INGRESS_IP"

# Get Kubeadmin Password
KUBE_PASS=$(az aro list-credentials -n $CLUSTER -g $RESOURCEGROUP --query "kubeadminPassword" -o tsv)
if [ -z "$KUBE_PASS" ]; then
  echo "❌ Error: Could not get kubeadmin password"
  exit 1
fi
echo "    Credentials retrieved."

# -----------------------------------------------------------------------------
# 2. CONFIGURE LOCAL DNS (/etc/hosts)
# -----------------------------------------------------------------------------
echo "--> [2/5] Updating /etc/hosts (Sudo required)..."

# FIX: Timestamped backup to prevent overwriting previous backups
BACKUP_NAME="/etc/hosts.bak-$(date +%Y%m%d%H%M%S)"
echo "    Backing up hosts to $BACKUP_NAME"
sudo cp /etc/hosts "$BACKUP_NAME"

clean_hosts_entry() {
  local domain=$1
  # Remove existing lines containing the domain (Mac/BSD sed syntax)
  sudo sed -i '' "/$domain/d" /etc/hosts
}

# Clean old entries
clean_hosts_entry "$API_DOMAIN"
clean_hosts_entry "$CONSOLE_DOMAIN"
clean_hosts_entry "$OAUTH_DOMAIN"

# Add new entries
echo "$API_IP $API_DOMAIN" | sudo tee -a /etc/hosts >/dev/null
echo "$INGRESS_IP $CONSOLE_DOMAIN" | sudo tee -a /etc/hosts >/dev/null
echo "$INGRESS_IP $OAUTH_DOMAIN" | sudo tee -a /etc/hosts >/dev/null

echo "    DNS updated locally."

# -----------------------------------------------------------------------------
# 3. LOG IN TO OPENSHIFT
# -----------------------------------------------------------------------------
echo "--> [3/5] Logging into OpenShift..."

# We use --insecure-skip-tls-verify because the certs aren't fixed yet!
oc login "https://$API_DOMAIN:6443" \
  --username=kubeadmin \
  --password=$KUBE_PASS \
  --insecure-skip-tls-verify=true

echo "    Login successful."

# -----------------------------------------------------------------------------
# 4. GENERATE & APPLY CERTIFICATES
# -----------------------------------------------------------------------------
###############################################################################
# CERT GENERATION (SAN-COMPLIANT – OpenShift 4.19+)
###############################################################################

echo "==> Generating Ingress CA..."
openssl req -x509 -new -nodes -sha256 -days 825 \
  -newkey rsa:4096 \
  -subj "/CN=Ingress-Root-CA/O=DevCluster/L=EastUS" \
  -keyout "$CERT_DIR/ingress-ca.key" \
  -out "$CERT_DIR/ingress-ca.crt"

###############################################################################
# Ingress / Apps cert (console + oauth + routes)
###############################################################################

echo "==> Generating Apps certificate with SANs..."

cat > "$CERT_DIR/apps.cnf" <<EOF
[ req ]
default_bits       = 4096
prompt             = no
default_md         = sha256
req_extensions     = req_ext
distinguished_name = dn

[ dn ]
CN = *.apps.${CLUSTER_DOMAIN}
O  = DevCluster
L  = EastUS

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = *.apps.${CLUSTER_DOMAIN}
DNS.2 = apps.${CLUSTER_DOMAIN}
DNS.3 = ${OAUTH_DOMAIN}
DNS.4 = ${CONSOLE_DOMAIN}
EOF

openssl req -new -nodes \
  -newkey rsa:4096 \
  -keyout "$CERT_DIR/ingress.key" \
  -out "$CERT_DIR/ingress.csr" \
  -config "$CERT_DIR/apps.cnf"

openssl x509 -req \
  -in "$CERT_DIR/ingress.csr" \
  -CA "$CERT_DIR/ingress-ca.crt" \
  -CAkey "$CERT_DIR/ingress-ca.key" \
  -CAcreateserial \
  -out "$CERT_DIR/ingress.crt" \
  -days 825 \
  -extensions req_ext \
  -extfile "$CERT_DIR/apps.cnf"

###############################################################################
# Apply Ingress cert
###############################################################################

echo "==> Applying Ingress certificate..."
oc delete secret custom-ingress-tls -n openshift-ingress --ignore-not-found=true
oc create secret tls custom-ingress-tls \
  --cert="$CERT_DIR/ingress.crt" \
  --key="$CERT_DIR/ingress.key" \
  -n openshift-ingress

oc patch ingresscontroller default \
  -n openshift-ingress-operator \
  --type=merge \
  --patch '{"spec":{"defaultCertificate":{"name":"custom-ingress-tls"}}}'

###############################################################################
# API CA + API cert (SAN-compliant)
###############################################################################

echo "==> Generating API CA..."
openssl req -x509 -new -nodes -sha256 -days 825 \
  -newkey rsa:4096 \
  -subj "/CN=API-Root-CA/O=DevCluster/L=EastUS" \
  -keyout "$CERT_DIR/api-ca.key" \
  -out "$CERT_DIR/api-ca.crt"

echo "==> Generating API certificate..."

cat > "$CERT_DIR/api.cnf" <<EOF
[ req ]
default_bits       = 4096
prompt             = no
default_md         = sha256
req_extensions     = req_ext
distinguished_name = dn

[ dn ]
CN = ${API_DOMAIN}
O  = DevCluster
L  = EastUS

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = ${API_DOMAIN}
EOF

openssl req -new -nodes \
  -newkey rsa:4096 \
  -keyout "$CERT_DIR/api.key" \
  -out "$CERT_DIR/api.csr" \
  -config "$CERT_DIR/api.cnf"

openssl x509 -req \
  -in "$CERT_DIR/api.csr" \
  -CA "$CERT_DIR/api-ca.crt" \
  -CAkey "$CERT_DIR/api-ca.key" \
  -CAcreateserial \
  -out "$CERT_DIR/api.crt" \
  -days 825 \
  -extensions req_ext \
  -extfile "$CERT_DIR/api.cnf"

###############################################################################
# Apply API cert
###############################################################################

echo "==> Applying API certificate..."
oc delete secret custom-api-tls -n openshift-config --ignore-not-found=true
oc create secret tls custom-api-tls \
  --cert="$CERT_DIR/api.crt" \
  --key="$CERT_DIR/api.key" \
  -n openshift-config

oc patch apiserver cluster --type=merge \
  --patch '{
    "spec":{
      "servingCerts":{
        "namedCertificates":[{
          "names":["'"$API_DOMAIN"'"],
          "servingCertificate":{"name":"custom-api-tls"}
        }]
      }
    }
  }'

###############################################################################
# TRUSTED CA (Cluster-wide)
###############################################################################

echo "==> Configuring cluster trusted CA..."

cat "$CERT_DIR/ingress-ca.crt" "$CERT_DIR/api-ca.crt" > "$BUNDLE_FILE"

oc create configmap custom-ca \
  --from-file=ca-bundle.crt="$BUNDLE_FILE" \
  -n openshift-config \
  --dry-run=client -o yaml | oc apply -f -

oc patch proxy/cluster --type=merge \
  --patch '{"spec":{"trustedCA":{"name":"custom-ca"}}}'

###############################################################################
# RESTART SYSTEM COMPONENTS (ORDER MATTERS)
###############################################################################

echo "==> Restarting OAuth / Console / Ingress..."
oc delete pod -n openshift-authentication --all
oc delete pod -n openshift-console --all
oc delete pod -n openshift-ingress --all


echo "================================================================"
echo "✅ SUCCESS!"
echo "Cluster URL: https://$CONSOLE_DOMAIN"
echo "Username:    kubeadmin"
echo "Password:    $KUBE_PASS"
echo "Certs Path:  $CERT_DIR"
echo "================================================================"


echo Trust the Ingress CA (for the Console)
echo "sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ~/dev/aro-bootstrap/certs/ingress-ca.crt"

echo Trust the API CA (for CLI tools)
echo "sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ~/dev/aro-bootstrap/certs/api-ca.crt"

echo "================================================================"
#  Configure private DNS Zone
echo "--> Creating Private DNS Zone '$DOMAIN'..."
az network private-dns zone create \
  -g $RESOURCEGROUP \
  -n $DOMAIN

echo "--> Linking DNS Zone to Cluster VNet..."
# This link tells the VNet to check this private zone before the public internet
az network private-dns link vnet create \
  -g $RESOURCEGROUP \
  -n "${CLUSTER}-dns-link" \
  -z $DOMAIN \
  -v $VNET_NAME \
  -e false

echo "--> Adding A Records..."
# 1. API Record
az network private-dns record-set a add-record \
  -g $RESOURCEGROUP \
  -z $DOMAIN \
  -n "api" \
  -a $API_IP

# 2. Apps Wildcard Record (*.apps)
az network private-dns record-set a add-record \
  -g $RESOURCEGROUP \
  -z $DOMAIN \
  -n "*.apps" \
  -a $INGRESS_IP

echo "✅ DNS Fixed. The cluster nodes can now resolve your domain."