K8S 202602 (Minikube on Ubuntu) Notes

readme.md

Minikube on Ubuntu

External URL Ref

• Hubble traffic monitoring: http://192.168.1.219:40001/
• Headlamp dashhboard: http://192.168.1.219:41000/
• ArgoCD: http://192.168.1.219:8080/

TODOs

• Enable TLS Service Mesh Likely needs a Cilium Reinstall: https://docs.cilium.io/en/stable/network/servicemesh/mutual-authentication/mutual-authentication/

Installation

Installing Docker

https://docs.docker.com/engine/install/ubuntu/

1. Clean up old docker versions

sudo apt remove $(dpkg --get-selections docker.io docker-compose docker-compose-v2 docker-doc podman-docker containerd runc | cut -f1)

2. Set up repo

# Add Docker's official GPG key:
sudo apt update
sudo apt install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
sudo tee /etc/apt/sources.list.d/docker.sources <<EOF
Types: deb
URIs: https://download.docker.com/linux/ubuntu
Suites: $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}")
Components: stable
Signed-By: /etc/apt/keyrings/docker.asc
EOF

sudo apt update

3. Install docker

sudo apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y

4. Add self to user group

sudo usermod -aG docker $USER && newgrp docker

Installing Minikube

1. Install Minikube: Download the Minikube binary and move it to your local executable path.

curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube

2. Start cluster

export NODE_IP=$(ip -o addr show dev $(ip r | awk '/default/ {print $5}') | awk '/inet / {print $4}' | cut -d/ -f1)
minikube start \
  --driver=docker \
  --network-plugin=cni \
  --cni=false

3. Check status

minikube status

Installing CLI Tools

1. Install kubectl

curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

2. Install helm

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4
chmod 700 get_helm.sh
./get_helm.sh

3. Install cilium CLI

CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}

4. Install Hubble CLI

HUBBLE_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt)
HUBBLE_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then HUBBLE_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz{,.sha256sum}
sha256sum --check hubble-linux-${HUBBLE_ARCH}.tar.gz.sha256sum
sudo tar xzvfC hubble-linux-${HUBBLE_ARCH}.tar.gz /usr/local/bin
rm hubble-linux-${HUBBLE_ARCH}.tar.gz{,.sha256sum}

5. Install Argo CLI

curl -sSL -o argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
sudo install -m 555 argocd-linux-amd64 /usr/local/bin/argocd
rm argocd-linux-amd64

6. Install k9s

curl -sSL -o k9s.tar.gz https://github.com/derailed/k9s/releases/latest/download/k9s_Linux_amd64.tar.gz
tar -xvf k9s.tar.gz k9s
sudo install -m 555 k9s /usr/local/bin/k9s
rm k9s

Installing Cilium/Hubble

1. Install Cilium

# Note: cilium install --list-versions will NOT always give the latest version
# Check https://github.com/cilium/cilium/releases
cilium install --version 1.19.1 \
  --set kubeProxyReplacement=true \
  --set ingressController.enabled=true \
  --set ingressController.loadbalancerMode=dedicated \
  --set ingressController.default=true \
  --set ingressController.hostNetwork.enabled=true \
  --set loadBalancer.l7.backend=envoy \
  --set certgen.ttlSecondsAfterFinished=60 \
  --set l2announcements.enabled=true \
  --set k8sClientRateLimit.qps=50 \
  --set k8sClientRateLimit.burst=100
cilium status --wait

2. Enable hubble observability

cilium hubble enable
cilium status --wait

3. Enable Hubble web UI

cilium hubble enable --ui
cilium status --wait

4. Make Hubble UI accessible

cat > ~/.config/systemd/user/hubble-ui-forward.service << 'EOF'
[Unit]
Description=Hubble UI kubectl port-forward
After=network.target

[Service]
Environment=KUBECONFIG=/home/shawn/.kube/config
Environment=HOME=/home/shawn
ExecStart=/usr/local/bin/kubectl port-forward svc/hubble-ui -n kube-system --address 0.0.0.0 40001:80
Restart=always
RestartSec=5

[Install]
WantedBy=default.target
EOF

systemctl --user daemon-reload
systemctl --user enable --now hubble-ui-forward

Installing Headlamp

1. Install repo and create stack

helm repo add headlamp https://kubernetes-sigs.github.io/headlamp/
helm install my-headlamp headlamp/headlamp --namespace kube-system

2. Get URL / Set up nodeport

export POD_NAME=$(kubectl get pods --namespace kube-system -l "app.kubernetes.io/name=headlamp,app.kubernetes.io/instance=my-headlamp" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace kube-system $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
export NODE_IP=$(ip -o addr show dev $(ip r | awk '/default/ {print $5}') | awk '/inet / {print $4}' | cut -d/ -f1)
export EXTERNAL_PORT=31000

# If it created a clusterip, this will clean it up
kubectl delete svc my-headlamp

# We use a manual manifest since we can't otherwise set the external port number non-interactively
kubectl apply -n kube-system -f - <<EOF
apiVersion: v1
kind: Service
metadata:
  name: my-headlamp-nodeport
spec:
  type: NodePort
  selector:
    app.kubernetes.io/instance: my-headlamp
    app.kubernetes.io/name: headlamp
  ports:
    - port: 8080
      name: http
      targetPort: ${CONTAINER_PORT}
      nodePort: ${EXTERNAL_PORT}
EOF

# Report the actual URL
echo "Visit http://${NODE_IP}:${EXTERNAL_PORT} to use Headlamp"

3. Get token

kubectl create token my-headlamp --namespace kube-system

4. Set up a system user unit to port forward

mkdir -p ~/.config/systemd/user/

cat > ~/.config/systemd/user/headlamp-forward.service << 'EOF'
[Unit]
Description=Headlamp kubectl port-forward
After=network.target

[Service]
Environment=KUBECONFIG=/home/shawn/.kube/config
Environment=HOME=/home/shawn
ExecStart=/usr/local/bin/kubectl port-forward svc/my-headlamp-nodeport -n kube-system --address 0.0.0.0 41000:8080
Restart=always
RestartSec=5

[Install]
WantedBy=default.target
EOF

systemctl --user daemon-reload
systemctl --user enable --now headlamp-forward

systemctl --user status headlamp-forward

sudo loginctl enable-linger shawn

Install ArgoCD

1. Install ArgoCD

kubectl create namespace argocd
kubectl apply -n argocd --server-side --force-conflicts -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

2. Set up a system user unit to port forward

mkdir -p ~/.config/systemd/user/

cat > ~/.config/systemd/user/argo-forward.service << 'EOF'
[Unit]
Description=Argo kubectl port-forward
After=network.target

[Service]
Environment=KUBECONFIG=/home/shawn/.kube/config
Environment=HOME=/home/shawn
ExecStart=/usr/local/bin/kubectl port-forward svc/argocd-server -n argocd --address 0.0.0.0 8080:443
Restart=always
RestartSec=5

[Install]
WantedBy=default.target
EOF

systemctl --user daemon-reload
systemctl --user enable --now argo-forward

systemctl --user status argo-forward

sudo loginctl enable-linger shawn

3. Get password, then change it and delete the OBE secret

argocd admin initial-password -n argocd
argocd login 127.0.0.1:8080
argocd account change-password
kubectl delete secret -n argocd argocd-initial-admin-secret

4. Add the cluster to Argo

export CLUSTER_NAME=$(kubectl config get-contexts -o name)
argocd cluster add ${CLUSTER_NAME}

5. Test deploy the guestbook app

kubectl config set-context --current --namespace=argocd
argocd app create guestbook --repo https://github.com/argoproj/argocd-example-apps.git --path guestbook --dest-server https://kubernetes.default.svc --dest-namespace default
argocd app get guestbook
argocd app sync guestbook

Windows Tools

Accessing remote cluster

Easiest option is to use an SSH tunnel tunnel 127.0.0.1:8443 -> $(minikube ip):8443 update config to use https://127.0.0.1:8443

You REALLY want this from WSL or some other bash-esque env. Otherwise, a lot of shell syntax breaks.