Serializable Reads Demo

etcd-serial-read-demo.sh

#!/usr/bin/env bash
# =============================================================================
# etcd Serializable Reads Demo
#
# Showcases that serializable reads work:
#   1. On a LEARNER node (not yet promoted to voting member)
#   2. On a cluster WITHOUT QUORUM (one node killed in a 2-node cluster)
#
# Requirements: etcd and etcdctl must be in PATH
# Usage:        ./etcd-serializable-demo.sh
# Cleanup:      ./etcd-serializable-demo.sh cleanup
# =============================================================================

set -euo pipefail

# --- Config ------------------------------------------------------------------

NODE1_CLIENT="http://127.0.0.1:2379"
NODE1_PEER="http://127.0.0.1:2380"
NODE2_CLIENT="http://127.0.0.1:2381"
NODE2_PEER="http://127.0.0.1:2382"

DATA_DIR_1="/tmp/etcd-demo-node1"
DATA_DIR_2="/tmp/etcd-demo-node2"

LOG_DIR="/tmp/etcd-demo-logs"
PID_FILE_1="/tmp/etcd-demo-node1.pid"
PID_FILE_2="/tmp/etcd-demo-node2.pid"

ETCDCTL="etcdctl"

# --- Helpers -----------------------------------------------------------------

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
BOLD='\033[1m'
NC='\033[0m' # No Color

section() {
    echo ""
    echo -e "${BOLD}${BLUE}══════════════════════════════════════════════════${NC}"
    echo -e "${BOLD}${BLUE}  $1${NC}"
    echo -e "${BOLD}${BLUE}══════════════════════════════════════════════════${NC}"
}

info()    { echo -e "  ${BLUE}ℹ${NC}  $*"; }
ok()      { echo -e "  ${GREEN}✔${NC}  $*"; }
warn()    { echo -e "  ${YELLOW}⚠${NC}  $*"; }
fail()    { echo -e "  ${RED}✘${NC}  $*"; }

run_read() {
    local label="$1"
    local endpoint="$2"
    local key="$3"
    local consistency="$4"   # "l" (linearizable, default) or "s" (serializable)

    local flag=""
    local type_label=""
    if [[ "$consistency" == "s" ]]; then
        flag="--consistency=s"
        type_label="serializable"
    else
        type_label="linearizable"
    fi

    printf "  %-14s  %-40s  " "[$type_label]" "get $key from $endpoint"

    local output
    # We intentionally don't use set -e here so we can capture failures
    if output=$($ETCDCTL --endpoints="$endpoint" get "$key" $flag 2>&1); then
        local value
        value=$(echo "$output" | grep -v '^$' | tail -1)
        ok "value = '${value}'"
    else
        local err
        err=$(echo "$output" | grep -v '^{' | tail -1)  # strip JSON log lines
        fail "FAILED — $err"
    fi
}

wait_for_endpoint() {
    local endpoint="$1"
    local name="$2"
    local retries=20
    info "Waiting for $name to be ready..."
    for i in $(seq 1 $retries); do
        if $ETCDCTL --endpoints="$endpoint" endpoint health &>/dev/null; then
            ok "$name is up"
            return 0
        fi
        sleep 0.5
    done
    fail "$name did not come up in time"
    exit 1
}

cleanup() {
    section "Cleanup"
    for pidfile in "$PID_FILE_1" "$PID_FILE_2"; do
        if [[ -f "$pidfile" ]]; then
            local pid
            pid=$(cat "$pidfile")
            if kill -0 "$pid" 2>/dev/null; then
                kill "$pid" 2>/dev/null && ok "Killed PID $pid"
            fi
            rm -f "$pidfile"
        fi
    done
    rm -rf "$DATA_DIR_1" "$DATA_DIR_2" "$LOG_DIR"
    ok "Data dirs and logs removed"
}

# --- Cleanup shortcut --------------------------------------------------------

if [[ "${1:-}" == "cleanup" ]]; then
    cleanup
    exit 0
fi

# --- Pre-flight --------------------------------------------------------------

section "Pre-flight checks"

if ! command -v etcd &>/dev/null; then
    fail "'etcd' not found in PATH"; exit 1
fi
if ! command -v etcdctl &>/dev/null; then
    fail "'etcdctl' not found in PATH"; exit 1
fi

ETCD_VERSION=$(etcd --version | head -1)
ok "Found: $ETCD_VERSION"

cleanup 2>/dev/null || true   # clean up any leftover state from previous runs
mkdir -p "$LOG_DIR"

# =============================================================================
# PART 1 — Start a single-node cluster and add a LEARNER
# =============================================================================

section "Part 1 — Start single-node cluster"

etcd \
    --name node1 \
    --data-dir "$DATA_DIR_1" \
    --listen-client-urls "$NODE1_CLIENT" \
    --advertise-client-urls "$NODE1_CLIENT" \
    --listen-peer-urls "$NODE1_PEER" \
    --initial-advertise-peer-urls "$NODE1_PEER" \
    --initial-cluster "node1=${NODE1_PEER}" \
    --initial-cluster-token etcd-demo \
    --initial-cluster-state new \
    --log-level warn \
    > "$LOG_DIR/node1.log" 2>&1 &
echo $! > "$PID_FILE_1"

wait_for_endpoint "$NODE1_CLIENT" "node1"

# --- Write test data ---------------------------------------------------------

section "Write test data to node1"

$ETCDCTL --endpoints="$NODE1_CLIENT" put /demo/key1 "hello"   > /dev/null
$ETCDCTL --endpoints="$NODE1_CLIENT" put /demo/key2 "world"   > /dev/null
$ETCDCTL --endpoints="$NODE1_CLIENT" put /demo/key3 "etcd"    > /dev/null
ok "Written: /demo/key1=hello, /demo/key2=world, /demo/key3=etcd"

# --- Add learner -------------------------------------------------------------

section "Add node2 as a LEARNER"

MEMBER_ADD_OUTPUT=$(
    $ETCDCTL --endpoints="$NODE1_CLIENT" \
        member add node2 \
        --learner \
        --peer-urls="$NODE2_PEER"
)
echo "$MEMBER_ADD_OUTPUT" | sed 's/^/  /'

# Extract ID from "Member <ID> added to cluster ..." line in the add output.
# We can't use `member list` here because the node is still "unstarted" at
# this point and has no name yet — it only appears as the peer URL.
LEARNER_ID=$(
    echo "$MEMBER_ADD_OUTPUT" \
    | grep "^Member" \
    | awk '{print $2}'
)
ok "Learner member ID: $LEARNER_ID"

# --- Start learner node ------------------------------------------------------

section "Start learner node2"

etcd \
    --name node2 \
    --data-dir "$DATA_DIR_2" \
    --listen-client-urls "$NODE2_CLIENT" \
    --advertise-client-urls "$NODE2_CLIENT" \
    --listen-peer-urls "$NODE2_PEER" \
    --initial-advertise-peer-urls "$NODE2_PEER" \
    --initial-cluster "node1=${NODE1_PEER},node2=${NODE2_PEER}" \
    --initial-cluster-token etcd-demo \
    --initial-cluster-state existing \
    --log-level warn \
    > "$LOG_DIR/node2.log" 2>&1 &
echo $! > "$PID_FILE_2"

# Learner won't pass health check (it rejects linearizable reads), so poll
# endpoint status instead
info "Waiting for node2 (learner) to start..."
for i in $(seq 1 30); do
    if $ETCDCTL --endpoints="$NODE2_CLIENT" \
        --write-out=json endpoint status &>/dev/null 2>&1; then
        ok "node2 (learner) is up"
        break
    fi
    sleep 0.5
    if [[ $i -eq 30 ]]; then
        fail "node2 did not come up in time"; exit 1
    fi
done

sleep 1  # let the learner replicate the existing data

# --- Show cluster state ------------------------------------------------------

section "Cluster member list"
$ETCDCTL --endpoints="$NODE1_CLIENT" member list --write-out=table | sed 's/^/  /'

section "Endpoint status"
$ETCDCTL \
    --endpoints="$NODE1_CLIENT,$NODE2_CLIENT" \
    endpoint status --write-out=table 2>/dev/null | sed 's/^/  /' || true

# =============================================================================
# SCENARIO 1 — Reads on the LEARNER node (both nodes alive)
# =============================================================================

section "Scenario 1 — Reads on the LEARNER (both nodes alive)"
info "node2 is a learner — it cannot vote and does not count toward quorum"
echo ""

run_read "node2/learner" "$NODE2_CLIENT" "/demo/key1" "l"
run_read "node2/learner" "$NODE2_CLIENT" "/demo/key1" "s"

echo ""
info "Result: linearizable reads are rejected on learners; serializable reads work."

# =============================================================================
# PROMOTE the learner so we have a proper 2-node cluster
# =============================================================================

section "Promote node2 to full voting member"

$ETCDCTL --endpoints="$NODE1_CLIENT" member promote "$LEARNER_ID" | sed 's/^/  /'
ok "node2 is now a full voting member"

sleep 1

section "Cluster member list after promotion"
$ETCDCTL --endpoints="$NODE1_CLIENT" member list --write-out=table | sed 's/^/  /'

# =============================================================================
# SCENARIO 2 — Reads WITHOUT QUORUM (kill one node in a 2-node cluster)
# =============================================================================

section "Scenario 2 — Kill node2 → quorum is lost"

NODE2_PID=$(cat "$PID_FILE_2")
kill "$NODE2_PID"
rm -f "$PID_FILE_2"
ok "Killed node2 (PID $NODE2_PID) — cluster now has 1/2 nodes, quorum lost"

sleep 1  # give node1 time to notice the peer is gone

echo ""
info "Attempting reads on node1 (surviving node, no quorum):"
echo ""

run_read "node1"  "$NODE1_CLIENT" "/demo/key1" "l"
run_read "node1"  "$NODE1_CLIENT" "/demo/key1" "s"
run_read "node1"  "$NODE1_CLIENT" "/demo/key2" "s"
run_read "node1"  "$NODE1_CLIENT" "/demo/key3" "s"

echo ""
info "Result: linearizable reads time out or fail; serializable reads still work."

# =============================================================================
# Summary
# =============================================================================

section "Summary"
cat <<EOF
  ┌─────────────────────────────────────────────────────────────────┐
  │  Scenario                   │ Linearizable │ Serializable       │
  ├─────────────────────────────────────────────────────────────────┤
  │  Learner node (not promoted)│      ✘       │      ✔             │
  │  No quorum (1 of 2 nodes up)│      ✘       │      ✔             │
  └─────────────────────────────────────────────────────────────────┘

  Serializable reads serve data from the node's local state,
  bypassing the need for a quorum round-trip. The trade-off:
  data may be slightly stale.

  Logs:  $LOG_DIR/
EOF

# --- Final cleanup -----------------------------------------------------------

section "Cleanup"
cleanup