Quickly create 1-node Pachyderm cluster in GKE

tiny-cluster.sh

#!/bin/bash

export RUNNING_CLUSTERS="$(
  gcloud container clusters list \
    | tail -n+2 \
    | awk '{print $1}'
)"
MACHINE_TYPE=n1-standard-4

eval "set -- $(getopt -l "no-deploy-pachyderm,create,machine-type:,delete:,delete-all" "--" "${0}" "${@}")"
deploy_pachyderm="true"
op=""
while true; do
  case "${1}" in
  --machine-type)
    if [[ "${CPU[$1]+is_set}" != "is_set" ]]; then
      echo "Unrecognized machine type: \"${2}\""
      exit 1
    fi
    MACHINE_TYPE="${2}"
    shift 2
    ;;

  --delete)
    if [[ "$(wc -l <<<"${RUNNING_CLUSTERS}")" -gt 0 ]]; then
      echo "You can delete all ${other_ct} other cluster(s) with --delete-all"
    fi
    echo "Deleting ${NAME}"
    NAME="${2}"
    gsutil -m rm -r "gs://${NAME}-bucket"
    gcloud container clusters delete --quiet "${NAME}"
    exit 0
    ;;

  --no-deploy-pachyderm)
    shift
    deploy_pachyderm="false"
    ;;

  --delete-all)
    echo "Deleting all clusters:"
    # space before escaped newline starts new arg/cmd, no space continues current cmd
    # There are two commands here: one three lines, one two lines
    sed -e "s#^#gs://#" -e "s/$/-bucket/" <<<"${RUNNING_CLUSTERS}" \
      | xargs gsutil -m rm -r
    xargs gcloud container clusters delete --quiet <<<"${RUNNING_CLUSTERS}"
    exit 0
    ;;

  --create)
    shift
    op="create"
    ;;

  --)
    shift
    if [[ -z "${op}" ]]; then
      echo "Need --create, --delete, --delete-all"
      echo "exiting..."
      exit 1
    fi
    break
    ;;

   *)
    echo "Unrecognized argument \"${1}\""
    echo "exiting..."
    exit 1
    ;;
  esac
done

if [[ $(wc -l <<<"${RUNNING_CLUSTERS}") -gt 0 ]]; then
  echo -e "You already have these clusters running:\n${RUNNING_CLUSTERS}"
fi

CLUSTER_NAME="tc-$(date +%Y%m%d-%H%M)"
BUCKET_NAME="${CLUSTER_NAME}-bucket"
BUCKET_NAME="${BUCKET_NAME,,}" # lowercase

set -x

make install && make docker-build && make push-images

# Create GCS bucket for objects
gsutil mb "gs://${BUCKET_NAME}"

# Create GKE cluster
gcloud container clusters create --quiet "${CLUSTER_NAME}" \
  --scopes=storage-rw \
  --machine-type="${MACHINE_TYPE}" \
  --num-nodes="1" \
  --disk-size="30"

# Create clusterrole binding to deploy pachyderm
kubectl create clusterrolebinding user-cluster-admin-binding \
  --clusterrole=cluster-admin \
  --user="$(gcloud config get-value account)"

if [[ "${deploy_pachyderm}" == "true" ]]; then
  PACHCTL=${PACHCTL:-pachctl}
  # Deploy pachyderm with the right toleration and scale the pachd Deployment to PACHD_NODES nodes
  ${PACHCTL} deploy google "${BUCKET_NAME}" "${PV_SIZE}" --dynamic-etcd-nodes=1 \
    --pachd-cpu-request="" \
    --pachd-memory-request=""
fi

set +x

echo "Run:"
echo "export PACHD_ADDRESS=\$(kc get nodes -o jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'):30650"