Publish host-level metrics from Databricks clusters to Prometheus/Grafana using Alloy

init-alloy.sh

#!/bin/bash
set -euo pipefail

# Init script to send host-level metrics from Databricks clusters to Prometheus/Grafana using Alloy agent. 
#
# 1. Add the init script to your cluster(s).
# 2. Add required environment variables in your cluster configuration, e.g.:
#   PROMETHEUS_HOST=10.0.0.1:9090
#   DB_CLUSTER_ID=cluster-or-job-name
# 3. Add more labels if needed (job name, etc)
# 4. Consider hosting the alloy binary in the internal repository :)
#
# Done!

ALLOY_VER="v1.10.1"
ARCH=$(uname -m)
case $ARCH in
  x86_64) ARCH=amd64 ;;
  aarch64|arm64) ARCH=arm64 ;;
esac

# download and unpack alloy
curl -fsSL \
  "https://github.com/grafana/alloy/releases/download/${ALLOY_VER}/alloy-linux-${ARCH}.zip" \
  -o /tmp/alloy.zip
unzip -p /tmp/alloy.zip alloy-linux-${ARCH} > /usr/local/bin/alloy
chmod +x /usr/local/bin/alloy
rm /tmp/alloy.zip

# alloy config
prometheusHost=$PROMETHEUS_HOST

mkdir -p /etc/alloy
cat >/etc/alloy/config.alloy <<EOF
prometheus.exporter.unix "host" {}

prometheus.scrape "host" {
  scrape_interval = "15s"
  targets         = prometheus.exporter.unix.host.targets
  forward_to      = [prometheus.remote_write.out.receiver]
}

prometheus.remote_write "out" {
  endpoint {
    url = "http://$prometheusHost/api/v1/write"
  }
  external_labels = { cluster_id = sys.env("DB_CLUSTER_ID") }
}
EOF

nohup alloy run /etc/alloy/config.alloy >>/var/log/alloy.log 2>&1 &