SkyPilot YAML that installs Prometheus and node_exporter for monitoring the remote cluster metrics

prometheus.sky.yaml

name: prometheus

resources:
  cpus: 1+

setup: |
  set -e
  
  PROMETHEUS_VERSION="3.5.0"
  NODE_EXPORTER_VERSION="1.9.1"
  INSTALL_DIR="/opt/prometheus"
  DATA_DIR="/var/lib/prometheus"
  USER="prometheus"
  
  echo "[INFO] Creating prometheus user..."
  if ! id "$USER" &>/dev/null; then
    sudo useradd --no-create-home --shell /bin/false $USER
  fi
  
  echo "[INFO] Creating directories..."
  sudo mkdir -p $INSTALL_DIR/{prometheus,node_exporter} $DATA_DIR /etc/prometheus /var/log/prometheus
  sudo chown -R $USER:$USER $INSTALL_DIR $DATA_DIR /etc/prometheus /var/log/prometheus
  
  echo "[INFO] Installing Prometheus..."
  cd /tmp
  wget -q https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz
  tar xzf prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz
  sudo cp prometheus-${PROMETHEUS_VERSION}.linux-amd64/prometheus $INSTALL_DIR/prometheus/
  sudo cp prometheus-${PROMETHEUS_VERSION}.linux-amd64/promtool $INSTALL_DIR/prometheus/
  sudo chown -R $USER:$USER $INSTALL_DIR/prometheus /etc/prometheus
  sudo ln -sf $INSTALL_DIR/prometheus/prometheus /usr/local/bin/prometheus
  rm -rf prometheus-${PROMETHEUS_VERSION}.linux-amd64*
  
  echo "[INFO] Installing Node Exporter..."
  wget -q https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
  tar xzf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
  sudo cp node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter $INSTALL_DIR/node_exporter/
  sudo chown -R $USER:$USER $INSTALL_DIR/node_exporter
  sudo ln -sf $INSTALL_DIR/node_exporter/node_exporter /usr/local/bin/node_exporter
  rm -rf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64*
  
  echo "[INFO] Creating config..."
  sudo tee /etc/prometheus/prometheus.yml << 'EOF'
  global:
    scrape_interval: 15s

  scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']

    - job_name: 'node_exporter'
      static_configs:
        - targets: ['localhost:9100']
  EOF
  sudo chown $USER:$USER /etc/prometheus/prometheus.yml
  
  echo "[INFO] Creating services..."
  sudo tee /etc/systemd/system/prometheus.service << EOF
  [Unit]
  Description=Prometheus
  After=network.target

  [Service]
  User=$USER
  Group=$USER
  ExecStart=$INSTALL_DIR/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=$DATA_DIR

  [Install]
  WantedBy=multi-user.target
  EOF
  
  sudo tee /etc/systemd/system/node_exporter.service << EOF
  [Unit]
  Description=Node Exporter
  After=network.target

  [Service]
  User=$USER
  Group=$USER
  ExecStart=$INSTALL_DIR/node_exporter/node_exporter

  [Install]
  WantedBy=multi-user.target
  EOF
  
  echo "[INFO] Starting services..."
  sudo systemctl daemon-reload
  sudo systemctl enable prometheus node_exporter
  sudo systemctl start prometheus node_exporter
  
  sleep 10
  sudo systemctl status prometheus node_exporter
  
  echo "=== Monitoring Setup Complete ==="
  echo "Prometheus UI: http://localhost:9090"
  echo "Node Exporter: http://localhost:9100/metrics"

run: |
  echo "Setup complete!"