maas-debug-gather.sh

#!/usr/bin/env bash

set -euo pipefail

TAIL_LINES="${TAIL_LINES:-200}"
OUT_FILE="${OUT_FILE:-./maas-debug-$(date +%Y%m%d%H%M%S).txt}"

require_cmd() {
  if ! command -v "$1" >/dev/null 2>&1; then
    echo "Missing required command: $1" >&2
    exit 1
  fi
}

banner() {
  echo "=========================================="
  echo "🔍 MaaS Debug Data Collection Started"
  echo "=========================================="
  echo "Collecting diagnostic data for MaaS/Kuadrant/KServe policy enforcement issues..."
  echo "Output will be written to: ${OUT_FILE}"
  echo "Started at: $(date)"
  echo ""
}

require_cmd kubectl

if command -v oc >/dev/null 2>&1; then
  HAS_OC=1
else
  HAS_OC=0
fi

# Show startup banner
banner

log_block() {
  local title="$1"
  local cmd="$2"

  {
    echo "## ${title}"
    echo "\$ ${cmd}"
    bash -lc "${cmd}" 2>&1 || echo "[warn] command failed: ${cmd}"
    echo ""
  } >>"${OUT_FILE}"
}

header() {
  {
    echo "MaaS Kuadrant/KServe token & rate limit debug gather"
    echo "Started: $(date -Iseconds)"
    echo "Output file: ${OUT_FILE}"
    echo ""
  } >"${OUT_FILE}"
}

api_supported() {
  local kind="$1"
  kubectl api-resources --no-headers | awk '{print $1}' | grep -qx "${kind}"
}

describe_all() {
  local kind="$1"
  local label="$2"

  if ! api_supported "${kind}"; then
    log_block "${label} (not installed)" "echo '${kind} resource not found on this cluster'"
    return
  fi

  log_block "${label}" "kubectl get ${kind} -A -o wide"

  if ! kubectl get "${kind}" -A >/dev/null 2>&1; then
    return
  fi

  mapfile -t resources < <(kubectl get "${kind}" -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}')
  if [ ${#resources[@]} -eq 0 ]; then
    log_block "${label} details" "echo 'No ${kind} resources found'"
    return
  fi

  log_block "${label} (yaml)" "kubectl get ${kind} -A -o yaml"

  for res in "${resources[@]}"; do
    local ns name
    ns="${res%%/*}"
    name="${res##*/}"
    log_block "${label} describe: ${ns}/${name}" "kubectl describe ${kind} ${name} -n ${ns}"
  done
}

describe_if_exists() {
  local kind="$1"
  local name="$2"
  local ns="$3"

  if api_supported "${kind}" && kubectl get "${kind}" "${name}" -n "${ns}" >/dev/null 2>&1; then
    log_block "${kind} ${ns}/${name}" "kubectl describe ${kind} ${name} -n ${ns}"
  fi
}

header

log_block "kubectl version" "kubectl version --client --short 2>/dev/null || kubectl version --short 2>/dev/null || kubectl version --client 2>/dev/null || kubectl version"
log_block "current context" "kubectl config current-context"
if [ "${HAS_OC}" -eq 1 ]; then
  log_block "oc user" "oc whoami"
fi
log_block "cluster domain" "kubectl get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}'"

# ========================================
# KUADRANT INSTALLATION STATUS
# ========================================
log_block "Kuadrant installation check" "kubectl api-resources | grep kuadrant"
if api_supported "kuadrants"; then
  log_block "Kuadrant CRs status" "kubectl get kuadrant -A -o wide"
  log_block "Kuadrant CRs detailed status" "kubectl get kuadrant -A -o yaml"
else
  log_block "Kuadrant CRs check" "echo 'ERROR: Kuadrant CRD not found - this explains the policy enforcement failures'"
fi

# ========================================
# MAAS GATEWAY STATUS
# ========================================
log_block "Gateway classes" "kubectl get gatewayclass -o wide"
log_block "MaaS Default Gateway status" "kubectl get gateway maas-default-gateway -n openshift-ingress -o wide || echo 'ERROR: maas-default-gateway not found'"
log_block "MaaS Default Gateway detailed status" "kubectl describe gateway maas-default-gateway -n openshift-ingress || echo 'ERROR: maas-default-gateway not found'"

# ========================================
# MAAS HTTPROUTES STATUS
# ========================================
log_block "MaaS HTTPRoutes status" "kubectl get httproutes -n openshift-ingress -o wide"

log_block "MaaS API Route details" "kubectl describe httproute maas-api-route -n openshift-ingress || echo 'WARNING: maas-api-route not found'"
log_block "Model Route details" "kubectl describe httproute model-route -n openshift-ingress || echo 'WARNING: model-route not found'"
log_block "HTTPRoute listener summary" "kubectl get httproutes -n openshift-ingress -o jsonpath='{range .items[*]}{.metadata.name}: {.status.parents[*].conditions[*].type}={.status.parents[*].conditions[*].status} - {.status.parents[*].conditions[*].message}{\"\\n\"}{end}'"

# ========================================
# POLICY ENFORCEMENT STATUS
# ========================================
log_block "AuthPolicy status overview" "kubectl get authpolicies -A -o wide || echo 'No AuthPolicies found'"
log_block "AuthPolicy enforcement status" "kubectl get authpolicies -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}: {.status.conditions[*].type}={.status.conditions[*].status} - {.status.conditions[*].message}{\"\\n\"}{end}' || echo 'No AuthPolicies found'"

log_block "TokenRateLimitPolicy status overview" "kubectl get tokenratelimitpolicies -A -o wide || echo 'No TokenRateLimitPolicies found'"
log_block "TokenRateLimitPolicy enforcement status" "kubectl get tokenratelimitpolicies -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}: {.status.conditions[*].type}={.status.conditions[*].status} - {.status.conditions[*].message}{\"\\n\"}{end}' || echo 'No TokenRateLimitPolicies found'"

log_block "RateLimitPolicy status overview" "kubectl get ratelimitpolicies -A -o wide || echo 'No RateLimitPolicies found'"

# Detailed policy status
if api_supported "authpolicies"; then
  log_block "AuthPolicy detailed status (YAML)" "kubectl get authpolicies -A -o yaml"
else
  log_block "AuthPolicy CRD check" "echo 'ERROR: AuthPolicy CRD not found - Kuadrant may not be properly installed'"
fi

if api_supported "tokenratelimitpolicies"; then
  log_block "TokenRateLimitPolicy detailed status (YAML)" "kubectl get tokenratelimitpolicies -A -o yaml"
else
  log_block "TokenRateLimitPolicy CRD check" "echo 'ERROR: TokenRateLimitPolicy CRD not found - Kuadrant may not be properly installed'"
fi

if api_supported "ratelimitpolicies"; then
  log_block "RateLimitPolicy detailed status (YAML)" "kubectl get ratelimitpolicies -A -o yaml"
fi

# ========================================
# KSERVE/LLMINFERENCESERVICE INTEGRATION
# ========================================
if api_supported "llminferenceservices"; then
  log_block "LLMInferenceServices status" "kubectl get llminferenceservices -A -o wide"
  log_block "LLMInferenceServices gateway configuration" "kubectl get llminferenceservices -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}: gateway={.spec.gatewayService} serviceUrl={.status.serviceUrl} conditions={.status.conditions[*].type}={.status.conditions[*].status}{\"\\n\"}{end}'"
  log_block "LLMInferenceServices detailed status" "kubectl get llminferenceservices -A -o yaml"
else
  log_block "LLMInferenceService CRD check" "echo 'ERROR: LLMInferenceService CRD not found - KServe/ODH may not be installed'"
fi

if api_supported "inferenceservices"; then
  log_block "InferenceServices status" "kubectl get inferenceservices -A -o wide"
fi

# ========================================
# KUADRANT SYSTEM STATUS
# ========================================
if kubectl get ns kuadrant-system >/dev/null 2>&1; then
  log_block "Kuadrant system pods status" "kubectl get pods -n kuadrant-system -o wide"
  log_block "Kuadrant system pod events" "kubectl get events -n kuadrant-system --sort-by=.lastTimestamp | tail -20"

  # Kuadrant Operator Status
  log_block "Kuadrant operator status" "kubectl get deployment kuadrant-operator-controller-manager -n kuadrant-system -o wide"
  log_block "Kuadrant operator logs (recent)" "kubectl logs -n kuadrant-system deployment/kuadrant-operator-controller-manager --tail=${TAIL_LINES} --since=1h || kubectl logs -n kuadrant-system deployment/kuadrant-operator-controller-manager --tail=${TAIL_LINES}"

  # Limitador Status (Rate Limiting Engine)
  log_block "Limitador status" "kubectl get limitadors -A -o wide || echo 'No Limitador CRs found'"
  log_block "Limitador pods status" "kubectl get pods -n kuadrant-system -l app.kubernetes.io/name=limitador -o wide || kubectl get pods -n kuadrant-system -l app=limitador -o wide || echo 'No Limitador pods found'"
  log_block "Limitador logs (recent)" "kubectl logs -n kuadrant-system -l app.kubernetes.io/name=limitador --tail=50 --since=1h || kubectl logs -n kuadrant-system -l app=limitador --tail=50 --since=1h || echo 'No Limitador logs available'"

  # Authorino Status (Auth Engine)
  log_block "Authorino status" "kubectl get authconfigs -A -o wide || echo 'No AuthConfigs found'"
  log_block "Authorino pods status" "kubectl get pods -n kuadrant-system -l app.kubernetes.io/name=authorino -o wide || kubectl get pods -n kuadrant-system -l app=authorino -o wide || echo 'No Authorino pods found'"
  log_block "Authorino logs (recent)" "kubectl logs -n kuadrant-system -l app.kubernetes.io/name=authorino --tail=50 --since=1h || kubectl logs -n kuadrant-system -l app=authorino --tail=50 --since=1h || echo 'No Authorino logs available'"
else
  log_block "Kuadrant namespace check" "echo 'ERROR: kuadrant-system namespace not found - Kuadrant not installed'"
fi

if api_supported "wasmplugins"; then
  log_block "WasmPlugins" "kubectl get wasmplugins -A -o wide"
  log_block "WasmPlugins (yaml)" "kubectl get wasmplugins -A -o yaml"
fi

# ========================================
# MAAS API STATUS
# ========================================
if kubectl get ns maas-api >/dev/null 2>&1; then
  log_block "MaaS API pods status" "kubectl get pods -n maas-api -o wide"
  log_block "MaaS API deployment status" "kubectl get deployment -n maas-api -o wide"
  log_block "MaaS API services" "kubectl get svc -n maas-api"
  log_block "MaaS API configmaps" "kubectl get configmaps -n maas-api"
  log_block "MaaS API tier mapping config" "kubectl get configmap tier-to-group-mapping -n maas-api -o yaml || echo 'Tier mapping config not found'"
  log_block "MaaS API logs (recent)" "kubectl logs -n maas-api -l app=maas-api --tail=${TAIL_LINES} --since=1h || echo 'No MaaS API logs available'"
else
  log_block "MaaS API namespace check" "echo 'ERROR: maas-api namespace not found'"
fi

# ========================================
# MODEL SERVING STATUS
# ========================================
if kubectl get ns llm >/dev/null 2>&1; then
  log_block "Model serving pods (llm namespace)" "kubectl get pods -n llm -o wide"
  log_block "Model serving services (llm namespace)" "kubectl get svc -n llm"
else
  log_block "LLM namespace check" "echo 'WARNING: llm namespace not found - no models deployed'"
fi

if kubectl get ns redhat-ods-applications >/dev/null 2>&1; then
  log_block "ODH/RHOAI applications status" "kubectl get pods -n redhat-ods-applications | grep -E '(kserve|model-serving)' || echo 'No KServe components found in ODH'"
else
  log_block "ODH namespace check" "echo 'WARNING: redhat-ods-applications namespace not found - ODH/RHOAI may not be installed'"
fi

# ========================================
# MAAS-API TO KUADRANT INTEGRATION ANALYSIS
# ========================================
log_block "MaaS API to Kuadrant Integration Check" "echo '=== Checking MaaS API integration with Kuadrant policies ==='; echo 'Testing tier lookup endpoint that Authorino calls:'; kubectl exec -n maas-api deployment/maas-api -- curl -s -X POST http://localhost:8080/v1/tiers/lookup -H 'Content-Type: application/json' -d '{\"groups\": [\"system:authenticated\"]}' || echo 'WARNING: Tier lookup endpoint not responding'; echo 'Checking if MaaS API service is reachable from kuadrant-system:'; kubectl exec -n kuadrant-system deployment/kuadrant-operator-controller-manager -- nslookup maas-api.maas-api.svc.cluster.local || echo 'WARNING: MaaS API service not resolvable from kuadrant-system'"

log_block "Tier mapping configuration check" "kubectl get configmap tier-to-group-mapping -n maas-api -o yaml || echo 'ERROR: Tier mapping configuration missing'"

# ========================================
# AUTHORINO INTEGRATION DIAGNOSIS
# ========================================
log_block "Authorino configuration status" "echo '=== Checking Authorino integration ==='; kubectl get authconfigs -A -o wide || echo 'No AuthConfigs found'; kubectl describe authconfigs -A || echo 'No AuthConfigs to describe'"

log_block "Authorino service connectivity" "echo 'Checking if Authorino can reach MaaS API:'; kubectl exec -n kuadrant-system deployment/authorino-operator -- nslookup maas-api.maas-api.svc.cluster.local || echo 'WARNING: Authorino cannot resolve MaaS API service'"

# ========================================
# KUADRANT INSTALLATION DIAGNOSIS
# ========================================
log_block "Kuadrant Installation Detection Check" "echo '=== Checking why policies show kuadrant not installed ==='; echo 'Kuadrant Operator Status:'; kubectl get pods -n kuadrant-system -l control-plane=controller-manager || echo 'Kuadrant operator pods not found'; echo 'Kuadrant CRD Installation:'; kubectl get crd kuadrants.kuadrant.io && echo 'Kuadrant CRD exists' || echo 'ERROR: Kuadrant CRD missing'; echo 'Kuadrant Instance Status:'; kubectl get kuadrant -A || echo 'ERROR: No Kuadrant instances found - this is likely the root cause'"

log_block "Kuadrant operator environment check" "echo 'Checking Kuadrant operator environment and Gateway Controller configuration:'; kubectl get deployment kuadrant-operator-controller-manager -n kuadrant-system -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"ISTIO_GATEWAY_CONTROLLER_NAMES\")]}' || echo 'Gateway controller env var not set'; kubectl get gatewayclass || echo 'No GatewayClasses found'"

log_block "Kuadrant instance creation check" "echo 'If no Kuadrant instances found above, create one with:'; echo 'kubectl apply -f - <<EOF'; echo 'apiVersion: kuadrant.io/v1beta1'; echo 'kind: Kuadrant'; echo 'metadata:'; echo '  name: kuadrant'; echo '  namespace: kuadrant-system'; echo 'spec: {}'; echo 'EOF'"

# ========================================
# INTEGRATION FLOW TEST
# ========================================
log_block "End-to-end integration test" "echo '=== Testing complete policy enforcement flow ==='; if kubectl get httproute -n openshift-ingress maas-api-route &>/dev/null; then echo 'Testing MaaS API endpoint accessibility:'; CLUSTER_DOMAIN=\$(kubectl get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}' 2>/dev/null); if [ -n \"\$CLUSTER_DOMAIN\" ]; then echo \"Testing: https://maas.\${CLUSTER_DOMAIN}/maas-api/v1/tiers/lookup\"; curl -sk -X POST \"https://maas.\${CLUSTER_DOMAIN}/maas-api/v1/tiers/lookup\" -H 'Content-Type: application/json' -d '{\"groups\": [\"system:authenticated\"]}' -w 'HTTP Status: %{http_code}\\n' -o /dev/null --max-time 5 || echo 'External tier lookup test failed - this may indicate policy enforcement issues'; else echo 'Cannot determine cluster domain'; fi; else echo 'maas-api-route HTTPRoute not found - MaaS API not accessible externally'; fi"

log_block "Policy attachment verification" "echo '=== Verifying policy attachment to Gateway ==='; kubectl get authpolicy gateway-auth-policy -n openshift-ingress -o jsonpath='{.spec.targetRef.name}' 2>/dev/null | xargs echo 'AuthPolicy targets gateway:' || echo 'AuthPolicy not found'; kubectl get tokenratelimitpolicy gateway-token-rate-limits -n openshift-ingress -o jsonpath='{.spec.targetRef.name}' 2>/dev/null | xargs echo 'TokenRateLimitPolicy targets gateway:' || echo 'TokenRateLimitPolicy not found'"

# ========================================
# SERVICE MESH AND GATEWAY CONTROLLER STATUS
# ========================================
log_block "Gateway Controller and Service Mesh status" "echo '=== Checking Gateway Controller and Service Mesh ==='; kubectl get gatewayclass -o wide; echo 'Service Mesh components:'; kubectl get pods -n istio-system 2>/dev/null | head -5 || echo 'No Service Mesh pods found'; kubectl get crd istios.sailoperator.io &>/dev/null && echo 'Service Mesh CRD exists' || echo 'Service Mesh CRD not found'"

# ========================================
# OPERATOR STATUS AND VERSIONS
# ========================================
log_block "Operator versions and status" "echo '=== Checking operator versions ==='; kubectl get csv -n kuadrant-system --no-headers | awk '{print \"CSV: \" \$1 \" - \" \$8}' || echo 'No CSVs found in kuadrant-system'; echo 'Deployment status:'; kubectl get deployments -n kuadrant-system -o wide"

# ========================================
# DIAGNOSTIC SUMMARY
# ========================================
log_block "Policy enforcement diagnostic summary" "echo '=== Policy Enforcement Issue Summary ==='; echo 'AuthPolicies found:' \$(kubectl get authpolicies -A --no-headers | wc -l); echo 'TokenRateLimitPolicies found:' \$(kubectl get tokenratelimitpolicies -A --no-headers | wc -l); echo 'Kuadrant CRs found:' \$(kubectl get kuadrant -A --no-headers 2>/dev/null | wc -l |
          | echo '0'); echo 'Key issues to check:'; echo '1. If Kuadrant CRs = 0, that explains the \"kuadrant is not installed\" error'; echo '2. If policies exist but show not enforced, check Kuadrant instance status'; echo '3. If gateway traffic not routing, check HTTPRoute and Gateway status above'; echo '4. If MaaS API tier lookup fails, policies cannot determine user tier'; echo '5. Check operator logs above for specific error messages'"

echo "Debug report written to ${OUT_FILE}"