sosreport gathering automation (NIDS focus)

get_nids_sos_report.sh

#!/bin/bash

# Configuration
OUTPUT_DIR="./sos-diagnostics"
IMAGE="registry.redhat.io/rhel9/support-tools:latest"
NAMESPACE="default"

mkdir -p "$OUTPUT_DIR"
TIMESTAMP=$(date +%Y%m%d-%H%M)

echo "--- OpenShift NIDS SOS Report Generator (Final Fix) ---"

# Target Worker Nodes
NODES=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[*].metadata.name}')
IFS=' ' read -r -a NODE_ARRAY <<< "$NODES"

for NODE in "${NODE_ARRAY[@]}"; do
    echo "================================================================"
    echo "Node: $NODE"
    
    # 1. CLEANUP
    echo "  > Cleaning up previous sessions..."
    oc debug node/"$NODE" -n "$NAMESPACE" --quiet=true -- \
        chroot /host podman rm -f sos-collector > /dev/null 2>&1

    # 2. RUN SOS REPORT
    echo "  > Starting SOS collection (Streaming logs)..."
    
    oc debug node/"$NODE" -n "$NAMESPACE" -- \
        chroot /host /bin/bash -c "
        
        if ! podman image exists $IMAGE; then
            echo '  ! Image not found locally. Attempting pull...'
            podman pull $IMAGE
        fi

        podman run --name sos-collector \
            --privileged \
            --ipc=host \
            --net=host \
            --pid=host \
            -e HOST=/host \
            -v /:/host \
            -v /var/log:/var/log \
            -v /run:/run \
            -v /var/tmp:/var/tmp \
            $IMAGE \
            sos report --batch \
              --sysroot /host \
              --tmp-dir /var/tmp \
              --name $NODE-$TIMESTAMP \
              --all-logs \
              -o openshift,openshift_ovn,openvswitch,networking,crio,podman \
              -k crio.all=on -k crio.logs=on \
              --plugin-timeout=300
        "

    # 3. CHECK & RETRIEVE (FIXED)
    echo "  > Checking for report..."
    
    # FIX: We list ALL sosreports sorted by time (newest first) and grab the top one.
    # We no longer match on the exact filename string because sos prepends hostnames.
    FILE_PATH=$(oc debug node/"$NODE" -n "$NAMESPACE" -- \
        chroot /host sh -c "ls -t /var/tmp/sosreport*.tar.xz 2>/dev/null | head -n 1")
    
    CLEAN_PATH=$(echo "$FILE_PATH" | grep -o "/var/tmp/sosreport.*.tar.xz")

    if [ -n "$CLEAN_PATH" ]; then
        FILENAME=$(basename "$CLEAN_PATH")
        echo "  > Found: $FILENAME"
        echo "  > Downloading..."
        
        oc debug node/"$NODE" -n "$NAMESPACE" -- \
            chroot /host cat "$CLEAN_PATH" > "$OUTPUT_DIR/$FILENAME"
            
        if [ -s "$OUTPUT_DIR/$FILENAME" ]; then
            echo "  > SUCCESS: Saved to $OUTPUT_DIR/$FILENAME"
            # Cleanup remote file
            oc debug node/"$NODE" -n "$NAMESPACE" -- chroot /host rm -f "$CLEAN_PATH" > /dev/null 2>&1
        else
            echo "  ! ERROR: File downloaded but is 0 bytes."
        fi
    else
        echo "  ! ERROR: No sosreport found in /var/tmp/."
    fi

    # Final Cleanup
    oc debug node/"$NODE" -n "$NAMESPACE" --quiet=true -- \
        chroot /host podman rm -f sos-collector > /dev/null 2>&1

done

echo "================================================================"

Extract the flows on MacOS:

#!/bin/bash

OUT_DIR="./sos-diagnostics/extracted_flows"
mkdir -p "$OUT_DIR"

echo "--- Extracting OVS Flows (MacOS Compatible) ---"

for report in sos-diagnostics/sosreport-*.tar.xz; do
    # 1. Get the Node IP (e.g. ip-10-0-22-95)
    # We grab the first matching pattern from the filename
    NODE_IP=$(echo "$report" | grep -oE "ip-10-0-[0-9]+-[0-9]+" | head -n 1)
    
    if [ -z "$NODE_IP" ]; then
        echo "Skipping $report (No IP detected in filename)"
        continue
    fi

    echo "Processing $NODE_IP..."

    # 2. Find the EXACT path of the flow files inside the tarball
    # BSD tar doesn't do wildcards well, so we ask it to list (-t) and we grep for what we want.
    
    # Target 1: Logic Flows (OpenFlow rules)
    LOGIC_FILE_PATH=$(tar -tf "$report" | grep "sos_commands/openvswitch/ovs-ofctl_dump-flows_br-int" | head -n 1)
    
    # Target 2: Datapath Flows (Kernel state)
    # We match "dpctl_dump-flows" generically to catch variations like "dump-flows_-m"
    DATAPATH_FILE_PATH=$(tar -tf "$report" | grep "sos_commands/openvswitch/ovs-appctl_dpctl_dump-flows" | head -n 1)

    # 3. Extract and Rename
    # We extract to a temporary location to handle the directory structure, then move.
    TEMP_DIR="./tmp_extract_$NODE_IP"
    mkdir -p "$TEMP_DIR"

    if [ -n "$LOGIC_FILE_PATH" ]; then
        tar -xf "$report" -C "$TEMP_DIR" "$LOGIC_FILE_PATH"
        # Move and Flatten
        mv "$TEMP_DIR/$LOGIC_FILE_PATH" "$OUT_DIR/${NODE_IP}_logic_flows.txt"
        echo "  > Extracted Logic Flows"
    else
        echo "  ! Missing Logic Flows in archive"
    fi

    if [ -n "$DATAPATH_FILE_PATH" ]; then
        tar -xf "$report" -C "$TEMP_DIR" "$DATAPATH_FILE_PATH"
        # Move and Flatten
        mv "$TEMP_DIR/$DATAPATH_FILE_PATH" "$OUT_DIR/${NODE_IP}_datapath_flows.txt"
        echo "  > Extracted Datapath Flows"
    else
        echo "  ! Missing Datapath Flows in archive"
    fi

    # Cleanup temp storage for this node
    rm -rf "$TEMP_DIR"
done

echo "------------------------------------------------"
echo "Done. Files are in $OUT_DIR"
ls -lh "$OUT_DIR"

Extracting OVS flow from an sosreport on MacOS

#!/bin/bash

OUT_DIR="./sos-diagnostics/extracted_flows"
mkdir -p "$OUT_DIR"

echo "--- Extracting OVS Flows (MacOS Compatible) ---"

for report in sos-diagnostics/sosreport-*.tar.xz; do
    # 1. Get the Node IP (e.g. ip-10-0-22-95)
    # We grab the first matching pattern from the filename
    NODE_IP=$(echo "$report" | grep -oE "ip-10-0-[0-9]+-[0-9]+" | head -n 1)
    
    if [ -z "$NODE_IP" ]; then
        echo "Skipping $report (No IP detected in filename)"
        continue
    fi

    echo "Processing $NODE_IP..."

    # 2. Find the EXACT path of the flow files inside the tarball
    # BSD tar doesn't do wildcards well, so we ask it to list (-t) and we grep for what we want.
    
    # Target 1: Logic Flows (OpenFlow rules)
    LOGIC_FILE_PATH=$(tar -tf "$report" | grep "sos_commands/openvswitch/ovs-ofctl_dump-flows_br-int" | head -n 1)
    
    # Target 2: Datapath Flows (Kernel state)
    # We match "dpctl_dump-flows" generically to catch variations like "dump-flows_-m"
    DATAPATH_FILE_PATH=$(tar -tf "$report" | grep "sos_commands/openvswitch/ovs-appctl_dpctl_dump-flows" | head -n 1)

    # 3. Extract and Rename
    # We extract to a temporary location to handle the directory structure, then move.
    TEMP_DIR="./tmp_extract_$NODE_IP"
    mkdir -p "$TEMP_DIR"

    if [ -n "$LOGIC_FILE_PATH" ]; then
        tar -xf "$report" -C "$TEMP_DIR" "$LOGIC_FILE_PATH"
        # Move and Flatten
        mv "$TEMP_DIR/$LOGIC_FILE_PATH" "$OUT_DIR/${NODE_IP}_logic_flows.txt"
        echo "  > Extracted Logic Flows"
    else
        echo "  ! Missing Logic Flows in archive"
    fi

    if [ -n "$DATAPATH_FILE_PATH" ]; then
        tar -xf "$report" -C "$TEMP_DIR" "$DATAPATH_FILE_PATH"
        # Move and Flatten
        mv "$TEMP_DIR/$DATAPATH_FILE_PATH" "$OUT_DIR/${NODE_IP}_datapath_flows.txt"
        echo "  > Extracted Datapath Flows"
    else
        echo "  ! Missing Datapath Flows in archive"
    fi

    # Cleanup temp storage for this node
    rm -rf "$TEMP_DIR"
done

echo "------------------------------------------------"
echo "Done. Files are in $OUT_DIR"
ls -lh "$OUT_DIR"

A haproxy-gather.sh for MacOS

#!/bin/zsh
# HAProxy Gather Script (Fixed Zsh Version)

NAMESPACE="openshift-ingress"
SELECTOR="-l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default"

DATE=$(date +"%Y-%m-%d-%H-%M-%S")
TARGETDIR="./haproxy-gather-${DATE}"

# Create directories
mkdir -p "$TARGETDIR/raw_stats"
mkdir -p "$TARGETDIR/haproxy_info"

print -P "%F{green}--- Starting HAProxy Gather for OCP 4.19 ---%f"
print "Target Dir: $TARGETDIR"

# 1. Get List of Running Router Pods
# Splitting output into Zsh array
pods=($(oc get pods -n "$NAMESPACE" $=SELECTOR --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}'))

if (( ${#pods} == 0 )); then
    print -P "%F{red}ERROR: No running router pods found in $NAMESPACE.%f"
    exit 1
fi

print "Found ${#pods} pods."

# 2. Gather General Overview
oc get pods -n "$NAMESPACE" -o wide > "$TARGETDIR/pod_overview.out"

# 3. Main Collection Loop
for pod in "${pods[@]}"; do
    print -P "%F{cyan}Processing $pod...%f"

    # --- RAW STATS ---
    # FIXED: We use 'bash' inside the pod, not zsh
    oc exec "$pod" -n "$NAMESPACE" -- bash -c "echo 'show stat' | socat - UNIX-CONNECT:/var/lib/haproxy/run/haproxy.sock" > "$TARGETDIR/${pod}_rawstats.csv"
    
    # Create a pretty version
    if [[ -s "$TARGETDIR/${pod}_rawstats.csv" ]]; then
        column -s, -t < "$TARGETDIR/${pod}_rawstats.csv" > "$TARGETDIR/raw_stats/${pod}_cleaned.out"
        mv "$TARGETDIR/${pod}_rawstats.csv" "$TARGETDIR/raw_stats/"
    fi

    # --- INFO & LOGS ---
    # FIXED: Renamed 'commands' -> 'haproxy_cmds' to avoid Zsh reserved word conflict
    typeset -A haproxy_cmds
    haproxy_cmds[info]="show info"
    haproxy_cmds[errors]="show errors"
    haproxy_cmds[pools]="show pools"
    haproxy_cmds[sessions]="show sess all"
    haproxy_cmds[activity]="show activity"

    for key val in "${(@kv)haproxy_cmds}"; do
        # FIXED: Using bash remotely
        oc exec "$pod" -n "$NAMESPACE" -- bash -c "echo '$val' | socat - UNIX-CONNECT:/var/lib/haproxy/run/haproxy.sock" > "$TARGETDIR/haproxy_info/${pod}_${key}.out"
    done
    
    # --- CONFIG ---
    # Copy configuration
    oc cp "$NAMESPACE/$pod":/var/lib/haproxy/conf/haproxy.config "$TARGETDIR/${pod}_haproxy.config"
done

# 4. Tarball
print "Compressing results..."
tar czf "${TARGETDIR}.tar.gz" "$TARGETDIR/"
print -P "%F{green}Done: ${TARGETDIR}.tar.gz%f"

A haproxy-gather.sh for MacOS:

#!/bin/zsh
# HAProxy Gather Script (Fixed Zsh Version)

NAMESPACE="openshift-ingress"
SELECTOR="-l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default"

DATE=$(date +"%Y-%m-%d-%H-%M-%S")
TARGETDIR="./haproxy-gather-${DATE}"

# Create directories
mkdir -p "$TARGETDIR/raw_stats"
mkdir -p "$TARGETDIR/haproxy_info"

print -P "%F{green}--- Starting HAProxy Gather for OCP 4.19 ---%f"
print "Target Dir: $TARGETDIR"

# 1. Get List of Running Router Pods
# Splitting output into Zsh array
pods=($(oc get pods -n "$NAMESPACE" $=SELECTOR --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}'))

if (( ${#pods} == 0 )); then
    print -P "%F{red}ERROR: No running router pods found in $NAMESPACE.%f"
    exit 1
fi

print "Found ${#pods} pods."

# 2. Gather General Overview
oc get pods -n "$NAMESPACE" -o wide > "$TARGETDIR/pod_overview.out"

# 3. Main Collection Loop
for pod in "${pods[@]}"; do
    print -P "%F{cyan}Processing $pod...%f"

    # --- RAW STATS ---
    # FIXED: We use 'bash' inside the pod, not zsh
    oc exec "$pod" -n "$NAMESPACE" -- bash -c "echo 'show stat' | socat - UNIX-CONNECT:/var/lib/haproxy/run/haproxy.sock" > "$TARGETDIR/${pod}_rawstats.csv"
    
    # Create a pretty version
    if [[ -s "$TARGETDIR/${pod}_rawstats.csv" ]]; then
        column -s, -t < "$TARGETDIR/${pod}_rawstats.csv" > "$TARGETDIR/raw_stats/${pod}_cleaned.out"
        mv "$TARGETDIR/${pod}_rawstats.csv" "$TARGETDIR/raw_stats/"
    fi

    # --- INFO & LOGS ---
    # FIXED: Renamed 'commands' -> 'haproxy_cmds' to avoid Zsh reserved word conflict
    typeset -A haproxy_cmds
    haproxy_cmds[info]="show info"
    haproxy_cmds[errors]="show errors"
    haproxy_cmds[pools]="show pools"
    haproxy_cmds[sessions]="show sess all"
    haproxy_cmds[activity]="show activity"

    for key val in "${(@kv)haproxy_cmds}"; do
        # FIXED: Using bash remotely
        oc exec "$pod" -n "$NAMESPACE" -- bash -c "echo '$val' | socat - UNIX-CONNECT:/var/lib/haproxy/run/haproxy.sock" > "$TARGETDIR/haproxy_info/${pod}_${key}.out"
    done
    
    # --- CONFIG ---
    # Copy configuration
    oc cp "$NAMESPACE/$pod":/var/lib/haproxy/conf/haproxy.config "$TARGETDIR/${pod}_haproxy.config"
done

# 4. Tarball
print "Compressing results..."
tar czf "${TARGETDIR}.tar.gz" "$TARGETDIR/"
print -P "%F{green}Done: ${TARGETDIR}.tar.gz%f"