#!/bin/bash

###############################################
# Detect CLI
###############################################

if command -v kubectl >/dev/null 2>&1; then
KCTL="kubectl"
elif command -v oc >/dev/null 2>&1; then
KCTL="oc"
else
echo "ERROR: kubectl or oc not found"
exit 1
fi

###############################################
# Namespace Input
###############################################

read -p "Enter ITSM namespace, or type No in case there is no ITSM OnPrem deployed: " ITSM_NS
read -p "Enter ITOM namespace (Required): " ITOM_NS

###############################################
# Validate namespaces
###############################################

validate_namespace() {
NS=$1
if ! $KCTL get ns "$NS" >/dev/null 2>&1; then
echo "ERROR: Namespace '$NS' does not exist"
exit 1
fi
}

if [[ -z "$ITOM_NS" ]]; then
echo "ERROR: ITOM namespace is required."
exit 1
fi

validate_namespace "$ITOM_NS"

if [[ "$ITSM_NS" != "No" && "$ITSM_NS" != "no" && -n "$ITSM_NS" ]]; then
validate_namespace "$ITSM_NS"
fi

###############################################
# Output directory
###############################################

TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
OUTDIR="k8s_support_bundle_${TIMESTAMP}"
mkdir -p "$OUTDIR"

NAMESPACES=""

if [[ "$ITSM_NS" != "No" && "$ITSM_NS" != "no" && -n "$ITSM_NS" ]]; then
NAMESPACES="$ITSM_NS"
fi

NAMESPACES="$NAMESPACES $ITOM_NS"

echo "Namespaces to collect:$NAMESPACES"

###############################################
# Cluster info
###############################################

echo "Collecting cluster information..."

$KCTL version > "$OUTDIR/cluster_version.txt" 2>&1
$KCTL cluster-info > "$OUTDIR/cluster_info.txt" 2>&1
$KCTL get nodes -o wide > "$OUTDIR/nodes.txt"
$KCTL describe nodes > "$OUTDIR/nodes_describe.txt"
$KCTL top nodes > "$OUTDIR/top_nodes.txt" 2>&1

###############################################
# Namespace Loop
###############################################

for NAMESPACE in $NAMESPACES
do

echo ""
echo "Processing namespace: $NAMESPACE"

if [[ "$NAMESPACE" == "$ITSM_NS" ]]; then
NSDIR="$OUTDIR/ITSM_${NAMESPACE}"
else
NSDIR="$OUTDIR/ITOM_${NAMESPACE}"
fi

mkdir -p "$NSDIR"

###############################################
# Resource inventory
###############################################

$KCTL get pods -n $NAMESPACE -o wide > "$NSDIR/pods.txt"
$KCTL get deployments -n $NAMESPACE > "$NSDIR/deployments.txt" 2>/dev/null
$KCTL get statefulsets -n $NAMESPACE > "$NSDIR/statefulsets.txt" 2>/dev/null
$KCTL get daemonsets -n $NAMESPACE > "$NSDIR/daemonsets.txt" 2>/dev/null
$KCTL get svc -n $NAMESPACE > "$NSDIR/services.txt" 2>/dev/null
$KCTL get pvc -n $NAMESPACE > "$NSDIR/pvc.txt" 2>/dev/null

###############################################
# Pod usage
###############################################

echo "Collecting pod resource usage..."

$KCTL top pods -n $NAMESPACE > "$NSDIR/kubectl_top_pods.txt" 2>/dev/null

if command -v oc >/dev/null 2>&1; then
oc adm top pods -n $NAMESPACE > "$NSDIR/oc_adm_top_pods.txt" 2>/dev/null
fi

###############################################
# High utilization detection
###############################################

echo "Checking pods with CPU or Memory >=90%"

TOP_DATA=$($KCTL top pods -n $NAMESPACE --no-headers 2>/dev/null)

echo "$TOP_DATA" | while read LINE
do

POD=$(echo $LINE | awk '{print $1}')
CPU=$(echo $LINE | awk '{print $2}')
MEM=$(echo $LINE | awk '{print $3}')

CPU_USAGE=$(echo $CPU | sed 's/m//')
MEM_USAGE=$(echo $MEM | sed 's/Mi//' | sed 's/Gi/*1024/' | bc 2>/dev/null)

CPU_LIMIT=$($KCTL describe pod $POD -n $NAMESPACE 2>/dev/null | grep -A3 Limits | grep cpu | head -1 | awk '{print $2}')
MEM_LIMIT=$($KCTL describe pod $POD -n $NAMESPACE 2>/dev/null | grep -A3 Limits | grep memory | head -1 | awk '{print $2}')

CPU_LIMIT=$(echo $CPU_LIMIT | sed 's/m//')
MEM_LIMIT=$(echo $MEM_LIMIT | sed 's/Mi//' | sed 's/Gi/*1024/' | bc 2>/dev/null)

CPU_PERCENT=0
MEM_PERCENT=0

if [[ ! -z "$CPU_LIMIT" && "$CPU_LIMIT" -gt 0 ]]; then
CPU_PERCENT=$(( CPU_USAGE * 100 / CPU_LIMIT ))
fi

if [[ ! -z "$MEM_LIMIT" && "$MEM_LIMIT" -gt 0 ]]; then
MEM_PERCENT=$(( MEM_USAGE * 100 / MEM_LIMIT ))
fi

if [[ "$CPU_PERCENT" -ge 90 || "$MEM_PERCENT" -ge 90 ]]; then

echo "High utilization pod detected: $POD"

UTILDIR="$NSDIR/High_Utilization_Pods"
mkdir -p "$UTILDIR"

$KCTL describe pod $POD -n $NAMESPACE > "$UTILDIR/${POD}_describe.txt"

fi

done

###############################################
# Problematic pod detection
###############################################

echo "Checking pods with error states..."

PROBDIR="$NSDIR/Problematic_Pods"
mkdir -p "$PROBDIR"

PROBLEM_STATES="ImagePullBackOff|ErrImagePull|CrashLoopBackOff|CreateContainerError|Init:CrashLoopBackOff|Pending"

PROBLEM_PODS=$($KCTL get pods -n $NAMESPACE --no-headers | grep -E "$PROBLEM_STATES" | awk '{print $1}')

ALL_PODS=$($KCTL get pods -n $NAMESPACE -o jsonpath='{.items[*].metadata.name}')

for POD in $ALL_PODS
do

REASONS=$($KCTL get pod $POD -n $NAMESPACE -o jsonpath='{.status.containerStatuses[*].state.waiting.reason} {.status.initContainerStatuses[*].state.waiting.reason}')

if echo "$REASONS" | grep -E "ImagePullBackOff|ErrImagePull|CrashLoopBackOff|CreateContainerError" >/dev/null
then
PROBLEM_PODS="$PROBLEM_PODS $POD"
fi

done

PROBLEM_PODS=$(echo $PROBLEM_PODS | tr ' ' '\n' | sort -u)

for POD in $PROBLEM_PODS
do

echo "Collecting diagnostics for problematic pod: $POD"

PODDIR="$PROBDIR/$POD"
mkdir -p "$PODDIR"

$KCTL describe pod $POD -n $NAMESPACE > "$PODDIR/${POD}_describe.txt"
$KCTL get pod $POD -n $NAMESPACE -o yaml > "$PODDIR/${POD}_yaml.txt"

CONTAINERS=$($KCTL get pod $POD -n $NAMESPACE -o jsonpath='{.spec.containers[*].name}')

for C in $CONTAINERS
do
$KCTL logs $POD -n $NAMESPACE -c $C > "$PODDIR/${C}.log" 2>/dev/null
$KCTL logs -p $POD -n $NAMESPACE -c $C > "$PODDIR/${C}_previous.log" 2>/dev/null
done

done

###############################################
# Restarted containers last 2 days
###############################################

echo "Checking containers restarted in last 2 days..."

PODS=$($KCTL get pods -n $NAMESPACE -o jsonpath='{.items[*].metadata.name}')

for POD in $PODS
do

CONTAINERS=$($KCTL get pod $POD -n $NAMESPACE -o jsonpath='{.spec.containers[*].name}')

for C in $CONTAINERS
do

RESTART=$($KCTL get pod $POD -n $NAMESPACE -o jsonpath="{.status.containerStatuses[?(@.name=='$C')].restartCount}")

FINISHED=$($KCTL get pod $POD -n $NAMESPACE -o jsonpath="{.status.containerStatuses[?(@.name=='$C')].lastState.terminated.finishedAt}")

if [[ "$RESTART" -gt 0 && ! -z "$FINISHED" ]]; then

LAST_TS=$(date -d "$FINISHED" +%s 2>/dev/null)
NOW_TS=$(date +%s)

DIFF=$(( (NOW_TS - LAST_TS) / 86400 ))

if [[ "$DIFF" -le 2 ]]; then

echo "Collecting logs for $POD / $C"

PODDIR="$NSDIR/Restarted_Pods/$POD"
mkdir -p "$PODDIR"

$KCTL describe pod $POD -n $NAMESPACE > "$PODDIR/describe.txt"
$KCTL logs $POD -c $C -n $NAMESPACE > "$PODDIR/${C}.log"
$KCTL logs -p $POD -c $C -n $NAMESPACE > "$PODDIR/${C}_previous.log"

fi
fi

done
done

###############################################
# PostgreSQL diagnostics
###############################################

echo "Checking postgres pods..."

PGDIR="$NSDIR/Postgres_Diagnostics"
mkdir -p "$PGDIR"

POSTGRES_PODS=$($KCTL get pods -n $NAMESPACE | grep postgres-bmc-pg-ha | grep -v pool | grep -v monitor | awk '{print $1}')

LEADER=$($KCTL get pods -n $NAMESPACE --show-labels | grep postgres-bmc-pg-ha | grep role=primary | awk '{print $1}')

echo "Postgres leader pod: $LEADER" | tee "$PGDIR/leader.txt"

for POD in $POSTGRES_PODS
do

echo "Collecting postgres diagnostics from $POD"

PODDIR="$PGDIR/$POD"
mkdir -p "$PODDIR"

$KCTL exec -n $NAMESPACE $POD -- df -h > "$PODDIR/df.txt"
$KCTL exec -n $NAMESPACE $POD -- patronictl list > "$PODDIR/patronictl_list.txt"

done

if [[ ! -z "$LEADER" ]]; then

$KCTL exec -n $NAMESPACE $LEADER -- psql -c "select * from pg_replication_slots;" \
> "$PGDIR/pg_replication_slots.txt"

fi

done

###############################################
# Compress bundle
###############################################

ZIPFILE="${OUTDIR}.zip"

echo ""
echo "Compressing support bundle..."

zip -r "$ZIPFILE" "$OUTDIR" >/dev/null

rm -rf "$OUTDIR"

echo ""
echo "Support bundle created:"
echo "$ZIPFILE"
echo ""