客户hami调度器握手失败,脚本检测和重启

脚本

cat << EOF > /usr/local/bin/hami-scheduler-heal.sh
#!/bin/bash
# /usr/local/bin/hami-scheduler-heal.sh
# crontab: */2 * * * * /usr/local/bin/hami-scheduler-heal.sh >> /var/log/hami-scheduler-heal.log 2>&1
export KUBECONFIG=/etc/kubernetes/admin.conf
set -euo pipefail
NS=topke-system
STATE=/var/lib/hami-handshake-heal.state
THRESHOLD=300

mkdir -p "$(dirname "$STATE")"
declare -A prev_hs prev_at
if [ -f "$STATE" ]; then
  while IFS='|' read -r n h t; do
    prev_hs["$n"]=$h
    prev_at["$n"]=$t
  done < "$STATE"
fi

: > "$STATE.tmp"
need_heal=0
now=$(date +%s)

while IFS=$'\t' read -r node hs; do
  [ -z "${node:-}" ] && continue
  [ -z "${hs:-}" ] && continue

  case "$hs" in
    Requesting_*)
      if [ "${prev_hs[$node]:-}" = "$hs" ]; then
        first=${prev_at[$node]}
        stuck=$((now - first))
        echo "$(date '+%F %T') STUCK node=$node handshake=$hs unchanged=${stuck}s"
        echo "$node|$hs|$first" >> "$STATE.tmp"
        if [ "$stuck" -gt "$THRESHOLD" ]; then
          kubectl annotate node "$node" hami.io/node-handshake- --overwrite || true
          need_heal=1
        fi
      else
        echo "$(date '+%F %T') skip node=$node handshake=$hs (timestamp moving, scheduler alive)"
        echo "$node|$hs|$now" >> "$STATE.tmp"
      fi
      ;;
    *)
      echo "$(date '+%F %T') ok node=$node $hs"
      ;;
  esac
done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.annotations.hami\.io/node-handshake}{"\n"}{end}')

mv -f "$STATE.tmp" "$STATE"

if [ "$need_heal" -eq 1 ]; then
  kubectl -n "$NS" get pod -o name | grep hami-scheduler | xargs -r kubectl -n "$NS" delete --wait=false
  echo "$(date '+%F %T') deleted hami-scheduler (handshake stuck)"
fi
EOF

安装

chmod +x /usr/local/bin/hami-scheduler-heal.sh
crontab -e
# 加一行：
*/2 * * * * /usr/local/bin/hami-scheduler-heal.sh >> /var/log/hami-scheduler-heal.log 2>&1