#!/bin/bash
#set -x

IPVSADMFILE="/etc/sysconfig/ipvsadm"
CHECKS="2"

IPS="$(cat $IPVSADMFILE |grep "-A" | cut -d" " -f 3)"
NODEIPS="$(cat $IPVSADMFILE |grep "-a" | cut -d" " -f 5 | sort | uniq)"
MAXDEL="$(echo "$(echo $NODEIPS | wc -w) - 1" | bc)"



reachable=
unreachable=
newunreach=

check() {
#  wget --no-check-certificate --timeout=5s $1 -S 2>&1 |grep -q " connected."
  IPPART="$(echo $1 | cut -d":" -f 1)"
  PORTPART="$(echo $1 | cut -d":" -f 2)"
  nc -ndz -w10 $IPPART $PORTPART >>/dev/null 2>&1
  return $?
}

if [ ! -e "/var/run/ipvsadm/unreachable" ]; then
  mkdir -p /var/run/ipvsadm
  touch /var/run/ipvsadm/unreachable
fi

for node in $NODEIPS; do
  failed=0

  for i in $(seq 1 $CHECKS); do
    check "$node" || failed=$(($failed + 1))
  done

  if [ $failed -ge $CHECKS ]; then
    unreachable="$unreachable $node"
  fi
done

# Remove unreachable servers from ipvsadm (max $MAXDEL servers)
count=0
for node in $unreachable; do
  test $count -eq $MAXDEL && break
  count=$(($count + 1))
  for ip in $IPS; do
    ipvsadm -d -t $ip -r $node > /dev/null 2>&1
  done
done

# Send an email if servers that couldn't be reached before are up again
for node in $(cat /var/run/ipvsadm/unreachable); do
  up=1

  for unreach in $unreachable; do
    test "$node" = "$unreach" && up=0
  done

  if [ $up -eq 1 ]; then
    reachable="$reachable $node"

    for ip in $IPS; do
      ipvsadm -a -t $ip -r $node -m -w 1
    done
  fi
done

if [ "$reachable" ]; then
  echo "Unreachable servers up again: $reachable" | logger -p user.notice -t checknodes
fi

## Email unreachable servers
mylist=
for node in $unreachable; do
  exists=0
  for unreach in $(cat /var/run/ipvsadm/unreachable); do
    test "$node" = "$unreach" && exists=1
  done

  if [ $exists -eq 0 ]; then
    mylist="$mylist $node"
  fi
done

if [ "$mylist" ]; then
  echo "Unreachable servers: $mylist"  | logger -p user.crit -t checknodes
fi

# Clear list
:> /var/run/ipvsadm/unreachable
for node in $unreachable; do
  echo "$node" >> /var/run/ipvsadm/unreachable
done