#!/bin/sh -e
#
# Exim monitoring job
#
# $Cambridge: hermes/conf/exim/sbin/exicheck,v 1.30 2011/10/11 10:55:37 fanf2 Exp $

# wait a little while so that we don't interfere with the stats gathering script
sleep 10

PATH=/opt/clamav/bin:/opt/spamd/bin:/opt/exim/bin:/opt/exim/sbin:$PATH

case `id` in
uid=0*)	: ok
	;;
*)	echo "$0: must be run as root"
	exit 1
esac

UX=/spool/exim/unexpected
find /spool/exim 2>/dev/null | perl -ne '
$m = qr{[A-Za-z0-9]{6}-[A-Za-z0-9]{6}[A-Za-z0-9]{2}};
print unless m{
^/spool/exim(/(input(/[A-Za-z0-9](/$m)?)?
              |scan(/$m(/$m.eml)?)?
              |log(/(main|reject)log([.][0-2][0-9]([.]gz)?)?)?
              |exim-daemon[.]pid
              |exim-process.info
              |server_rates_90
              |server_rates_99
              |frozen
              |unexpected
))?}xo' >$UX
if [ -s $UX ]
then
	echo "***"
	echo "***" `hostname` unexpected files!
	echo "***"
	cat $UX
	echo
fi

if [ -f /spool/exim/log/paniclog ]
then
	echo "***"
	echo "***" `hostname` PANICLOG
	echo "***"
	cat /spool/exim/log/paniclog*
	rm  /spool/exim/log/paniclog*
	echo
fi

OLD=/spool/exim/frozen
NEW=$OLD.$$
rm -f $NEW
exim -bp |
	sed '/[*][*][*] frozen [*][*][*]/!d;/ <> /d;s/.* <//;s/> .*//' |
	uniq -c >$NEW
if [ -s $NEW ]
then
	if diff $OLD $NEW >/dev/null 2>&1
	then	: ok
	else
		echo "***"
		echo "***" `hostname` frozen messages
		echo "***"
		echo "count / sender"
		cat $NEW
		echo
	fi
fi
mv $NEW $OLD

# rest of script only applies to ppsw

case `hostname` in
ppsw-*)	: ok
	;;
*)	exit 0
	;;
esac

for db in callout ratelimit
do
	if ! /opt/exim/bin/exim_lockdb /spool/exim/db/$db.lockfile
	then
		echo "**"
		echo "**" `hostname` zapping wedged $db db
		echo "**"
		rm -f /spool/exim/db/$db /spool/exim/db/$db.lockfile
	fi
done

if e=$(/opt/exim/bin/smtpc -t10 -v localhost root@ppsw.cam.ac.uk 2>&1)
then
	: ok
else
	echo "**"
	echo "**" `hostname` exim daemon missing!
	echo "**"
	echo "$e"
	/opt/exim/etc/rc.exim quickstart
	echo "**"
	echo "**" restarted
	echo "**"
	echo
fi

# get a list of servers near or at their rate limits
OLD_90=/spool/exim/server_rates_90
OLD_99=/spool/exim/server_rates_99
NEW_90=$OLD_90.$$
NEW_99=$OLD_99.$$
NOISE=/spool/exim/server_rate_report.$$
# ensure files are present / empty as expected
touch $OLD_90 $OLD_99
>$NEW_90 >$NEW_90.brief
>$NEW_99 >$NEW_99.brief
>$NOISE
grep mail/d /opt/exim/etc/tables/client_params |
while read host param_mail_d param_mail_h param_rcpt_d param_rcpt_h rest
do
	for addr in $(dig +short A $host)
	do
		# get rates all on one line
		echo $( ratelimit list $addr |
			sed 's/record .* not found/2 sender rate: 0/;
				s/^\*\* Failed .*/2 sender rate: 0/;
				/2 sender rate: */!d;s///;s/[.].*//'
			) |
		while read rate_mail_d rate_mail_h rate_rcpt_d rate_rcpt_h
		do
			for v in mail_d mail_h rcpt_d rcpt_h
			do
				# look up variable values
				eval param=\$param_$v
				eval rate=\$rate_$v
				# extract numeric limit
				max=${param##*=}
				# reduce to 90%
				warn=$((max*9/10))
				# allow for jitter
				limit=$((max-1))
				# check
				if [ "$rate" -ge "$limit" ]
				then
					file=$NEW_99
				elif [ "$rate" -ge "$warn" ]
				then
					file=$NEW_90
				else
					file=/dev/null
				fi
				printf "%5d %5d %5d $v %s %s\n" $warn $rate $max $addr $host >>$file
				printf "$v %s %s\n" $addr $host >>$file.brief
			done
		done
	done
done
# compare new and old lists to work out what we need to report
cat $NEW_90.brief |
while read line
do
	# if not present last time...
	if ! grep "$line" $OLD_90 $OLD_99 >/dev/null
	then
		grep -h "$line" $NEW_90 >>$NOISE
	fi
done
cat $NEW_99.brief |
while read line
do
	if ! grep "$line" $OLD_99 >/dev/null
	then
		grep -h "$line" $NEW_99 >>$NOISE
	fi
done
if [ -s $NOISE ]
then
	echo "***"
	echo "***" `hostname` rate limit warning
	echo "***"
	echo
	cat $NOISE
	echo
fi
mv $NEW_90 $OLD_90
mv $NEW_99 $OLD_99
rm $NEW_90.brief
rm $NEW_99.brief
rm $NOISE

exit 0
