#!/bin/sh -e
#
# rough program for graphing all the data we have accumulated
#
# $Cambridge: hermes/src/stats/scripts/rrdgraphall,v 1.44 2004/11/30 13:45:06 fanf2 Exp $

PATH=$PATH:/opt/stats/bin:/opt/stats/scripts

RRDDIR=/data/stats/ppsw/graphs
IMGDIR=/data/www/stats/ppsw/img
HTMLDIR=/data/www/stats/ppsw/

# extract the list of data sets from an rrd file
#
rrd_ds () {
	rrdtool info $1 | sed '/ds[[]\([^]]*\)[]][.]type.*/!d;s//\1/'
}

# basename of an rrd file
#
rrd_base () {
	base=${1#$RRDDIR/}
	echo ${base%.rrd}
}

# basic list of arguments for RRD::GRAPH
# (see above for relative paths)
#
RRDARGS="--lazy --width 720 --height 360 --lower-limit 0 --imginfo '<IMG SRC=\"img/%s\">'"
RRDLOG="--lazy --width 720 --height 360 --logarithmic --imginfo '<IMG SRC=\"img/%s\">'"

# define all the data sets for RRD::GRAPH
#
for file in $RRDDIR/*.rrd
do
	base=`rrd_base $file`
	for ds in `rrd_ds $file`
	do
		RRDARGS="$RRDARGS DEF:$base$ds=$file:$ds:AVERAGE"
		RRDLOG="$RRDLOG DEF:$base$ds=$file:$ds:AVERAGE"
	done
done

cd $HTMLDIR

rm -f index.html
cat <<-EOF >>index.html
	<html>
	<head><title>ppsw activity graphs</title></head>
	<body>   <h1>ppsw activity graphs</h1>
	<hr>
	<p><a href="/">home page</a></p>
	<ul>
EOF

#	/4	/3	/2	/1
#	/4ish	/3	/2ish	/1
#	/2	/1
#	/4	/2	/1
#	/4
for period in \
	6hours	8hours	12hours	1day	\
	3days	5days	7days	15days	\
	30days	60days	\
	360days 720days	1440days \
	2160days
do
	echo $period
	cgi=index.$period.cgi
	cat <<-EOF >>index.html
		<li><a href="$cgi">last $period</a></li>
	EOF
	rm -f $cgi
	# page prefix
	cat <<-EOF >>$cgi
		#!/usr/bin/rrdcgi
		<html>
		<head><title>ppsw activity for the last $period</title></head>
		<body>   <h1>ppsw activity for the last $period</h1>
		<hr><a name="top"></a>
		<p><a href="index.html">main page</a></p>
		<p>Contents
		: <a href="#overview">overview of incoming message classifications</a>
		: <a href="#propreject">proportional view of message classifications</a>
		: <a href="#compreject">proportional comparison of rejection criteria</a>
		: <a href="#propmua">proportional view of user agent protocols</a>
		: <a href="#altroles">alternative view of roles</a>
		: <a href="#propfwd">proportion of traffic forwarded by Hermes</a>
		: <a href="#queueload">comparison of various load indicators</a>
		: <a href="#delay">mean time between acceptance and delivery</a>
		: <a href="#tmprej">temporary rejection comparison</a>
		</p>
		<p>Raw data sets
	EOF
	for file in $RRDDIR/*.rrd
	do
		base=`rrd_base $file`
		cat <<-EOF >>$cgi
			: <a href="#raw$base">$base</a>
		EOF
	done
	# hand-drawn grapgs
	cat <<-EOF >>$cgi
		</p>
		<p>Most of the graphs below are measuring messages
		(or other events) per second. The volume measurements
		are in bytes per second.</p>
		<hr>
		<ul>
		<li>(<a href="#top">top</a>)
		<a name="overview">overview of incoming message classifications:</a><br>
		<RRD::GRAPH $IMGDIR/overview.$period.gif
		--start -$period --end now $RRDARGS
		CDEF:RelayUnauth=protocolRelay,protocolUnauth,+
		CDEF:ServerBL=dnsbl2HeloNS,dnsbl2FromNS,dnsbl2FromMX,+,+
		AREA:rejectAll#000000:"total rejections"
		STACK:trafficMsgIn#00ff00:"accepted messages"
		AREA:rejectAll
		STACK:virusClamAV#ff0000:"discarded viruses"
		STACK:spam2HiSpam#00ffff:"high-scoring spam"
		STACK:spam2LoSpam#0000ff:"low-scoring spam"
		AREA:protocolSync#3f3f3f:"pump-and-dump"
		STACK:RelayUnauth#ffffff:"attempted unauthorized use"
		STACK:dnsblSpamhaus#bf003f:"blacklisted (Spamhaus)"
		STACK:dnsblRBLplus#3f00bf:"blacklisted (RBL+)"
		STACK:ServerBL#ff7f00:"blacklisted servers"
		STACK:protocolHello#7f0000:"bad HELO domain"
		STACK:senderPerm#007fff:"invalid sender address"
		STACK:rejectTemp#007f00:"temporary rejections"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="propreject">proportional view of message classifications:</a><br>
		<RRD::GRAPH $IMGDIR/propreject.$period.gif
		--start -$period --end now --upper-limit 1 --rigid $RRDARGS
		CDEF:RelayUnauth=protocolRelay,protocolUnauth,+
		CDEF:ServerBL=dnsbl2HeloNS,dnsbl2FromNS,dnsbl2FromMX,+,+
		CDEF:Total=rejectAll,trafficMsgIn,+
		CDEF:PrejectAll=rejectAll,Total,/
		CDEF:PtrafficMsgIn=trafficMsgIn,Total,/
		CDEF:PvirusClamAV=virusClamAV,Total,/
		CDEF:Pspam2HiSpam=spam2HiSpam,Total,/
		CDEF:Pspam2LoSpam=spam2LoSpam,Total,/
		CDEF:PprotocolSync=protocolSync,Total,/
		CDEF:PRelayUnauth=RelayUnauth,Total,/
		CDEF:PdnsblSpamhaus=dnsblSpamhaus,Total,/
		CDEF:PdnsblRBLplus=dnsblRBLplus,Total,/
		CDEF:PServerBL=ServerBL,Total,/
		CDEF:PprotocolHello=protocolHello,Total,/
		CDEF:PsenderPerm=senderPerm,Total,/
		CDEF:PrejectTemp=rejectTemp,Total,/
		AREA:PrejectAll#000000:"total rejections"
		STACK:PtrafficMsgIn#00ff00:"accepted messages"
		AREA:PrejectAll
		STACK:PvirusClamAV#ff0000:"discarded viruses"
		STACK:Pspam2HiSpam#00ffff:"high-scoring spam"
		STACK:Pspam2LoSpam#0000ff:"low-scoring spam"
		AREA:PprotocolSync#3f3f3f:"pump-and-dump"
		STACK:PRelayUnauth#ffffff:"attempted unauthorized use"
		STACK:PdnsblSpamhaus#bf003f:"blacklisted (Spamhaus)"
		STACK:PdnsblRBLplus#3f00bf:"blacklisted (RBL+)"
		STACK:PServerBL#ff7f00:"blacklisted servers"
		STACK:PprotocolHello#7f0000:"bad HELO domain"
		STACK:PsenderPerm#007fff:"invalid sender address"
		STACK:PrejectTemp#007f00:"temporary rejections"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="compreject">proportional comparison of rejection criteria:</a><br>
		the vertical axis for most of the lines is what
		proportion of rejected email is caused by this test
		(all rejections together being 100%) whereas the spam
		and virus lines are relative to accepted email.
		<RRD::GRAPH $IMGDIR/compreject.$period.gif
		--start -$period --end now $RRDARGS
		CDEF:RelayUnauth=protocolRelay,protocolUnauth,+
		CDEF:ServerBL=dnsbl2HeloNS,dnsbl2FromNS,dnsbl2FromMX,+,+
		CDEF:Sender=senderTemp,senderPerm,+
		CDEF:PdnsblSpamhaus=dnsblSpamhaus,rejectAll,/
		CDEF:PdnsblRBLplus=dnsblRBLplus,rejectAll,/
		CDEF:PprotocolHello=protocolHello,rejectAll,/
		CDEF:PprotocolSync=protocolSync,rejectAll,/
		CDEF:PRelayUnauth=RelayUnauth,rejectAll,/
		CDEF:PServerBL=ServerBL,rejectAll,/
		CDEF:PSender=Sender,rejectAll,/
		CDEF:PspamSpam=spamSpam,trafficMsgIn,/
		CDEF:PvirusClamAV=virusClamAV,trafficMsgIn,/
		LINE1:PspamSpam#7f7f7f:"spam"
		LINE1:PdnsblRBLplus#bf3f00:"blacklisted (RBL+)"
		LINE1:PdnsblSpamhaus#ff7f00:"blacklisted (Spamhaus)"
		LINE1:PServerBL#dfbf00:"blacklisted servers"
		LINE1:PprotocolHello#00bf00:"bad HELO domain"
		LINE1:PprotocolSync#bf00bf:"pump-and-dump"
		LINE1:PRelayUnauth#000000:"attempted unauthorized use"
		LINE1:PSender#007fff:"invalid sender address"
		LINE1:PvirusClamAV#ff0000:"discarded viruses"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="propmua">proportional view of user agent protocols:</a><br>
		(see also <a href="#rawprocesses">the raw processes graph</a>)<br>
		(this graph is incorrect before December 2004)<br>
		<RRD::GRAPH $IMGDIR/propimap.$period.gif
		--start -$period --end now --upper-limit 1 --rigid $RRDARGS
		CDEF:FromHermes=fromhermesHermes1,fromhermesHermes2,+
		CDEF:Total=processesproxy_imaps,processesproxy_imap,processesproxy_pops,processesproxy_pop,+,+,+
		CDEF:Pimaps=processesproxy_imaps,Total,/
		CDEF:Pimap=processesproxy_imap,Total,/
		CDEF:Ppops=processesproxy_pops,Total,/
		CDEF:Ppop=processesproxy_pop,Total,/
		CDEF:Hermes=rolesHermes25,rolesHermes465,rolesHermes587,+,+
		CDEF:SMTPS=smtptypesmtps,smtptypeesmtpa,smtptypeesmtps,smtptypeesmtpsa,+,+,+
		CDEF:Phermes=FromHermes,Hermes,/
		CDEF:Psmtps=SMTPS,Hermes,/
		AREA:Pimaps#ff0000:"secure IMAP"
		STACK:Pimap#7f0000:"insecure IMAP"
		STACK:Ppops#0000ff:"secure POP"
		STACK:Ppop#00007f:"insecure POP"
		LINE1:Phermes#cfcfcf:"SMTP from Hermes"
		LINE1:Psmtps#000000:"all secure SMTP"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="altroles">alternative view of roles:</a><br>
		note that the total is sometimes larger than the sum of
		the others since it also includes mailing list messages
		(see also <a href="#rawroles">the raw roles graph</a>)<br>
		<RRD::GRAPH $IMGDIR/altroles.$period.gif
		--start -$period --end now $RRDARGS
		CDEF:FromHermes=fromhermesHermes1,fromhermesHermes2,+
		AREA:trafficMsgIn#bfbfbf:"total incoming messages"
		LINE1:FromHermes#003fbf:"from Hermes webmail & pine"
		LINE1:rolesppsw#7f007f:"via ppsw host name"
		STACK:rolesVppsw#7f7f00:"via ppsw virtual IP address"
		STACK:rolesHermes25#ff0000:"via Hermes port 25"
		STACK:rolesHermes465#00ff00:"via Hermes port 465"
		STACK:rolesHermes587#0000ff:"via Hermes port 587"
		STACK:rolesMX#000000:"via mx.cam.ac.uk"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="propfwd">proportional view of roles:</a><br>
		the forwarding counts to not cover all aliasing/forwarding,
		for example addresses handled entirely by ppsw are not counted,
		nor is forwarding via a college or departmental server<br>
		(this graph is incorrect before December 2004)<br>
		<RRD::GRAPH $IMGDIR/propfwd.$period.gif
		--start -$period --end now --upper-limit 1 --rigid $RRDARGS
		CDEF:FromHermes=fromhermesHermes1,fromhermesHermes2,+
		CDEF:Phermes25=rolesHermes25,trafficMsgIn,/
		CDEF:Phermes465=rolesHermes465,trafficMsgIn,/
		CDEF:Phermes587=rolesHermes587,trafficMsgIn,/
		CDEF:PhermesMUA=FromHermes,trafficMsgIn,/
		CDEF:PVppsw=rolesVppsw,trafficMsgIn,/
		CDEF:Pppsw=rolesppsw,trafficMsgIn,/
		CDEF:Pfwdvol=forwardedFwdVol,volumeVolIn,/
		CDEF:Pfwdmsg=forwardedForwarded,trafficMsgIn,/
		CDEF:Ifwdvol=1,Pfwdvol,-
		CDEF:Ifwdmsg=1,Pfwdmsg,-
		CDEF:ISVppsw=Ifwdmsg,PVppsw,-
		CDEF:ISppsw=ISVppsw,Pppsw,-
		CDEF:Pmx=rolesMX,trafficMsgIn,/
		LINE1:Pmx#dfdfdf:"via mx.cam.ac.uk"
		LINE1:Phermes587#0000ff:"via hermes port 587"
		STACK:Phermes465#00ff00:"via hermes port 465"
		STACK:Phermes25#ff0000:"via hermes port 25"
		LINE1:PhermesMUA#3f3f3f:"via hermes webmail & pine"
		LINE1:Ifwdvol#ff7f7f:"forwarded volume"
		LINE1:Ifwdmsg#007f7f:"forwarded messages"
		LINE1:ISVppsw#7f7f00:"via ppsw virtual IP address"
		LINE1:ISppsw#7f007f:"via ppsw host name"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="queueload">comparison of various load indicators:</a><br>
		<RRD::GRAPH $IMGDIR/queueload.$period.gif
		--start -$period --end now $RRDARGS
		CDEF:Zppsw0=loadppsw0,UN,0,loadppsw0,IF
		CDEF:Zppsw1=loadppsw1,UN,0,loadppsw1,IF
		CDEF:Zppsw2=loadppsw2,UN,0,loadppsw2,IF
		CDEF:Zppsw3=loadppsw3,UN,0,loadppsw3,IF
		CDEF:Zppsw4=loadppsw4,UN,0,loadppsw4,IF
		CDEF:Zppsw5=loadppsw5,UN,0,loadppsw5,IF
		CDEF:Zppsw6=loadppsw6,UN,0,loadppsw6,IF
		CDEF:Zppsw7=loadppsw7,UN,0,loadppsw7,IF
		CDEF:Zppsw8=loadppsw8,UN,0,loadppsw8,IF
		CDEF:Zppsw9=loadppsw9,UN,0,loadppsw9,IF
		CDEF:Load0=Zppsw0,Zppsw1,Zppsw2,Zppsw3,Zppsw4,+,+,+,+
		CDEF:Load1=Zppsw5,Zppsw6,Zppsw7,Zppsw8,Zppsw9,+,+,+,+
		CDEF:TotalLoad=Load0,Load1,+
		CDEF:Connrefuse=connrefuseHighLoad,connrefuseHighCount,connrefuse2TooMany,+,+
		CDEF:EximIn=processesexim_in,100,/
		CDEF:EximOut=processesexim_out,100,/
		LINE1:TotalLoad#7f0000:"Total load average"
		LINE1:inqueueInQueueSz#007f00:"Incoming queue size"
		LINE1:trafficMsgIn#00007f:"Incoming message rate"
		LINE1:Connrefuse#7f007f:"Connection refused"
		LINE1:EximIn#bfbf00:"Exim in ('00)"
		LINE1:EximOut#7f7f00:"Exim out ('00)"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="delay">mean time between acceptance and delivery:</a><br>
		Note that the median would give a better indication of the typical
		delivery delay, but I haven't yet persuaded the graphing tool to
		calculate it. In the mean time a logarithmic view of the data
		gives a slightly better picture.
		(compare <a href="#rawdelay">the raw delay graph</a>)<br>
		<RRD::GRAPH $IMGDIR/delay.$period.gif
		--start -$period --end now $RRDLOG
		LINE1:delayDelay#7f0000:"delay"
		></li>
		<li>(<a href="#top">top</a>)
		<a name="tmprej">temporary rejection comparison:</a><br>
		This is just to show that most temporary rejections
		are due to sender address verification.
		<RRD::GRAPH $IMGDIR/tmprej.$period.gif
		--start -$period --end now $RRDARGS
		LINE1:rejectTemp#7f0000:"all temporary rejections"
		LINE1:senderTemp#00007f:"temporary sender rejections"
		></li>
		</ul>
		<hr>
		<ul>
	EOF
	# do the raw graphs
	for file in $RRDDIR/*.rrd
	do
		base=`rrd_base $file`
		# list of colours to use
		set --	'#7f0000' '#00007f' '#007f00' '#7f007f' '#3f3f3f' \
			'#0000ff' '#00ff00' '#ff0000' '#ff00ff' '#000000'
		lines="$RRDARGS"
		for ds in `rrd_ds $file`
		do
			lines="$lines LINE1:$base$ds$1:$ds"
			shift || true
		done
		cat <<-EOF >>$cgi
			<li>(<a href="#top">top</a>)
			<a name="raw$base">$base:</a><br>
			<RRD::GRAPH $IMGDIR/raw$base.$period.gif
			--start -$period --end now $lines
			></li>
		EOF
	done
	# page suffix
	cat <<-EOF >>$cgi
		</ul>
		<hr>
		<p><a href="index.html">main page</a></p>
		</body>
		</html>
	EOF
	chmod +x $cgi
done

cat <<-EOF >>index.html
	</ul>
	<hr>
	<p>If you have any questions or comments about the graphs, please email
	<a href="mailto:mail-support@ucs.cam.ac.uk>mail-support@ucs.cam.ac.uk</a>.</p>
	</body>
	</html>
EOF

exit
