#!/usr/bin/perl -ws
#
# Extract information about particular hosts from the logs,
# for populating a blacklist.
#
# XXX: what about hosts that do not retry after 4xx?
#
# $Cambridge: hermes/src/stats/scripts/log2bl,v 1.18 2004/09/29 18:15:55 fanf2 Exp $

use strict;

use DB_File;
use POSIX;

use vars qw( $v $r $d );

my $DIR = '/data/stats/ppsw/blacklist/';
my $PIDFILE = "$DIR/log2bl.pid";
my $DBFILE = "$DIR/ipaddrs";
my $TMPFILE = "$DBFILE.tmp";
my $DUMPFILE = "$DBFILE.dump";

if (@ARGV == 1) {
	if($ARGV[1] =~ m{^[/.]}) {
		$DBFILE = "$ARGV[1]";
	} else {
		$DBFILE = "$DIR/$ARGV[1]";
	}
} elsif (@ARGV > 1) {
	print "usage: $0 [-drv] [dbfile]\n",
	    "	-d	dump database\n",
	    "	-r	rotate database at midnight\n",
	    "	-v	be verbose\n";
	exit 1;
}

my %id2ip;
my %ipaddrs;
my $DB;

sub opendb () {
	$DB = tie %ipaddrs, 'DB_File', "$DBFILE.db",
	    $d ? O_RDONLY : O_RDWR|O_CREAT|O_EXCL, 0666, $DB_HASH;
	die "dbopen $DBFILE: $!\n" unless defined $DB;
}

sub closedb () {
	undef $DB;
	untie %ipaddrs;
}

opendb;

$SIG{TERM} = $SIG{INT} = sub { closedb; exit; };

if ($d) {
	open DUMP, "> $TMPFILE"
	    or die "open $TMPFILE: $!\n";
	while (my ($k, $v) = each %ipaddrs) {
		print DUMP "$k: $v\n"
	}
	rename $TMPFILE, $DUMPFILE
	    or die "rename $TMPFILE $DUMPFILE: $!\n";
	exit;
}

open PID, "> $PIDFILE"
    or die "open $PIDFILE: $!\n";
print PID "$$\n";
close PID
    or die "close $PIDFILE: $!\n";

my $count = 0;

sub inc ($$;$) {
	my $field = shift;
	my $ipaddr = shift;
	my $thresh = shift;
	print "$ipaddr $field: $_"
	    if ($v);
	if (not defined $ipaddrs{$ipaddr}) {
		$ipaddrs{$ipaddr} = " $field=1 ";
	} elsif ($ipaddrs{$ipaddr} =~ s/ $field=([0-9]+) //) {
		if (defined $thresh and $thresh <= $1+1
		    and $ipaddrs{$ipaddr} eq "") {
			delete $ipaddrs{$ipaddr};
		} else {
			$ipaddrs{$ipaddr} .= " $field=".($1+1)." ";
		}
	} else {
		$ipaddrs{$ipaddr} .= " $field=1 ";
	}
	$DB->sync() if ++$count % 500 == 0;
}

my $midnight = 0;

while (<>) {
	if ($r and not $midnight and
	    m/^([A-Z][a-z][a-z]) ([0-9][0-9]) 00:00:00 /) {
		my $keepfile = "$DBFILE." . (localtime[5] + 1900) .".".
		    { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03',
		      'Apr' => '04', 'May' => '05', 'Jun' => '06',
		      'Jul' => '07', 'Aug' => '08', 'Sep' => '09',
		      'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }->{$1}
			  . ".$2.db";
		closedb;
		rename "$DBFILE.db", $keepfile
		    or die "rename $DBFILE.db $keepfile: $!\n";
		opendb;
		$midnight = 1;
	} else {
		$midnight = 0;
	}
	if (m/( H=|from )([^ ]* ){1,2}[[]([0-9.]*)[]](:[0-9]*)? /) {
		my $ipaddr = $3;
		if (/ ([0-9A-Za-z-]{16}) <= /) {
			$id2ip{$1} = $ipaddr;
			inc 'accept', $ipaddr, 10;
		} elsif (/ ([0-9A-Za-z-]{16}) Completed /) {
			delete $id2ip{$1};
		} elsif (/Relaying is not permitted/) {
			inc 'relay', $ipaddr;
		} elsif (/No SMTP service for unauthorized users/) {
			inc 'unauth', $ipaddr;
		} elsif (/Please use your name when saying HELO/) {
			inc 'HELO', $ipaddr;
		} elsif (m|rejected "POST / HTTP/1.0"|) {
			inc 'proxy', $ipaddr;
		} elsif (/SMTP protocol violation: synchronization error/) {
			inc 'sync', $ipaddr;
		} elsif (/SMTP syntax error/) {
			inc 'syntax', $ipaddr;
		} elsif (/too many nonmail commands/) {
			inc 'nonmail', $ipaddr;
		} elsif (/rejected RCPT/
			 and not /sender verify/
			 and not /listed at/) {
			inc 'RCPT', $ipaddr;
		}
	} else {
		if (/Message ([0-9A-Za-z-]{16}) from .* SpamAssassin .score=([0-9.]+)/) {
			inc 'spam', $id2ip{$1}
			    if defined $id2ip{$1} and $2 > 5;
			inc 'hispam', $id2ip{$1}
			    if defined $id2ip{$1} and $2 > 10;
		} elsif (m{/([0-9A-Za-z-]{16})/.* (FOUND|!!!) *$}) {
			inc 'virus', $id2ip{$1}
			    if defined $id2ip{$1};
		}
	}
}
