#!/usr/bin/perl
use strict;
#use warnings;
#use diagnostics;
# the output of warnings and diagnostics should not be enabled in production.
# the SNMP daemon depends on the output of maillogs, so we need to return
# nothing or valid counters.

#
# $Id: Logs.pm, Exp $
#

package Mail::Toaster::Logs;

use Carp;
use Getopt::Std;
use Params::Validate qw( :all);
use File::Path;
use English qw( -no_match_vars );
use Pod::Usage;

use vars qw( $VERSION $spam_ref $count_ref );
$VERSION = "5.02";

use lib "lib";

use Mail::Toaster::Utility 5; 
my $utility = Mail::Toaster::Utility->new;

sub new {
    my $class = shift;
    # to create an object of this class, you must pass in a hashref which
    # should contains the contents of toaster.conf
    my %p = validate(@_, { 'conf'=>HASHREF, }, );
    my $self = { 
        class => $class, 
        conf  => $p{'conf'}, 
        debug => $p{'conf'}->{'logs_debug'} || 0,
    };
    bless( $self, $class );
    return $self;
}

sub report_yesterdays_activity {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate(@_, { test_ok => {type=>BOOLEAN, optional=>1}, } );

    if ( defined $p{'test_ok'} ) { return $p{'test_ok'}; }

    my $email = $conf->{'toaster_admin_email'} || "postmaster";

    my $qmailanalog_dir = $self->find_qmailanalog();
    return unless ( $qmailanalog_dir );

    my $log = $self->get_yesterdays_send_log();
    if ( ! -s $log ) {
        carp "no log file for yesterday found!\n";
        return;
    };

    print "processing log: $log\n" if $debug;

    my $cat = $log =~ m/\.bz2$/ ? $utility->find_the_bin( bin => "bzcat",debug=>0 )
            : $log =~ m/\.gz$/  ? $utility->find_the_bin( bin => "gzcat",debug=>0 )
            : $utility->find_the_bin(bin=>"cat", debug=>0)
            ;

    print "calculating overall stats with:\n" if $debug;
    print "`$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zoverall`\n"
      if $debug;
    my $overall = `$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zoverall`;

    print "calculating failure stats with:\n" if $debug;
    print "`$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zfailures`\n"
      if $debug;
    my $failures =
      `$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zfailures`;
      
    print "calculating deferral stats\n" if $debug;
    print "`$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zdeferrals`\n"
      if $debug;
    my $deferrals =
      `$cat $log | $qmailanalog_dir/matchup 5>/dev/null | $qmailanalog_dir/zdeferrals`;

    my ( $dd, $mm, $yy ) = $utility->get_the_date(bump=>0,debug=>$debug);
    my $date = "$yy.$mm.$dd";
    print "date: $yy.$mm.$dd\n" if $debug;

    #require Mail::Toaster::Perl;
    #my $perl = Mail::Toaster::Perl->new;
    #if ( $perl->has_module("Perl6::Form") ) {
    #    $self->pretty_yesterday($date, $overall, $failures, $deferrals);
    #    return;
    #};

    ## no critic
    open my $EMAIL, "| /var/qmail/bin/qmail-inject";
    ## use critic
    print $EMAIL <<"EO_EMAIL";
To: $email
From: postmaster
Subject: Daily Mail Toaster Report for $date

 ==================================================================== 
               OVERALL MESSAGE DELIVERY STATISTICS                  
 ____________________________________________________________________

$overall


 ==================================================================== 
                        MESSAGE FAILURE REPORT                         
 ____________________________________________________________________
$failures


 ==================================================================== 
                      MESSAGE DEFERRAL REPORT                       
 ____________________________________________________________________
$deferrals
EO_EMAIL

    close $EMAIL;

    print "all done!\n" if $debug;

    return 1;
}

sub pretty_yesterday {

    my $self = shift;
    my $conf = $self->{'conf'};

    my ($date, $overall, $failures, $deferrals) = @_;

    my $email = $conf->{'toaster_admin_email'} || "postmaster";
    my $host  = $conf->{'toaster_hostname'} || "localhost";

    #use Perl6::Form;
    #Perl6::Form->import;

    #my $overall = format_me($overall);

    #print form,
    print 
        "To: $email
From: postmaster\@$host
Subject: Daily Mail Toaster Report for $date",
        '',
        '==================================================================== ',
        '               OVERALL MESSAGE DELIVERY STATISTICS                  |',
        '--------------------------------------------------------------------|',
        '|   {[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}    |',
        $overall,

        ' ==================================================================== ',
        '|                     MESSAGE FAILURE REPORT                         |',
        '|--------------------------------------------------------------------|',
        '|   {[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}     |',
        $failures,

        ' ==================================================================== ',
        '|                      MESSAGE DEFERRAL REPORT                       |',
        '|--------------------------------------------------------------------|',
        '|   {[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}     |',
        $deferrals,
        ' ==================================================================== ';


    ## no critic
    #open my $EMAIL, "| /var/qmail/bin/qmail-inject";
    ## use critic
    #print $mess;

    sub format_me {
        my $overall = shift;
        my @lines = split(/\n/, $overall);
        print join("\t", @lines) . "\n";
    }

};

sub get_yesterdays_send_log {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    if ( $conf->{'send_log_method'} && $conf->{'send_log_method'} eq "syslog" ) {
        # freebsd's maillog is rotated daily
        if ( $OSNAME eq "freebsd" ) {
             my $file = "/var/log/maillog.0";

             return -e "$file.gz" ? "$file.gz"
                  : -e "$file.bz2" ? "$file.bz2" 
                  : croak "could not find your mail logs from yesterday! "
                    . "Check your send_log_method setting in toaster-watcher.conf "
                    . "and make sure it is configured appropriately for your system.";
        } 
        elsif ( $OSNAME eq "darwin" ) {
            # logs are rotated weekly.
            return "/var/log/mail.log";
        }
        
        my $file = "/var/log/mail.log.0";

        return  -e "$file.gz" ? "$file.gz"
              : -e "$file.bz2" ? "$file.bz2" 
              : croak "could not find your mail logs from yesterday!\n";
    };

    # some form of multilog logging
    my $logbase = $conf->{'logs_base'}
                || $conf->{'qmail_log_base'}
                || "/var/log/mail";

    # set up our date variables for today
    my ( $dd, $mm, $yy ) = $utility->get_the_date(bump=>0,debug=>$debug);

    # where todays logs are being archived
    my $log = "$logbase/$yy/$mm/$dd/sendlog";

    print "report_yesterdays_activity: updating todays symlink for sendlogs\n"
        if $debug;
    unlink("$logbase/sendlog") if ( -l "$logbase/sendlog" );
    symlink( $log, "$logbase/sendlog" );

    # where yesterdays logs are being archived
    print "report_yesterdays_activity: updating yesterdays symlink for sendlogs\n"
        if $debug;
    ( $dd, $mm, $yy ) = $utility->get_the_date(bump=>1,debug=>$debug);
    $log = "$logbase/$yy/$mm/$dd/sendlog.gz";

    unlink("$logbase/sendlog.gz") if ( -l "$logbase/sendlog.gz" );
    symlink( $log, "$logbase/sendlog.gz" );

    return $log;
};

sub find_qmailanalog {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $qmailanalog_dir = $conf->{'qmailanalog_bin'} || "/var/qmail/qmailanalog/bin";

    # the port location changed, if toaster.conf hasn't been updated, this 
    # will catch it.
    if ( ! -d $qmailanalog_dir ) {
        carp <<"EO_QMADIR_MISSING";
  ERROR: the location of qmailanalog programs is missing! Make sure you have
  qmailanalog installed and the path to the binaries is set correctly in 
  toaster.conf. The current setting is $qmailanalog_dir
EO_QMADIR_MISSING


        if ( -d "/usr/local/qmailanalog/bin" ) {
            $qmailanalog_dir = "/usr/local/qmailanalog/bin";

            carp <<"EO_QMADIR_FOUND";

  YAY!  I found your qmailanalog programs in /usr/local/qmailanalog/bin. You 
  should update toaster.conf so you stop getting this error message.
EO_QMADIR_FOUND
        };
    };

    # make sure that the matchup program is in there
    unless ( -x "$qmailanalog_dir/matchup" ) {
        carp <<"EO_NO_MATCHUP";

   report_yesterdays_activity: ERROR! The 'maillogs yesterday' feature only 
   works if qmailanalog is installed. I am unable to find the binaries for 
   it. Please make sure it is installed and the qmailanalog_bin setting in
   toaster.conf is configured correctly.
EO_NO_MATCHUP

        return;
    }

    return $qmailanalog_dir;
};

sub verify_settings {

    my $self = shift;

    my $conf     = $self->{'conf'};
    my $logbase  = $conf->{'logs_base'}     || $conf->{'qmail_log_base'};
    my $counters = $conf->{'logs_counters'} || "counters" || "/var/log/mail";

    my $user  = $conf->{'logs_user'}  || "qmaill";
    my $group = $conf->{'logs_group'} || "qnofiles";
    
    my $uid   = getpwnam($user);
    my $gid   = getgrnam($group);

    unless ( $uid && $gid ) {
        croak "
     The log user ($user)  or group ($group) is not installed!\n\n";
    };

    unless ( -e $logbase ) {
        mkpath( $logbase, 0, oct('0755') )
            or croak "Couldn't create $logbase: $!\n";

        chown( $uid, $gid, $logbase ) 
            or croak "Couldn't chown $logbase to $uid: $!\n";
    };

    if ( -w $logbase ) {
        chown( $uid, $gid, $logbase )
            or croak "Couldn't chown $logbase to $uid: $!\n";
    }

    my $dir = "$logbase/$counters";

    unless ( -e $dir ) {
        mkpath( $dir, 0, oct('0755') ) or croak "Couldn't create $dir: $!\n";
        chown( $uid, $gid, $dir ) or croak "Couldn't chown $dir to $uid: $!\n";
    }
    else {
        unless ( -d $dir ) {
            croak"$dir should be a directory and is not! \n";
        }
    }

    my $script = "/usr/local/sbin/maillogs";

    print "WARNING: $script must be installed!\n"  unless ( -e $script );
    print "WARNING: $script must be executable!\n" unless ( -x $script );
}

sub parse_cmdline_flags {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate(@_, {
            'prot' => {type=>SCALAR|UNDEF, optional=>1, },
            'debug'=> {type=>BOOLEAN, optional=>1, default=>$debug},
    } );

    my $prot  = $p{'prot'};
       $debug = $p{'debug'};

    my %valid_prots = (
        'smtp' => 1,   'yesterday'    => 1,
        'send' => 1,   'qmailscanner' => 1,
        'imap' => 1,   'spamassassin' => 1,
        'pop3' => 1,   'webmail' => 1,
        'rbl'  => 1,   'test'    => 1,
    );

    pod2usage() if !$prot;

    if ( $prot eq "test" ) { return 1; };
    
    print "parse_cmdline_flags: prot is $prot\n" if $debug;


    # in case it was passed to us via $conf
    if ( $conf->{'debug'} ) { $debug  = $conf->{'debug'} };

    print "working on protocol: $prot\n" if $debug;
 
      $prot eq "smtp" ? $self->smtp_auth_count()
    : $prot eq "rbl"  ? $self->rbl_count  ()
    : $prot eq "send" ? $self->send_count ()
    : $prot eq "pop3" ? $self->pop3_count ()
    : $prot eq "imap" ? $self->imap_count ()
    : $prot eq "spamassassin" ? $self->spama_count()
    : $prot eq "qmailscanner" ? $self->qms_count()
    : $prot eq "webmail"      ? $self->webmail_count()
    : $prot eq "yesterday"    ? $self->report_yesterdays_activity() 
    : pod2usage();
}

sub what_am_i {

    my $self  = shift;
    my $debug = $self->{'debug'};

    print "what_am_i: $0 \n" if $debug;
    $0 =~ /([a-zA-Z0-9\.]*)$/;
    print "what_am_i: returning $1\n" if $debug;
    return $1;
}


sub rbl_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $countfile = $self->set_countfile(prot=>"rbl");
    $spam_ref     = $self->counter_read( file=>$countfile );

    my $logbase   = $conf->{'logs_base'} || "/var/log/mail";

    $self->process_rbl_logs( 
        files => $self->check_log_files( ["$logbase/smtp/current"] ),
    );

    print "      Spam Counts\n\n" if $debug;

    my $i = 0;
    while ( my ($description,$count) =  each %$spam_ref ) {
        print ":" if ( $i > 0 );
        print "$description:$count";
        $i++;
    }
    print "\n" if $i > 0;

    my $supervise = $conf->{'logs_supervise'} || "/var/qmail/supervise";

    $self->rotate_supervised_logs("$supervise/smtp/log");

    $self->compress_yesterdays_logs( file=>"smtplog" );

    return 1;

  #  do not write out the counters, they get updated when we rotate the logs
  #	$self->counter_write(log=>$countfile, values=>$spam_ref );
}

sub smtp_auth_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $countfile = $self->set_countfile(prot=>"smtp");
    my $count_ref = $self->counter_read( file=>$countfile );

    print "      SMTP Counts\n\n" if $debug;

    my $logfiles = $self->check_log_files( [ $self->syslog_locate() ] );
    if ( $logfiles->[0] eq "" ) {
        carp "\nsmtp_auth_count: Ack, no logfiles! You may want to see why?";
        return 1;
    }

    my (%new_entries, $lines);

    # we could have one log file, or dozens (multilog)
    # so we go through them all adding their entries to the new_entries hash.
    foreach (@$logfiles) {
        open my $LOGF, "<", $_;

        while ( my $log_line = <$LOGF> ) {
            next
              unless ( $log_line =~ /vchkpw-smtp/ || $log_line =~ /vchkpw-submission/ );

            $lines++;
            $new_entries{'connect'}++;
            $new_entries{'success'}++ if ( $log_line =~ /success/ );
        }
    }

    # no point in all the following if there were no log entries.
    return 1 unless $lines;

    if ( $new_entries{'success'} ) {

        # because rrdutil needs ever increasing counters (ie, not starting new
        # each day), we keep track of when the counts suddenly reset (ie, after a
        # syslog gets rotated). To reliably know when this happens, we save the
        # last counter in a _last count. If the new count is greater the last
        # count, add the difference, which is how many authentications
        # happened since we last checked.

        if ( $new_entries{'success'} >= $count_ref->{'success_last'} ) {
            $count_ref->{'success'} +=
            ( $new_entries{'success'} - $count_ref->{'success_last'} );
        }
        else { 
            # If the counters are lower, then the logs were just rolled and we 
            # need only to add them to the new count. 
            $count_ref->{'success'} += $new_entries{'success'};
        };

        $count_ref->{'success_last'} = $new_entries{'success'};
    };

    if ( $new_entries{'connect'} ) {
        if ( $new_entries{'connect'} >= $count_ref->{'connect_last'} ) {
            $count_ref->{'connect'} += 
                ( $new_entries{'connect'} - $count_ref->{'connect_last'} );
        }
        else { 
            $count_ref->{'connect'} += $new_entries{'connect'} 
        };

        $count_ref->{'connect_last'} = $new_entries{'connect'};
    };

    print "smtp_auth_connect:$count_ref->{'connect'}:"
         ."smtp_auth_success:$count_ref->{'success'}\n";

    return $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0, debug=>$debug );
}

sub send_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $logbase   = $conf->{'logs_base'} || "/var/log/mail";
    my $countfile = $self->set_countfile(prot=>"send");
       $count_ref = $self->counter_read( file=>$countfile );

    print "processing send logs\n" if $debug;

    $self->process_send_logs(
        roll  => 0,
        files => $self->check_log_files( ["$logbase/send/current"] ),
    );

    if ( $count_ref->{'status_remotep'} && $count_ref->{'status'} ) {
        $count_ref->{'concurrencyremote'} =
          ( $count_ref->{'status_remotep'} / $count_ref->{'status'} ) * 100;
    }

    print "      Counts\n\n" if $debug;

    my $i = 0;
    while ( my ($description, $count) = each %$count_ref ) {
        print ":" if ( $i > 0 );
        print "$description:$count";
        $i++;
    }
    print "\n";

    # rotate the multilog files
    my $supervise = $conf->{'logs_supervise'}  || "/var/qmail/supervise";
    if ( -d "$supervise/send/log" ) {
        $self->rotate_supervised_logs("$supervise/send/log");
    };

    # update multilog (this is normally off on toaster.conf)
    if ( $conf->{'logs_isoqlog'} && $UID == 0 ) {
        $utility->syscmd( command => "isoqlog", debug=>$debug, fatal=>0 );
    }

    $self->compress_yesterdays_logs( file=>"sendlog" );
    $self->purge_last_months_logs() if $conf->{'logs_archive_purge'};

    return 1;
}

sub imap_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my ( $imap_success, $imap_connect, $imap_ssl_success, $imap_ssl_connect );

    my $countfile = $self->set_countfile(prot=>"imap");
       $count_ref = $self->counter_read( file=>$countfile );

    my $logfiles  = $self->check_log_files( [ $self->syslog_locate() ] );
    if ( @$logfiles[0] eq "" ) {
        carp "\n   imap_count ERROR: no logfiles!";
        return;
    }

    my $lines;
    foreach (@$logfiles) {
        open my $LOGF, "<", $_;

        while ( my $line = <$LOGF> ) {
            next if $line !~ /imapd/;
            $lines++;

            if ( $line =~ /ssl: LOGIN/ ) { 
                $imap_ssl_success++;
                next;
            }

            if ( $line =~ /LOGIN/ ) { 
                $imap_success++; 
                next;
            }
            
            # elsif ( $line =~ /ssl: Connection/     ) { $imap_ssl_connect++; }
            # elsif ( $line =~ /Connection/          ) { $imap_connect++; }
        }
        close $LOGF;
    }

    unless ( $lines ) {
        print "imap_success:$count_ref->{'imap_success'}"
            . ":imap_ssl_success:$count_ref->{'imap_ssl_success'}\n";
        carp "imap_count: no log entries to process. I'm done!" if $debug;
        return 1;
    };

    if ( $imap_success ) {
        if ( $imap_success >= $count_ref->{'imap_success_last'} ) {
            $count_ref->{'imap_success'} 
                += ( $imap_success - $count_ref->{'imap_success_last'} );
        }
        else { 
            $count_ref->{'imap_success'} += $imap_success;
        }

        $count_ref->{'imap_success_last'}     = $imap_success;
    };

    if ( $imap_ssl_success ) {
        if ( $imap_ssl_success >= $count_ref->{'imap_ssl_success_last'} ) {
            $count_ref->{'imap_ssl_success'} += 
            ( $imap_ssl_success - $count_ref->{'imap_ssl_success_last'} );
        }
        else {
            $count_ref->{'imap_ssl_success'} += $imap_ssl_success;
        }

        $count_ref->{'imap_ssl_success_last'} = $imap_ssl_success;
    };

#    Courier no longer logs this information
#    if ( $imap_connect >= $count_ref->{'imap_connect_last'} ) {
#        $count_ref->{'imap_connect'} += 
#          ( $imap_connect - $count_ref->{'imap_connect_last'} );
#    }
#    else { $count_ref->{'imap_connect'} += $imap_connect }
#
#    if ( $imap_ssl_connect >= $count_ref->{'imap_ssl_connect_last'} ) {
#        $count_ref->{'imap_ssl_connect'} += 
#          ( $imap_ssl_connect - $count_ref->{'imap_ssl_connect_last'} );
#    }
#    else {
#        $count_ref->{'imap_ssl_connect'} += $imap_ssl_connect;
#    }
#
#    $count_ref->{'imap_connect_last'}     = $imap_connect;
#    $count_ref->{'imap_ssl_connect_last'} = $imap_ssl_connect;
#
#    print "connect_imap:$count_ref->{'imap_connect'}:connect_imap_ssl" 
#        . ":$count_ref->{'imap_ssl_connect'}:"

    print "imap_success:$count_ref->{'imap_success'}"
        . ":imap_ssl_success:$count_ref->{'imap_ssl_success'}\n";

    return $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0 );
}

sub pop3_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    # read our counters from disk
    my $countfile = $self->set_countfile(prot=>"pop3");

    print "pop3_count: reading counters from $countfile.\n" if $debug;
       $count_ref = $self->counter_read( file=>$countfile );

    # get the location of log files to process
    print "finding the log files to process.\n" if $debug;
    my $logfiles  = $self->check_log_files( [$self->syslog_locate()] );
    if ( $logfiles->[0] eq "" ) {
        carp "    pop3_count: ERROR: no logfiles to process!";
        return;
    }

    print "pop3_count: processing files @$logfiles.\n" if $debug;

    my $lines;
    my %new_entries = ( 
        'connect'     => 0,
        'success'     => 0,
        'ssl_connect' => 0,
        'ssl_success' => 0,
    );

    my %valid_counters = (
        'pop3_success'          => 1,    # successful authentication
        'pop3_success_last'     => 1,    # last success count
        'pop3_connect'          => 1,    # total connections
        'pop3_connect_last'     => 1,    # last total connections
        'pop3_ssl_success'      => 1,    # ssl successful auth
        'pop3_ssl_success_last' => 1,    # last ssl success auths
        'pop3_ssl_connect'      => 1,    # ssl connections
        'pop3_ssl_connect_last' => 1,    # last ssl connects
    );

    foreach my $key ( keys %valid_counters ) {
        if ( ! defined $count_ref->{$key} ) {
             carp "pop3_count: missing key $key in count_ref!" if $debug;
             $count_ref->{$key} = 0;
        };
    };

    print "processing...\n" if $debug;
    foreach (@$logfiles) {
        open my $LOGF, "<", $_;

        LINE:
        while ( my $line = <$LOGF> ) {
            next unless ( $line =~ /pop3/ );
            $lines++;

            if ( $line =~ /vchkpw-pop3:/ ) {    # qmail-pop3d
                $new_entries{'connect'}++;
                $new_entries{'success'}++ if ( $line =~ /success/ );
            }
            elsif ( $line =~ /pop3d: / ) {      # courier pop3d
                $new_entries{'connect'}++ if ( $line =~ /Connection/ );
                $new_entries{'success'}++ if ( $line =~ /LOGIN/ );
            }
            elsif ( $line =~ /pop3d-ssl: / ) {    # courier pop3d-ssl
                if ( $line =~ /LOGIN/ ) {
                    $new_entries{'ssl_success'}++;
                    next LINE;
                };
                $new_entries{'ssl_connect'}++ if ( $line =~ /Connection/ );
            }
        }
        close $LOGF;
    }

    if ( ! $lines ) {
        pop3_report();
        carp "pop3_count: no log entries, I'm done!" if $debug;
        return 1;
    };

    if ( $new_entries{'success'} ) {
        if ( $new_entries{'success'} >= $count_ref->{'pop3_success_last'} ) {
            $count_ref->{'pop3_success'} += 
                ( $new_entries{'success'} - $count_ref->{'pop3_success_last'} );
        }
        else { 
            $count_ref->{'pop3_success'} += $new_entries{'success'};
        }

        $count_ref->{'pop3_success_last'} = $new_entries{'success'};
    };

    if ( $new_entries{'connect'} ) {
        if ( $new_entries{'connect'} >= $count_ref->{'pop3_connect_last'} ) {
            $count_ref->{'pop3_connect'} += 
            ( $new_entries{'connect'} - $count_ref->{'pop3_connect_last'} );
        }
        else { $count_ref->{'pop3_connect'} += $new_entries{'connect'} }

        $count_ref->{'pop3_connect_last'}     = $new_entries{'connect'};
    };

    if ( $new_entries{'ssl_success'} ) {
        if ( $new_entries{'ssl_success'} >= $count_ref->{'pop3_ssl_success_last'} ) {
            $count_ref->{'pop3_ssl_success'} += 
            ( $new_entries{'ssl_success'} - $count_ref->{'pop3_ssl_success_last'} );
        }
        else {
            $count_ref->{'pop3_ssl_success'} += $new_entries{'ssl_success'};
        }

        $count_ref->{'pop3_ssl_success_last'} = $new_entries{'ssl_success'};
    };

    if ( $new_entries{'ssl_connect'} ) {
        if ( $new_entries{'ssl_connect'} >= $count_ref->{'pop3_ssl_connect_last'} ) {
            $count_ref->{'pop3_ssl_connect'} 
                += ( $new_entries{'ssl_connect'} - $count_ref->{'pop3_ssl_connect_last'} );
        }
        else {
            $count_ref->{'pop3_ssl_connect'} += $new_entries{'ssl_connect'};
        }

        $count_ref->{'pop3_ssl_connect_last'} = $new_entries{'ssl_connect'};
    };

    pop3_report();

    my $pop3_logs = $conf->{'logs_pop3d'} || "courier";

    if ( $pop3_logs eq "qpop3d" ) {
        my $supervise = $conf->{'logs_supervise'} || "/var/qmail/supervise";
        $self->rotate_supervised_logs("$supervise/pop3/log");
        $self->compress_yesterdays_logs( file=>"pop3log", fatal=>0 );
    }

    return $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0 );
}

sub pop3_report {

    $count_ref->{'pop3_connect'}     || 0;
    $count_ref->{'pop3_ssl_connect'} || 0;
    $count_ref->{'pop3_success'}     || 0;
    $count_ref->{'pop3_ssl_success'} || 0;

    print "pop3_connect:"     . $count_ref->{'pop3_connect'}
       . ":pop3_ssl_connect:" . $count_ref->{'pop3_ssl_connect'}
       . ":pop3_success:"     . $count_ref->{'pop3_success'}
       . ":pop3_ssl_success:" . $count_ref->{'pop3_ssl_success'}
       . "\n";
};

sub webmail_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $countfile = $self->set_countfile(prot=>"web");
       $count_ref = $self->counter_read( file=>$countfile );

    my $logfiles  = $self->check_log_files( [$self->syslog_locate()] );
    if ( @$logfiles[0] eq "" ) {
        carp "\n    ERROR: no logfiles!";
        return 0;
    }

# sample log entries
# Feb 21 10:24:41 cadillac sqwebmaild: LOGIN, user=matt@cadillac.net, ip=[66.227.213.209]
# Feb 21 10:27:00 cadillac sqwebmaild: LOGIN FAILED, user=matt@cadillac.net, ip=[66.227.213.209]

    my %temp;

    foreach (@$logfiles) {
        open my $LOGF, "<", $_;

        while ( my $line = <$LOGF> ) {
            next if $line =~ /spamd/;    # typically half the syslog file
            next if $line =~ /pop3/;     # another 1/3 to 1/2

            if ( $line =~ /Successful webmail login/ ) { # squirrelmail w/plugin
                $temp{'success'}++;
                $temp{'connect'}++;
            }
            elsif ( $line =~ /sqwebmaild/ ) {            # sqwebmail
                $temp{'connect'}++;
                $temp{'success'}++ if ( $line !~ /FAILED/ );
            }
            elsif ( $line =~ /imapd: LOGIN/ && $line =~ /127\.0\.0\.1/ )
            {    # IMAP connections from localhost are webmail
                $temp{'success'}++;
            }
        }
        close $LOGF;
    }

    if ( !$temp{'connect'} ) {
        carp "webmail_count: No webmail logins! I'm all done." if $debug;
        return 1;
    };

    if ( $temp{'success'} ) {
        if ( $temp{'success'} >= $count_ref->{'success_last'} ) {
            $count_ref->{'success'} =
            $count_ref->{'success'} + ( $temp{'success'} - $count_ref->{'success_last'} );
        }
        else { $count_ref->{'success'} = $count_ref->{'success'} + $temp{'success'} }

        $count_ref->{'success_last'} = $temp{'success'};
    };

    if ( $temp{'connect'} ) {
        if ( $temp{'connect'} >= $count_ref->{'connect_last'} ) {
            $count_ref->{'connect'} =
            $count_ref->{'connect'} + ( $temp{'connect'} - $count_ref->{'connect_last'} );
        }
        else { $count_ref->{'connect'} = $count_ref->{'connect'} + $temp{'connect'} }

        $count_ref->{'connect_last'} = $temp{'connect'};
    };

    if ( ! $count_ref->{'connect'} ) {
        $count_ref->{'connect'} = 0;
    };

    if ( ! $count_ref->{'success'} ) {
        $count_ref->{'success'} = 0;
    };

    print "webmail_connect:$count_ref->{'connect'}"
        . ":webmail_success:$count_ref->{'success'}"
        . "\n";

    return $self->counter_write( 
        log    => $countfile, 
        values => $count_ref, 
        fatal  => 0, 
    );
}

sub spama_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $countfile = $self->set_countfile(prot=>"spam");
       $count_ref = $self->counter_read( file=>$countfile );

    my $logfiles  = $self->check_log_files( [$self->syslog_locate()] );
    if ( @$logfiles[0] eq "" ) {
        carp "\n   spamassassin_count ERROR: no logfiles!";
        return;
    }

    my %temp = ( spam => 1, ham => 1 );

    foreach (@$logfiles) {
        open my $LOGF, "<", $_;

        while ( my $line = <$LOGF> ) {
            next unless $line =~ /spamd/;
           	$temp{spamd_lines}++;

            if ( $line =~
/clean message \(([0-9-\.]+)\/([0-9\.]+)\) for .* in ([0-9\.]+) seconds, ([0-9]+) bytes/
              )
            {
                $temp{ham}++;
                $temp{ham_scores}  += $1;
                $temp{threshhold}  += $2;
                $temp{ham_seconds} += $3;
                $temp{ham_bytes}   += $4;
            }
            elsif ( $line =~
/identified spam \(([0-9-\.]+)\/([0-9\.]+)\) for .* in ([0-9\.]+) seconds, ([0-9]+) bytes/
              )
            {
                $temp{spam}++;
                $temp{spam_scores}  += $1;
                $temp{threshhold}   += $2;
                $temp{spam_seconds} += $3;
                $temp{spam_bytes}   += $4;
            }
            else {
                $temp{other}++;
            }
        }

        close $LOGF;
    }

    unless ( $temp{'spamd_lines'} ) {
        carp "spamassassin_count: no log file entries for spamd!" if $debug;
        return 1;
    };

    my $ham_count  = $temp{'ham'} || 0;
    my $spam_count = $temp{'spam'} || 0;

    if ( $ham_count ) {
        if ( $ham_count >= $count_ref->{'sa_ham_last'} ) {
            $count_ref->{'sa_ham'} =
            $count_ref->{'sa_ham'} + ( $ham_count - $count_ref->{'sa_ham_last'} );
        }
        else {
            $count_ref->{'sa_ham'} = $count_ref->{'sa_ham'} + $ham_count;
        }
    };

    if ( $spam_count ) {
        if ( $spam_count >= $count_ref->{'sa_spam_last'} ) {
            $count_ref->{'sa_spam'} =
            $count_ref->{'sa_spam'} + ( $spam_count - $count_ref->{'sa_spam_last'} );
        }
        else {
            $count_ref->{'sa_spam'} = $count_ref->{'sa_spam'} + $spam_count;
        }
    };

    require POSIX;    # needed for floor()
    $count_ref->{'avg_spam_score'} = (defined $temp{'spam_scores'} && $spam_count ) 
        ? POSIX::floor( $temp{'spam_scores'} / $spam_count * 100 ) : 0;

    $count_ref->{'avg_ham_score'}  = (defined $temp{'ham_scores'} && $ham_count ) 
        ? POSIX::floor( $temp{'ham_scores'} / $ham_count * 100 )   : 0;

    $count_ref->{'threshhold'}     = ( $temp{'threshhold'} && ($ham_count || $spam_count) )
        ? POSIX::floor( $temp{'threshhold'} / ( $ham_count + $spam_count ) * 100 ) : 0;

    $count_ref->{'sa_ham_last'}     = $ham_count;
    $count_ref->{'sa_spam_last'}    = $spam_count;

    $count_ref->{'sa_ham_seconds'}  = (defined $temp{'ham_seconds'} && $ham_count )
        ? POSIX::floor( $temp{'ham_seconds'} / $ham_count * 100 ) : 0;

    $count_ref->{'sa_spam_seconds'} = (defined $temp{spam_seconds} && $spam_count)
        ? POSIX::floor( $temp{spam_seconds} / $spam_count * 100 ) : 0;

    $count_ref->{'sa_ham_bytes'}  = (defined $temp{'ham_bytes'} && $ham_count ) 
        ? POSIX::floor( $temp{'ham_bytes'} / $ham_count * 100 ) : 0;

    $count_ref->{'sa_spam_bytes'} = (defined $temp{'ham_bytes'} && $spam_count ) 
        ? POSIX::floor( $temp{'spam_bytes'} / $spam_count * 100 ) : 0;

    print "sa_spam:$count_ref->{'sa_spam'}"
        . ":sa_ham:$count_ref->{'sa_ham'}"
        . ":spam_score:$count_ref->{'avg_spam_score'}"
        . ":ham_score:$count_ref->{'avg_ham_score'}"
        . ":threshhold:$count_ref->{'threshhold'}"
        . ":ham_seconds:$count_ref->{'sa_ham_seconds'}"
        . ":spam_seconds:$count_ref->{'sa_spam_seconds'}"
        . ":ham_bytes:$count_ref->{'sa_ham_bytes'}"
        . ":spam_bytes:$count_ref->{'sa_spam_bytes'}"
        . "\n";

    return $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0 );
}

sub qms_count {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my ( $qs_clean, $qs_virus, $qs_all );

    my $countfile = $self->set_countfile(prot=>"virus");
    my $count_ref = $self->counter_read( file=>$countfile );

    my $logfiles  = $self->check_log_files( [$self->syslog_locate()] );
    if ( !$logfiles || @$logfiles[0] eq "" ) {
        carp "    qms_count: ERROR: no logfiles!";
        return 1;
    }

    my $grep = $utility->find_the_bin(bin=>"grep", debug=>0);
    my $wc   = $utility->find_the_bin(bin=>"wc", debug=>0);

    $qs_clean = `$grep " qmail-scanner" @$logfiles | $grep "Clear:" | $wc -l`;
    $qs_clean = $qs_clean * 1;
    $qs_all   = `$grep " qmail-scanner" @$logfiles | $wc -l`;
    $qs_all   = $qs_all * 1;
    $qs_virus = $qs_all - $qs_clean;

    if ( $qs_all == 0 ) {
        carp "qms_count: no log files for qmail-scanner found!" if $debug;
        return 1;
    };

    if ( $qs_clean ) {
        if ( $qs_clean >= $count_ref->{'qs_clean_last'} ) {
            $count_ref->{'qs_clean'} =
                $count_ref->{'qs_clean'} + ( $qs_clean - $count_ref->{'qs_clean_last'} );
        }
        else { $count_ref->{'qs_clean'} = $count_ref->{'qs_clean'} + $qs_clean }

        $count_ref->{'qs_clean_last'} = $qs_clean;
    };

    if ( $qs_virus ) {
        if ( $qs_virus >= $count_ref->{'qs_virus_last'} ) {
            $count_ref->{'qs_virus'} =
                $count_ref->{'qs_virus'} + ( $qs_virus - $count_ref->{'qs_virus_last'} );
        }
        else { $count_ref->{'qs_virus'} = $count_ref->{'qs_virus'} + $qs_virus }

        $count_ref->{'qs_virus_last'} = $qs_virus;
    };

    print "qs_clean:$qs_clean:qs_virii:$qs_virus\n";

    if ( !$count_ref ) {
        $count_ref = { qs_clean=>0, qs_virii=>0 };
    };

    return $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0 );
}


sub roll_send_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $logbase  = $conf->{'logs_base'} || "/var/log/mail";
    print "roll_send_logs: logging base is $logbase.\n" if $debug;

    my $countfile = $self->set_countfile(prot=>"send");
       $count_ref = $self->counter_read( file=>$countfile );

    $self->process_send_logs( 
        roll  => 1,
        files => $self->check_log_files( ["$logbase/send/current"] ),
    );

    $self->counter_write( log=>$countfile, values=>$count_ref, fatal=>0 );
}

sub roll_rbl_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my $logbase = $conf->{'logs_base'} || "/var/log/mail";

    my $countfile = $self->set_countfile(prot=>"rbl");
    unless ( -r $countfile ) {
        carp "WARNING: roll_rbl_logs could not read $countfile!: $!";
        return;
    }

    $spam_ref = $self->counter_read( file=>$countfile );

    $self->process_rbl_logs( 
        roll  => 1,
        files => $self->check_log_files( ["$logbase/smtp/current"] ),
    );

    $self->counter_write( log=>$countfile, values=>$spam_ref, fatal=>0 );
}

sub roll_pop3_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    #	my $countfile = "$logbase/$counters/$qpop_log";
    #	%count        = $self->counter_read( file=>$countfile );

    my $logbase = $conf->{'logs_base'} || "/var/log/mail";

    $self->process_pop3_logs( 
        roll  => 1, 
        files => $self->check_log_files( [ "$logbase/pop3/current" ] ),
    );

    #	$self->counter_write(log=>$countfile, values=>\%count);
    #$self->rotate_supervised_logs( "$supervise/pop3/log" );
    $self->compress_yesterdays_logs( file=>"pop3log" );
}

sub compress_yesterdays_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate( @_, {
            'file'    => { type=>SCALAR,  },
            'fatal'   => { type=>BOOLEAN, optional=>1, default=>1 },
            'debug'   => { type=>BOOLEAN, optional=>1, default=>$debug },
        },
    );

    my ( $file, $fatal ) = ( $p{'file'}, $p{'fatal'} );

    $debug = $p{'debug'};

    my ( $dd, $mm, $yy ) = $utility->get_the_date(bump=>1, debug=>$debug);

    my $logbase = $conf->{'logs_base'} || "/var/log/mail";
    my $log     = "$logbase/$yy/$mm/$dd/$file";
    
    if ( -e "$log.gz" ) {
        print "   $log is already compressed\n\n" if $debug;
        return 1;
    }

    if ( ! -e $log ) {
        print "   $log does not exist.\n\n" if $debug;
        return 1;
    };

    if ( ! $utility->is_writable(file=>"$log.gz",fatal=>0,debug=>$debug) ) {
        carp "could not compress $log because I do not have write permissions!";
        return;
    };

    print "   Compressing the logfile $log..." if $debug;
    
    if ( $utility->syscmd( command=>"gzip $log", debug=>$debug, fatal=>$fatal) ) {
        print "done.\n\n" if $debug;
        return 1;
    };

    carp "compressing $log FAILED." if $debug;
    return;
}

sub purge_last_months_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate( @_, {
            'fatal'   => { type=>BOOLEAN, optional=>1, default=>1 },
        },
    );

    if ( ! $conf->{'logs_archive_purge'} ) {
        print "purge_last_months_logs is disabled in toaster.conf, skipping.\n";
        return 1;
    };

    my $fatal = $p{'fatal'};

    my ( $dd, $mm, $yy ) = $utility->get_the_date(bump=>31, debug=>$debug);

    if ( ! $mm || !$yy ) {
        carp "purge_last_months_logs: ERROR: get_the_date returned invalid values!";
        return;
    };

    my $logbase    = $conf->{'logs_base'} || "/var/log/mail";

    unless ( $logbase && -d $logbase ) {
        carp "purge_last_months_logs: no log directory $logbase. I'm done!";
        return 1;
    };

    my $last_m_log = "$logbase/$yy/$mm";

    if ( ! -d $last_m_log ) {
        print "purge_last_months_logs: log dir $last_m_log doesn't exist. I'm done.\n" if $debug;
        return 1;
    };
    
    print "\nI'm about to delete $last_m_log...." if $debug;
    if ( rmtree($last_m_log) ) {
        print "done.\n\n" if $debug;
        return 1;
    };

    return;
}

sub rotate_supervised_logs {

    my $self  = shift;
    my $debug = $self->{'debug'};

    my (@dirs) = @_;

    unless ( $UID == 0 ) {
        carp "rotate_supervsed_logs: root privs are needed to restart daemons."
            . " log rotate FAILED." if $debug;
        return;
    }

    DIR:
    for (@dirs) { 
        if ( ! -d $_ ) {
            carp "rotate_supervised_logs: the log directory $_ is missing!";
            next DIR;
        };
        if ( ! -e "$_/run" ) {
            carp  "the run file in the log direcory $_ is missing!";
            next  DIR;
        };
        $utility->syscmd( command => "svc -a $_", debug=>0, fatal=>0 );
    };
    return 1;
}

sub check_log_files {

    my $self  = shift;
    my $check = shift;;
 
    my @exists;

    foreach my $file ( @$check ) {
        next if !$file;
        if ( -e $file ) { push @exists, $file; };
    };

    return \@exists;
}

sub process_pop3_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate(@_, {
            'roll'  => { type=>BOOLEAN, optional=>1, default=>0 },
            'files' => { type=>ARRAYREF, optional=>1 }
         }
    );

    my $files_ref = $p{'files'};

    my $skip_archive;

    if ( $p{'roll'} ) {

        # no log file(s)! 
        if ( !$files_ref->[0] ) {
            $skip_archive++;
        };

        my $PIPE_TO_CRONOLOG;
        if ( ! $skip_archive ) {
            $PIPE_TO_CRONOLOG = $self->get_cronolog_handle(file=>"pop3log");
            if ( ! $PIPE_TO_CRONOLOG ) {
                $skip_archive++;
            };
        };

        while (<STDIN>) {
            print                   $_ if $conf->{'logs_taifiles'};
            print $PIPE_TO_CRONOLOG $_ if ! $skip_archive;
        }
        close $PIPE_TO_CRONOLOG if ! $skip_archive;
        $skip_archive ? return : return 1;
    }

    # these logfiles are empty unless debugging is enabled
    foreach my $file ( @$files_ref ) {
        print "process_pop3_logs: reading file $file..." if $debug;

        my $MULTILOG_FILE;
        if ( ! open $MULTILOG_FILE, "<", $file ) {
            carp "couldn't read $file: $!";
            $skip_archive++;
            next;
        };

        while (<$MULTILOG_FILE>) {
            chomp;
            #count_pop3_line( $_ );
        }
        close($MULTILOG_FILE);
        print "done.\n" if $debug;
    }

    $skip_archive ? return : return 1;
}

sub process_rbl_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate( @_, {
            'roll'    => { type=>BOOLEAN, optional=>1, default=>0 },
            'files'   => { type=>ARRAYREF,optional=>1,  },
        },
    );

    my $files_ref = $p{'files'};

    my $skip_archive = 0;

    if ( $p{'roll'} ) {

        # no log file(s)! 
        $skip_archive++ if !$files_ref->[0];

        my $PIPE_TO_CRONOLOG;
        if ( ! $skip_archive ) {
            $PIPE_TO_CRONOLOG = $self->get_cronolog_handle(file=>"smtplog");
            $skip_archive++ if ! $PIPE_TO_CRONOLOG;
        };

        while (<STDIN>) {
            $self->count_rbl_line ( $_ );
            print                   $_ if $conf->{'logs_taifiles'};
            print $PIPE_TO_CRONOLOG $_ if ! $skip_archive;
        }
        close $PIPE_TO_CRONOLOG if ! $skip_archive;
        $skip_archive ? return : return 1;
    }

    foreach my $file ( @$files_ref ) {
        print "process_rbl_logs: reading file $file..." if $debug;

        my $MULTILOG_FILE;
        if ( ! open $MULTILOG_FILE, "<", $file ) {
            carp "couldn't read $file: $!";
            $skip_archive++;
            next;
        };

        while (<$MULTILOG_FILE>) {
            $self->count_rbl_line( $_ );
        }
        close($MULTILOG_FILE);
        print "done.\n" if $debug;
    }

    $skip_archive ? return : return 1;
}

sub count_rbl_line {

    my $self = shift;
    my $line = shift;

    # comment out print lines
    return unless $line;
    chomp $line;

    if ( $line =~ /rblsmtpd/ ) {
        # match the most common entries earliest
          $line =~ /spamhaus/     ? $spam_ref->{'spamhaus'}++
        : $line =~ /spamcop/      ? $spam_ref->{'spamcop'}++
        : $line =~ /dsbl\.org/    ? $spam_ref->{'dsbl'}++
        : $line =~ /services/     ? $spam_ref->{'services'}++
        : $line =~ /rfc-ignorant/ ? $spam_ref->{'ignorant'}++
        : $line =~ /sorbs/        ? $spam_ref->{'sorbs'}++
        : $line =~ /njabl/        ? $spam_ref->{'njabl'}++
        : $line =~ /ORDB/         ? $spam_ref->{'ordb'}++
        : $line =~ /mail-abuse/   ? $spam_ref->{'maps'}++
        : $line =~ /monkeys/      ? $spam_ref->{'monkeys'}++
        : $line =~ /visi/         ? $spam_ref->{'visi'}++
        #: print $line;
        : $spam_ref->{'other'}++
    }
    elsif ( $line =~ /CHKUSER/ ) {
          $line =~ /CHKUSER acce/ ? $spam_ref->{'ham'}++
        : $line =~ /CHKUSER reje/ ? $spam_ref->{'chkuser'}++
        #: print $line;
        : $spam_ref->{'other'}++;
    }
    elsif ( $line =~ /simscan/ ) {
           lc($line) =~ /clean/   ? $spam_ref->{'ham'}++
        : lc($line) =~ /virus:/   ? $spam_ref->{'virus'}++
        : lc($line) =~ /spam rej/ ? $spam_ref->{'spamassassin'}++
        #: print $line;    
        : $spam_ref->{'other'}++;
    }
    else {
          $line =~ /badhelo:/       ? $spam_ref->{'badhelo'}++
        : $line =~ /badmailfrom:/   ? $spam_ref->{'badmailfrom'}++
        : $line =~ /badmailto:/     ? $spam_ref->{'badmailto'}++
        : $line =~ /Reverse/        ? $spam_ref->{'dns'}++
        #: print $line;
        : $spam_ref->{'other'}++
    }

    $spam_ref->{'count'}++;
    return 1;
}

sub process_send_logs {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate( @_, {
            'roll'    => { type=>SCALAR,  optional=>1, default=>0 },
            'files'   => { type=>ARRAYREF,optional=>1,  },
        },
    );

    my $files_ref = $p{'files'};

    my $skip_archive = 0;

    if ( $p{'roll'} ) {

        print "process_send_logs: log rolling is enabled.\n" if $debug;

        # no log file(s)! 
        $skip_archive++ if !$files_ref->[0];

        my $PIPE_TO_CRONOLOG;
        if ( ! $skip_archive ) {
            unless ( $PIPE_TO_CRONOLOG = $self->get_cronolog_handle(file=>"sendlog") ) {
                $skip_archive++;
            };
        };

        while (<STDIN>) {
            $self->count_send_line( $_ );
            print                   $_ if $conf->{'logs_taifiles'};
            print $PIPE_TO_CRONOLOG $_ if ! $skip_archive;
        }
        close $PIPE_TO_CRONOLOG if ! $skip_archive;
        $skip_archive ? return : return 1;
    }

    print "process_send_logs: log rolling is disabled.\n" if $debug;

    foreach my $file ( @$files_ref ) {
        
        print "process_send_logs: reading file $file.\n" if ($debug);
        
        my $INFILE;
        if ( ! open( $INFILE, "<", $file ) ) {
            carp "process_send_logs couldn't read $file: $!";
            $skip_archive++;
            next;
        };

        while (<$INFILE>) {
            chomp;
            $self->count_send_line( $_ );
        }
        close($INFILE);
    }

    $skip_archive ? return : return 1;
}

sub count_send_line {

    my $self = shift;
    my $line = shift;

########## $line will have a log entry in this format ########
# @40000000450c020b32315f74 new msg 71198
# @40000000450c020b32356e84 info msg 71198: bytes 3042 from <doc-committers@FreeBSD.org> qp 44209 uid 89
# @40000000450c020b357ed10c starting delivery 196548: msg 71198 to localexample.org-user@example.org
# @40000000450c020b357f463c status: local 1/10 remote 0/100
# @40000000450c020c06ac5dcc delivery 196548: success: did_0+0+1/
# @40000000450c020c06b6122c status: local 0/10 remote 0/100
# @40000000450c020c06be6ae4 end msg 71198
################################################

    unless ($line) {
        $count_ref->{'message_other'}++;
        return; 
    };
    chomp $line;

    #carp "$line";

    # split the line into date and activity
    my ( $tai_date, $activity ) = $line =~ /\A@([a-z0-9]*)\s(.*)\z/xms;

    unless ($activity) {
        $count_ref->{'message_other'}++;
        return; 
    };

    # because we have to process these regexps for every line,
    # we use a simple regexp in the cascading if/elsif and then run a
    # more complete regexp to get the values we want after we match. This
    # considerably reduces the number of regexp operations required. 
    # There is likely a more efficient way to do this.

    if    ( $activity =~ /^new msg/ ) {
        # new msg 71512
        # the complete line match: /^new msg ([0-9]*)/ 
        $count_ref->{'message_new'}++; 
    }
    elsif ( $activity =~ /^info msg / ) {
        
        # sample line
        # info msg 71766: bytes 28420 from <elfer@club-internet.fr> qp 5419 uid 89
        
        # a bigger slower regexp, but we want to bytes
        $activity =~ /^info msg ([0-9]*): bytes ([0-9]*) from/;
        
        # the complete line match
        # /^info msg ([0-9]*): bytes ([0-9]*) from \<(.*)\> qp ([0-9]*)/

        $count_ref->{'message_bytes'} += $2;
        $count_ref->{'message_info'}++;
    }
    elsif ( $activity =~ /^starting delivery/ ) {
        
        # starting delivery 136986: msg 71766 to remote bbarnes@example.com
        
        # a more complete line match
        # /^starting delivery ([0-9]*): msg ([0-9]*) to ([a-z]*) ([a-zA-Z\@\._-])$/
        
        $activity =~ /^starting delivery ([0-9]*): msg ([0-9]*) to ([a-z]*) /;
        
          $3 eq "remote" ? $count_ref->{'start_delivery_remote'}++
        : $3 eq "local"  ? $count_ref->{'start_delivery_local'}++
        : print "count_send_line: error, unknown delivery line format\n";

        $count_ref->{'start_delivery'}++;
    }
    elsif ( $activity =~ /^status: local/ ) {
        # status: local 0/10 remote 3/100
        $activity =~ /^status: local ([0-9]*)\/([0-9]*) remote ([0-9]*)\/([0-9]*)/;
        
        $count_ref->{'status_localp'}  = $count_ref->{'status_localp'} +  ( $1 / $2 );
        $count_ref->{'status_remotep'} = $count_ref->{'status_remotep'} + ( $3 / $4 );
        
        $count_ref->{'status'}++;
    }
    elsif ( $activity =~ /^end msg/ ) {
        # end msg 71766
        # /^end msg ([0-9]*)$/
        
        # this line is useless, why was it here?
        #$count_ref->{'local'}++ if ( $3 && $3 eq "local" );

        $count_ref->{'message_end'}++;
    }
    elsif ( $activity =~ /^delivery/ ) {
        # delivery 136986: success: 67.109.54.82_accepted_message./Remote_host_said:
        #   _250_2.6.0__<000c01c6c92a$97f4a580$8a46c3d4@p3>_Queued_mail_for_delivery/
        
        $activity =~ /^delivery ([0-9]*): ([a-z]*): /;
        
          $2 eq "success"  ? $count_ref->{'delivery_success'}++
        : $2 eq "deferral" ? $count_ref->{'delivery_deferral'}++
        : $2 eq "failure"  ? $count_ref->{'delivery_failure'}++
        : print "unknown " . $activity . "\n";

        $count_ref->{'delivery'}++;
    }
    elsif ( $activity =~ /^bounce/ ) {
        # /^bounce msg ([0-9]*) [a-z]* ([0-9]*)/
        $count_ref->{'message_bounce'}++;
    }
    else {
        #warn "other: $activity";
        $count_ref->{'other'}++;
    }

    return 1;
}


sub counter_read {

    my $self  = shift;
    my $debug = $self->{'debug'};

    my %p = validate(@_, { 
            'file' => SCALAR, 
            'debug' => {type=>BOOLEAN, optional=>1, default=>$debug},
        } 
    );

    my $file  = $p{'file'};
       $debug = $p{'debug'};

    my %hash;

    unless ( $file ) {
        croak "you must pass a filename!\n";
    };

    print "counter_read: fetching counters from $file..." if $debug;

    if ( ! -e $file ) {
        carp "\nWARN: the file $file is missing! I will try to create it." if $debug;

        if ( ! $utility->is_writable(file=>$file,debug=>0,fatal=>0) ) {
            carp "FAILED.\n $file does not exist and the user $UID has "
                . "insufficent privileges to create it!" if $debug;
            return;
        };

        $self->counter_write( 
            log    => $file, 
            values => { created => time(), },
        );

        my $user = $self->{'conf'}->{'logs_user'} || "qmaill";
        my $group = $self->{'conf'}->{'logs_group'} || "qnofiles";

        $utility->file_chown( file=>$file, uid=>$user, gid=>$group, debug=>0);

        print "done.\n";
        return;
    }
    
    my @lines = $utility->file_read( file => $file, debug=>$debug );

    foreach (@lines) {
        my ($description, $count) = split( ":", $_ );

        # sets key = $value in %hash
        $hash{ $description } = $count;
    }

    print "done.\n" if $debug;

    return \%hash;
}

sub counter_write {

    my $self  = shift;
    my $debug = $self->{'debug'};
    
    my %p = validate( @_, {
            'values' => HASHREF,
            'log'    => SCALAR,
            'fatal'  => { type=>BOOLEAN, optional=>1, default=>1 },
            'debug'  => { type=>BOOLEAN, optional=>1, default=>$debug },
        },
    );

    $debug = $p{'debug'};
    my ( $log, $values_ref, $fatal ) = ( $p{'log'}, $p{'values'}, $p{'fatal'} );

    my @lines;

    if ( -d $log ) { print "FAILURE: counter_write $log is a directory!\n"; }

    unless ( $utility->is_writable( file => $log, debug => 0, fatal=>$fatal ) ) {
        $utility->_formatted( "counter_write: $log is not writable", "FAILED" );
        return;
    }

    unless ( -e $log ) {
        print "WARNING: counter_write $log does not exist! Creating...";
    }

    # it might be necessary to wrap the counters
    #
    # if so, the 32 and 64 bit limits are listed below. Just
    # check the number, and subtract the maximum value for it.
    # rrdtool will continue to Do The Right Thing. :)

    while ( my ($key, $value) = each %$values_ref ) {
        print "key: $key  \t val: $value \n" if $debug;
        if ( $key && defined $value ) {
            # 32 bit - 4294967295
            # 64 bit - 18446744073709551615
            if ( $value > 4294967295 ) { $value = $value - 4294967295; };
            push @lines, "$key:$value";
        }
    }

    if ($debug) {
        #require Data::Dumper;
        #print Data::Dumper::Dumper ($values_ref); 
    };

    return $utility->file_write( 
        file  => $log, 
        lines => \@lines, 
        debug => $debug,
        fatal => $fatal,
    );
}

sub get_cronolog_handle {

    my $self  = shift;
    my $conf  = $self->{'conf'};
    my $debug = $self->{'debug'};
    
    my %p = validate(@_, { 'file' => SCALAR, },);

    my $file = $p{'file'};

    my $logbase = $conf->{'logs_base'} || "/var/log/mail";

    # archives disabled in toaster.conf
    if ( ! $conf->{'logs_archive'} ) {
        print "get_cronolog_handle: archives disabled, skipping cronolog handle.\n" if $debug; 
        return;
    };

    # $logbase is missing and we haven't permission to create it
    unless ( -w $logbase ) {
        carp "WARNING: could not write to $logbase. FAILURE!";
        return;
    }

    my $cronolog = $utility->find_the_bin( bin => "cronolog", debug=>0, fatal=>0 );
    if ( ! $cronolog || !-x $cronolog) {
        carp "cronolog could not be found. Please install it!";
        return;
    }

    my $tai64nlocal;

    if ( $conf->{'logs_archive_untai'} ) {
        my $taibin = $utility->find_the_bin( bin=>"tai64nlocal",debug=>0, fatal=>0 );

        if ( ! $taibin ) {
            carp "tai64nlocal is selected in toaster.conf but cannot be found!";
        };

        if ( $taibin && ! -x $taibin ) { 
            carp "tai64nlocal is not executable by you! ERROR!";
        }

        $tai64nlocal = $taibin;
    }


    my $cronolog_invocation = "| ";
    $cronolog_invocation .= "$tai64nlocal | " if $tai64nlocal;
    $cronolog_invocation .= "$cronolog $logbase/\%Y/\%m/\%d/$file";

    ## no critic
    open my $PIPE_TO_CRONOLOG, $cronolog_invocation or return;
    ## use critic

    return $PIPE_TO_CRONOLOG;
};

sub syslog_locate {

    my ( $self, $debug ) = @_;

    my $log = "/var/log/maillog";

    if ( -e $log ) {
        print "syslog_locate: using $log\n" if $debug;
        return "$log";
    }

    $log = "/var/log/mail.log";
    if ( -e $log ) {
        print "syslog_locate: Darwin detected...using $log\n" if $debug;
        return $log;
    }

    $log = "/var/log/messages";
    return $log if -e $log;

    croak "syslog_locate: can't find your syslog mail log\n";
}

sub set_countfile {

    my $self = shift;
    my $conf = $self->{'conf'};
    my $debug = $self->{'debug'};

    my %p = validate(@_, { prot=>SCALAR }, );
    my $prot = $p{'prot'};

    print "set_countfile: countfile for prot $prot is " if $debug;

    my $logbase  = $conf->{'logs_base'} || "/var/log/mail";
    my $counters = $conf->{'logs_counters'} || "counters";
    my $prot_file = $conf->{'logs_'.$prot.'_count'} || "$prot.txt";

    print "$logbase/$counters/$prot_file\n" if $debug;

    return "$logbase/$counters/$prot_file";
}

1;
__END__


=head1 NAME

Mail::Toaster::Logs - objects and functions for interacting with email logs

=head1 SYNOPSIS

    maillog <protocol> [-r] [-v]

    <protocol> is one of: 

         smtp - report SMTP AUTH attempts and successes
          rbl - report RBL, virus, and invalid format message blocks
         send - report qmail-send counters
         pop3 - report pop3 counters
         imap - report imap counters
 spamassassin - report spamassassin counters
 qmailscanner - report qmailscanner counters
      webmail - count webmail authentications

    yesterday - mail an activity report to the admin


This module contains functions related to mail logging and are used primarily in maillogs. Some functions are also used in toaster-watcher.pl and toaster_setup.pl.


=head1 METHODS

=over 8

=item new

Create a new Mail::Toaster::Logs object.

    use Mail::Toaster::Logs;
    $logs = Mail::Toaster::Logs->new;


=item report_yesterdays_activity

email a report of yesterdays email traffic.


=item verify_settings

Does some checks to make sure things are set up correctly.

    $logs->verify_settings(conf=>$conf);

tests: 

  logs base directory exists
  logs based owned by qmaill
  counters directory exists
  maillogs is installed


=item parse_cmdline_flags

Do the appropriate things based on what argument is passed on the command line.

	$logs->parse_cmdline_flags(prot=>$prot, debug=>0);

$conf is a hashref of configuration values, assumed to be pulled from toaster-watcher.conf.

$prot is the protocol we're supposed to work on. 


=item check_log_files

	$logs->check_log_files( [$check] );


=item compress_yesterdays_logs

	$logs->compress_yesterdays_logs(
	    file  => $file, 
	);

You'll have to guess what this does. ;)


 
=item count_rbl_line

    $logs->count_rbl_line($line);


=item count_send_line

 usage:
     $logs->count_send_line( $count, $line );
     
 arguments required:
      count - a hashref of counter values
      line  - an entry from qmail's send logs

 results:
     a hashref will be returned with updated counters


=item counter_read

	$logs->counter_read( file=>$file, debug=>$debug);

$file is the file to read from. $debug is optional, it prints out verbose messages during the process. The sub returns a hashref full of key value pairs.


=item counter_write

	$logs->counter_write(log=>$file, values=>$values);

 arguments required:
    file   - the logfile to write.
    values - a hashref of value=count style pairs.

 result:
   1 if written
   0 if not.

=cut

=item imap_count

	$logs->imap_count(conf=>$conf);

Count the number of connections and successful authentications via IMAP and IMAP-SSL.


=item pop3_count

	$logs->pop3_count(conf=>$conf);

Count the number of connections and successful authentications via POP3 and POP3-SSL.


=item process_pop3_logs


=item process_rbl_logs

    process_rbl_logs(
        roll  => 0,
        files => $self->check_log_files(["$logbase/smtp/current"]),
    );



=item process_send_logs



=item qms_count

	$logs->qms_count($conf);

Count statistics logged by qmail scanner.


=item purge_last_months_logs

	$logs->purge_last_months_logs(
        fatal   => 0,
	);

For a supplied protocol, cleans out last months email logs.



=item rotate_supervised_logs

	$logs->rotate_supervised_logs(@dirs);

Tell multilog to rotate the maillogs for the array of dirs supplied.


=item rbl_count

Count the number of connections we've blocked (via rblsmtpd) for each RBL that we use.

	$logs->rbl_count(conf=>$conf, $debug);

=item roll_rbl_logs

	$logs->roll_rbl_logs($conf, $debug);

Roll the qmail-smtpd logs (without 2>&1 output generated by rblsmtpd).

=item RollPOP3Logs

	$logs->RollPOP3Logs($conf);

These logs will only exist if tcpserver debugging is enabled. Rolling them is not likely to be necessary but the code is here should it ever prove necessary.


=item roll_send_logs

	$logs->roll_send_logs();

Roll the qmail-send multilog logs. Update the maillogs counter.


=item send_count

	$logs->send_count(conf=>$conf);

Count the number of messages we deliver, and a whole mess of stats from qmail-send.


=item smtp_auth_count

	$logs->smtp_auth_count(conf=>$conf);

Count the number of times users authenticate via SMTP-AUTH to our qmail-smtpd daemon.


=item spama_count

	$logs->spama_count($conf);

Count statistics logged by SpamAssassin.


=item syslog_locate

	$logs->syslog_locate($debug);

Determine where syslog.mail is logged to. Right now we just test based on the OS you're running on and assume you've left it in the default location. This is easy to expand later.

=cut

=item webmail_count

	$logs->webmail_count();

Count the number of webmail authentications.


=item what_am_i

	$logs->what_am_i()

Determine what the filename of this program is. This is used in maillogs, as maillogs gets renamed in order to function as a log post-processor for multilog.

=back

=head1 AUTHOR

Matt Simerson <matt@tnpi.net>


=head1 BUGS

None known. Report any to author.
Patches welcome.


=head1 SEE ALSO

The following are relevant man/perldoc pages: 

 maillogs
 Mail::Toaster 
 toaster.conf

 http://mail-toaster.org/


=head1 COPYRIGHT

Copyright (c) 2004-2006, The Network People, Inc. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

Neither the name of the The Network People, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

=cut
