#! /usr/bin/perl
#
# Copyright 2003 Kevin Miller
#
# Unrestricted use or distribution granted without warranty.
#
# idea:
# read the .toptalkers files and generate bytecount summaries per IP
#
# dump back into a .toptalkers format for looking
#

use strict;

use Data::Dumper;

my $Debug = 1;

# Changing the version will cause all the cache files to be regenerated..
# double-edged sword but lets us make sure there's no bad data in the system
my $Version = '0.3';

my $CONFIG_FILE = '/home/argus/etc/topt.conf';
my $TOPT = '/usr/ng/bin/topt';

## **************************************************************************

usage() if ($#ARGV == -1);

my $rConfig;

for my $i (0..$#ARGV) {

  my $arg = $ARGV[$i];

  usage() if ($arg eq '-h');

  if ($arg eq '-config') {
    if ($i == $#ARGV) {
      warn "Argument to -config required.\n";
      usage();
    }

    $i++;
    $CONFIG_FILE = $ARGV[$i];
    next;
  }

  # After this point in the argument processing, any other argument
  # will be acted upon immediately and then we'll exit;
  $rConfig = load_config($CONFIG_FILE);

  if ($arg eq '-daymail') {
    proc_general('%%yesterday%%', 1);
  }elsif($arg eq '-today') {
    proc_general('%%today%%', 0);
  }elsif($arg eq '-fiveday') {
    proc_general('%%last5days%%', 1);
  }else{
    proc_general($arg, 0);
  }
  exit 0;
}

# proc_general
# Process data (optionally directory is specified as %%today%% or 
# %%yesterday%% to automatically find those directories. Mail can be
# 0 or 1 and specifies if mail is to be sent
sub proc_general {
  my ($Directory, $Mail) = @_;
  print STDERR "proc_general($Directory, $Mail)\n" if ($Debug);

  my $ActiveConf;

  foreach my $Conf (keys %$rConfig) {
    print STDERR "proc_daymail :: processing $Conf\n" if ($Debug);
    $ActiveConf = $rConfig->{$Conf};

    my $LMail = $Mail;
    if ($Directory =~ /^\%\%([^\%]+)\%\%$/) {
      $LMail = 0 unless (defined $ActiveConf->{"mail_$1"} &&
			 $ActiveConf->{"mail_$1"});
    }

    validate_conf($Conf, $ActiveConf);
    eval $ActiveConf->{filespec};
    my @Dirs = ();
    my ($CurrDir, $Date, $TimeStr);
    if ($Directory eq '%%today%%') {
      ($CurrDir, $Date) = get_nday($ActiveConf, 0);
      @Dirs = ($CurrDir);
      $TimeStr = 'Daily';
    }elsif($Directory eq '%%yesterday%%') {
      ($CurrDir, $Date) = get_nday($ActiveConf, 1);
      @Dirs = ($CurrDir);
      $TimeStr = 'Daily';
    }elsif($Directory eq '%%last5days%%') {
      $TimeStr = '5 Day';
      my ($X);
      ($CurrDir, $Date) = get_nday($ActiveConf, 1);
      push(@Dirs, $CurrDir);
      for my $i (2..5) {
	($CurrDir, $X) = get_nday($ActiveConf, $i);
	push(@Dirs, $CurrDir);
      }
    }else{
      $CurrDir = $Directory;
      @Dirs = ($CurrDir);
      $TimeStr = 'Daily';
    }
    aggr_down_mult(\@Dirs, $ActiveConf);

    mail_dir_topt(\@Dirs, $Date, $TimeStr, $ActiveConf) if ($LMail);
  }
}

# validate_conf
# Report an error if the configuration is invalid (mostly things not specified)
sub validate_conf {
  my ($Name, $Conf) = @_;

  my @REQUIRED_VARS = qw/email_address CFILE_SPEC FBASE filespec
			 topt_options subject mail_notes/;

  foreach my $V (@REQUIRED_VARS) {
    die "Conf $Name doesn't specify required var: $V" 
      unless (defined $Conf->{$V} && $Conf->{$V} ne '');
  }
}

# load_config
# Load the configuration information
sub load_config {
  my ($ConfigFile) = @_;

  die "Cannot read config file: $ConfigFile" unless (-r $ConfigFile);
  my $res = do $ConfigFile;
  return $res;
}

## ***********************************************************************
# usage
# Program usage
sub usage {
  print STDERR "$0 [-h] [-daymail] [-today] [-fiveday] [directory]\n\t".
    "-daymail: Generate yesterday's statistics and send mail\n".
      "\t-today: Generate today's statistics\n".
	"\tdirectory: Generate the counts files under this dir\n";
  exit 1;
}

# mail_dir_topt
# Mails out the counts to the bboard
#
sub mail_dir_topt {
  my ($rDirs, $Date, $MailStr, $ActiveConf) = @_;
  
  my $Samples = 0;
  my $NDays = 0;
  my $AggrFile = '';
  print STDERR "mail_dir_topt(rDirs, $Date, $MailStr, ActiveConf)\n" if ($Debug);
  foreach my $Dir (@$rDirs) {
    my $CountsFile = "$Dir/.counts/$ActiveConf->{CFILE_SPEC}.counts";
    return unless (-r $CountsFile);
    
    open(FILE, $CountsFile) or die "Cannot open file: $CountsFile";
    while(<FILE>) {
      if ($_ =~ /\%5minSlices\%\s+(\d+)/) {
	$Samples += $1;
      }
    }
    close(FILE);
    $AggrFile .= "$CountsFile ";
    $NDays++;
  }
 
  my $Output = `$TOPT $ActiveConf->{topt_options} $AggrFile`;
  my $HN = `hostname`;
  chomp($HN);
  my $DayMax = 12*24*$NDays;

  my $Addr = $ActiveConf->{email_address};
  print "Sending mail to $Addr\n"; 
  open(MAIL, "|/usr/local/bin/mail -s \"$Date $MailStr $ActiveConf->{subject}\" $Addr");
  print MAIL "$MailStr Toptalkers Report\nFile Spec: $ActiveConf->{CFILE_SPEC}\n";
  print MAIL "Hostname: $HN\n";
  print MAIL "Counts File(s): $AggrFile\n";
  print MAIL "Notes: $ActiveConf->{mail_notes}\n\n";
  printf MAIL "$Samples/$DayMax samples = %.1f%% day reporting\n", $Samples/$DayMax*100;
  print MAIL "\nNote that wireless/authbridge addresses might have the incorrect DNS name\n".
	"as resolution is done as this report is generated.\n";
  print MAIL $Output;
  close(MAIL);
}

# topt_matchIP
# Match campus IP addresses; everything else is ignored
sub topt_matchIP {
  return ($_[0] =~ /^128\.2\./ || $_[0] =~ /^128\.237/ ||
	  $_[0] =~ /^192\.58\.107/ || $_[0] =~ /^192\.12\.32/ ||
	  $_[0] =~ /^204\.194\.28\./ || $_[0] =~ /^204\.194\.29\.0/ ||
	  $_[0] =~ /^204\.194\.30\./ || $_[0] =~ /^204\.194\.31\.0/ ||
	  $_[0] =~ /^128\.182\./);
}

sub topt_matchMAC {
	return ( $_[0] =~ /^(\w{1,2}:){5}\w{1,2}$/ && $_[0] ne '0:2:7e:21:8f:a0');
}

# get_nday
# figure out the path to X days ago's argus/toptalkers files
sub get_nday {
  my ($ActiveConf, $DaysAgo) = @_;
  return get_time(time() - (86400*$DaysAgo), $ActiveConf);
}

sub get_time {
  my ($Time, $ActiveConf) = @_;
  my ($filename);

  my ($sec, $min, $hour, $mday, $mon, 
      $year, $wday, $yday, $isdst) = localtime($Time);

  $year = $year + 1900;
  $mon++;

  if ($mon <= 9) {
    $mon = "0$mon";
  }
  if ($mday <= 9) {
    $mday = "0$mday";
  }

  for ($min) {
    /^0/ and do { if ($min < 5) { $min = "00"; } else { $min = "05"; } last; };
    /^1/ and do { if ($min < 15) { $min = "10"; } else { $min = "15"; } last; };
    /^2/ and do { if ($min < 25) { $min = "20"; } else { $min = "25"; } last; };
    /^3/ and do { if ($min < 35) { $min = "30"; } else { $min = "35"; } last; };
    /^4/ and do { if ($min < 45) { $min = "40"; } else { $min = "45"; } last; };
    /^5/ and do { if ($min < 55) { $min = "50"; } else { $min = "55"; } last; };
  }

  $filename = "$ActiveConf->{FBASE}/$year/$mon/$mday";
  my $Date = "$mon/$mday/$year";
  return ($filename, $Date);
}

# aggr_down_mult
# Run aggr_down on multiple directories
sub aggr_down_mult {
  my ($rDirs, $ActiveConf) = @_;
  foreach my $Dir (@$rDirs) {
    aggr_down($Dir, $ActiveConf);
  }
}

# aggr_down
# From the specified directory (first argument), verify that .counts
# directories exist all the way down the directory hierarchy, updating/
# creating if needed.
sub aggr_down {
  my ($Dir, $ActiveConf) = @_;

  print STDERR "aggr_down($Dir)\n" if ($Debug);
  die "Unable to write/not a dir: $Dir" unless (-d $Dir && -w $Dir);

  opendir(DIR, $Dir);
  my @SubDirectories = grep { !/^\./ && -d "$Dir/$_" } readdir(DIR);
  closedir(DIR);

  # Process all the subdirectories of this one
  foreach my $SDir (@SubDirectories) {
    aggr_down("$Dir/$SDir", $ActiveConf);
  }
 
  if (-e "$Dir/.counts") {
    die ".counts not accessible: $Dir/.counts" unless (-d "$Dir/.counts" &&
						       -w "$Dir/.counts");
  }else{
    mkdir("$Dir/.counts");
  }

  my $Result = counts_check($Dir, $ActiveConf);
  if ($Result == 0) {
    # We need to do/re-do the counts
    my $rCounts = counts_make($Dir, $ActiveConf);
  }
  return 1;
}

# counts_make
# This builds the counts information file
sub counts_make {
  my ($Dir, $ActiveConf) = @_;

  my %DataCounts;
  my %MetaVars;
  my @INCR_VARS = qw/5minSlices/;

  my @Files = counts_subfiles($Dir, $ActiveConf);
  foreach my $File (@Files) {
    my %MetaVarsPresent = ();

    open(FILE, $File) or die "Cannot open file: $File";
    while(my $Line = <FILE>) {
      # Empty lines or lines beginning with # will be ignored completely
      next if ($Line =~ /^\s*$/ or $Line =~ /^\s*\#/);

      # MetaVars are lines beginning with %variable-name%, followed by
      # space, and the value of the variable.
      if ($Line =~ /^\s*\%([^\%]+)\%\s+(\S+)/) {
		if (grep($1, @INCR_VARS)) {
	  	$MetaVars{$1} += $2;
		}
		$MetaVarsPresent{$1} = 1;
		next;
      }

      my ($IP, $BytesI, $BytesO, $PackI, $PackO, $Flows) = 
	split(/\s+/, $Line);

	  if(defined $MetaVarsPresent{'Type-MAC'}) {
		  next unless (topt_matchMAC($IP));
	  } else {
      	next unless (topt_matchIP($IP));
	  }
      $DataCounts{$IP}->[0] += $BytesI;
      $DataCounts{$IP}->[1] += $BytesO;
      $DataCounts{$IP}->[2] += $PackI;
      $DataCounts{$IP}->[3] += $PackO;
      $DataCounts{$IP}->[4] += $Flows;
    }
    close(FILE);

    # The base .toptalkers files won't have some of the metavars that
    # we want to set
    unless (defined $MetaVarsPresent{'5minSlices'}) {
      $MetaVars{'5minSlices'}++;
    }
  }

  my $Res = counts_write($Dir, \%DataCounts, \%MetaVars, $ActiveConf);
  die "Unable to write DataCounts" if ($Res != 1);

  my $CurrCache = counts_makecachefile($Dir, \@Files, $ActiveConf);
  open(FILE, ">$Dir/.counts/$ActiveConf->{CFILE_SPEC}.cacheinfo");
  print FILE $CurrCache;
  close(FILE);
}

# counts_write
# Write out the DataCount information file. Very trivial..
#
sub counts_write {
  my ($Dir, $rDataCounts, $rMetaVars, $ActiveConf) = @_;

  print STDERR "Writing file $Dir/.counts/$ActiveConf->{CFILE_SPEC}.counts\n"
    if ($Debug);

  unless (-d "$Dir/.counts") {
    mkdir("$Dir/.counts");
  }

  open(FILE, ">$Dir/.counts/$ActiveConf->{CFILE_SPEC}.counts")
    or die "Cannot open file $Dir/.counts/$ActiveConf->{CFILE_SPEC}.counts";
  foreach my $MVar (keys %$rMetaVars) {
    print FILE '%'.$MVar.'% '.$rMetaVars->{$MVar}."\n";
  }

  foreach my $RD (keys %$rDataCounts) {
    my $A = $RD;
    for my $i (0..4) {
      $A .= "\t".$rDataCounts->{$RD}->[$i];
    }
    print FILE "$A\n";
  }
  close(FILE);
  return 1;
}

# counts_check
# This verifies that the output in .counts reflects the current
# downstream data.
sub counts_check {
  my ($Dir, $ActiveConf) = @_;

  unless (-r "$Dir/.counts/$ActiveConf->{CFILE_SPEC}.cacheinfo") {
    print STDERR "counts_check($Dir) :: .cacheinfo not found\n" 
      if ($Debug);
    return 0;
  }

  open(FILE, "$Dir/.counts/$ActiveConf->{CFILE_SPEC}.cacheinfo")
    or die "Cannot open $Dir/.counts/$ActiveConf->{CFILE_SPEC}/.cacheinfo";
  my @Data = <FILE>;
  close(FILE);
  my $OInfo = join('', @Data);

  my @Files = counts_subfiles($Dir, $ActiveConf);
  my $CurrCache = counts_makecachefile($Dir, \@Files, $ActiveConf);

  if ($OInfo ne $CurrCache) {
    print STDERR "counts_checks($Dir) :: CurrCache/OInfo different\n"
      if ($Debug);
    return 0;
  }

  print STDERR "counts_check($Dir) :: cache matches\n"
    if ($Debug);
  return 1;
}

# counts_subfiles
# From the specified directory, tell us about every file that
# we need to include in this directory's counts file. This will include
# the counts files of all subdirectories and any .toptalkers files
# in this directory (matching the topt specification)
sub counts_subfiles {
  my ($Dir, $ActiveConf) = @_;

  opendir(DIR, $Dir) or die "Cannot open directory $Dir";
  my @Files = grep { !/^\./ } readdir(DIR);
  closedir(DIR);

  my @NFiles = map { (topt_filespec($_) ? "$Dir/$_" : ()) } @Files;

  foreach my $SubDir ( map { ((-e "$Dir/$_" ) ? "$Dir/$_" : ()) } @Files) {
    if (-r "$SubDir/.counts/$ActiveConf->{CFILE_SPEC}.counts") {
      push(@NFiles, "$SubDir/.counts/$ActiveConf->{CFILE_SPEC}.counts");
    }
  }

  if ($Debug) {
    print STDERR "counts_subfiles($Dir) :: ".
      join(',', @NFiles)."\n";
  }
  return @NFiles;
}

# counts_makecachefile
# This generates the cachefile which will let us avoid duplicate
# work.
sub counts_makecachefile {
  my ($Dir, $rFiles, $ActiveConf) = @_;

  my $CacheData = "# Do not modify or delete this file! (argus-bytecounts $Version)\n".
    "# FileNum\tFileName\tInode\tSize\tMTime\n";

  # We're going to cache the inode, mtime, and file size
  # We might want to MD5 the file but to save effort, we won't..
  my $FileNum = 0;
  foreach my $File (@$rFiles) {
    my @Info = stat($File);
    my ($Inode, $Size, $MTime) = ($Info[1], $Info[7], $Info[9]);
    $CacheData .= join("\t", $FileNum++, $File, $Inode, $Size, $MTime)."\n";
  }

  return $CacheData;
}