#!/usr/bin/perl

#######################################################################
#
# runstats.pl
#
# Produces daily, weekly, monthly, and long term IBM-SP job
# runtime stats.
#
# HISTORY:
#   09 Sep 1998    Brent A. Gordon   -- Implemented script.
#   11 Sep 1998    Brent A. Gordon   -- converted to Perl
#   17 Sep 1998    Brent A. Gordon   -- Modified to get rid of short
#                                       runs and added the average start
#                                       time and sum of squares.
#   03 Jun 1999    Brent A. Gordon   -- Converted to run on IBM-SP.
#   19 May 2000    Brent A. Gordon   -- Added 30 day average output.
#
#######################################################################

$DATA = $ENV{'DATA'};
$PDY = $ENV{'PDY'};
$PDYm1 = $ENV{'PDYm1'};
$com = $ENV{'com'};
chop($DOW = `date -u +%w`);
chop($WEEK = `date -u +%W`);
chop($YR = `date -u +%Y`);
chop($mon = `date -u +%m`);

$aborted = "$DATA/aborted.new";
$active = "$DATA/active.new";
$complete = "$DATA/complete.new";
$limbo = "$com/limbo.hold";
$logfile = "$com/$PDY.log";
$daily = "$com/daily/$PDY.daily";
$weekly = "$com/weekly/${YR}_$WEEK.weekly";
$weekday = "$com/weekday/$DOW.weekday";
$monthly = "$com/monthly/${YR}_$mon.monthly";
$total = "$com/total/runstats.all";
$ave30dfull = "$com/ave30/ave30day.full";
$ave30day = "$com/ave30/ave30day.short";

#
# Define all hash tables
#
%abo_time = ();
%abo_date = ();
%com_time = ();
%com_date = ();
%act_time = ();
%act_date = ();
%runtim = ();
%is_complete = ();
%lweek = ();
%sweek = ();
%aweek = ();
%nweek = ();
%astweek = ();
%n2week = ();
%ldow = ();
%sdow = ();
%adow = ();
%ndow = ();
%astdow = ();
%n2dow = ();
%lmonth = ();
%smonth = ();
%amonth = ();
%nmonth = ();
%astmonth = ();
%n2month = ();
%ltotal = ();
%stotal = ();
%atotal = ();
%ntotal = ();
%asttotal = ();
%n2total = ();
%a30start = ();
%a30stop = ();
%start30 = ();
%stop30 = ();
%job30 = ();
%new30start = ();
%new30stop = ();

#
# Read in the data from the aborted, complete, limbo, and active files.
#

open( ABORTED, "$aborted" );
while( <ABORTED> ) {
    chop;
    #
    # Parse line using whitespace as delimiter
    #
    ( $dum1, $dum2, $time, $date, $jid ) = split( /\s+/ );
    ( $dum3, $jobid ) = split( /:/, $jid );
    ( $dum3, $abo_time{$jobid} ) = split( /\[/, $time );
    ( $abo_date{$jobid}, $dum3 ) = split( /\]/, $date );
}
close ABORTED;

open( COMPLETE, "$complete" );
while( <COMPLETE> ) {
    chop;
    #    
    # Parse line using whitespace as delimiter
    #
    ( $dum1, $dum2, $time, $date, $jid ) = split( /\s+/ );
    ( $dum3, $jobid, $dum4 ) = split( /:/, $jid );
    ( $DD, $MM, $YYYY ) = split( /\./, $date );
    $datePDY=sprintf("%4d%02d%02d",$YYYY,$MM,$DD);
    if ( $datePDY == $PDY || $datePDY == $PDYm1 ) {
	if ( $jobid =~ m#jruc2a_post# && $dum4 ne "grb_processing_complete") {
	     $dum5=1;
	 } else {
	     ( $dum3, $com_time{$jobid} ) = split( /\[/, $time );
	     ( $com_date{$jobid}, $dum3 ) = split( /\]/, $date );
	 }
    }
}
close COMPLETE;

open( COMPLETE_OLD, "$logfile" );
while( <COMPLETE_OLD> ) {
    chop;
    #    
    # Parse line using whitespace as delimiter
    #
    ( $jid, $dum, $dum, $dum, $dum ) = split( /\s+/ );
    $is_complete{$jid} = $jid;
}
close COMPLETE_OLD;

open( LIMBO, "$limbo" );
while( <LIMBO> ) {
    chop;
    #    
    # Parse line using whitespace as delimiter
    #
    ( $jobid, $date, $time ) = split( /\s+/ );
    $act_time{$jobid} = $time;
    $act_date{$jobid} = $date;
}
close LIMBO;

open( ACTIVE, "$active" );
while( <ACTIVE> ) {
    chop;
    #    
    # Parse line using whitespace as delimiter
    #
    ( $dum1, $dum2, $time, $date, $jid ) = split( /\s+/ );
    ( $dum3, $jobid ) = split( /:/, $jid );
    ( $dum3, $act_time{$jobid} ) = split( /\[/, $time );
    ( $act_date{$jobid}, $dum3 ) = split( /\]/, $date );
}
close ACTIVE;

#
# Read in the weekly, weekday, monthly, total, and 30-day average files.
#

open( STATS, "$weekly" );
while ( <STATS> ) {
   chop;

   #
   # Parse line using whitespace as delimiter
   #
   ( $jid, $sw, $lw, $aw, $nw, $astw, $n2w ) = split( /\s+/ );
   $sweek{$jid} = $sw;
   $lweek{$jid} = $lw;
   $aweek{$jid} = $aw;
   $nweek{$jid} = $nw;
   $astweek{$jid} = $astw;
   $n2week{$jid} = $n2w;
}
close STATS;

open( STATS, "$weekday" );
while ( <STATS> ) {
   chop;

   #
   # Parse line using whitespace as delimiter
   #
   ( $jid, $sw, $lw, $aw, $nw, $astw, $n2w ) = split( /\s+/ );
   $sdow{$jid} = $sw;
   $ldow{$jid} = $lw;
   $adow{$jid} = $aw;
   $ndow{$jid} = $nw;
   $astdow{$jid} = $astw;
   $n2dow{$jid} = $n2w;
}
close STATS;

open( STATS, "$monthly" );
while ( <STATS> ) {
   chop;

   #
   # Parse line using whitespace as delimiter
   #
   ( $jid, $sm, $lm, $am, $nm, $astm, $n2m ) = split( /\s+/ );
   $smonth{$jid} = $sm;
   $lmonth{$jid} = $lm;
   $amonth{$jid} = $am;
   $nmonth{$jid} = $nm;
   $astmonth{$jid} = $astm;
   $n2month{$jid} = $n2m;
}
close STATS;

open( STATS, "$total" );
while ( <STATS> ) {
   chop;

   #
   # Parse line using whitespace as delimiter
   #
   ( $jid, $st, $lt, $at, $nt, $astt, $n2t ) = split( /\s+/ );
   $stotal{$jid} = $st;
   $ltotal{$jid} = $lt;
   $atotal{$jid} = $at;
   $ntotal{$jid} = $nt;
   $asttotal{$jid} = $astt;
   $n2total{$jid} = $n2t;
}
close STATS;

open( STATS, "$ave30dfull" );
while ( <STATS> ) {
    chop;

    #
    # Parse line using whitespace as delimiter
    #
    ( $jid, $avestart, $avestop, $a1, $b1, $a2, $b2, $a3, $b3, $a4, $b4, 
     $a5, $b5, $a6, $b6, $a7, $b7, $a8, $b8, $a9, $b9, $a10, $b10,
     $a11, $b11, $a12, $b12, $a13, $b13, $a14, $b14, $a15, $b15, $a16, $b16,
     $a17, $b17, $a18, $b18, $a19, $b19, $a20, $b20, $a21, $b21, $a22, $b22,
     $a23, $b23, $a24, $b24, $a25, $b25, $a26, $b26, $a27, $b27, $a28, $b28,
     $a29, $b29, $a30, $b30 ) = split( /\s+/ );

    $job30{$jid} = $jid;
    $a30start{$jid} = $avestart;
    $a30stop{$jid} = $avestop;
    $start30{$jid,1} = $a1;
    $stop30{$jid,1} = $b1;
    $start30{$jid,2} = $a2;
    $stop30{$jid,2} = $b2;
    $start30{$jid,3} = $a3;
    $stop30{$jid,3} = $b3;
    $start30{$jid,4} = $a4;
    $stop30{$jid,4} = $b4;
    $start30{$jid,5} = $a5;
    $stop30{$jid,5} = $b5;
    $start30{$jid,6} = $a6;
    $stop30{$jid,6} = $b6;
    $start30{$jid,7} = $a7;
    $stop30{$jid,7} = $b7;
    $start30{$jid,8} = $a8;
    $stop30{$jid,8} = $b8;
    $start30{$jid,9} = $a9;
    $stop30{$jid,9} = $b9;
    $start30{$jid,10} = $a10;
    $stop30{$jid,10} = $b10;
    $start30{$jid,11} = $a11;
    $stop30{$jid,11} = $b11;
    $start30{$jid,12} = $a12;
    $stop30{$jid,12} = $b12;
    $start30{$jid,13} = $a13;
    $stop30{$jid,13} = $b13;
    $start30{$jid,14} = $a14;
    $stop30{$jid,14} = $b14;
    $start30{$jid,15} = $a15;
    $stop30{$jid,15} = $b15;
    $start30{$jid,16} = $a16;
    $stop30{$jid,16} = $b16;
    $start30{$jid,17} = $a17;
    $stop30{$jid,17} = $b17;
    $start30{$jid,18} = $a18;
    $stop30{$jid,18} = $b18;
    $start30{$jid,19} = $a19;
    $stop30{$jid,19} = $b19;
    $start30{$jid,20} = $a20;
    $stop30{$jid,20} = $b20;
    $start30{$jid,21} = $a21;
    $stop30{$jid,21} = $b21;
    $start30{$jid,22} = $a22;
    $stop30{$jid,22} = $b22;
    $start30{$jid,23} = $a23;
    $stop30{$jid,23} = $b23;
    $start30{$jid,24} = $a24;
    $stop30{$jid,24} = $b24;
    $start30{$jid,25} = $a25;
    $stop30{$jid,25} = $b25;
    $start30{$jid,26} = $a26;
    $stop30{$jid,26} = $b26;
    $start30{$jid,27} = $a27;
    $stop30{$jid,27} = $b27;
    $start30{$jid,28} = $a28;
    $stop30{$jid,28} = $b28;
    $start30{$jid,29} = $a29;
    $stop30{$jid,29} = $b29;
    $start30{$jid,30} = $a30;
    $stop30{$jid,30} = $b30;

	
}
close STATS;

# Find out which jobs in the active and limbo files have completed or aborted.
# Write start and stop time data of those files that have completed, write
# all other job start times into the limbo file.

# Open the daily output files and reopen the limbo file here
open( LOGFILE, ">>$logfile" );
open( DAILY, ">>$daily" );
open( LIMBO, ">$limbo" );

foreach $jid (keys %act_time) {
    ( $shr, $smn, $ssc ) = split( /:/, $act_time{$jid} );
    ( $sdy, $smo, $syr ) = split( /./, $act_date{$jid} );
    $starttime = $shr * 3600 + $smn * 60 + $ssc;

    # Check to see if the job has already completed today (i.e. check for
    # reruns).  If this is a rerun do not use these numbers.
    $already_run = 0;
    if ( $is_complete{$jid} ne "" ) { $already_run = 1; }

    # See if job has aborted if so did it abort before or after the start of 
    # the current job.
    $abortflag = 0;
    if ( $abo_time{$jid} ne "" ) {
	( $ahr, $amn, $asc ) = split( /:/, $abo_time{$jid} );
	( $ady, $amo, $ayr ) = split( /./, $abo_date{$jid} );
	$aborttime = $ahr * 3600 + $amn * 60 + $asc;
	if ( $starttime < $aborttime || ( $sdy < $ady || $smo < $amo || $syr < $ayr ) ) {
            # Do not use this run as an abort in this job has occurred after
            # the start of the current job.
	    $abortflag=1;
	}
    }
    if ( $abortflag == 0 && $already_run == 0 && $com_time{$jid} ne "" ) {
	#  An ending time has been found.  Store the data, write the two
        #  log files and calculate the weekly, monthly and total numbers.
	select( LOGFILE );
	print "$jid   $act_date{$jid} $act_time{$jid}  $com_date{$jid} $com_time{$jid}\n";
	#
	# Account for a job starting BEFORE 00 UTC when it should have
	# started after 00 UTC.
	#
	if ( $asttotal{$jid} ne "" && $asttotal{$jid} < 21600 && $starttime > 64800 ) {
	    $starttime = $starttime - 86400;
	}
	#
	# Account for a job starting AFTER 00 UTC when it should have
	# started before 00 UTC.
	#
	if ( $asttotal{$jid} ne "" && $asttotal{$jid} > 64800 && $starttime < 21600 ) {
	    $starttime =  $starttime + 86400;
	}
	( $ehr, $emn, $esc ) = split( /:/, $com_time{$jid} );
	if ( $avestop{$jid} ne "" && $avestop{$jid} > 64800 && $avestop{$jid} < 21600 ) {
	    $ehr = $ehr + 24;
	}
	$stoptime=$ehr * 60 + $emn + $esc / 60;

	#
	# Fill the hash tables with appropriate values
	#
	$runtim{$jid} = ( $stoptime ) - ( $shr * 60 + $smn + $ssc / 60 );

	$too_short=0;
	$too_long=0;
	if ( $ntotal{$jid} ne "" ) {
	    if ( $runtim{$jid} < ( $atotal{$jid} / 5 ) ) {
		$too_short=1;
	    }
	    if ( $runtim{$jid} > ( $atotal{$jid} * 5 ) ) {
		$too_long=1;
	    }
	}

	if ( $too_short == 0 && $too_long == 0 ) {
	    select( DAILY );
	    print "$jid   $runtim{$jid}\n";

	    if ( $nweek{$jid} ne "" ) {
		if ( $runtim{$jid} < $sweek{$jid} ) { $sweek{$jid} = $runtim{$jid}; }
		if ( $runtim{$jid} > $lweek{$jid} ) { $lweek{$jid} = $runtim{$jid}; }
		$oldave = $aweek{$jid} * $nweek{$jid};
		$oldn2 = $n2week{$jid} * $nweek{$jid};
		$oldast = $astweek{$jid} * $nweek{$jid};
		$nweek{$jid}++;
		$aweek{$jid} = ( $oldave + $runtim{$jid} ) / $nweek{$jid};
		$n2week{$jid} = ( $oldn2 + $runtim{$jid}**2 ) / $nweek{$jid};
		$astweek{$jid} = ( $oldast + $starttime ) / $nweek{$jid};
	    } else {
		$sweek{$jid} = $runtim{$jid};
		$lweek{$jid} = $runtim{$jid};
		$aweek{$jid} = $runtim{$jid};
		$nweek{$jid} = 1;
		$n2week{$jid} = $runtim{$jid}**2;
		$astweek{$jid} = $starttime;
	    }
       
	    if ( $ndow{$jid} ne "" ) {
		if ( $runtim{$jid} < $sdow{$jid} ) { $sdow{$jid} = $runtim{$jid}; }
		if ( $runtim{$jid} > $ldow{$jid} ) { $ldow{$jid} = $runtim{$jid}; }
		$oldave = $adow{$jid} * $ndow{$jid};
		$oldn2 = $n2dow{$jid} * $ndow{$jid};
		$oldast = $astdow{$jid} * $ndow{$jid};
		$ndow{$jid}++;
		$adow{$jid} = ( $oldave + $runtim{$jid} ) / $ndow{$jid};
		$n2dow{$jid} = ( $oldn2 + $runtim{$jid}**2 ) / $ndow{$jid};
		$astdow{$jid} = ( $oldast + $starttime ) / $ndow{$jid};
	    } else {
		$sdow{$jid} = $runtim{$jid};
		$ldow{$jid} = $runtim{$jid};
		$adow{$jid} = $runtim{$jid};
		$ndow{$jid} = 1;
		$n2dow{$jid} = $runtim{$jid}**2;
		$astdow{$jid} = $starttime;
	    }

	    if ( $nmonth{$jid} ne "" ) {
		if ( $runtim{$jid} < $smonth{$jid} ) { $smonth{$jid} = $runtim{$jid}; }
		if ( $runtim{$jid} > $lmonth{$jid} ) { $lmonth{$jid} = $runtim{$jid}; }
		$oldave = $amonth{$jid} * $nmonth{$jid};
		$oldn2 = $n2month{$jid} * $nmonth{$jid};
		$oldast = $astmonth{$jid} * $nmonth{$jid};
		$nmonth{$jid}++;
		$amonth{$jid} = ( $oldave + $runtim{$jid} ) / $nmonth{$jid};
		$n2month{$jid} = ( $oldn2 + $runtim{$jid}**2 ) / $nmonth{$jid};
		$astmonth{$jid} = ( $oldast + $starttime ) / $nmonth{$jid};
	    } else {
		$smonth{$jid} = $runtim{$jid};
		$lmonth{$jid} = $runtim{$jid};
		$amonth{$jid} = $runtim{$jid};
		$nmonth{$jid} = 1;
		$n2month{$jid} = $runtim{$jid}**2;
		$astmonth{$jid} = $starttime;
	    }

	    if ( $ntotal{$jid} ne "" ) {
		if ( $runtim{$jid} < $stotal{$jid} ) { $stotal{$jid} = $runtim{$jid}; }
		if ( $runtim{$jid} > $ltotal{$jid} ) { $ltotal{$jid} = $runtim{$jid}; }
		$oldave = $atotal{$jid} * $ntotal{$jid};
		$oldn2 = $n2total{$jid} * $ntotal{$jid};
		$oldast = $asttotal{$jid} * $ntotal{$jid};
		$ntotal{$jid}++;
		$atotal{$jid} = ( $oldave + $runtim{$jid} ) / $ntotal{$jid};
		$n2total{$jid} = ( $oldn2 + $runtim{$jid}**2 ) / $ntotal{$jid};
		$asttotal{$jid} = ( $oldast + $starttime ) / $ntotal{$jid};
	    } else {
		$stotal{$jid} = $runtim{$jid};
		$ltotal{$jid} = $runtim{$jid};
		$atotal{$jid} = $runtim{$jid};
		$ntotal{$jid} = 1;
		$n2total{$jid} = $runtim{$jid}**2;
		$asttotal{$jid} = $starttime;
	    }

	    if ( $a30start{$jid} ne "" ) {
		#
		# get the oldest (day 30) data out of the average
		# add in the new data and re calc the average
		#
		$new30start{$jid} = $starttime / 60;
		$new30stop{$jid} = $stoptime;
		if ( $a30start{$jid} < 360 && $new30start{$jid} > 1080 ) {
		    $new30start{$jid} = $new30start{$jid} - 1440;
		}
		if ( $a30stop{$jid} < 360 && $new30stop{$jid} > 1080 ) {
		    $new30stop{$jid} = $new30stop{$jid} - 1440;
		}
		if ( $a30start{$jid} > 1080 && $new30start{$jid} < 360 ) {
		    $new30start{$jid} = $new30start{$jid} + 1440;
		}
		if ( $a30stop{$jid} > 1080 && $new30stop{$jid} < 360 ) {
		    $new30stop{$jid} = $new30stop{$jid} + 1440;
		}
		$tempavestart = $new30start{$jid};
		$tempavestop = $new30stop{$jid};
		for ( $i = 1; $i <= 29; $i++ ) {
                    if ( $start30{$jid,$i} eq "" || $start30{$jid,$i} eq " " ) { 
			$start30{$jid,$i} = $new30start{$jid};
		    }
		    if ( $stop30{$jid,$i} eq "" || $stop30{$jid,$i} eq " " ) { 
			$stop30{$jid,$i} = $new30stop{$jid}; 
		    }
		    $tempavestart = $tempavestart + $start30{$jid,$i};
		    $tempavestop = $tempavestop + $stop30{$jid,$i};
		}
		$a30start{$jid} = $tempavestart / 30;
		$a30stop{$jid} = $tempavestop / 30;
	    } else {
		#
		# This is the first time this job has
		# appeared.  Set the averages and 30 day
		# history to the initial value 
		#
		$job30{$jid} = $jid;
		$a30start{$jid} = $starttime / 60;
		$a30stop{$jid} = $stoptime;
		$new30start{$jid} = $starttime / 60;
                $new30stop{$jid} = $stoptime;
		for ( $i = 1; $i <= 29; $i++ ) {
		    $start30{$jid,$i} = $starttime / 60;
		    $stop30{$jid,$i} = $stoptime;
		}
	    }
	}
    } elsif ( $already_run == 0 ) {
	#  Assume that job has not finished, therefore write data into
	#  limbo file
	select( LIMBO );
	print "$jid $act_date{$jid} $act_time{$jid}\n";
    }
}
close LOGFILE;
close DAILY;
close LIMBO;

#
# Finally, write the statistical output files.
#
open( WEEKLY, ">$weekly" );
select ( WEEKLY );
@keys = keys %sweek;
@sv = values %sweek;
@lv = values %lweek;
@av = values %aweek;
@nv = values %nweek;
@astv = values %astweek;
@n2v = values %n2week;
while (@keys) {
   print pop(@keys),'   ',pop(@sv),' ',pop(@lv),' ',pop(@av),' ',pop(@nv),' ',pop(@astv),' ',pop(@n2v),"\n";
}
close WEEKLY;

open( DOW, ">$weekday" );
select ( DOW );
@keys = keys %sdow;
@sv = values %sdow;
@lv = values %ldow;
@av = values %adow;
@nv = values %ndow;
@astv = values %astdow;
@n2v = values %n2dow;
while (@keys) {
   print pop(@keys),'   ',pop(@sv),' ',pop(@lv),' ',pop(@av),' ',pop(@nv),' ',pop(@astv),' ',pop(@n2v),"\n";
}
close DOW;

open( MONTHLY, ">$monthly" );
select ( MONTHLY );
@keys = keys %smonth;
@sv = values %smonth;
@lv = values %lmonth;
@av = values %amonth;
@nv = values %nmonth;
@astv = values %astmonth;
@n2v = values %n2month;
while (@keys) {
   print pop(@keys),'   ',pop(@sv),' ',pop(@lv),' ',pop(@av),' ',pop(@nv),' ',pop(@astv),' ',pop(@n2v),"\n";
}
close MONTHLY;

open( TOTAL, ">$total" );
select ( TOTAL );
@keys = keys %stotal;
@sv = values %stotal;
@lv = values %ltotal;
@av = values %atotal;
@nv = values %ntotal;
@astv = values %asttotal;
@n2v = values %n2total;
while (@keys) {
   print pop(@keys),'   ',pop(@sv),' ',pop(@lv),' ',pop(@av),' ',pop(@nv),' ',pop(@astv),' ',pop(@n2v),"\n";
}
close TOTAL;

#
# Write out 30-day average files
#
open( AVELONG, ">$ave30dfull" );
open( AVESHORT, ">$ave30day" );
@keys = keys %job30;
while (@keys) {
    $temp=pop(@keys);
    select ( AVELONG );
    if ( $new30start{$temp} ne "" ) {
	print "$temp $a30start{$temp} $a30stop{$temp} $new30start{$temp} $new30stop{$temp}";
	for ( $i =1; $i <= 29; $i++ ) {
	    print " $start30{$temp,$i} $stop30{$temp,$i}";
	}
    } else {
	print "$temp $a30start{$temp} $a30stop{$temp}";
	for ( $i =1; $i <= 30; $i++ ) {
            print " $start30{$temp,$i} $stop30{$temp,$i}";
        }
    }
    print "\n";
    select ( AVESHORT );
    print $temp,' ',$a30start{$temp},' ',$a30stop{$temp},' ',$a30stop{$temp}-$a30start{$temp},"\n";
}
close AVELONG;
close AVESHORT;